From 88bcd4d0a44bcdd8061a9252c8744091ec9f6d5a Mon Sep 17 00:00:00 2001 From: A R R R Associates Date: Wed, 9 Sep 2026 11:59:54 +0530 Subject: [PATCH] Fix mirror master persistence and add group diagnostics --- .../templates/accounting/tally.html | 21 +++++- app/modules/documents/agent_package.py | 2 +- .../erp_local_agent/__init__.py | 2 +- .../erp_local_agent/accounting_mirror.py | 58 ++++++++++++++++- .../erp_local_agent/commands.py | 4 +- .../mirror_runtime/build_act_v3_0_1.py | 64 +++++++++++++++++-- 6 files changed, 139 insertions(+), 12 deletions(-) diff --git a/app/modules/accounting/templates/accounting/tally.html b/app/modules/accounting/templates/accounting/tally.html index 68893fb..56862ec 100644 --- a/app/modules/accounting/templates/accounting/tally.html +++ b/app/modules/accounting/templates/accounting/tally.html @@ -339,7 +339,8 @@ @@ -574,7 +575,23 @@ const j=d.job||{}, p=Math.max(0,Math.min(100,Number(j.percent)||0)); box.classList.remove('hidden'); setText('fullExportStage',j.stage||j.status||'Working…'); setText('fullExportMessage',j.message||''); setText('fullExportPct',Math.round(p)+'%'); document.getElementById('fullExportBar').style.width=p+'%'; - const rows=j.rows||((j.mirror||{}).counts)||{}; setText('fullExportLedgers',rows.ledger_master||0); setText('fullExportVouchers',rows.voucher||rows.voucher_header||0); setText('fullExportLines',rows.voucher_ledger_entry||rows.voucher_ledger||0); setText('fullExportInventory',rows.inventory_movement||rows.inventory||0); + const rows=j.rows||((j.mirror||{}).counts)||{}; + setText('fullExportLedgers',rows.ledger_master||0); + setText('fullExportGroups',rows.group_master||0); + setText('fullExportVouchers',rows.voucher||rows.voucher_header||0); + setText('fullExportLines',rows.voucher_ledger_entry||rows.voucher_ledger||0); + setText('fullExportInventory',rows.inventory_movement||rows.inventory||0); + const diag=j.import_diagnostics||{}; + const ledgerDiag=diag.ledgers||{}, groupDiag=diag.groups||{}; + const diagEl=document.getElementById('fullExportMasterDiagnostics'); + if(diagEl && (ledgerDiag.extracted_rows!=null || groupDiag.extracted_rows!=null)){ + const parts=[]; + if(ledgerDiag.extracted_rows!=null) parts.push('Ledgers: '+Number(ledgerDiag.extracted_rows||0)+' extracted → '+Number(ledgerDiag.stored_rows||0)+' stored'); + if(groupDiag.extracted_rows!=null) parts.push('Groups: '+Number(groupDiag.extracted_rows||0)+' extracted → '+Number(groupDiag.stored_rows||0)+' stored'); + if(diag.flat_master_rows!=null) parts.push('Flat master rows: '+Number(diag.flat_master_rows||0)); + diagEl.textContent=parts.join(' • '); + diagEl.classList.remove('hidden'); + } const paths=[]; if(j.mirror_db_path)paths.push('Mirror: '+j.mirror_db_path); if(j.accounting_db_path)paths.push('Client DB: '+j.accounting_db_path); setText('fullExportPaths',paths.join(' • ')); const er=document.getElementById('fullExportError'); if(j.error){er.textContent=j.error;er.classList.remove('hidden')}else{er.classList.add('hidden')} const status=String(j.status||'').toLowerCase(); diff --git a/app/modules/documents/agent_package.py b/app/modules/documents/agent_package.py index d79f096..0e695ce 100644 --- a/app/modules/documents/agent_package.py +++ b/app/modules/documents/agent_package.py @@ -4,7 +4,7 @@ import io from pathlib import Path import zipfile -ERP_LOCAL_AGENT_VERSION = "1.26.8" +ERP_LOCAL_AGENT_VERSION = "1.26.9" ERP_LOCAL_AGENT_NAME = "ERP Local Agent" RUNTIME_ROOT = Path(__file__).resolve().parent / "local_agent_runtime" _DETERMINISTIC_ZIP_TIMESTAMP = (2026, 1, 1, 0, 0, 0) diff --git a/app/modules/documents/local_agent_runtime/erp_local_agent/__init__.py b/app/modules/documents/local_agent_runtime/erp_local_agent/__init__.py index e0f55b2..624b92a 100644 --- a/app/modules/documents/local_agent_runtime/erp_local_agent/__init__.py +++ b/app/modules/documents/local_agent_runtime/erp_local_agent/__init__.py @@ -1,2 +1,2 @@ -__version__ = "1.26.8" +__version__ = "1.26.9" AGENT_NAME = "ERP Local Agent" diff --git a/app/modules/documents/local_agent_runtime/erp_local_agent/accounting_mirror.py b/app/modules/documents/local_agent_runtime/erp_local_agent/accounting_mirror.py index 47894db..9411ff4 100644 --- a/app/modules/documents/local_agent_runtime/erp_local_agent/accounting_mirror.py +++ b/app/modules/documents/local_agent_runtime/erp_local_agent/accounting_mirror.py @@ -154,7 +154,7 @@ class AccountingMirrorEngine: tables = [ "ledger_master","stock_item_master","stock_item_hsn_detail","stock_item_gst_detail", "voucher","voucher_ledger_entry","inventory_movement", - "group_master","voucher_type_master","stock_group_master","stock_category_master", + "group_master","accounting_master_flat","voucher_type_master","stock_group_master","stock_category_master", "unit_master","cost_centre_master","cost_category_master", ] counts = {} @@ -191,7 +191,7 @@ class AccountingMirrorEngine: db.close() @staticmethod - def _add_auxiliary_masters(db_path: Path, output_dir: Path) -> None: + def _add_auxiliary_masters(db_path: Path, output_dir: Path) -> dict[str, Any]: specs = { "group_master": ("group_master.csv", [ ("master_guid","TEXT"),("name","TEXT"),("parent","TEXT"),("reserved_name","TEXT"), @@ -218,6 +218,7 @@ class AccountingMirrorEngine: ("master_guid","TEXT"),("name","TEXT"),("parent","TEXT"),("is_active","TEXT") ]), } + diagnostics: dict[str, Any] = {} db = sqlite3.connect(db_path, timeout=30) try: for table, (filename, cols) in specs.items(): @@ -225,6 +226,7 @@ class AccountingMirrorEngine: db.execute(f"CREATE TABLE {table} ({', '.join(name+' '+typ for name,typ in cols)})") rows = AccountingMirrorEngine._read_csv(output_dir / filename) if not rows: + diagnostics[table] = {"source_rows": 0, "stored_rows": 0} continue names = [name for name,_ in cols] placeholders = ",".join("?" for _ in names) @@ -245,6 +247,8 @@ class AccountingMirrorEngine: value = _text(value) values.append(value) db.execute(f"INSERT INTO {table} ({','.join(names)}) VALUES ({placeholders})", values) + stored = int(db.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0]) + diagnostics[table] = {"source_rows": len(rows), "stored_rows": stored} # Common flat accounting master table. Keep the original # ledger_master and group_master physical tables intact. db.execute("DROP TABLE IF EXISTS accounting_master_flat") @@ -292,7 +296,10 @@ class AccountingMirrorEngine: "CREATE INDEX IF NOT EXISTS idx_accounting_master_flat_parent " "ON accounting_master_flat(parent)" ) + flat_count = int(db.execute("SELECT COUNT(*) FROM accounting_master_flat").fetchone()[0]) + diagnostics["accounting_master_flat"] = {"stored_rows": flat_count} db.commit() + return diagnostics finally: db.close() @@ -433,9 +440,53 @@ class AccountingMirrorEngine: if not candidate.is_file(): raise RuntimeError("Accounting Mirror database was not produced.") report(93, "Finalising master tables", "Preparing auxiliary master tables and validating the SQLite mirror.", rows=dict(rows)) - self._add_auxiliary_masters(candidate, output_dir) + auxiliary_diagnostics = self._add_auxiliary_masters(candidate, output_dir) summary = self._summary(candidate) + counts = summary.get("counts") or {} + # Cross-check the extractor rows against what is actually persisted. + extracted_ledgers = int(rows.get("ledger_master") or 0) + stored_ledgers = int(counts.get("ledger_master") or 0) + extracted_groups = int(rows.get("group_master") or 0) + stored_groups = int(counts.get("group_master") or 0) + + import_diagnostics = { + "ledgers": { + "extracted_rows": extracted_ledgers, + "stored_rows": stored_ledgers, + }, + "groups": { + "extracted_rows": extracted_groups, + "stored_rows": stored_groups, + }, + "flat_master_rows": int(counts.get("accounting_master_flat") or 0), + "auxiliary": auxiliary_diagnostics, + } + + if extracted_ledgers > 0 and stored_ledgers == 0: + raise RuntimeError( + "Ledger Master persistence failed after extraction: " + f"Tally ODBC extracted {extracted_ledgers} ledger rows, but the local " + "Accounting Mirror SQLite ledger_master table contains 0 rows. " + "The incomplete mirror was NOT published." + ) + if extracted_groups > 0 and stored_groups == 0: + raise RuntimeError( + "Group Master persistence failed after extraction: " + f"Tally ODBC extracted {extracted_groups} group rows, but the local " + "Accounting Mirror SQLite group_master table contains 0 rows. " + "The incomplete mirror was NOT published." + ) + + report( + 94, + "Validating master persistence", + f"Ledgers {extracted_ledgers} extracted / {stored_ledgers} stored; " + f"Groups {extracted_groups} extracted / {stored_groups} stored.", + rows=dict(rows), + import_diagnostics=import_diagnostics, + mirror=summary, + ) company = summary.get("company") or {} actual_name = _text(company.get("company_name")) actual_guid = _text(company.get("company_guid")) @@ -494,6 +545,7 @@ class AccountingMirrorEngine: "odbc_company_guid": actual_guid, "company_cache_path": str(cache), "validation_warnings": validation_warnings, + "import_diagnostics": import_diagnostics, "extractor_log_tail": log[-1500:], "loader_log_tail": llog[-1500:], **self.status(client_id), } diff --git a/app/modules/documents/local_agent_runtime/erp_local_agent/commands.py b/app/modules/documents/local_agent_runtime/erp_local_agent/commands.py index bff1c71..92dfdae 100644 --- a/app/modules/documents/local_agent_runtime/erp_local_agent/commands.py +++ b/app/modules/documents/local_agent_runtime/erp_local_agent/commands.py @@ -4089,7 +4089,9 @@ class AgentCommandProcessor: message=("Full Accounting Export completed. ERP analysis tools can now read the refreshed SQLite data without re-extracting from Tally." + ((" " + validation_warnings[0]) if validation_warnings else "")), finished_at_utc=datetime.now(timezone.utc).isoformat(), error="", warnings=validation_warnings, - rows=counts, mirror=mirror_summary, mirror_db_path=str(mirror_result.get("mirror_db_path") or ""), + rows=counts, mirror=mirror_summary, + import_diagnostics=mirror_result.get("import_diagnostics") or {}, + mirror_db_path=str(mirror_result.get("mirror_db_path") or ""), accounting_db_path=str(self.store.db_path(client_id)), transaction_sync=transaction_sync, ) diff --git a/app/modules/documents/local_agent_runtime/erp_local_agent/mirror_runtime/build_act_v3_0_1.py b/app/modules/documents/local_agent_runtime/erp_local_agent/mirror_runtime/build_act_v3_0_1.py index 0d877f8..bfb18e6 100644 --- a/app/modules/documents/local_agent_runtime/erp_local_agent/mirror_runtime/build_act_v3_0_1.py +++ b/app/modules/documents/local_agent_runtime/erp_local_agent/mirror_runtime/build_act_v3_0_1.py @@ -46,6 +46,42 @@ def csv_rows(src, name): for row in r: yield norm(row) +def csv_rows_expected(src, name, expected_fields): + """Read a Tally ODBC CSV by header first, then by proven SELECT position. + + Some Tally ODBC builds return valid row values while exposing blank or + expression-style column names. The extractor therefore reports 986 rows, + but a name-based SQLite loader can see GUID/Name as blank and skip every + row. Positional fallback preserves the exact SELECT order without changing + the Tally extraction itself. + """ + p=src/name + if not p.exists(): + raise SystemExit(f"Missing {p}. Run run_v3_0.ps1 first.") + with p.open("r",encoding="utf-8-sig",newline="") as f: + raw=csv.reader(f) + try: + headers=next(raw) + except StopIteration: + return + clean=[ck(x) for x in headers] + print(f"{name} headers:",clean) + header_keys={x.casefold() for x in clean if x} + expected_keys={str(x).casefold() for x in expected_fields} + use_headers=bool(expected_keys.intersection(header_keys)) and "name" in header_keys + for values in raw: + if use_headers: + row={} + for idx,key in enumerate(clean): + if key: + row[key]=values[idx] if idx < len(values) else "" + yield norm(row) + else: + row={} + for idx,key in enumerate(expected_fields): + row[key]=values[idx] if idx < len(values) else "" + yield row + def company_instance_id(guid, name, starting_from, books_from): raw="|".join([t(guid),t(name),t(starting_from),t(books_from)]).encode("utf-8") return hashlib.sha256(raw).hexdigest()[:24] @@ -349,19 +385,39 @@ def main(): "stock_item_gst_detail","stock_item_gst_rate_detail"): con.execute(f"DELETE FROM {table} WHERE company_instance_id=?",(instance,)) - ledger_count=0; max_ledger_alter=0 - for r in csv_rows(src,"ledger_master.csv"): + ledger_count=0; ledger_source_rows=0; ledger_skipped_rows=0; max_ledger_alter=0 + ledger_fields=[ + "GUID","AlterID","Name","Parent","OpeningBalance","ClosingBalance", + "IsRevenue","PartyGSTIN","GSTRegistrationType" + ] + for r in csv_rows_expected(src,"ledger_master.csv",ledger_fields): + ledger_source_rows+=1 lg=t(r.get("GUID")) - if not lg: continue + lname=t(r.get("Name")) + if not lg or not lname: + ledger_skipped_rows+=1 + continue aid=integer(r.get("AlterID")) or 0 max_ledger_alter=max(max_ledger_alter,aid) con.execute("""INSERT OR REPLACE INTO ledger_master VALUES(?,?,?,?,?,?,?,?,?,?,?,?)""", - (instance,guid,lg,aid,t(r.get("Name")),t(r.get("Parent")), + (instance,guid,lg,aid,lname,t(r.get("Parent")), num(r.get("OpeningBalance")),num(r.get("ClosingBalance")),yes(r.get("IsRevenue")), t(r.get("PartyGSTIN")),t(r.get("GSTRegistrationType")),now)) ledger_count+=1 + ledger_stored_rows=int(con.execute( + "SELECT COUNT(*) FROM ledger_master WHERE company_instance_id=?",(instance,) + ).fetchone()[0]) + print("LEDGER IMPORT - SOURCE ROWS:",ledger_source_rows) + print("LEDGER IMPORT - STORED ROWS:",ledger_stored_rows) + print("LEDGER IMPORT - SKIPPED ROWS:",ledger_skipped_rows) + if ledger_source_rows > 0 and ledger_stored_rows == 0: + raise SystemExit( + "Ledger Master persistence failed: ledger_master.csv contains " + f"{ledger_source_rows} rows but SQLite ledger_master contains 0 rows." + ) + # Read stock tax data first so effective histories are available when stock and # inventory rows are inserted. tax_by_name={}