Fix mirror master persistence and add group diagnostics

app/modules/documents/agent_package.py

@@ -4,7 +4,7 @@ import io
 from pathlib import Path
 import zipfile
 
-ERP_LOCAL_AGENT_VERSION = "1.26.8"
+ERP_LOCAL_AGENT_VERSION = "1.26.9"
 ERP_LOCAL_AGENT_NAME = "ERP Local Agent"
 RUNTIME_ROOT = Path(__file__).resolve().parent / "local_agent_runtime"
 _DETERMINISTIC_ZIP_TIMESTAMP = (2026, 1, 1, 0, 0, 0)

app/modules/documents/local_agent_runtime/erp_local_agent/__init__.py

@@ -1,2 +1,2 @@
-__version__ = "1.26.8"
+__version__ = "1.26.9"
 AGENT_NAME = "ERP Local Agent"

app/modules/documents/local_agent_runtime/erp_local_agent/accounting_mirror.py

@@ -154,7 +154,7 @@ class AccountingMirrorEngine:
             tables = [
                 "ledger_master","stock_item_master","stock_item_hsn_detail","stock_item_gst_detail",
                 "voucher","voucher_ledger_entry","inventory_movement",
-                "group_master","voucher_type_master","stock_group_master","stock_category_master",
+                "group_master","accounting_master_flat","voucher_type_master","stock_group_master","stock_category_master",
                 "unit_master","cost_centre_master","cost_category_master",
             ]
             counts = {}
@@ -191,7 +191,7 @@ class AccountingMirrorEngine:
             db.close()
 
     @staticmethod
-    def _add_auxiliary_masters(db_path: Path, output_dir: Path) -> None:
+    def _add_auxiliary_masters(db_path: Path, output_dir: Path) -> dict[str, Any]:
         specs = {
             "group_master": ("group_master.csv", [
                 ("master_guid","TEXT"),("name","TEXT"),("parent","TEXT"),("reserved_name","TEXT"),
@@ -218,6 +218,7 @@ class AccountingMirrorEngine:
                 ("master_guid","TEXT"),("name","TEXT"),("parent","TEXT"),("is_active","TEXT")
             ]),
         }
+        diagnostics: dict[str, Any] = {}
         db = sqlite3.connect(db_path, timeout=30)
         try:
             for table, (filename, cols) in specs.items():
@@ -225,6 +226,7 @@ class AccountingMirrorEngine:
                 db.execute(f"CREATE TABLE {table} ({', '.join(name+' '+typ for name,typ in cols)})")
                 rows = AccountingMirrorEngine._read_csv(output_dir / filename)
                 if not rows:
+                    diagnostics[table] = {"source_rows": 0, "stored_rows": 0}
                     continue
                 names = [name for name,_ in cols]
                 placeholders = ",".join("?" for _ in names)
@@ -245,6 +247,8 @@ class AccountingMirrorEngine:
                             value = _text(value)
                         values.append(value)
                     db.execute(f"INSERT INTO {table} ({','.join(names)}) VALUES ({placeholders})", values)
+                stored = int(db.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0])
+                diagnostics[table] = {"source_rows": len(rows), "stored_rows": stored}
             # Common flat accounting master table. Keep the original
             # ledger_master and group_master physical tables intact.
             db.execute("DROP TABLE IF EXISTS accounting_master_flat")
@@ -292,7 +296,10 @@ class AccountingMirrorEngine:
                 "CREATE INDEX IF NOT EXISTS idx_accounting_master_flat_parent "
                 "ON accounting_master_flat(parent)"
             )
+            flat_count = int(db.execute("SELECT COUNT(*) FROM accounting_master_flat").fetchone()[0])
+            diagnostics["accounting_master_flat"] = {"stored_rows": flat_count}
             db.commit()
+            return diagnostics
         finally:
             db.close()
 
@@ -433,9 +440,53 @@ class AccountingMirrorEngine:
         if not candidate.is_file():
             raise RuntimeError("Accounting Mirror database was not produced.")
         report(93, "Finalising master tables", "Preparing auxiliary master tables and validating the SQLite mirror.", rows=dict(rows))
-        self._add_auxiliary_masters(candidate, output_dir)
+        auxiliary_diagnostics = self._add_auxiliary_masters(candidate, output_dir)
 
         summary = self._summary(candidate)
+        counts = summary.get("counts") or {}
+        # Cross-check the extractor rows against what is actually persisted.
+        extracted_ledgers = int(rows.get("ledger_master") or 0)
+        stored_ledgers = int(counts.get("ledger_master") or 0)
+        extracted_groups = int(rows.get("group_master") or 0)
+        stored_groups = int(counts.get("group_master") or 0)
+
+        import_diagnostics = {
+            "ledgers": {
+                "extracted_rows": extracted_ledgers,
+                "stored_rows": stored_ledgers,
+            },
+            "groups": {
+                "extracted_rows": extracted_groups,
+                "stored_rows": stored_groups,
+            },
+            "flat_master_rows": int(counts.get("accounting_master_flat") or 0),
+            "auxiliary": auxiliary_diagnostics,
+        }
+
+        if extracted_ledgers > 0 and stored_ledgers == 0:
+            raise RuntimeError(
+                "Ledger Master persistence failed after extraction: "
+                f"Tally ODBC extracted {extracted_ledgers} ledger rows, but the local "
+                "Accounting Mirror SQLite ledger_master table contains 0 rows. "
+                "The incomplete mirror was NOT published."
+            )
+        if extracted_groups > 0 and stored_groups == 0:
+            raise RuntimeError(
+                "Group Master persistence failed after extraction: "
+                f"Tally ODBC extracted {extracted_groups} group rows, but the local "
+                "Accounting Mirror SQLite group_master table contains 0 rows. "
+                "The incomplete mirror was NOT published."
+            )
+
+        report(
+            94,
+            "Validating master persistence",
+            f"Ledgers {extracted_ledgers} extracted / {stored_ledgers} stored; "
+            f"Groups {extracted_groups} extracted / {stored_groups} stored.",
+            rows=dict(rows),
+            import_diagnostics=import_diagnostics,
+            mirror=summary,
+        )
         company = summary.get("company") or {}
         actual_name = _text(company.get("company_name"))
         actual_guid = _text(company.get("company_guid"))
@@ -494,6 +545,7 @@ class AccountingMirrorEngine:
             "odbc_company_guid": actual_guid,
             "company_cache_path": str(cache),
             "validation_warnings": validation_warnings,
+            "import_diagnostics": import_diagnostics,
             "extractor_log_tail": log[-1500:], "loader_log_tail": llog[-1500:],
             **self.status(client_id),
         }

app/modules/documents/local_agent_runtime/erp_local_agent/commands.py

@@ -4089,7 +4089,9 @@ class AgentCommandProcessor:
                 message=("Full Accounting Export completed. ERP analysis tools can now read the refreshed SQLite data without re-extracting from Tally."
                          + ((" " + validation_warnings[0]) if validation_warnings else "")),
                 finished_at_utc=datetime.now(timezone.utc).isoformat(), error="", warnings=validation_warnings,
-                rows=counts, mirror=mirror_summary, mirror_db_path=str(mirror_result.get("mirror_db_path") or ""),
+                rows=counts, mirror=mirror_summary,
+                import_diagnostics=mirror_result.get("import_diagnostics") or {},
+                mirror_db_path=str(mirror_result.get("mirror_db_path") or ""),
                 accounting_db_path=str(self.store.db_path(client_id)),
                 transaction_sync=transaction_sync,
             )

app/modules/documents/local_agent_runtime/erp_local_agent/mirror_runtime/build_act_v3_0_1.py

@@ -46,6 +46,42 @@ def csv_rows(src, name):
         for row in r:
             yield norm(row)
 
+def csv_rows_expected(src, name, expected_fields):
+    """Read a Tally ODBC CSV by header first, then by proven SELECT position.
+
+    Some Tally ODBC builds return valid row values while exposing blank or
+    expression-style column names. The extractor therefore reports 986 rows,
+    but a name-based SQLite loader can see GUID/Name as blank and skip every
+    row. Positional fallback preserves the exact SELECT order without changing
+    the Tally extraction itself.
+    """
+    p=src/name
+    if not p.exists():
+        raise SystemExit(f"Missing {p}. Run run_v3_0.ps1 first.")
+    with p.open("r",encoding="utf-8-sig",newline="") as f:
+        raw=csv.reader(f)
+        try:
+            headers=next(raw)
+        except StopIteration:
+            return
+        clean=[ck(x) for x in headers]
+        print(f"{name} headers:",clean)
+        header_keys={x.casefold() for x in clean if x}
+        expected_keys={str(x).casefold() for x in expected_fields}
+        use_headers=bool(expected_keys.intersection(header_keys)) and "name" in header_keys
+        for values in raw:
+            if use_headers:
+                row={}
+                for idx,key in enumerate(clean):
+                    if key:
+                        row[key]=values[idx] if idx < len(values) else ""
+                yield norm(row)
+            else:
+                row={}
+                for idx,key in enumerate(expected_fields):
+                    row[key]=values[idx] if idx < len(values) else ""
+                yield row
+
 def company_instance_id(guid, name, starting_from, books_from):
     raw="|".join([t(guid),t(name),t(starting_from),t(books_from)]).encode("utf-8")
     return hashlib.sha256(raw).hexdigest()[:24]
@@ -349,19 +385,39 @@ def main():
                   "stock_item_gst_detail","stock_item_gst_rate_detail"):
         con.execute(f"DELETE FROM {table} WHERE company_instance_id=?",(instance,))
 
-    ledger_count=0; max_ledger_alter=0
-    for r in csv_rows(src,"ledger_master.csv"):
+    ledger_count=0; ledger_source_rows=0; ledger_skipped_rows=0; max_ledger_alter=0
+    ledger_fields=[
+        "GUID","AlterID","Name","Parent","OpeningBalance","ClosingBalance",
+        "IsRevenue","PartyGSTIN","GSTRegistrationType"
+    ]
+    for r in csv_rows_expected(src,"ledger_master.csv",ledger_fields):
+        ledger_source_rows+=1
         lg=t(r.get("GUID"))
-        if not lg: continue
+        lname=t(r.get("Name"))
+        if not lg or not lname:
+            ledger_skipped_rows+=1
+            continue
         aid=integer(r.get("AlterID")) or 0
         max_ledger_alter=max(max_ledger_alter,aid)
         con.execute("""INSERT OR REPLACE INTO ledger_master
           VALUES(?,?,?,?,?,?,?,?,?,?,?,?)""",
-          (instance,guid,lg,aid,t(r.get("Name")),t(r.get("Parent")),
+          (instance,guid,lg,aid,lname,t(r.get("Parent")),
            num(r.get("OpeningBalance")),num(r.get("ClosingBalance")),yes(r.get("IsRevenue")),
            t(r.get("PartyGSTIN")),t(r.get("GSTRegistrationType")),now))
         ledger_count+=1
 
+    ledger_stored_rows=int(con.execute(
+        "SELECT COUNT(*) FROM ledger_master WHERE company_instance_id=?",(instance,)
+    ).fetchone()[0])
+    print("LEDGER IMPORT - SOURCE ROWS:",ledger_source_rows)
+    print("LEDGER IMPORT - STORED ROWS:",ledger_stored_rows)
+    print("LEDGER IMPORT - SKIPPED ROWS:",ledger_skipped_rows)
+    if ledger_source_rows > 0 and ledger_stored_rows == 0:
+        raise SystemExit(
+            "Ledger Master persistence failed: ledger_master.csv contains "
+            f"{ledger_source_rows} rows but SQLite ledger_master contains 0 rows."
+        )
+
     # Read stock tax data first so effective histories are available when stock and
     # inventory rows are inserted.
     tax_by_name={}