Rakseeibra/arrr-erp
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix mirror master persistence and add group diagnostics

Browse Source
This commit is contained in:
A R R R Associates
2026-09-09 11:59:54 +05:30
parent 8bbdd3be30
commit 88bcd4d0a4
6 changed files with 139 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/modules/documents/local_agent_runtime/erp_local_agent/__init__.py
+1 -1
View File
@@ -1,2 +1,2 @@
__version__ = "1.26.8"
__version__ = "1.26.9"
AGENT_NAME = "ERP Local Agent"
app/modules/documents/local_agent_runtime/erp_local_agent/accounting_mirror.py
+55 -3
View File
@@ -154,7 +154,7 @@ class AccountingMirrorEngine:
tables = [
"ledger_master","stock_item_master","stock_item_hsn_detail","stock_item_gst_detail",
"voucher","voucher_ledger_entry","inventory_movement",
"group_master","voucher_type_master","stock_group_master","stock_category_master",
"group_master","accounting_master_flat","voucher_type_master","stock_group_master","stock_category_master",
"unit_master","cost_centre_master","cost_category_master",
]
counts = {}
@@ -191,7 +191,7 @@ class AccountingMirrorEngine:
db.close()
@staticmethod
def _add_auxiliary_masters(db_path: Path, output_dir: Path) -> None:
def _add_auxiliary_masters(db_path: Path, output_dir: Path) -> dict[str, Any]:
specs = {
"group_master": ("group_master.csv", [
("master_guid","TEXT"),("name","TEXT"),("parent","TEXT"),("reserved_name","TEXT"),
@@ -218,6 +218,7 @@ class AccountingMirrorEngine:
("master_guid","TEXT"),("name","TEXT"),("parent","TEXT"),("is_active","TEXT")
]),
}
diagnostics: dict[str, Any] = {}
db = sqlite3.connect(db_path, timeout=30)
try:
for table, (filename, cols) in specs.items():
@@ -225,6 +226,7 @@ class AccountingMirrorEngine:
db.execute(f"CREATE TABLE {table} ({', '.join(name+' '+typ for name,typ in cols)})")
rows = AccountingMirrorEngine._read_csv(output_dir / filename)
if not rows:
diagnostics[table] = {"source_rows": 0, "stored_rows": 0}
continue
names = [name for name,_ in cols]
placeholders = ",".join("?" for _ in names)
@@ -245,6 +247,8 @@ class AccountingMirrorEngine:
value = _text(value)
values.append(value)
db.execute(f"INSERT INTO {table} ({','.join(names)}) VALUES ({placeholders})", values)
stored = int(db.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0])
diagnostics[table] = {"source_rows": len(rows), "stored_rows": stored}
# Common flat accounting master table. Keep the original
# ledger_master and group_master physical tables intact.
db.execute("DROP TABLE IF EXISTS accounting_master_flat")
@@ -292,7 +296,10 @@ class AccountingMirrorEngine:
"CREATE INDEX IF NOT EXISTS idx_accounting_master_flat_parent "
"ON accounting_master_flat(parent)"
)
flat_count = int(db.execute("SELECT COUNT(*) FROM accounting_master_flat").fetchone()[0])
diagnostics["accounting_master_flat"] = {"stored_rows": flat_count}
db.commit()
return diagnostics
finally:
db.close()
@@ -433,9 +440,53 @@ class AccountingMirrorEngine:
if not candidate.is_file():
raise RuntimeError("Accounting Mirror database was not produced.")
report(93, "Finalising master tables", "Preparing auxiliary master tables and validating the SQLite mirror.", rows=dict(rows))
self._add_auxiliary_masters(candidate, output_dir)
auxiliary_diagnostics = self._add_auxiliary_masters(candidate, output_dir)
summary = self._summary(candidate)
counts = summary.get("counts") or {}
# Cross-check the extractor rows against what is actually persisted.
extracted_ledgers = int(rows.get("ledger_master") or 0)
stored_ledgers = int(counts.get("ledger_master") or 0)
extracted_groups = int(rows.get("group_master") or 0)
stored_groups = int(counts.get("group_master") or 0)
import_diagnostics = {
"ledgers": {
"extracted_rows": extracted_ledgers,
"stored_rows": stored_ledgers,
},
"groups": {
"extracted_rows": extracted_groups,
"stored_rows": stored_groups,
},
"flat_master_rows": int(counts.get("accounting_master_flat") or 0),
"auxiliary": auxiliary_diagnostics,
}
if extracted_ledgers > 0 and stored_ledgers == 0:
raise RuntimeError(
"Ledger Master persistence failed after extraction: "
f"Tally ODBC extracted {extracted_ledgers} ledger rows, but the local "
"Accounting Mirror SQLite ledger_master table contains 0 rows. "
"The incomplete mirror was NOT published."
)
if extracted_groups > 0 and stored_groups == 0:
raise RuntimeError(
"Group Master persistence failed after extraction: "
f"Tally ODBC extracted {extracted_groups} group rows, but the local "
"Accounting Mirror SQLite group_master table contains 0 rows. "
"The incomplete mirror was NOT published."
)
report(
94,
"Validating master persistence",
f"Ledgers {extracted_ledgers} extracted / {stored_ledgers} stored; "
f"Groups {extracted_groups} extracted / {stored_groups} stored.",
rows=dict(rows),
import_diagnostics=import_diagnostics,
mirror=summary,
)
company = summary.get("company") or {}
actual_name = _text(company.get("company_name"))
actual_guid = _text(company.get("company_guid"))
@@ -494,6 +545,7 @@ class AccountingMirrorEngine:
"odbc_company_guid": actual_guid,
"company_cache_path": str(cache),
"validation_warnings": validation_warnings,
"import_diagnostics": import_diagnostics,
"extractor_log_tail": log[-1500:], "loader_log_tail": llog[-1500:],
**self.status(client_id),
}
app/modules/documents/local_agent_runtime/erp_local_agent/commands.py
+3 -1
View File
@@ -4089,7 +4089,9 @@ class AgentCommandProcessor:
message=("Full Accounting Export completed. ERP analysis tools can now read the refreshed SQLite data without re-extracting from Tally."
+ ((" " + validation_warnings[0]) if validation_warnings else "")),
finished_at_utc=datetime.now(timezone.utc).isoformat(), error="", warnings=validation_warnings,
rows=counts, mirror=mirror_summary, mirror_db_path=str(mirror_result.get("mirror_db_path") or ""),
rows=counts, mirror=mirror_summary,
import_diagnostics=mirror_result.get("import_diagnostics") or {},
mirror_db_path=str(mirror_result.get("mirror_db_path") or ""),
accounting_db_path=str(self.store.db_path(client_id)),
transaction_sync=transaction_sync,
)
app/modules/documents/local_agent_runtime/erp_local_agent/mirror_runtime/build_act_v3_0_1.py
+60 -4
View File
@@ -46,6 +46,42 @@ def csv_rows(src, name):
for row in r:
yield norm(row)
def csv_rows_expected(src, name, expected_fields):
"""Read a Tally ODBC CSV by header first, then by proven SELECT position.
Some Tally ODBC builds return valid row values while exposing blank or
expression-style column names. The extractor therefore reports 986 rows,
but a name-based SQLite loader can see GUID/Name as blank and skip every
row. Positional fallback preserves the exact SELECT order without changing
the Tally extraction itself.
"""
p=src/name
if not p.exists():
raise SystemExit(f"Missing {p}. Run run_v3_0.ps1 first.")
with p.open("r",encoding="utf-8-sig",newline="") as f:
raw=csv.reader(f)
try:
headers=next(raw)
except StopIteration:
return
clean=[ck(x) for x in headers]
print(f"{name} headers:",clean)
header_keys={x.casefold() for x in clean if x}
expected_keys={str(x).casefold() for x in expected_fields}
use_headers=bool(expected_keys.intersection(header_keys)) and "name" in header_keys
for values in raw:
if use_headers:
row={}
for idx,key in enumerate(clean):
if key:
row[key]=values[idx] if idx < len(values) else ""
yield norm(row)
else:
row={}
for idx,key in enumerate(expected_fields):
row[key]=values[idx] if idx < len(values) else ""
yield row
def company_instance_id(guid, name, starting_from, books_from):
raw="|".join([t(guid),t(name),t(starting_from),t(books_from)]).encode("utf-8")
return hashlib.sha256(raw).hexdigest()[:24]
@@ -349,19 +385,39 @@ def main():
"stock_item_gst_detail","stock_item_gst_rate_detail"):
con.execute(f"DELETE FROM {table} WHERE company_instance_id=?",(instance,))
ledger_count=0; max_ledger_alter=0
for r in csv_rows(src,"ledger_master.csv"):
ledger_count=0; ledger_source_rows=0; ledger_skipped_rows=0; max_ledger_alter=0
ledger_fields=[
"GUID","AlterID","Name","Parent","OpeningBalance","ClosingBalance",
"IsRevenue","PartyGSTIN","GSTRegistrationType"
]
for r in csv_rows_expected(src,"ledger_master.csv",ledger_fields):
ledger_source_rows+=1
lg=t(r.get("GUID"))
if not lg: continue
lname=t(r.get("Name"))
if not lg or not lname:
ledger_skipped_rows+=1
continue
aid=integer(r.get("AlterID")) or 0
max_ledger_alter=max(max_ledger_alter,aid)
con.execute("""INSERT OR REPLACE INTO ledger_master
VALUES(?,?,?,?,?,?,?,?,?,?,?,?)""",
(instance,guid,lg,aid,t(r.get("Name")),t(r.get("Parent")),
(instance,guid,lg,aid,lname,t(r.get("Parent")),
num(r.get("OpeningBalance")),num(r.get("ClosingBalance")),yes(r.get("IsRevenue")),
t(r.get("PartyGSTIN")),t(r.get("GSTRegistrationType")),now))
ledger_count+=1
ledger_stored_rows=int(con.execute(
"SELECT COUNT(*) FROM ledger_master WHERE company_instance_id=?",(instance,)
).fetchone()[0])
print("LEDGER IMPORT - SOURCE ROWS:",ledger_source_rows)
print("LEDGER IMPORT - STORED ROWS:",ledger_stored_rows)
print("LEDGER IMPORT - SKIPPED ROWS:",ledger_skipped_rows)
if ledger_source_rows > 0 and ledger_stored_rows == 0:
raise SystemExit(
"Ledger Master persistence failed: ledger_master.csv contains "
f"{ledger_source_rows} rows but SQLite ledger_master contains 0 rows."
)
# Read stock tax data first so effective histories are available when stock and
# inventory rows are inserted.
tax_by_name={}