Add Phase 3 Tally accounting master synchronization

app/core/startup.py

@@ -336,6 +336,7 @@ ROLE_PERMISSION_MAP = {
         "accounting.tally.view",
         "accounting.tally.connect",
         "accounting.tally.map_company",
+        "accounting.tally.sync_masters",
         "accounting.act.initialize",
         "clients.view.own_only",
         "employees.dashboard.view",
@@ -969,3 +970,4 @@ def on_startup(app: FastAPI) -> None:
     # Phase 7O: start alert notification/escalation automation after schema and seed checks.
     start_notification_scheduler()
 
+

app/modules/accounting/act_store.py

@@ -8,7 +8,7 @@ import os
 import sqlite3
 from typing import Iterator, Sequence
 
-ACT_SCHEMA_VERSION = 2
+ACT_SCHEMA_VERSION = 3
 
 
 class AccountingActStoreError(RuntimeError):
@@ -31,8 +31,8 @@ class AccountingActStore:
 
     Phase 1 provides accounting storage and read-only Tally discovery.
     Phase 2 adds durable client/registration -> Tally company mapping keyed by
-    Tally GUID. Accounting master/transaction sync and write-back remain out of
+    Tally GUID. Phase 3 adds read-only Tally accounting master snapshot tables.
-    scope.
+    Voucher/transaction sync and write-back remain out of scope.
     """
 
     def __init__(self, root: str | Path) -> None:
@@ -136,6 +136,44 @@ class AccountingActStore:
                     details_json TEXT NOT NULL DEFAULT '{}'
                 );
 
+                CREATE TABLE IF NOT EXISTS tally_groups (
+                    id INTEGER PRIMARY KEY AUTOINCREMENT, tally_guid TEXT NOT NULL, company_name TEXT NOT NULL, master_guid TEXT NOT NULL DEFAULT '', name TEXT NOT NULL, parent TEXT NOT NULL DEFAULT '', synced_at_utc TEXT NOT NULL, payload_json TEXT NOT NULL DEFAULT '{}'
+                );
+                CREATE TABLE IF NOT EXISTS tally_ledgers (
+                    id INTEGER PRIMARY KEY AUTOINCREMENT, tally_guid TEXT NOT NULL, company_name TEXT NOT NULL, master_guid TEXT NOT NULL DEFAULT '', name TEXT NOT NULL, parent TEXT NOT NULL DEFAULT '', opening_balance REAL NOT NULL DEFAULT 0, closing_balance REAL NOT NULL DEFAULT 0, is_billwise_on TEXT NOT NULL DEFAULT '', gst_applicable TEXT NOT NULL DEFAULT '', synced_at_utc TEXT NOT NULL, payload_json TEXT NOT NULL DEFAULT '{}'
+                );
+                CREATE TABLE IF NOT EXISTS tally_voucher_types (
+                    id INTEGER PRIMARY KEY AUTOINCREMENT, tally_guid TEXT NOT NULL, company_name TEXT NOT NULL, master_guid TEXT NOT NULL DEFAULT '', name TEXT NOT NULL, parent TEXT NOT NULL DEFAULT '', numbering_method TEXT NOT NULL DEFAULT '', synced_at_utc TEXT NOT NULL, payload_json TEXT NOT NULL DEFAULT '{}'
+                );
+                CREATE TABLE IF NOT EXISTS tally_stock_groups (
+                    id INTEGER PRIMARY KEY AUTOINCREMENT, tally_guid TEXT NOT NULL, company_name TEXT NOT NULL, master_guid TEXT NOT NULL DEFAULT '', name TEXT NOT NULL, parent TEXT NOT NULL DEFAULT '', synced_at_utc TEXT NOT NULL, payload_json TEXT NOT NULL DEFAULT '{}'
+                );
+                CREATE TABLE IF NOT EXISTS tally_stock_categories (
+                    id INTEGER PRIMARY KEY AUTOINCREMENT, tally_guid TEXT NOT NULL, company_name TEXT NOT NULL, master_guid TEXT NOT NULL DEFAULT '', name TEXT NOT NULL, parent TEXT NOT NULL DEFAULT '', synced_at_utc TEXT NOT NULL, payload_json TEXT NOT NULL DEFAULT '{}'
+                );
+                CREATE TABLE IF NOT EXISTS tally_stock_items (
+                    id INTEGER PRIMARY KEY AUTOINCREMENT, tally_guid TEXT NOT NULL, company_name TEXT NOT NULL, master_guid TEXT NOT NULL DEFAULT '', name TEXT NOT NULL, parent TEXT NOT NULL DEFAULT '', category TEXT NOT NULL DEFAULT '', base_units TEXT NOT NULL DEFAULT '', hsn_code TEXT NOT NULL DEFAULT '', synced_at_utc TEXT NOT NULL, payload_json TEXT NOT NULL DEFAULT '{}'
+                );
+                CREATE TABLE IF NOT EXISTS tally_units (
+                    id INTEGER PRIMARY KEY AUTOINCREMENT, tally_guid TEXT NOT NULL, company_name TEXT NOT NULL, master_guid TEXT NOT NULL DEFAULT '', name TEXT NOT NULL, original_name TEXT NOT NULL DEFAULT '', synced_at_utc TEXT NOT NULL, payload_json TEXT NOT NULL DEFAULT '{}'
+                );
+                CREATE TABLE IF NOT EXISTS tally_cost_categories (
+                    id INTEGER PRIMARY KEY AUTOINCREMENT, tally_guid TEXT NOT NULL, company_name TEXT NOT NULL, master_guid TEXT NOT NULL DEFAULT '', name TEXT NOT NULL, synced_at_utc TEXT NOT NULL, payload_json TEXT NOT NULL DEFAULT '{}'
+                );
+                CREATE TABLE IF NOT EXISTS tally_cost_centres (
+                    id INTEGER PRIMARY KEY AUTOINCREMENT, tally_guid TEXT NOT NULL, company_name TEXT NOT NULL, master_guid TEXT NOT NULL DEFAULT '', name TEXT NOT NULL, parent TEXT NOT NULL DEFAULT '', category TEXT NOT NULL DEFAULT '', synced_at_utc TEXT NOT NULL, payload_json TEXT NOT NULL DEFAULT '{}'
+                );
+
+                CREATE INDEX IF NOT EXISTS ix_tally_groups_company ON tally_groups(tally_guid, name);
+                CREATE INDEX IF NOT EXISTS ix_tally_ledgers_company ON tally_ledgers(tally_guid, name);
+                CREATE INDEX IF NOT EXISTS ix_tally_voucher_types_company ON tally_voucher_types(tally_guid, name);
+                CREATE INDEX IF NOT EXISTS ix_tally_stock_groups_company ON tally_stock_groups(tally_guid, name);
+                CREATE INDEX IF NOT EXISTS ix_tally_stock_categories_company ON tally_stock_categories(tally_guid, name);
+                CREATE INDEX IF NOT EXISTS ix_tally_stock_items_company ON tally_stock_items(tally_guid, name);
+                CREATE INDEX IF NOT EXISTS ix_tally_units_company ON tally_units(tally_guid, name);
+                CREATE INDEX IF NOT EXISTS ix_tally_cost_categories_company ON tally_cost_categories(tally_guid, name);
+                CREATE INDEX IF NOT EXISTS ix_tally_cost_centres_company ON tally_cost_centres(tally_guid, name);
+
                 CREATE INDEX IF NOT EXISTS ix_tally_companies_last_seen
                     ON tally_companies(last_seen_at_utc);
                 CREATE INDEX IF NOT EXISTS ix_tally_company_mapping_client_active

app/modules/accounting/templates/accounting/tally.html

@@ -1,11 +1,12 @@
 {% extends "ui/templates/base/layout.html" %}
 {% block content %}
+{% if synced %}<div class="mb-4 rounded-xl border border-emerald-200 bg-emerald-50 px-4 py-3 text-sm text-emerald-800">Tally accounting masters synchronized successfully.</div>{% endif %}
 <div class="space-y-6">
   <div class="flex flex-wrap items-start justify-between gap-4">
     <div>
       <p class="text-xs font-semibold uppercase tracking-[0.16em] text-brand-600">Tools · Accounting</p>
-      <h1 class="mt-1 text-2xl font-semibold text-slate-900">Tally Company Mapping</h1>
+      <h1 class="mt-1 text-2xl font-semibold text-slate-900">Tally Accounting Masters</h1>
-      <p class="mt-1 text-sm text-slate-500">Phase 2 permanently maps an ERP client or registration to a loaded TallyPrime company using the Tally GUID.</p>
+      <p class="mt-1 text-sm text-slate-500">Phase 3 permanently maps an ERP client or registration to a loaded TallyPrime company using the Tally GUID.</p>
     </div>
     <a href="/tools/tally{% if selected_client %}?client_id={{ selected_client.id }}&refresh=1{% else %}?refresh=1{% endif %}" class="rounded-xl bg-brand-600 px-4 py-2 text-sm font-semibold text-white shadow-soft hover:bg-brand-700">Refresh Tally Companies</a>
   </div>
@@ -122,7 +123,7 @@
       </label>
 
       <label class="text-sm font-medium text-slate-700">
-        Loaded Tally Company
+        Currently Open in TallyPrime
         <select name="tally_guid" required class="mt-1 w-full rounded-xl border border-slate-300 bg-white px-3 py-2 text-sm">
           <option value="">Select Tally company</option>
           {% for company in tally.companies %}
@@ -138,7 +139,7 @@
         <input type="checkbox" name="allow_gstin_mismatch" value="true" class="mt-1 rounded border-slate-300" />
         <span class="text-sm text-slate-700">
           Allow GSTIN mismatch after manual verification.
-          <span class="block text-xs text-slate-500">For a GSTIN registration, Phase 2 blocks a mapping when both ERP and Tally contain different GSTINs unless this override is explicitly selected.</span>
+          <span class="block text-xs text-slate-500">For a GSTIN registration, Phase 3 blocks a mapping when both ERP and Tally contain different GSTINs unless this override is explicitly selected.</span>
         </span>
       </label>
 
@@ -149,6 +150,47 @@
   </section>
   {% endif %}
 
+  {% if accounting and accounting.exists and tally and tally.connected %}
+  <section class="rounded-2xl bg-white p-5 shadow-soft">
+    <div class="flex flex-wrap items-start justify-between gap-4">
+      <div>
+        <h2 class="font-semibold text-slate-900">Phase 3 · Accounting Master Sync</h2>
+        <p class="mt-1 text-sm text-slate-500">Select the company that is currently open/loaded in TallyPrime. The Local Agent verifies that the selected company is already mapped to this ERP client before reading masters.</p>
+      </div>
+      <span class="rounded-full bg-blue-50 px-3 py-1 text-xs font-semibold text-blue-700">Read-only</span>
+    </div>
+
+    <form method="post" action="/tools/tally/sync-masters" class="mt-5 grid gap-4 lg:grid-cols-[1fr_auto] lg:items-end">
+      <input type="hidden" name="csrf_token" value="{{ csrf_token }}" />
+      <input type="hidden" name="client_id" value="{{ selected_client.id }}" />
+      <label class="text-sm font-medium text-slate-700">
+        Currently Open in TallyPrime
+        <select name="tally_guid" required class="mt-1 w-full rounded-xl border border-slate-300 bg-white px-3 py-2 text-sm">
+          <option value="">Select open Tally company</option>
+          {% for company in tally.companies %}
+          {% if company.guid %}
+          <option value="{{ company.guid }}">{{ company.name }}{% if company.gstin %} · {{ company.gstin }}{% endif %}</option>
+          {% endif %}
+          {% endfor %}
+        </select>
+        <span class="mt-1 block text-xs font-normal text-slate-500">This dropdown is populated live from the companies currently loaded/open in the local TallyPrime instance.</span>
+      </label>
+      <button type="submit" class="rounded-xl bg-brand-600 px-4 py-2 text-sm font-semibold text-white hover:bg-brand-700">Sync Accounting Masters</button>
+    </form>
+
+    <div class="mt-5 grid gap-3 sm:grid-cols-2 lg:grid-cols-4">
+      {% set last_sync = accounting.latest_master_sync if accounting else none %}
+      <div class="rounded-xl border border-slate-200 p-3"><div class="text-xs font-semibold uppercase tracking-wide text-slate-500">Latest Sync</div><div class="mt-1 text-sm font-medium text-slate-900">{{ last_sync.completed_at_utc if last_sync and last_sync.completed_at_utc else 'Not synced yet' }}</div></div>
+      <div class="rounded-xl border border-slate-200 p-3"><div class="text-xs font-semibold uppercase tracking-wide text-slate-500">Company</div><div class="mt-1 text-sm font-medium text-slate-900">{{ last_sync.company_name if last_sync else '-' }}</div></div>
+      <div class="rounded-xl border border-slate-200 p-3"><div class="text-xs font-semibold uppercase tracking-wide text-slate-500">Rows Stored</div><div class="mt-1 text-sm font-medium text-slate-900">{{ last_sync.rows_processed if last_sync else 0 }}</div></div>
+      <div class="rounded-xl border border-slate-200 p-3"><div class="text-xs font-semibold uppercase tracking-wide text-slate-500">Status</div><div class="mt-1 text-sm font-medium text-slate-900">{{ last_sync.status|title if last_sync else '-' }}</div></div>
+    </div>
+
+    <div class="mt-4 rounded-xl bg-slate-50 px-4 py-3 text-xs text-slate-600">
+      Synced into the selected client's .act database: Groups, Ledgers, Voucher Types, Stock Groups, Stock Categories, Stock Items, Units, Cost Categories and Cost Centres. No voucher or transaction data is imported in Phase 3.
+    </div>
+  </section>
+
   {% if accounting and accounting.exists %}
   <section class="overflow-hidden rounded-2xl bg-white shadow-soft">
     <div class="border-b border-slate-100 px-5 py-4">
@@ -236,7 +278,7 @@
   {% endif %}
 
   <div class="rounded-xl border border-blue-200 bg-blue-50 px-4 py-3 text-sm text-blue-800">
-    Phase 2 only stores company mappings. Ledger masters, vouchers, inventory and Tally write-back remain disabled.
+    Phase 3 synchronizes accounting masters only. Voucher/transaction sync and Tally write-back remain disabled and are reserved for later phases.
   </div>
 </div>
 {% endblock %}

app/modules/accounting/ui.py

@@ -114,6 +114,7 @@ def tally_tool(
     initialized: int = 0,
     mapped: int = 0,
     unmapped: int = 0,
+    synced: int = 0,
     error: str = "",
 ):
     db = CommonSessionLocal()
@@ -148,7 +149,7 @@ def tally_tool(
             try:
                 response_data = request_agent_command(
                     node.node_code,
-                    "phase2_status",
+                    "phase3_status",
                     payload,
                     timeout_seconds=20,
                 )
@@ -163,7 +164,7 @@ def tally_tool(
             request,
             db,
             user,
-            title="Tally Company Mapping",
+            title="Tally Accounting Masters",
             clients=clients,
             selected_client=selected_client,
             registrations=registrations,
@@ -173,6 +174,7 @@ def tally_tool(
             initialized=bool(initialized),
             mapped=bool(mapped),
             unmapped=bool(unmapped),
+            synced=bool(synced),
             command_error=command_error,
         )
     finally:
@@ -358,3 +360,51 @@ def unmap_tally_company(
         )
     finally:
         db.close()
+
+
+@router.post("/sync-masters")
+def sync_tally_masters(
+    request: Request,
+    client_id: int = Form(...),
+    tally_guid: str = Form(...),
+    csrf_token: str = Form(...),
+):
+    validate_csrf(request, csrf_token)
+    db = CommonSessionLocal()
+    try:
+        user, response = _require_partner(request, db, "accounting.tally.sync_masters")
+        if response:
+            return response
+        client, _clients, scope = _find_visible_client(db, request, user, client_id)
+        if not client:
+            return _denied()
+        node = get_active_storage_node_for_branch(db, scope.tenant_id, scope.branch_id)
+        if not node or not _node_online(node):
+            return RedirectResponse(
+                url=f"/tools/tally?client_id={client.id}&error={quote('ERP Local Agent is offline for the active branch.')}",
+                status_code=303,
+            )
+        try:
+            result = request_agent_command(
+                node.node_code,
+                "accounting_sync_masters",
+                {
+                    "client_id": int(client.id),
+                    "tally_guid": str(tally_guid or "").strip(),
+                    "requested_by_user_id": int(user.id),
+                },
+                timeout_seconds=120,
+            )
+            if not result.get("ok"):
+                raise RuntimeError(str(result.get("error") or "Tally master synchronization failed."))
+        except Exception as exc:
+            return RedirectResponse(
+                url=f"/tools/tally?client_id={client.id}&error={quote(str(exc))}",
+                status_code=303,
+            )
+        return RedirectResponse(
+            url=f"/tools/tally?client_id={client.id}&refresh=1&synced=1",
+            status_code=303,
+        )
+    finally:
+        db.close()

app/modules/core/rbac/permissions_registry.py

@@ -150,6 +150,7 @@ PERMISSIONS = {
     "accounting.tally.view": "View Tally Accounting Tool",
     "accounting.tally.connect": "Connect to Tally Through Local Agent",
     "accounting.tally.map_company": "Map ERP Client or Registration to Tally Company",
+    "accounting.tally.sync_masters": "Synchronize Tally Accounting Masters",
     "accounting.act.initialize": "Initialize Client Accounting ACT Storage",
 
     "notice_cases.view": "View Notice and Case Management",
@@ -179,3 +180,4 @@ def expand_permission_codes(code: str) -> list[str]:
             codes.append(alias)
     return codes
 
+

app/modules/documents/agent_package.py

@@ -4,14 +4,9 @@ import io
 from pathlib import Path
 import zipfile
 
-ERP_LOCAL_AGENT_VERSION = "1.3.0"
+ERP_LOCAL_AGENT_VERSION = "1.4.0"
 ERP_LOCAL_AGENT_NAME = "ERP Local Agent"
 RUNTIME_ROOT = Path(__file__).resolve().parent / "local_agent_runtime"
-
-# ZIP metadata is part of the ZIP byte stream. If writestr() is allowed to use
-# the current clock time, two packages built from identical source files have
-# different SHA256 hashes. Keep update packages reproducible so the SHA256
-# returned by update-manifest is exactly the SHA256 of update-package.
 _DETERMINISTIC_ZIP_TIMESTAMP = (2026, 1, 1, 0, 0, 0)
 
 
@@ -23,24 +18,11 @@ def build_agent_env(*, erp_base_url: str, node_code: str, node_secret: str, stor
             erp_base_url = "https://" + erp_base_url[7:]
     storage_root = (storage_root or r"D:\AuditFirmStorage").strip()
     return (
-        f"ERP_BASE_URL={erp_base_url}\n"
+        f"ERP_BASE_URL={erp_base_url}\n" f"NODE_CODE={(node_code or '').strip()}\n" f"NODE_SECRET={(node_secret or '').strip()}\n"
-        f"NODE_CODE={(node_code or '').strip()}\n"
+        f"STORAGE_ROOT={storage_root}\n" f"TENANT_ID={'' if tenant_id is None else tenant_id}\n" f"AUDIT_FIRM_ID={'' if tenant_id is None else tenant_id}\n"
-        f"NODE_SECRET={(node_secret or '').strip()}\n"
+        f"BRANCH_ID={'' if branch_id is None else branch_id}\n" f"SYNC_INTERVAL_SECONDS={int(sync_interval_seconds or 30)}\n" f"POLL_INTERVAL_SECONDS={int(sync_interval_seconds or 30)}\n"
-        f"STORAGE_ROOT={storage_root}\n"
+        f"REQUEST_TIMEOUT_SECONDS={int(request_timeout_seconds or 60)}\n" f"TUNNEL_ENABLED={str(bool(tunnel_enabled)).lower()}\n" f"TUNNEL_RECONNECT_SECONDS={int(tunnel_reconnect_seconds or 10)}\n"
-        f"TENANT_ID={'' if tenant_id is None else tenant_id}\n"
+        "AUTO_UPDATE=true\nAUTO_INSTALL_UPDATES=false\nUPDATE_CHECK_INTERVAL_SECONDS=300\nDASHBOARD_ENABLED=true\nDASHBOARD_HOST=127.0.0.1\nDASHBOARD_PORT=8788\n"
-        f"AUDIT_FIRM_ID={'' if tenant_id is None else tenant_id}\n"
-        f"BRANCH_ID={'' if branch_id is None else branch_id}\n"
-        f"SYNC_INTERVAL_SECONDS={int(sync_interval_seconds or 30)}\n"
-        f"POLL_INTERVAL_SECONDS={int(sync_interval_seconds or 30)}\n"
-        f"REQUEST_TIMEOUT_SECONDS={int(request_timeout_seconds or 60)}\n"
-        f"TUNNEL_ENABLED={str(bool(tunnel_enabled)).lower()}\n"
-        f"TUNNEL_RECONNECT_SECONDS={int(tunnel_reconnect_seconds or 10)}\n"
-        "AUTO_UPDATE=true\n"
-        "AUTO_INSTALL_UPDATES=false\n"
-        "UPDATE_CHECK_INTERVAL_SECONDS=300\n"
-        "DASHBOARD_ENABLED=true\n"
-        "DASHBOARD_HOST=127.0.0.1\n"
-        "DASHBOARD_PORT=8788\n"
     )
 
 
@@ -48,8 +30,6 @@ def _zip_info(name: str) -> zipfile.ZipInfo:
     info = zipfile.ZipInfo(filename=name, date_time=_DETERMINISTIC_ZIP_TIMESTAMP)
     info.compress_type = zipfile.ZIP_DEFLATED
     info.create_system = 3
-    # Regular file with rw-r--r-- permissions. Stable metadata keeps ZIP bytes
-    # reproducible on every request and across Linux deployments.
     info.external_attr = (0o100644 & 0xFFFF) << 16
     return info
 
@@ -61,50 +41,28 @@ def _write_zip_bytes(dst: zipfile.ZipFile, name: str, payload: bytes) -> None:
 def _build_zip(*, env_text: str | None, include_env: bool, include_admin_readme: bool) -> bytes:
     if not RUNTIME_ROOT.exists():
         raise RuntimeError(f"ERP Local Agent runtime is missing: {RUNTIME_ROOT}")
-
     buffer = io.BytesIO()
-    with zipfile.ZipFile(
+    with zipfile.ZipFile(buffer, "w", compression=zipfile.ZIP_DEFLATED, compresslevel=9) as dst:
-        buffer,
-        "w",
-        compression=zipfile.ZIP_DEFLATED,
-        compresslevel=9,
-    ) as dst:
         for path in sorted(RUNTIME_ROOT.rglob("*"), key=lambda item: item.as_posix()):
             if not path.is_file() or "__pycache__" in path.parts:
                 continue
-            name = path.relative_to(RUNTIME_ROOT).as_posix()
+            _write_zip_bytes(dst, path.relative_to(RUNTIME_ROOT).as_posix(), path.read_bytes())
-            _write_zip_bytes(dst, name, path.read_bytes())
-
         if include_env and env_text is not None:
             _write_zip_bytes(dst, ".env", env_text.encode("utf-8"))
-
         if include_admin_readme:
-            readme = (
+            text = (
                 f"ERP Local Agent {ERP_LOCAL_AGENT_VERSION}\n"
-                "Existing storage, WebSocket tunnel, Tally, accounting .act and local dashboard functionality are preserved.\n"
+                "Existing storage, WebSocket tunnel, dashboard, Tally mapping and client .act functionality are preserved.\n"
-                "Local dashboard: http://127.0.0.1:8788\n"
+                "Phase 3 adds read-only Tally accounting master synchronization.\n"
-                "The agent checks for updates automatically but installation is always initiated by the local user.\n"
+                "Dashboard: http://127.0.0.1:8788\n"
-                "The existing .env, data, logs, .venv and client storage are preserved during updates.\n"
             )
-            _write_zip_bytes(dst, "README_ERP_LOCAL_AGENT.txt", readme.encode("utf-8"))
+            _write_zip_bytes(dst, "README_ERP_LOCAL_AGENT.txt", text.encode("utf-8"))
-
     return buffer.getvalue()
 
 
 def build_preconfigured_agent_zip(*, env_text: str, include_admin_readme: bool = False) -> bytes:
-    return _build_zip(
+    return _build_zip(env_text=env_text, include_env=True, include_admin_readme=include_admin_readme)
-        env_text=env_text,
-        include_env=True,
-        include_admin_readme=include_admin_readme,
-    )
 
 
 def build_agent_update_zip() -> bytes:
-    # Update package deliberately excludes .env, .venv, data, logs and client files.
+    return _build_zip(env_text=None, include_env=False, include_admin_readme=False)
-    # The ZIP is deterministic so update-manifest and update-package always agree
-    # on SHA256 for the same deployed runtime/version.
-    return _build_zip(
-        env_text=None,
-        include_env=False,
-        include_admin_readme=False,
-    )

app/modules/documents/local_agent_runtime/README_ERP_LOCAL_AGENT.txt

@@ -1,4 +1,4 @@
-ERP Local Agent 1.3.0
+ERP Local Agent 1.4.0
 
 Existing storage, WebSocket tunnel, Tally and client .act functionality are preserved.
 
@@ -18,3 +18,5 @@ Local operational database:
 Client accounting .act databases remain separate under the configured STORAGE_ROOT.
 
 Phase 2: client/registration to Tally company mapping is supported using Tally GUID.
+
+Phase 3: read-only Tally accounting master sync (Groups, Ledgers, Voucher Types, Stock Groups/Categories/Items, Units and Cost Centres/Categories) into client .act storage.

app/modules/documents/local_agent_runtime/erp_local_agent/__init__.py

@@ -1,2 +1,2 @@
-__version__ = "1.3.0"
+__version__ = "1.4.0"
 AGENT_NAME = "ERP Local Agent"

app/modules/documents/local_agent_runtime/erp_local_agent/accounting_store.py

@@ -7,7 +7,19 @@ import sqlite3
 from typing import Sequence
 
 
-SCHEMA_VERSION = "2"
+SCHEMA_VERSION = "3"
+
+MASTER_TABLES = {
+    "groups": "tally_groups",
+    "ledgers": "tally_ledgers",
+    "voucher_types": "tally_voucher_types",
+    "stock_groups": "tally_stock_groups",
+    "stock_categories": "tally_stock_categories",
+    "stock_items": "tally_stock_items",
+    "units": "tally_units",
+    "cost_categories": "tally_cost_categories",
+    "cost_centres": "tally_cost_centres",
+}
 
 
 def _utc_now_iso() -> str:
@@ -15,11 +27,10 @@ def _utc_now_iso() -> str:
 
 
 class LocalAccountingStore:
-    """Client-scoped SQLite .act storage under the existing branch storage root.
+    """Client-scoped SQLite .act accounting store.
 
-    Phase 2 persists ERP client/registration -> Tally company mappings using the
+    Phase 3 preserves Phase 1/2 metadata and mappings and adds company-scoped
-    Tally GUID as the durable identifier. Existing Phase 1 databases are upgraded
+    read-only Tally master snapshots. Existing .act files are upgraded in place.
-    in place without deleting accounting history.
     """
 
     def __init__(self, storage_root: Path):
@@ -43,7 +54,7 @@ class LocalAccountingStore:
     def connect(self, client_id: int):
         path = self.db_path(client_id)
         path.parent.mkdir(parents=True, exist_ok=True)
-        db = sqlite3.connect(path, timeout=30)
+        db = sqlite3.connect(path, timeout=60)
         db.row_factory = sqlite3.Row
         db.execute("PRAGMA foreign_keys=ON")
         db.execute("PRAGMA journal_mode=WAL")
@@ -69,81 +80,102 @@ class LocalAccountingStore:
             if name not in columns:
                 db.execute(f"ALTER TABLE tally_company_mapping ADD COLUMN {name} {ddl}")
 
-    def initialize(
+    @staticmethod
-        self,
+    def _master_table_ddl(table: str) -> str:
-        client_id: int,
+        return f"""
-        client_name: str = "",
+        CREATE TABLE IF NOT EXISTS {table} (
-        tenant_id: int | None = None,
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
-        created_by_user_id: int | None = None,
+            tally_guid TEXT NOT NULL,
-    ) -> Path:
+            company_name TEXT NOT NULL,
+            master_guid TEXT NOT NULL DEFAULT '',
+            name TEXT NOT NULL,
+            parent TEXT NOT NULL DEFAULT '',
+            category TEXT NOT NULL DEFAULT '',
+            reserved_name TEXT NOT NULL DEFAULT '',
+            base_units TEXT NOT NULL DEFAULT '',
+            additional_units TEXT NOT NULL DEFAULT '',
+            original_name TEXT NOT NULL DEFAULT '',
+            opening_balance REAL NOT NULL DEFAULT 0,
+            closing_balance REAL NOT NULL DEFAULT 0,
+            opening_value REAL NOT NULL DEFAULT 0,
+            opening_rate TEXT NOT NULL DEFAULT '',
+            numbering_method TEXT NOT NULL DEFAULT '',
+            tax_type TEXT NOT NULL DEFAULT '',
+            gst_applicable TEXT NOT NULL DEFAULT '',
+            gst_registration_type TEXT NOT NULL DEFAULT '',
+            gst_type_of_supply TEXT NOT NULL DEFAULT '',
+            hsn_code TEXT NOT NULL DEFAULT '',
+            is_revenue TEXT NOT NULL DEFAULT '',
+            is_deemed_positive TEXT NOT NULL DEFAULT '',
+            is_billwise_on TEXT NOT NULL DEFAULT '',
+            is_simple_unit TEXT NOT NULL DEFAULT '',
+            conversion TEXT NOT NULL DEFAULT '',
+            is_active TEXT NOT NULL DEFAULT '',
+            synced_at_utc TEXT NOT NULL,
+            payload_json TEXT NULL
+        );
+        CREATE INDEX IF NOT EXISTS ix_{table}_company ON {table}(tally_guid, name);
+        CREATE INDEX IF NOT EXISTS ix_{table}_master_guid ON {table}(tally_guid, master_guid);
+        """
+
+    def initialize(self, client_id: int, client_name: str = "", tenant_id: int | None = None, created_by_user_id: int | None = None) -> Path:
         path = self.db_path(client_id)
         with self.connect(client_id) as db:
             db.executescript(
                 """
                 CREATE TABLE IF NOT EXISTS act_meta (
-                    key TEXT PRIMARY KEY,
+                    key TEXT PRIMARY KEY, value TEXT NOT NULL, updated_at_utc TEXT NOT NULL
-                    value TEXT NOT NULL,
-                    updated_at_utc TEXT NOT NULL
                 );
                 CREATE TABLE IF NOT EXISTS tally_companies (
                     id INTEGER PRIMARY KEY AUTOINCREMENT,
-                    tally_guid TEXT NOT NULL DEFAULT '',
+                    tally_guid TEXT NOT NULL DEFAULT '', company_name TEXT NOT NULL,
-                    company_name TEXT NOT NULL,
+                    gstin TEXT NOT NULL DEFAULT '', first_seen_at_utc TEXT NOT NULL,
-                    gstin TEXT NOT NULL DEFAULT '',
+                    last_seen_at_utc TEXT NOT NULL, is_currently_loaded INTEGER NOT NULL DEFAULT 0
-                    first_seen_at_utc TEXT NOT NULL,
-                    last_seen_at_utc TEXT NOT NULL,
-                    is_currently_loaded INTEGER NOT NULL DEFAULT 0
                 );
                 CREATE INDEX IF NOT EXISTS ix_tally_companies_guid ON tally_companies(tally_guid);
                 CREATE INDEX IF NOT EXISTS ix_tally_companies_name ON tally_companies(company_name);
 
                 CREATE TABLE IF NOT EXISTS tally_company_mapping (
-                    id INTEGER PRIMARY KEY AUTOINCREMENT,
+                    id INTEGER PRIMARY KEY AUTOINCREMENT, client_id INTEGER NOT NULL,
-                    client_id INTEGER NOT NULL,
+                    registration_id INTEGER NULL, registration_type_code TEXT NOT NULL DEFAULT '',
-                    registration_id INTEGER NULL,
+                    registration_number TEXT NOT NULL DEFAULT '', registration_legal_name TEXT NOT NULL DEFAULT '',
-                    registration_type_code TEXT NOT NULL DEFAULT '',
+                    registration_trade_name TEXT NOT NULL DEFAULT '', business_unit_id INTEGER,
-                    registration_number TEXT NOT NULL DEFAULT '',
+                    client_branch_id INTEGER, tally_guid TEXT NOT NULL, company_name TEXT NOT NULL,
-                    registration_legal_name TEXT NOT NULL DEFAULT '',
+                    gstin TEXT NOT NULL DEFAULT '', is_active INTEGER NOT NULL DEFAULT 1,
-                    registration_trade_name TEXT NOT NULL DEFAULT '',
+                    created_at_utc TEXT NOT NULL, updated_at_utc TEXT NOT NULL,
-                    business_unit_id INTEGER,
+                    mapped_by_user_id INTEGER, unmapped_at_utc TEXT, unmapped_by_user_id INTEGER
-                    client_branch_id INTEGER,
-                    tally_guid TEXT NOT NULL,
-                    company_name TEXT NOT NULL,
-                    gstin TEXT NOT NULL DEFAULT '',
-                    is_active INTEGER NOT NULL DEFAULT 1,
-                    created_at_utc TEXT NOT NULL,
-                    updated_at_utc TEXT NOT NULL,
-                    mapped_by_user_id INTEGER,
-                    unmapped_at_utc TEXT,
-                    unmapped_by_user_id INTEGER
                 );
-                CREATE INDEX IF NOT EXISTS ix_tally_company_mapping_client_active
+                CREATE INDEX IF NOT EXISTS ix_tally_company_mapping_client_active ON tally_company_mapping(client_id, is_active);
-                    ON tally_company_mapping(client_id, is_active);
+                CREATE INDEX IF NOT EXISTS ix_tally_company_mapping_guid ON tally_company_mapping(tally_guid, is_active);
-                CREATE INDEX IF NOT EXISTS ix_tally_company_mapping_guid
-                    ON tally_company_mapping(tally_guid, is_active);
 
                 CREATE TABLE IF NOT EXISTS tally_connection_history (
-                    id INTEGER PRIMARY KEY AUTOINCREMENT,
+                    id INTEGER PRIMARY KEY AUTOINCREMENT, checked_at_utc TEXT NOT NULL,
-                    checked_at_utc TEXT NOT NULL,
+                    connected INTEGER NOT NULL, tally_url TEXT NOT NULL DEFAULT '',
-                    connected INTEGER NOT NULL,
+                    company_count INTEGER NOT NULL DEFAULT 0, error_message TEXT NULL, payload_json TEXT NULL
-                    tally_url TEXT NOT NULL DEFAULT '',
-                    company_count INTEGER NOT NULL DEFAULT 0,
-                    error_message TEXT NULL,
-                    payload_json TEXT NULL
                 );
                 CREATE TABLE IF NOT EXISTS tally_sync_runs (
                     id INTEGER PRIMARY KEY AUTOINCREMENT,
-                    sync_type TEXT NOT NULL,
+                    sync_type TEXT NOT NULL, tally_guid TEXT NOT NULL DEFAULT '', company_name TEXT NOT NULL DEFAULT '',
-                    status TEXT NOT NULL,
+                    mapping_id INTEGER, requested_by_user_id INTEGER,
-                    started_at_utc TEXT NOT NULL,
+                    status TEXT NOT NULL, started_at_utc TEXT NOT NULL, completed_at_utc TEXT NULL,
-                    completed_at_utc TEXT NULL,
+                    rows_processed INTEGER NOT NULL DEFAULT 0, error_message TEXT NULL, details_json TEXT NULL
-                    rows_processed INTEGER NOT NULL DEFAULT 0,
-                    error_message TEXT NULL,
-                    details_json TEXT NULL
                 );
                 """
             )
             self._ensure_mapping_columns(db)
+            # Upgrade older tally_sync_runs created in Phase 1 without dropping history.
+            sync_columns = {row["name"] for row in db.execute("PRAGMA table_info(tally_sync_runs)").fetchall()}
+            for name, ddl in {
+                "tally_guid": "TEXT NOT NULL DEFAULT ''",
+                "company_name": "TEXT NOT NULL DEFAULT ''",
+                "mapping_id": "INTEGER",
+                "requested_by_user_id": "INTEGER",
+            }.items():
+                if name not in sync_columns:
+                    db.execute(f"ALTER TABLE tally_sync_runs ADD COLUMN {name} {ddl}")
+            for table in MASTER_TABLES.values():
+                db.executescript(self._master_table_ddl(table))
+
             now = _utc_now_iso()
             meta = {
                 "schema_version": SCHEMA_VERSION,
@@ -175,8 +207,7 @@ class LocalAccountingStore:
                 guid = str(company.get("guid") or "").strip()
                 gstin = str(company.get("gstin") or "").strip().upper()
                 existing = db.execute(
-                    "SELECT id FROM tally_companies WHERE tally_guid=? AND company_name=? LIMIT 1",
+                    "SELECT id FROM tally_companies WHERE tally_guid=? AND company_name=? LIMIT 1", (guid, name)
-                    (guid, name),
                 ).fetchone()
                 if existing:
                     db.execute(
@@ -192,26 +223,10 @@ class LocalAccountingStore:
             db.execute(
                 """INSERT INTO tally_connection_history(checked_at_utc, connected, tally_url, company_count, error_message, payload_json)
                    VALUES (?, ?, ?, ?, ?, ?)""",
-                (
+                (now, 1 if status.get("connected") else 0, str(status.get("url") or ""), int(status.get("company_count") or 0), str(status.get("error") or "") or None, json.dumps(status, ensure_ascii=False, separators=(",", ":"))),
-                    now,
-                    1 if status.get("connected") else 0,
-                    str(status.get("url") or ""),
-                    int(status.get("company_count") or 0),
-                    str(status.get("error") or "") or None,
-                    json.dumps(status, ensure_ascii=False, separators=(",", ":")),
-                ),
             )
 
-    def map_company(
+    def map_company(self, client_id: int, *, tally_guid: str, company_name: str, gstin: str = "", registration: dict | None = None, mapped_by_user_id: int | None = None) -> dict:
-        self,
-        client_id: int,
-        *,
-        tally_guid: str,
-        company_name: str,
-        gstin: str = "",
-        registration: dict | None = None,
-        mapped_by_user_id: int | None = None,
-    ) -> dict:
         if not self.exists(client_id):
             raise ValueError("Accounting storage is not initialized for this client.")
         guid = str(tally_guid or "").strip()
@@ -220,136 +235,145 @@ class LocalAccountingStore:
             raise ValueError("Tally company GUID is required for permanent mapping.")
         if not name:
             raise ValueError("Tally company name is required.")
-
         registration = registration or {}
         registration_id = registration.get("id")
         registration_id = int(registration_id) if registration_id not in (None, "") else None
         now = _utc_now_iso()
-
         with self.connect(client_id) as db:
             if registration_id is None:
-                db.execute(
+                db.execute("UPDATE tally_company_mapping SET is_active=0, updated_at_utc=?, unmapped_at_utc=? WHERE client_id=? AND registration_id IS NULL AND is_active=1", (now, now, int(client_id)))
-                    """UPDATE tally_company_mapping
-                       SET is_active=0, updated_at_utc=?, unmapped_at_utc=?
-                       WHERE client_id=? AND registration_id IS NULL AND is_active=1""",
-                    (now, now, int(client_id)),
-                )
             else:
-                db.execute(
+                db.execute("UPDATE tally_company_mapping SET is_active=0, updated_at_utc=?, unmapped_at_utc=? WHERE client_id=? AND registration_id=? AND is_active=1", (now, now, int(client_id), registration_id))
-                    """UPDATE tally_company_mapping
+            db.execute("UPDATE tally_company_mapping SET is_active=0, updated_at_utc=?, unmapped_at_utc=? WHERE client_id=? AND tally_guid=? AND is_active=1", (now, now, int(client_id), guid))
-                       SET is_active=0, updated_at_utc=?, unmapped_at_utc=?
-                       WHERE client_id=? AND registration_id=? AND is_active=1""",
-                    (now, now, int(client_id), registration_id),
-                )
-
-            db.execute(
-                """UPDATE tally_company_mapping
-                   SET is_active=0, updated_at_utc=?, unmapped_at_utc=?
-                   WHERE client_id=? AND tally_guid=? AND is_active=1""",
-                (now, now, int(client_id), guid),
-            )
-
             cur = db.execute(
-                """INSERT INTO tally_company_mapping(
+                """INSERT INTO tally_company_mapping(client_id, registration_id, registration_type_code, registration_number,
-                       client_id, registration_id, registration_type_code,
+                       registration_legal_name, registration_trade_name, business_unit_id, client_branch_id,
-                       registration_number, registration_legal_name,
+                       tally_guid, company_name, gstin, is_active, created_at_utc, updated_at_utc, mapped_by_user_id)
-                       registration_trade_name, business_unit_id, client_branch_id,
+                   VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?, ?, ?)""",
-                       tally_guid, company_name, gstin, is_active,
+                (int(client_id), registration_id, str(registration.get("registration_type_code") or ""), str(registration.get("registration_number") or ""), str(registration.get("legal_name") or ""), str(registration.get("trade_name") or ""), registration.get("business_unit_id"), registration.get("client_branch_id"), guid, name, str(gstin or "").strip().upper(), now, now, mapped_by_user_id),
-                       created_at_utc, updated_at_utc, mapped_by_user_id
-                   ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?, ?, ?)""",
-                (
-                    int(client_id),
-                    registration_id,
-                    str(registration.get("registration_type_code") or ""),
-                    str(registration.get("registration_number") or ""),
-                    str(registration.get("legal_name") or ""),
-                    str(registration.get("trade_name") or ""),
-                    registration.get("business_unit_id"),
-                    registration.get("client_branch_id"),
-                    guid,
-                    name,
-                    str(gstin or "").strip().upper(),
-                    now,
-                    now,
-                    mapped_by_user_id,
-                ),
             )
             mapping_id = int(cur.lastrowid)
         return self.get_mapping(client_id, mapping_id)
 
     def get_mapping(self, client_id: int, mapping_id: int) -> dict:
         with self.connect(client_id) as db:
-            row = db.execute(
+            row = db.execute("SELECT * FROM tally_company_mapping WHERE id=? AND client_id=? LIMIT 1", (int(mapping_id), int(client_id))).fetchone()
-                "SELECT * FROM tally_company_mapping WHERE id=? AND client_id=? LIMIT 1",
-                (int(mapping_id), int(client_id)),
-            ).fetchone()
         if not row:
             raise ValueError("Tally company mapping was not found.")
         return dict(row)
 
+    def get_active_mapping_by_guid(self, client_id: int, tally_guid: str) -> dict:
+        with self.connect(client_id) as db:
+            row = db.execute(
+                "SELECT * FROM tally_company_mapping WHERE client_id=? AND tally_guid=? AND is_active=1 ORDER BY id DESC LIMIT 1",
+                (int(client_id), str(tally_guid or "").strip()),
+            ).fetchone()
+        if not row:
+            raise ValueError("The selected open Tally company is not mapped to this ERP client. Complete Phase 2 mapping first.")
+        return dict(row)
+
     def unmap_company(self, client_id: int, mapping_id: int, unmapped_by_user_id: int | None = None) -> dict:
         if not self.exists(client_id):
             raise ValueError("Accounting storage is not initialized for this client.")
         now = _utc_now_iso()
         with self.connect(client_id) as db:
-            row = db.execute(
+            row = db.execute("SELECT id FROM tally_company_mapping WHERE id=? AND client_id=? AND is_active=1", (int(mapping_id), int(client_id))).fetchone()
-                "SELECT id FROM tally_company_mapping WHERE id=? AND client_id=? AND is_active=1",
-                (int(mapping_id), int(client_id)),
-            ).fetchone()
             if not row:
                 raise ValueError("Active Tally company mapping was not found.")
-            db.execute(
+            db.execute("UPDATE tally_company_mapping SET is_active=0, updated_at_utc=?, unmapped_at_utc=?, unmapped_by_user_id=? WHERE id=? AND client_id=?", (now, now, unmapped_by_user_id, int(mapping_id), int(client_id)))
-                """UPDATE tally_company_mapping
-                   SET is_active=0, updated_at_utc=?, unmapped_at_utc=?, unmapped_by_user_id=?
-                   WHERE id=? AND client_id=?""",
-                (now, now, unmapped_by_user_id, int(mapping_id), int(client_id)),
-            )
         return {"mapping_id": int(mapping_id), "unmapped": True}
 
+    def replace_master_snapshot(self, client_id: int, *, mapping: dict, masters: dict[str, list[dict]], requested_by_user_id: int | None = None) -> dict:
+        if not self.exists(client_id):
+            raise ValueError("Accounting storage is not initialized for this client.")
+        tally_guid = str(mapping.get("tally_guid") or "").strip()
+        company_name = str(mapping.get("company_name") or "").strip()
+        mapping_id = int(mapping.get("id"))
+        if not tally_guid or not company_name:
+            raise ValueError("Active Tally mapping is incomplete.")
+        started = _utc_now_iso()
+        counts: dict[str, int] = {}
+        run_id = None
+        try:
+            with self.connect(client_id) as db:
+                cur = db.execute(
+                    """INSERT INTO tally_sync_runs(sync_type, tally_guid, company_name, mapping_id, requested_by_user_id, status, started_at_utc, rows_processed)
+                       VALUES ('masters', ?, ?, ?, ?, 'running', ?, 0)""",
+                    (tally_guid, company_name, mapping_id, requested_by_user_id, started),
+                )
+                run_id = int(cur.lastrowid)
+                total = 0
+                synced_at = _utc_now_iso()
+                for key, table in MASTER_TABLES.items():
+                    rows = masters.get(key) or []
+                    db.execute(f"DELETE FROM {table} WHERE tally_guid=?", (tally_guid,))
+                    for row in rows:
+                        payload = json.dumps(row, ensure_ascii=False, separators=(",", ":"))
+                        db.execute(
+                            f"""INSERT INTO {table}(
+                                tally_guid, company_name, master_guid, name, parent, category, reserved_name,
+                                base_units, additional_units, original_name, opening_balance, closing_balance,
+                                opening_value, opening_rate, numbering_method, tax_type, gst_applicable,
+                                gst_registration_type, gst_type_of_supply, hsn_code, is_revenue,
+                                is_deemed_positive, is_billwise_on, is_simple_unit, conversion, is_active,
+                                synced_at_utc, payload_json
+                            ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
+                            (tally_guid, company_name, str(row.get("guid") or ""), str(row.get("name") or ""), str(row.get("parent") or ""), str(row.get("category") or ""), str(row.get("reserved_name") or ""), str(row.get("base_units") or ""), str(row.get("additional_units") or ""), str(row.get("original_name") or ""), float(row.get("opening_balance") or 0), float(row.get("closing_balance") or 0), float(row.get("opening_value") or 0), str(row.get("opening_rate") or ""), str(row.get("numbering_method") or ""), str(row.get("tax_type") or ""), str(row.get("gst_applicable") or ""), str(row.get("gst_registration_type") or ""), str(row.get("gst_type_of_supply") or ""), str(row.get("hsn_code") or ""), str(row.get("is_revenue") or ""), str(row.get("is_deemed_positive") or ""), str(row.get("is_billwise_on") or ""), str(row.get("is_simple_unit") or ""), str(row.get("conversion") or ""), str(row.get("is_active") or ""), synced_at, payload),
+                        )
+                    counts[key] = len(rows)
+                    total += len(rows)
+                details = {"counts": counts, "schema_version": SCHEMA_VERSION}
+                db.execute(
+                    "UPDATE tally_sync_runs SET status='completed', completed_at_utc=?, rows_processed=?, details_json=? WHERE id=?",
+                    (_utc_now_iso(), total, json.dumps(details, ensure_ascii=False, separators=(",", ":")), run_id),
+                )
+            return {"sync_run_id": run_id, "status": "completed", "company_name": company_name, "tally_guid": tally_guid, "counts": counts, "rows_processed": sum(counts.values())}
+        except Exception as exc:
+            if run_id is not None:
+                try:
+                    with self.connect(client_id) as db:
+                        db.execute("UPDATE tally_sync_runs SET status='failed', completed_at_utc=?, error_message=? WHERE id=?", (_utc_now_iso(), str(exc), run_id))
+                except Exception:
+                    pass
+            raise
+
+    def _master_counts(self, db: sqlite3.Connection, tally_guid: str) -> dict[str, int]:
+        result = {}
+        for key, table in MASTER_TABLES.items():
+            result[key] = int(db.execute(f"SELECT COUNT(*) FROM {table} WHERE tally_guid=?", (tally_guid,)).fetchone()[0])
+        return result
+
     def snapshot(self, client_id: int) -> dict:
         path = self.db_path(client_id)
         if not path.is_file():
-            return {
+            return {"exists": False, "db_path": str(path), "metadata": {}, "latest_connection": None, "companies": [], "mappings": [], "latest_master_sync": None}
-                "exists": False,
-                "db_path": str(path),
-                "metadata": {},
-                "latest_connection": None,
-                "companies": [],
-                "mappings": [],
-            }
-
-        # initialize() upgrades older Phase 1 schema in-place
         self.initialize(client_id)
         with self.connect(client_id) as db:
             meta_rows = db.execute("SELECT key, value FROM act_meta ORDER BY key").fetchall()
-            companies = db.execute(
+            companies = db.execute("SELECT tally_guid AS guid, company_name AS name, gstin, first_seen_at_utc, last_seen_at_utc, is_currently_loaded FROM tally_companies ORDER BY is_currently_loaded DESC, company_name COLLATE NOCASE").fetchall()
-                """SELECT tally_guid AS guid, company_name AS name, gstin,
-                          first_seen_at_utc, last_seen_at_utc, is_currently_loaded
-                   FROM tally_companies
-                   ORDER BY is_currently_loaded DESC, company_name COLLATE NOCASE"""
-            ).fetchall()
             mappings = db.execute(
-                """SELECT id, client_id, registration_id, registration_type_code,
+                """SELECT id, client_id, registration_id, registration_type_code, registration_number,
-                          registration_number, registration_legal_name,
+                          registration_legal_name, registration_trade_name, business_unit_id, client_branch_id,
-                          registration_trade_name, business_unit_id, client_branch_id,
+                          tally_guid, company_name, gstin, is_active, created_at_utc AS mapped_at_utc,
-                          tally_guid, company_name, gstin, is_active,
+                          updated_at_utc, mapped_by_user_id, unmapped_at_utc, unmapped_by_user_id
-                          created_at_utc AS mapped_at_utc, updated_at_utc,
+                   FROM tally_company_mapping WHERE is_active=1
-                          mapped_by_user_id, unmapped_at_utc, unmapped_by_user_id
+                   ORDER BY CASE WHEN registration_id IS NULL THEN 0 ELSE 1 END, registration_type_code, registration_number, id"""
-                   FROM tally_company_mapping
-                   WHERE is_active=1
-                   ORDER BY CASE WHEN registration_id IS NULL THEN 0 ELSE 1 END,
-                            registration_type_code, registration_number, id"""
             ).fetchall()
-            latest = db.execute(
+            latest = db.execute("SELECT checked_at_utc, connected, tally_url, company_count, error_message FROM tally_connection_history ORDER BY id DESC LIMIT 1").fetchone()
-                "SELECT checked_at_utc, connected, tally_url, company_count, error_message FROM tally_connection_history ORDER BY id DESC LIMIT 1"
+            latest_sync = db.execute("SELECT id, sync_type, tally_guid, company_name, mapping_id, requested_by_user_id, status, started_at_utc, completed_at_utc, rows_processed, error_message, details_json FROM tally_sync_runs WHERE sync_type='masters' ORDER BY id DESC LIMIT 1").fetchone()
-            ).fetchone()
-
             loaded_guids = {str(row["guid"] or "") for row in companies if int(row["is_currently_loaded"] or 0)}
             mapped = []
             for row in mappings:
                 item = dict(row)
                 item["currently_loaded"] = bool(item.get("tally_guid") and item["tally_guid"] in loaded_guids)
+                item["master_counts"] = self._master_counts(db, str(item.get("tally_guid") or ""))
                 mapped.append(item)
+        latest_sync_dict = dict(latest_sync) if latest_sync else None
+        if latest_sync_dict and latest_sync_dict.get("details_json"):
+            try:
+                latest_sync_dict["details"] = json.loads(latest_sync_dict["details_json"])
+            except Exception:
+                latest_sync_dict["details"] = {}
         return {
             "exists": True,
             "db_path": str(path),
@@ -357,4 +381,5 @@ class LocalAccountingStore:
             "latest_connection": dict(latest) if latest else None,
             "companies": [dict(row) for row in companies],
             "mappings": mapped,
+            "latest_master_sync": latest_sync_dict,
         }

app/modules/documents/local_agent_runtime/erp_local_agent/commands.py

@@ -23,7 +23,7 @@ class AgentCommandProcessor:
         error: str | None = None
         ok = False
         try:
-            if action in {"tally_status", "phase1_status", "phase2_status"}:
+            if action in {"tally_status", "phase1_status", "phase2_status", "phase3_status"}:
                 result = self._status(payload)
             elif action == "accounting_initialize":
                 result = self._initialize(payload)
@@ -31,28 +31,21 @@ class AgentCommandProcessor:
                 result = self._map_company(payload)
             elif action == "accounting_unmap_company":
                 result = self._unmap_company(payload)
+            elif action == "accounting_sync_masters":
+                result = self._sync_masters(payload)
             else:
                 raise ValueError(f"Unsupported local-agent command: {action}")
             ok = True
         except Exception as exc:
             error = str(exc)
             self.logger.exception("Agent command failed action=%s command_id=%s: %s", action, command_id, exc)
-        return {
+        return {"type": "command_result", "command_id": command_id, "ok": ok, "result": result, "error": error, "agent_time_utc": datetime.now(timezone.utc).isoformat()}
-            "type": "command_result",
-            "command_id": command_id,
-            "ok": ok,
-            "result": result,
-            "error": error,
-            "agent_time_utc": datetime.now(timezone.utc).isoformat(),
-        }
 
     def _agent_info(self) -> dict[str, Any]:
         return {
-            "name": "ERP Local Agent",
+            "name": "ERP Local Agent", "version": __version__,
-            "version": __version__,
+            "tally_capability": True, "accounting_act_capability": True,
-            "tally_capability": True,
+            "tally_mapping_capability": True, "tally_master_sync_capability": True,
-            "accounting_act_capability": True,
-            "tally_mapping_capability": True,
         }
 
     def _status(self, payload: dict[str, Any]) -> dict[str, Any]:
@@ -64,32 +57,17 @@ class AgentCommandProcessor:
             if self.store.exists(client_id):
                 self.store.record_tally_status(client_id, tally_status)
             accounting = self.store.snapshot(client_id)
-        return {
+        return {"agent": self._agent_info(), "tally": tally_status, "accounting": accounting}
-            "agent": self._agent_info(),
-            "tally": tally_status,
-            "accounting": accounting,
-        }
 
     def _initialize(self, payload: dict[str, Any]) -> dict[str, Any]:
         client_id = int(payload.get("client_id"))
         client_name = str(payload.get("client_name") or "").strip()
         tenant_id = payload.get("tenant_id")
         created_by_user_id = payload.get("requested_by_user_id")
-        path = self.store.initialize(
+        path = self.store.initialize(client_id, client_name, int(tenant_id) if tenant_id not in (None, "") else None, int(created_by_user_id) if created_by_user_id not in (None, "") else None)
-            client_id,
-            client_name,
-            int(tenant_id) if tenant_id not in (None, "") else None,
-            int(created_by_user_id) if created_by_user_id not in (None, "") else None,
-        )
         tally_status = self.tally.status()
         self.store.record_tally_status(client_id, tally_status)
-        return {
+        return {"initialized": True, "db_path": str(path), "accounting": self.store.snapshot(client_id), "tally": tally_status, "agent": self._agent_info()}
-            "initialized": True,
-            "db_path": str(path),
-            "accounting": self.store.snapshot(client_id),
-            "tally": tally_status,
-            "agent": self._agent_info(),
-        }
 
     def _map_company(self, payload: dict[str, Any]) -> dict[str, Any]:
         client_id = int(payload.get("client_id"))
@@ -99,74 +77,54 @@ class AgentCommandProcessor:
         requested_guid = str(payload.get("tally_guid") or "").strip()
         registration = payload.get("registration") or None
         allow_gstin_mismatch = bool(payload.get("allow_gstin_mismatch"))
-
         if not requested_guid:
             raise ValueError("Select a Tally company before mapping.")
-
         if not self.store.exists(client_id):
-            self.store.initialize(
+            self.store.initialize(client_id, client_name, int(tenant_id) if tenant_id not in (None, "") else None, int(mapped_by_user_id) if mapped_by_user_id not in (None, "") else None)
-                client_id,
-                client_name,
-                int(tenant_id) if tenant_id not in (None, "") else None,
-                int(mapped_by_user_id) if mapped_by_user_id not in (None, "") else None,
-            )
-
         tally_status = self.tally.status()
         if not tally_status.get("connected"):
             raise ValueError(str(tally_status.get("error") or "TallyPrime is not connected."))
-
+        company = next((row for row in (tally_status.get("companies") or []) if str(row.get("guid") or "").strip() == requested_guid), None)
-        companies = tally_status.get("companies") or []
-        company = next(
-            (row for row in companies if str(row.get("guid") or "").strip() == requested_guid),
-            None,
-        )
         if not company:
             raise ValueError("The selected Tally company is no longer loaded. Refresh Tally companies and try again.")
-
         company_name = str(company.get("name") or "").strip()
         company_gstin = str(company.get("gstin") or "").strip().upper()
         if not company_name:
             raise ValueError("Tally returned an invalid company name.")
-        if not requested_guid:
-            raise ValueError("Tally returned no GUID. Permanent mapping requires a Tally GUID.")
-
         if registration:
             reg_type = str(registration.get("registration_type_code") or "").strip().upper()
             reg_number = str(registration.get("registration_number") or "").strip().upper()
             if reg_type == "GSTIN" and reg_number and company_gstin and reg_number != company_gstin and not allow_gstin_mismatch:
-                raise ValueError(
+                raise ValueError(f"GSTIN mismatch: ERP registration is {reg_number}, but Tally company reports {company_gstin}. Verify the company or explicitly allow the mismatch.")
-                    f"GSTIN mismatch: ERP registration is {reg_number}, but Tally company reports {company_gstin}. "
-                    "Verify the company or explicitly allow the mismatch."
-                )
-
         self.store.record_tally_status(client_id, tally_status)
-        mapping = self.store.map_company(
+        mapping = self.store.map_company(client_id, tally_guid=requested_guid, company_name=company_name, gstin=company_gstin, registration=registration, mapped_by_user_id=int(mapped_by_user_id) if mapped_by_user_id not in (None, "") else None)
-            client_id,
+        return {"mapped": True, "mapping": mapping, "accounting": self.store.snapshot(client_id), "tally": tally_status, "agent": self._agent_info()}
-            tally_guid=requested_guid,
-            company_name=company_name,
-            gstin=company_gstin,
-            registration=registration,
-            mapped_by_user_id=int(mapped_by_user_id) if mapped_by_user_id not in (None, "") else None,
-        )
-        return {
-            "mapped": True,
-            "mapping": mapping,
-            "accounting": self.store.snapshot(client_id),
-            "tally": tally_status,
-            "agent": self._agent_info(),
-        }
 
     def _unmap_company(self, payload: dict[str, Any]) -> dict[str, Any]:
         client_id = int(payload.get("client_id"))
         mapping_id = int(payload.get("mapping_id"))
         unmapped_by_user_id = payload.get("unmapped_by_user_id")
-        result = self.store.unmap_company(
+        result = self.store.unmap_company(client_id, mapping_id, int(unmapped_by_user_id) if unmapped_by_user_id not in (None, "") else None)
-            client_id,
+        return {**result, "accounting": self.store.snapshot(client_id), "agent": self._agent_info()}
-            mapping_id,
+
-            int(unmapped_by_user_id) if unmapped_by_user_id not in (None, "") else None,
+    def _sync_masters(self, payload: dict[str, Any]) -> dict[str, Any]:
-        )
+        client_id = int(payload.get("client_id"))
-        return {
+        requested_guid = str(payload.get("tally_guid") or "").strip()
-            **result,
+        requested_by_user_id = payload.get("requested_by_user_id")
-            "accounting": self.store.snapshot(client_id),
+        if not requested_guid:
-            "agent": self._agent_info(),
+            raise ValueError("Select a currently open Tally company before synchronizing masters.")
-        }
+        if not self.store.exists(client_id):
+            raise ValueError("Accounting storage is not initialized for this client.")
+        mapping = self.store.get_active_mapping_by_guid(client_id, requested_guid)
+        tally_status = self.tally.status()
+        if not tally_status.get("connected"):
+            raise ValueError(str(tally_status.get("error") or "TallyPrime is not connected."))
+        company = next((row for row in (tally_status.get("companies") or []) if str(row.get("guid") or "").strip() == requested_guid), None)
+        if not company:
+            raise ValueError("The selected mapped Tally company is not currently open in TallyPrime. Open it in Tally and refresh the ERP page.")
+        company_name = str(company.get("name") or mapping.get("company_name") or "").strip()
+        masters = self.tally.fetch_accounting_masters(company_name)
+        self.store.record_tally_status(client_id, tally_status)
+        sync = self.store.replace_master_snapshot(client_id, mapping={**mapping, "company_name": company_name}, masters=masters, requested_by_user_id=int(requested_by_user_id) if requested_by_user_id not in (None, "") else None)
+        self.logger.info("Tally master sync completed client_id=%s company=%s rows=%s", client_id, company_name, sync.get("rows_processed"))
+        return {"synced": True, "sync": sync, "accounting": self.store.snapshot(client_id), "tally": tally_status, "agent": self._agent_info()}

app/modules/documents/local_agent_runtime/erp_local_agent/tally.py

@@ -6,11 +6,12 @@ import re
 import urllib.error
 import urllib.request
 import xml.etree.ElementTree as ET
+from typing import Iterable
 
 
 DEFAULT_TALLY_HOST = "127.0.0.1"
 DEFAULT_TALLY_PORT = 9000
-DEFAULT_TALLY_TIMEOUT_SECONDS = 8
+DEFAULT_TALLY_TIMEOUT_SECONDS = 20
 
 
 class TallyConnectionError(RuntimeError):
@@ -24,14 +25,35 @@ def _clean_xml_response(xml_text: str) -> str:
     return re.sub(r"[\x00-\x08\x0B\x0C\x0E-\x1F]", "", xml_text)
 
 
+def _tag(element: ET.Element) -> str:
+    return str(element.tag).split("}")[-1].upper()
+
+
 def _child_text(element: ET.Element, tag_name: str) -> str:
     wanted = tag_name.upper()
     for child in list(element):
-        if str(child.tag).split("}")[-1].upper() == wanted:
+        if _tag(child) == wanted:
             return (child.text or "").strip()
     return ""
 
 
+def _first_text(element: ET.Element, names: Iterable[str]) -> str:
+    wanted = {str(name).upper() for name in names}
+    for node in element.iter():
+        if _tag(node) in wanted and node.text:
+            return node.text.strip()
+    return ""
+
+
+def _to_number(value: str) -> float:
+    text = str(value or "").replace(",", "").strip()
+    text = re.sub(r"[^0-9.\-]", "", text)
+    try:
+        return float(text or 0)
+    except Exception:
+        return 0.0
+
+
 @dataclass(frozen=True)
 class TallyCompany:
     name: str
@@ -43,7 +65,69 @@ class TallyCompany:
 
 
 class TallyLiveConnector:
-    """Read-only TallyPrime XML/HTTP discovery connector."""
+    """Read-only TallyPrime XML/HTTP connector used by the ERP Local Agent.
+
+    Phase 3 adds master discovery for the currently loaded Tally companies. Every
+    master export is scoped with SVCURRENTCOMPANY and does not create or modify
+    any Tally data.
+    """
+
+    MASTER_SPECS = {
+        "groups": {
+            "collection": "ARRRAccountingGroups",
+            "type": "Group",
+            "tag": "GROUP",
+            "fetch": "Name,GUID,Parent,ReservedName,IsRevenue,IsDeemedPositive",
+        },
+        "ledgers": {
+            "collection": "ARRRAccountingLedgers",
+            "type": "Ledger",
+            "tag": "LEDGER",
+            "fetch": "Name,GUID,Parent,OpeningBalance,ClosingBalance,IsBillWiseOn,TaxType,GSTApplicable,GSTRegistrationType",
+        },
+        "voucher_types": {
+            "collection": "ARRRAccountingVoucherTypes",
+            "type": "VoucherType",
+            "tag": "VOUCHERTYPE",
+            "fetch": "Name,GUID,Parent,NumberingMethod,IsDeemedPositive,IsActive",
+        },
+        "stock_groups": {
+            "collection": "ARRRAccountingStockGroups",
+            "type": "StockGroup",
+            "tag": "STOCKGROUP",
+            "fetch": "Name,GUID,Parent,BaseUnits,IsAddable",
+        },
+        "stock_categories": {
+            "collection": "ARRRAccountingStockCategories",
+            "type": "StockCategory",
+            "tag": "STOCKCATEGORY",
+            "fetch": "Name,GUID,Parent",
+        },
+        "stock_items": {
+            "collection": "ARRRAccountingStockItems",
+            "type": "StockItem",
+            "tag": "STOCKITEM",
+            "fetch": "Name,GUID,Parent,Category,BaseUnits,AdditionalUnits,OpeningBalance,OpeningValue,OpeningRate,GSTApplicable,GSTTypeOfSupply,HSNCode",
+        },
+        "units": {
+            "collection": "ARRRAccountingUnits",
+            "type": "Unit",
+            "tag": "UNIT",
+            "fetch": "Name,GUID,OriginalName,IsSimpleUnit,BaseUnits,AdditionalUnits,Conversion",
+        },
+        "cost_categories": {
+            "collection": "ARRRAccountingCostCategories",
+            "type": "CostCategory",
+            "tag": "COSTCATEGORY",
+            "fetch": "Name,GUID,IsRevenue,IsNonRevenue",
+        },
+        "cost_centres": {
+            "collection": "ARRRAccountingCostCentres",
+            "type": "CostCentre",
+            "tag": "COSTCENTRE",
+            "fetch": "Name,GUID,Parent,Category",
+        },
+    }
 
     def __init__(self, host: str = DEFAULT_TALLY_HOST, port: int = DEFAULT_TALLY_PORT, timeout: int = DEFAULT_TALLY_TIMEOUT_SECONDS):
         self.host = (host or DEFAULT_TALLY_HOST).strip()
@@ -54,6 +138,10 @@ class TallyLiveConnector:
     def url(self) -> str:
         return f"http://{self.host}:{self.port}"
 
+    @staticmethod
+    def _xml_escape(value: str) -> str:
+        return html.escape(str(value or ""), quote=False)
+
     def _post_xml(self, xml_text: str) -> str:
         request = urllib.request.Request(
             self.url,
@@ -69,20 +157,23 @@ class TallyLiveConnector:
                 f"Could not connect to TallyPrime at {self.url}. Open TallyPrime and ensure its HTTP/XML server is available on port {self.port}. Details: {exc}"
             ) from exc
 
+    def _static_variables(self, company_name: str = "") -> str:
+        base = "<SVEXPORTFORMAT>$$SysName:XML</SVEXPORTFORMAT>"
+        if not company_name:
+            return base
+        return base + f"<SVCURRENTCOMPANY>{self._xml_escape(company_name)}</SVCURRENTCOMPANY>"
+
     def get_loaded_companies(self) -> list[TallyCompany]:
         xml = """<ENVELOPE>
   <HEADER>
-    <VERSION>1</VERSION>
+    <VERSION>1</VERSION><TALLYREQUEST>Export</TALLYREQUEST><TYPE>Collection</TYPE><ID>ARRRAccountingLoadedCompanies</ID>
-    <TALLYREQUEST>Export</TALLYREQUEST>
-    <TYPE>Collection</TYPE>
-    <ID>ARRRAccountingLoadedCompanies</ID>
   </HEADER>
-  <BODY>
+  <BODY><DESC>
-    <DESC>
     <STATICVARIABLES><SVEXPORTFORMAT>$$SysName:XML</SVEXPORTFORMAT></STATICVARIABLES>
-      <TDL><TDLMESSAGE><COLLECTION NAME="ARRRAccountingLoadedCompanies" ISMODIFY="No"><TYPE>Company</TYPE><FETCH>Name,GUID,GSTRegistrationNumber</FETCH></COLLECTION></TDLMESSAGE></TDL>
+    <TDL><TDLMESSAGE>
-    </DESC>
+      <COLLECTION NAME="ARRRAccountingLoadedCompanies" ISMODIFY="No"><TYPE>Company</TYPE><FETCH>Name,GUID,GSTRegistrationNumber</FETCH></COLLECTION>
-  </BODY>
+    </TDLMESSAGE></TDL>
+  </DESC></BODY>
 </ENVELOPE>"""
         return self._parse_companies(self._post_xml(xml))
 
@@ -109,6 +200,87 @@ class TallyLiveConnector:
                 "error": str(exc),
             }
 
+    def export_master_collection(self, company_name: str, master_key: str) -> list[dict]:
+        key = str(master_key or "").strip()
+        spec = self.MASTER_SPECS.get(key)
+        if not spec:
+            raise ValueError(f"Unsupported Tally master collection: {key}")
+        if not str(company_name or "").strip():
+            raise ValueError("Tally company name is required for master sync.")
+
+        xml = f"""<ENVELOPE>
+  <HEADER><VERSION>1</VERSION><TALLYREQUEST>Export</TALLYREQUEST><TYPE>Collection</TYPE><ID>{spec['collection']}</ID></HEADER>
+  <BODY><DESC>
+    <STATICVARIABLES>{self._static_variables(company_name)}</STATICVARIABLES>
+    <TDL><TDLMESSAGE>
+      <COLLECTION NAME="{spec['collection']}" ISMODIFY="No"><TYPE>{spec['type']}</TYPE><FETCH>{spec['fetch']}</FETCH></COLLECTION>
+    </TDLMESSAGE></TDL>
+  </DESC></BODY>
+</ENVELOPE>"""
+        raw = self._post_xml(xml)
+        return self._parse_master_rows(raw, spec["tag"])
+
+    def fetch_accounting_masters(self, company_name: str) -> dict[str, list[dict]]:
+        result: dict[str, list[dict]] = {}
+        for key in self.MASTER_SPECS:
+            result[key] = self.export_master_collection(company_name, key)
+        return result
+
+    @staticmethod
+    def _parse_master_rows(xml_text: str, expected_tag: str) -> list[dict]:
+        cleaned = _clean_xml_response(xml_text)
+        if not cleaned.strip():
+            return []
+        try:
+            root = ET.fromstring(cleaned.encode("utf-8"))
+        except Exception as exc:
+            raise ValueError(f"Tally returned invalid XML while reading {expected_tag} masters: {exc}") from exc
+
+        rows: list[dict] = []
+        seen: set[tuple[str, str]] = set()
+        for element in root.iter():
+            if _tag(element) != expected_tag.upper():
+                continue
+            name = (
+                element.attrib.get("NAME")
+                or element.attrib.get("name")
+                or _child_text(element, "NAME")
+            ).strip()
+            guid = _child_text(element, "GUID")
+            if not name:
+                continue
+            identity = (guid.upper(), name.upper())
+            if identity in seen:
+                continue
+            seen.add(identity)
+            rows.append({
+                "name": name,
+                "guid": guid,
+                "parent": _child_text(element, "PARENT"),
+                "reserved_name": _child_text(element, "RESERVEDNAME"),
+                "category": _child_text(element, "CATEGORY"),
+                "base_units": _child_text(element, "BASEUNITS"),
+                "additional_units": _child_text(element, "ADDITIONALUNITS"),
+                "original_name": _child_text(element, "ORIGINALNAME"),
+                "opening_balance": _to_number(_child_text(element, "OPENINGBALANCE")),
+                "closing_balance": _to_number(_child_text(element, "CLOSINGBALANCE")),
+                "opening_value": _to_number(_child_text(element, "OPENINGVALUE")),
+                "opening_rate": _child_text(element, "OPENINGRATE"),
+                "numbering_method": _child_text(element, "NUMBERINGMETHOD"),
+                "tax_type": _child_text(element, "TAXTYPE"),
+                "gst_applicable": _child_text(element, "GSTAPPLICABLE"),
+                "gst_registration_type": _child_text(element, "GSTREGISTRATIONTYPE"),
+                "gst_type_of_supply": _child_text(element, "GSTTYPEOFSUPPLY"),
+                "hsn_code": _first_text(element, ("HSNCODE", "GSTHSNCODE")),
+                "is_revenue": _child_text(element, "ISREVENUE"),
+                "is_deemed_positive": _child_text(element, "ISDEEMEDPOSITIVE"),
+                "is_billwise_on": _child_text(element, "ISBILLWISEON"),
+                "is_simple_unit": _child_text(element, "ISSIMPLEUNIT"),
+                "conversion": _child_text(element, "CONVERSION"),
+                "is_active": _child_text(element, "ISACTIVE"),
+            })
+        return rows
+
     @staticmethod
     def _parse_companies(xml_text: str) -> list[TallyCompany]:
         cleaned = _clean_xml_response(xml_text)
@@ -127,7 +299,7 @@ class TallyLiveConnector:
         companies: list[TallyCompany] = []
         seen: set[str] = set()
         for element in root.iter():
-            if str(element.tag).split("}")[-1].upper() != "COMPANY":
+            if _tag(element) != "COMPANY":
                 continue
             name = (
                 element.attrib.get("NAME")