From 3ad20087be8e73fe7dbf8166ed45aab91457da43 Mon Sep 17 00:00:00 2001 From: A R R R Associates Date: Fri, 4 Sep 2026 15:30:39 +0530 Subject: [PATCH] Fix opening balance scope and add async cash ledger loading --- app/modules/accounting/cash_payment_ui.py | 48 +++- .../accounting/opening_balance_service.py | 31 ++- app/modules/accounting/opening_balance_ui.py | 101 ++++++- .../accounting/cash_payment_review.html | 87 +++++- .../accounting/opening_balances.html | 6 +- app/modules/documents/agent_package.py | 2 +- .../erp_local_agent/__init__.py | 2 +- .../erp_local_agent/commands.py | 263 +++++++++++++++++- .../erp_local_agent/dashboard.py | 2 +- 9 files changed, 508 insertions(+), 34 deletions(-) diff --git a/app/modules/accounting/cash_payment_ui.py b/app/modules/accounting/cash_payment_ui.py index e7b594f..9bb7a1d 100644 --- a/app/modules/accounting/cash_payment_ui.py +++ b/app/modules/accounting/cash_payment_ui.py @@ -252,6 +252,7 @@ def cash_payment_review( @router.post("/ledgers") async def cash_payment_ledgers(request: Request): + """Return cached ledgers immediately or start a non-blocking Tally master job.""" form = await request.form() validate_csrf(request, str(form.get("csrf_token") or "")) db = CommonSessionLocal() @@ -269,6 +270,7 @@ async def cash_payment_ledgers(request: Request): return JSONResponse({"ok": False, "error": "Select a valid client."}, status_code=400) tally_guid = str(form.get("tally_guid") or "").strip() date_from = str(form.get("date_from") or date.today().isoformat()).strip() + force_refresh = str(form.get("force_refresh") or "").strip().lower() in {"1","true","yes","on"} if not tally_guid: return JSONResponse({"ok": False, "error": "Select a mapped Tally company."}, status_code=400) node = get_active_storage_node_for_branch(db, scope.tenant_id, scope.branch_id) @@ -278,14 +280,52 @@ async def cash_payment_ledgers(request: Request): fy = _financial_year_for_date(date.fromisoformat(date_from)) result = request_agent_command( node.node_code, - "accounting_cash_payment_ledgers", - {**_accounting_storage_payload(selected_client, fy), "tally_guid": tally_guid}, - timeout_seconds=45, + "accounting_cash_payment_ledgers_start", + { + **_accounting_storage_payload(selected_client, fy), + "tally_guid": tally_guid, + "force_refresh": force_refresh, + }, + timeout_seconds=20, ) except Exception as exc: return JSONResponse({"ok": False, "error": str(exc)}, status_code=502) if not result.get("ok"): - return JSONResponse({"ok": False, "error": str(result.get("error") or "Could not read Tally ledgers.")}, status_code=409) + return JSONResponse({"ok": False, "error": str(result.get("error") or "Could not start Tally ledger sync.")}, status_code=409) + return JSONResponse({"ok": True, **(result.get("result") or {})}) + finally: + db.close() + + +@router.get("/ledgers/progress") +def cash_payment_ledgers_progress(request: Request, client_id: int, job_id: str, date_from: str): + db = CommonSessionLocal() + try: + user, response = _require_partner(request, db, "accounting.tally.view") + if response: + return JSONResponse({"ok": False, "error": "Access denied."}, status_code=403) + clients, scope = _visible_clients(db, request, user) + selected_client = next((row for row in clients if int(row.id) == int(client_id)), None) + if not selected_client: + return JSONResponse({"ok": False, "error": "Client not found."}, status_code=404) + node = get_active_storage_node_for_branch(db, scope.tenant_id, scope.branch_id) + if not node or not _node_online(node): + return JSONResponse({"ok": False, "error": "ERP Local Agent is offline."}, status_code=409) + try: + fy = _financial_year_for_date(date.fromisoformat(date_from)) + result = request_agent_command( + node.node_code, + "accounting_cash_payment_ledgers_status", + { + **_accounting_storage_payload(selected_client, fy), + "job_id": str(job_id), + }, + timeout_seconds=15, + ) + except Exception as exc: + return JSONResponse({"ok": False, "error": str(exc)}, status_code=502) + if not result.get("ok"): + return JSONResponse({"ok": False, "error": str(result.get("error") or "Could not read ledger-sync progress.")}, status_code=409) return JSONResponse({"ok": True, **(result.get("result") or {})}) finally: db.close() diff --git a/app/modules/accounting/opening_balance_service.py b/app/modules/accounting/opening_balance_service.py index 106e88f..65d44bf 100644 --- a/app/modules/accounting/opening_balance_service.py +++ b/app/modules/accounting/opening_balance_service.py @@ -38,6 +38,33 @@ def _near(a, b, tolerance=0.01): return abs(float(a or 0) - float(b or 0)) <= tolerance + +def _is_balance_sheet_ledger(row): + """Return True only for ledger masters that belong to the Balance Sheet. + + Tally exports ISREVENUE for Ledger masters. Revenue ledgers (income, + expense, purchase and sales accounts) must not carry an opening balance + into a new financial year. Stock Items are handled separately below. + """ + flag = _s((row or {}).get("is_revenue")).casefold() + if flag in {"yes", "y", "true", "1"}: + return False + if flag in {"no", "n", "false", "0"}: + return True + + # Defensive fallback for older Tally responses that omitted ISREVENUE. + # Exclude only unmistakable P&L roots; retain Balance-Sheet masters. + parent = _norm((row or {}).get("parent")) + pnl_roots = ( + "direct expenses", "indirect expenses", "direct incomes", + "indirect incomes", "sales accounts", "purchase accounts", + ) + return parent not in pnl_roots + + +def _balance_sheet_ledgers(masters): + return [row for row in list((masters or {}).get("ledgers") or []) if _is_balance_sheet_ledger(row)] + def _mapping_index(db, *, tenant_id, client_id, master_type, previous_company_guid, current_company_guid): rows = list( db.execute( @@ -308,8 +335,8 @@ def create_comparison_run( ) ledger_matches = _match_ledgers( - list(previous_masters.get("ledgers") or []), - list(current_masters.get("ledgers") or []), + _balance_sheet_ledgers(previous_masters), + _balance_sheet_ledgers(current_masters), ledger_map, ) stock_matches = _match_stock( diff --git a/app/modules/accounting/opening_balance_ui.py b/app/modules/accounting/opening_balance_ui.py index 25a67ce..67b6877 100644 --- a/app/modules/accounting/opening_balance_ui.py +++ b/app/modules/accounting/opening_balance_ui.py @@ -1,10 +1,12 @@ from __future__ import annotations import json +from io import BytesIO from urllib.parse import urlencode +from datetime import datetime from fastapi import APIRouter, Form, Request -from fastapi.responses import RedirectResponse +from fastapi.responses import RedirectResponse, StreamingResponse from app.core.db.common import CommonSessionLocal from app.core.security.csrf import get_or_create_csrf_token, validate_csrf @@ -29,6 +31,8 @@ from app.modules.accounting.ui import ( ) from app.modules.core.rbac.deps import get_user_permissions, get_user_roles from app.modules.documents.services import get_active_storage_node_for_branch +from openpyxl import Workbook +from openpyxl.styles import Font, PatternFill, Alignment router = APIRouter( prefix="/tools/accounting/opening-balances", @@ -277,6 +281,101 @@ def page( db.close() + +@router.get("/download/{run_id}") +def download_report(request: Request, run_id: int, client_id: int | None = None): + """Download the complete opening-balance comparison as an Excel workbook.""" + db = CommonSessionLocal() + try: + user, denied = _require_partner(request, db, "accounting.tally.view") + if denied: + return denied + clients, scope = _visible_clients(db, request, user) + run = db.get(AccountingOpeningBalanceRun, int(run_id)) + if not run or run.tenant_id != scope.tenant_id: + return _go(error="Opening balance comparison run was not found.") + selected = next((row for row in clients if int(row.id) == int(run.client_id)), None) + if not selected or (client_id and int(client_id) != int(run.client_id)): + return _go(error="Opening balance comparison run is not visible.") + + ledger_rows = ledger_items(db, run_id=run.id) + stock_rows = stock_items(db, run_id=run.id) + wb = Workbook() + ws = wb.active + ws.title = "Balance Sheet Ledgers" + headers = [ + "Previous Ledger", "Previous Group", "PY Closing", "Current Ledger", + "Current Group", "CY Opening", "Difference", "Status", + "Match Method", "Confidence %", "Correction Status", "Correction Note", + ] + ws.append(headers) + for row in ledger_rows: + ws.append([ + row.previous_name or "", row.previous_group or "", float(row.previous_closing_balance or 0), + row.current_name or "", row.current_group or "", float(row.current_opening_balance or 0), + float(row.difference or 0), row.match_status or "", row.match_method or "", + int(row.confidence or 0), row.correction_status or "", row.correction_note or "", + ]) + + sws = wb.create_sheet("Stock Items") + stock_headers = [ + "Previous Stock Item", "Previous Group", "HSN", "Unit", "PY Closing Qty", + "PY Closing Value", "Current Stock Item", "Current Group", "Current HSN", + "Current Unit", "CY Opening Qty", "CY Opening Value", "Qty Difference", + "Value Difference", "Status", "Match Method", "Confidence %", + "Correction Status", "Correction Note", + ] + sws.append(stock_headers) + for row in stock_rows: + sws.append([ + row.previous_name or "", row.previous_group or "", row.previous_hsn or "", row.previous_unit or "", + float(row.previous_closing_qty or 0), float(row.previous_closing_value or 0), + row.current_name or "", row.current_group or "", row.current_hsn or "", row.current_unit or "", + float(row.current_opening_qty or 0), float(row.current_opening_value or 0), + float(row.quantity_difference or 0), float(row.value_difference or 0), + row.match_status or "", row.match_method or "", int(row.confidence or 0), + row.correction_status or "", row.correction_note or "", + ]) + + meta = wb.create_sheet("Summary", 0) + meta.append(["Opening Balance Review"]) + meta.append(["Client", getattr(selected, "client_name", "") or getattr(selected, "name", "") or str(selected.id)]) + meta.append(["Run ID", run.id]) + meta.append(["Previous Tally Company", run.previous_company_name]) + meta.append(["Current Tally Company", run.current_company_name]) + meta.append(["Scope", "Balance Sheet ledgers and Stock Items only"]) + meta.append(["Balance Sheet Ledger Rows", len(ledger_rows)]) + meta.append(["Stock Item Rows", len(stock_rows)]) + meta.append(["Generated", datetime.now().strftime("%Y-%m-%d %H:%M:%S")]) + + header_fill = PatternFill("solid", fgColor="DCE6F1") + for sheet in (ws, sws): + sheet.freeze_panes = "A2" + sheet.auto_filter.ref = sheet.dimensions + for cell in sheet[1]: + cell.font = Font(bold=True) + cell.fill = header_fill + cell.alignment = Alignment(vertical="center", wrap_text=True) + for col in sheet.columns: + letter = col[0].column_letter + width = min(38, max(10, max(len(str(c.value or "")) for c in col[:200]) + 2)) + sheet.column_dimensions[letter].width = width + meta["A1"].font = Font(bold=True, size=14) + meta.column_dimensions["A"].width = 28 + meta.column_dimensions["B"].width = 48 + + output = BytesIO() + wb.save(output) + output.seek(0) + filename = f"opening_balance_review_run_{run.id}.xlsx" + return StreamingResponse( + output, + media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + headers={"Content-Disposition": f'attachment; filename="{filename}"'}, + ) + finally: + db.close() + @router.post("/compare") def compare( request: Request, diff --git a/app/modules/accounting/templates/accounting/cash_payment_review.html b/app/modules/accounting/templates/accounting/cash_payment_review.html index fed7c77..c9a7d79 100644 --- a/app/modules/accounting/templates/accounting/cash_payment_review.html +++ b/app/modules/accounting/templates/accounting/cash_payment_review.html @@ -49,8 +49,13 @@ {% if selected_cash_ledger_name %}{% endif %} + + +

Loads cached Tally ledgers instantly when available. Refresh from Tally runs safely in the Local Agent background and shows progress here and on the Local Agent dashboard.

+ -

Reads the Tally ledger master and lets you confirm the exact Cash ledger.

@@ -200,27 +205,81 @@ const ledgerSelect = document.getElementById('cash-ledger-select'); const ledgerHelp = document.getElementById('cash-ledger-help'); - async function loadCashLedgers() { + const ledgerRefreshButton = document.getElementById('cash-refresh-ledgers'); + const ledgerProgress = document.getElementById('cash-ledger-progress'); + const ledgerProgressBar = document.getElementById('cash-ledger-progress-bar'); + const ledgerProgressPct = document.getElementById('cash-ledger-progress-pct'); + const ledgerProgressStage = document.getElementById('cash-ledger-progress-stage'); + let ledgerPollTimer = null; + + function populateLedgers(data) { + const previous = ledgerSelect.value; + ledgerSelect.innerHTML = ''; + (data.ledgers || []).forEach(function(row){ + const opt=document.createElement('option'); + opt.value=row.name; + opt.textContent=row.name + (row.parent ? ' — '+row.parent : '') + (row.is_cash_candidate ? ' • Cash candidate' : ''); + if ((previous && previous===row.name) || (!previous && row.is_cash_candidate && !ledgerSelect.value)) opt.selected=true; + ledgerSelect.appendChild(opt); + }); + if (!ledgerSelect.value && (data.cash_candidates||[]).length===1) ledgerSelect.value=data.cash_candidates[0].name; + ledgerHelp.textContent = String(data.ledger_count || 0)+' ledger(s) ready from '+(data.company_name || 'Tally')+(data.from_cache?' (local cache).':'.')+' Confirm the Cash ledger before extraction.'; + } + + function updateLedgerProgress(job) { + const p=Math.max(0,Math.min(100,Number(job.percent||0))); + ledgerProgress.classList.remove('hidden'); + ledgerProgressBar.style.width=p+'%'; + ledgerProgressPct.textContent=Math.round(p)+'%'; + ledgerProgressStage.textContent=job.stage||job.status||'Working…'; + } + + async function pollLedgers(jobId, clientId, dateFrom) { + try { + const url=new URL('/tools/accounting/cash-payments/ledgers/progress',window.location.origin); + url.searchParams.set('client_id',clientId); url.searchParams.set('job_id',jobId); url.searchParams.set('date_from',dateFrom); + const response=await fetch(url.toString(),{headers:{'Accept':'application/json'},cache:'no-store'}); + const data=await response.json(); + if(!response.ok||!data.ok) throw new Error(data.error||'Could not read Tally ledger progress.'); + const job=data.job||{}; updateLedgerProgress(job); + if(job.status==='completed'){ + populateLedgers(data); ledgerButton.disabled=false; if(ledgerRefreshButton) ledgerRefreshButton.disabled=false; + window.setTimeout(()=>ledgerProgress.classList.add('hidden'),1800); return; + } + if(job.status==='failed') throw new Error(job.error||'Tally ledger sync failed.'); + ledgerPollTimer=window.setTimeout(()=>pollLedgers(jobId,clientId,dateFrom),1500); + } catch(err) { + ledgerHelp.textContent=err&&err.message?err.message:String(err); ledgerButton.disabled=false; + if(ledgerRefreshButton) ledgerRefreshButton.disabled=false; + ledgerProgressStage.textContent=ledgerHelp.textContent; + } + } + + async function loadCashLedgers(forceRefresh) { const fd = new FormData(form); if (!fd.get('client_id') || !fd.get('tally_guid')) { ledgerHelp.textContent = 'Select a client and Tally company first.'; return; } fd.set('csrf_token', document.getElementById('cash-csrf-token').value); - ledgerButton.disabled = true; ledgerHelp.textContent = 'Reading Tally ledger master…'; + if(forceRefresh) fd.set('force_refresh','1'); + ledgerButton.disabled = true; if(ledgerRefreshButton) ledgerRefreshButton.disabled=true; + ledgerHelp.textContent = forceRefresh ? 'Starting background refresh from Tally…' : 'Loading cached ledgers or starting background Tally sync…'; try { const response = await fetch('/tools/accounting/cash-payments/ledgers', {method:'POST', body:fd, headers:{'Accept':'application/json'}}); const data = await response.json(); if (!response.ok || !data.ok) throw new Error(data.error || 'Could not load Tally ledgers.'); - const previous = ledgerSelect.value; ledgerSelect.innerHTML = ''; - (data.ledgers || []).forEach(function(row){ - const opt=document.createElement('option'); opt.value=row.name; opt.textContent=row.name + (row.parent ? ' — '+row.parent : '') + (row.is_cash_candidate ? ' • Cash candidate' : ''); - if ((previous && previous===row.name) || (!previous && row.is_cash_candidate && !ledgerSelect.value)) opt.selected=true; - ledgerSelect.appendChild(opt); - }); - if (!ledgerSelect.value && (data.cash_candidates||[]).length===1) ledgerSelect.value=data.cash_candidates[0].name; - ledgerHelp.textContent = String(data.ledger_count || 0)+' ledger(s) loaded from '+(data.company_name || 'Tally')+'. Confirm the Cash ledger before extraction.'; - } catch(err) { ledgerHelp.textContent = err && err.message ? err.message : String(err); } - finally { ledgerButton.disabled=false; } + if ((data.ledgers||[]).length) { + populateLedgers(data); ledgerButton.disabled=false; if(ledgerRefreshButton) ledgerRefreshButton.disabled=false; return; + } + const job=data.job||{}; + if(!job.job_id) throw new Error('Local Agent did not return a ledger-sync job id.'); + updateLedgerProgress(job); + pollLedgers(job.job_id,String(fd.get('client_id')||''),String(fd.get('date_from')||'')); + } catch(err) { + ledgerHelp.textContent = err && err.message ? err.message : String(err); + ledgerButton.disabled=false; if(ledgerRefreshButton) ledgerRefreshButton.disabled=false; + } } - if (ledgerButton) ledgerButton.addEventListener('click', loadCashLedgers); + if (ledgerButton) ledgerButton.addEventListener('click', function(){loadCashLedgers(false);}); + if (ledgerRefreshButton) ledgerRefreshButton.addEventListener('click', function(){loadCashLedgers(true);}); function show() { overlay.classList.remove('hidden'); diff --git a/app/modules/accounting/templates/accounting/opening_balances.html b/app/modules/accounting/templates/accounting/opening_balances.html index c55fc64..299f0eb 100644 --- a/app/modules/accounting/templates/accounting/opening_balances.html +++ b/app/modules/accounting/templates/accounting/opening_balances.html @@ -6,7 +6,7 @@

Accounting · Phase 21A v2

Opening Balance Review & Correction

- Summary-first review of previous-year closing balances against current-year openings. Matched and new zero-balance masters stay out of the main review; actionable differences are shown immediately and retain the existing controlled Tally correction workflow. + Summary-first review of previous-year closing balances against current-year openings. Only Balance Sheet ledgers and Stock Items are checked; revenue, expense, purchase and sales ledgers are excluded. Matched and new zero-balance masters stay out of the main review; actionable differences retain the existing controlled Tally correction workflow.

@@ -77,6 +77,10 @@ {% endif %} {% if selected_run %} +
+ Download Excel Report +
+
diff --git a/app/modules/documents/agent_package.py b/app/modules/documents/agent_package.py index b2f8a20..a72cda8 100644 --- a/app/modules/documents/agent_package.py +++ b/app/modules/documents/agent_package.py @@ -4,7 +4,7 @@ import io from pathlib import Path import zipfile -ERP_LOCAL_AGENT_VERSION = "1.22.14" +ERP_LOCAL_AGENT_VERSION = "1.22.15" ERP_LOCAL_AGENT_NAME = "ERP Local Agent" RUNTIME_ROOT = Path(__file__).resolve().parent / "local_agent_runtime" _DETERMINISTIC_ZIP_TIMESTAMP = (2026, 1, 1, 0, 0, 0) diff --git a/app/modules/documents/local_agent_runtime/erp_local_agent/__init__.py b/app/modules/documents/local_agent_runtime/erp_local_agent/__init__.py index 33a914f..f8005bb 100644 --- a/app/modules/documents/local_agent_runtime/erp_local_agent/__init__.py +++ b/app/modules/documents/local_agent_runtime/erp_local_agent/__init__.py @@ -1,2 +1,2 @@ -__version__ = "1.22.14" +__version__ = "1.22.15" AGENT_NAME = "ERP Local Agent" diff --git a/app/modules/documents/local_agent_runtime/erp_local_agent/commands.py b/app/modules/documents/local_agent_runtime/erp_local_agent/commands.py index c182f06..4426842 100644 --- a/app/modules/documents/local_agent_runtime/erp_local_agent/commands.py +++ b/app/modules/documents/local_agent_runtime/erp_local_agent/commands.py @@ -22,6 +22,8 @@ _CASH_CACHE_THREADS_LOCK = threading.Lock() _CASH_CACHE_CANCEL_REQUESTS: set[str] = set() _CASH_CACHE_STARTING: dict[str, dict[str, Any]] = {} _CASH_PROGRESS_FILE = Path(__file__).resolve().parents[1] / "data" / "cash_payment_progress.json" +_CASH_LEDGER_THREADS: dict[str, threading.Thread] = {} +_CASH_LEDGER_THREADS_LOCK = threading.Lock() def _cash_progress_snapshot(payload: dict[str, Any]) -> None: """Publish one lightweight cash-job snapshot for the local dashboard. @@ -106,6 +108,10 @@ class AgentCommandProcessor: result = self._bank_reconciliation_extract(payload) elif action == "accounting_cash_payment_ledgers": result = self._cash_payment_ledgers(payload) + elif action == "accounting_cash_payment_ledgers_start": + result = self._cash_payment_ledgers_start(payload) + elif action == "accounting_cash_payment_ledgers_status": + result = self._cash_payment_ledgers_status(payload) elif action == "accounting_cash_payment_cache_start": result = self._cash_payment_cache_start(payload) elif action == "accounting_cash_payment_cache_status": @@ -166,6 +172,9 @@ class AgentCommandProcessor: "cash_payment_background_sync_capability": True, "cash_payment_ledger_confirmation_capability": True, "cash_payment_dual_progress_capability": True, + "cash_payment_async_ledger_master_capability": True, + "cash_payment_ledger_cache_capability": True, + "opening_balance_balance_sheet_only_capability": True, "tally_writeback_capability": True, } @@ -706,18 +715,21 @@ class AgentCommandProcessor: } - def _cash_payment_ledgers(self, payload: dict[str, Any]) -> dict[str, Any]: - company, company_name = self._resolve_open_company(payload) - rows = self.tally.export_master_collection(company_name, "ledgers") or [] - ledgers = [] - for row in rows: + @staticmethod + def _cash_ledger_rows(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: + ledgers: list[dict[str, Any]] = [] + for row in rows or []: name = str(row.get("name") or "").strip() if not name: continue parent = str(row.get("parent") or "").strip() reserved = str(row.get("reserved_name") or "").strip() parent_key = parent.casefold().replace("-", " ") - is_cash = parent_key == "cash in hand" or reserved.casefold() == "cash" or name.casefold() == "cash" + is_cash = ( + parent_key == "cash in hand" + or reserved.casefold() == "cash" + or name.casefold() == "cash" + ) ledgers.append({ "name": name, "parent": parent, @@ -726,12 +738,245 @@ class AgentCommandProcessor: "is_cash_candidate": bool(is_cash), }) ledgers.sort(key=lambda x: (not x["is_cash_candidate"], x["name"].casefold())) - return { - "company": company, + return ledgers + + def _ensure_cash_ledger_schema(self, client_id: int) -> None: + with self.store.connect(client_id) as db: + db.executescript( + """ + CREATE TABLE IF NOT EXISTS cash_payment_ledger_cache ( + client_id INTEGER NOT NULL, + tally_guid TEXT NOT NULL, + company_name TEXT NOT NULL, + ledger_name TEXT NOT NULL, + parent_name TEXT NOT NULL DEFAULT '', + reserved_name TEXT NOT NULL DEFAULT '', + closing_balance REAL, + is_cash_candidate INTEGER NOT NULL DEFAULT 0, + synced_at_utc TEXT NOT NULL, + PRIMARY KEY (client_id, tally_guid, ledger_name) + ); + CREATE TABLE IF NOT EXISTS cash_payment_ledger_jobs ( + job_id TEXT PRIMARY KEY, + client_id INTEGER NOT NULL, + tally_guid TEXT NOT NULL, + company_name TEXT NOT NULL, + status TEXT NOT NULL, + stage TEXT NOT NULL DEFAULT '', + percent INTEGER NOT NULL DEFAULT 0, + ledger_count INTEGER NOT NULL DEFAULT 0, + error TEXT NOT NULL DEFAULT '', + started_at_utc TEXT NOT NULL, + finished_at_utc TEXT NOT NULL DEFAULT '' + ); + """ + ) + db.commit() + + def _cash_payment_cached_ledgers(self, client_id: int, tally_guid: str) -> list[dict[str, Any]]: + self._ensure_cash_ledger_schema(client_id) + with self.store.connect(client_id) as db: + rows = db.execute( + """SELECT ledger_name, parent_name, reserved_name, closing_balance, is_cash_candidate + FROM cash_payment_ledger_cache + WHERE client_id=? AND tally_guid=? + ORDER BY is_cash_candidate DESC, lower(ledger_name), ledger_name""", + (int(client_id), str(tally_guid or "")), + ).fetchall() + return [{ + "name": str(row[0] or ""), + "parent": str(row[1] or ""), + "reserved_name": str(row[2] or ""), + "closing_balance": row[3], + "is_cash_candidate": bool(row[4]), + } for row in rows] + + def _cash_payment_ledgers(self, payload: dict[str, Any]) -> dict[str, Any]: + """Backward-compatible non-blocking ledger request. + + If a cached ledger master exists it is returned immediately. Otherwise a + background master-sync job is started and the caller receives its job id. + """ + client_id = int(payload.get("client_id")) + company, company_name = self._resolve_open_company(payload) + tally_guid = str(company.get("guid") or payload.get("tally_guid") or "").strip() + cached = self._cash_payment_cached_ledgers(client_id, tally_guid) + if cached: + return { + "status": "cached", + "company": company, + "company_name": company_name, + "ledgers": cached, + "cash_candidates": [x for x in cached if x["is_cash_candidate"]], + "ledger_count": len(cached), + "from_cache": True, + "agent": self._agent_info(), + } + return self._cash_payment_ledgers_start(payload) + + def _cash_payment_ledgers_start(self, payload: dict[str, Any]) -> dict[str, Any]: + client_id = int(payload.get("client_id")) + company, company_name = self._resolve_open_company(payload) + tally_guid = str(company.get("guid") or payload.get("tally_guid") or "").strip() + self._ensure_cash_ledger_schema(client_id) + + cached = self._cash_payment_cached_ledgers(client_id, tally_guid) + force_refresh = bool(payload.get("force_refresh")) + if cached and not force_refresh: + return { + "status": "cached", + "company": company, + "company_name": company_name, + "ledgers": cached, + "cash_candidates": [x for x in cached if x["is_cash_candidate"]], + "ledger_count": len(cached), + "from_cache": True, + "agent": self._agent_info(), + } + + with self.store.connect(client_id) as db: + running = db.execute( + """SELECT job_id FROM cash_payment_ledger_jobs + WHERE client_id=? AND tally_guid=? AND status IN ('queued','running') + ORDER BY started_at_utc DESC LIMIT 1""", + (client_id, tally_guid), + ).fetchone() + if running: + return self._cash_payment_ledgers_status({**payload, "job_id": str(running[0])}) + + job_id = "CASHLEDGER-" + uuid.uuid4().hex[:16].upper() + now = datetime.now(timezone.utc).isoformat() + db.execute( + """INSERT INTO cash_payment_ledger_jobs + (job_id, client_id, tally_guid, company_name, status, stage, percent, ledger_count, error, started_at_utc, finished_at_utc) + VALUES (?,?,?,?,?,?,?,?,?,?,?)""", + (job_id, client_id, tally_guid, company_name, "queued", "Queued for Tally ledger master sync", 2, 0, "", now, ""), + ) + db.commit() + + _cash_progress_snapshot({ + "job_id": job_id, + "job_kind": "ledger_master", + "status": "queued", + "stage": "Queued for Tally ledger master sync", + "percent": 2, "company_name": company_name, + "ledger_count": 0, + }) + + def worker(): + try: + with self.store.connect(client_id) as db: + db.execute( + "UPDATE cash_payment_ledger_jobs SET status='running',stage=?,percent=? WHERE job_id=?", + ("Connected to Tally; reading ledger master", 15, job_id), + ) + db.commit() + _cash_progress_snapshot({ + "job_id": job_id, "job_kind": "ledger_master", "status": "running", + "stage": "Connected to Tally; reading ledger master", "percent": 15, + "company_name": company_name, "ledger_count": 0, + }) + # The Tally export is intentionally isolated in this background thread + # so websocket/tunnel keepalives and the HTTP command ACK remain responsive. + rows = self.tally.export_master_collection(company_name, "ledgers") or [] + ledgers = self._cash_ledger_rows(rows) + _cash_progress_snapshot({ + "job_id": job_id, "job_kind": "ledger_master", "status": "running", + "stage": f"Received {len(ledgers)} ledgers; saving local cache", "percent": 82, + "company_name": company_name, "ledger_count": len(ledgers), + }) + synced = datetime.now(timezone.utc).isoformat() + with self.store.connect(client_id) as db: + db.execute( + "DELETE FROM cash_payment_ledger_cache WHERE client_id=? AND tally_guid=?", + (client_id, tally_guid), + ) + db.executemany( + """INSERT INTO cash_payment_ledger_cache + (client_id,tally_guid,company_name,ledger_name,parent_name,reserved_name,closing_balance,is_cash_candidate,synced_at_utc) + VALUES (?,?,?,?,?,?,?,?,?)""", + [( + client_id, tally_guid, company_name, row["name"], row["parent"], + row["reserved_name"], row.get("closing_balance"), + 1 if row["is_cash_candidate"] else 0, synced, + ) for row in ledgers], + ) + db.execute( + """UPDATE cash_payment_ledger_jobs + SET status='completed',stage=?,percent=100,ledger_count=?,finished_at_utc=? + WHERE job_id=?""", + (f"{len(ledgers)} ledgers cached and ready", len(ledgers), synced, job_id), + ) + db.commit() + self.logger.info( + "Cash ledger master sync completed client_id=%s company=%s rows=%s", + client_id, company_name, len(ledgers), + ) + _cash_progress_snapshot({ + "job_id": job_id, "job_kind": "ledger_master", "status": "completed", + "stage": f"{len(ledgers)} ledgers cached and ready", "percent": 100, + "company_name": company_name, "ledger_count": len(ledgers), + }) + except Exception as exc: + finished = datetime.now(timezone.utc).isoformat() + try: + with self.store.connect(client_id) as db: + db.execute( + """UPDATE cash_payment_ledger_jobs + SET status='failed',stage='Ledger master sync failed',error=?,finished_at_utc=? + WHERE job_id=?""", + (str(exc), finished, job_id), + ) + db.commit() + except Exception: + pass + self.logger.exception("Cash ledger master background sync failed job_id=%s: %s", job_id, exc) + _cash_progress_snapshot({ + "job_id": job_id, "job_kind": "ledger_master", "status": "failed", + "stage": "Ledger master sync failed", "percent": 100, + "company_name": company_name, "ledger_count": 0, "error": str(exc), + }) + finally: + with _CASH_LEDGER_THREADS_LOCK: + _CASH_LEDGER_THREADS.pop(job_id, None) + + thread = threading.Thread(target=worker, name=f"cash-ledgers-{job_id[-6:]}", daemon=True) + with _CASH_LEDGER_THREADS_LOCK: + _CASH_LEDGER_THREADS[job_id] = thread + thread.start() + return self._cash_payment_ledgers_status({**payload, "job_id": job_id}) + + def _cash_payment_ledgers_status(self, payload: dict[str, Any]) -> dict[str, Any]: + client_id = int(payload.get("client_id")) + job_id = str(payload.get("job_id") or "").strip() + if not job_id: + raise ValueError("Cash ledger master job id is required.") + self._ensure_cash_ledger_schema(client_id) + with self.store.connect(client_id) as db: + row = db.execute( + """SELECT job_id,tally_guid,company_name,status,stage,percent,ledger_count,error,started_at_utc,finished_at_utc + FROM cash_payment_ledger_jobs WHERE job_id=? AND client_id=?""", + (job_id, client_id), + ).fetchone() + if not row: + raise ValueError("Cash ledger master job was not found.") + job = { + "job_id": str(row[0]), "tally_guid": str(row[1]), "company_name": str(row[2]), + "status": str(row[3]), "stage": str(row[4]), "percent": int(row[5] or 0), + "ledger_count": int(row[6] or 0), "error": str(row[7] or ""), + "started_at_utc": str(row[8] or ""), "finished_at_utc": str(row[9] or ""), + } + ledgers: list[dict[str, Any]] = [] + if job["status"] == "completed": + ledgers = self._cash_payment_cached_ledgers(client_id, job["tally_guid"]) + return { + "job": job, "ledgers": ledgers, "cash_candidates": [x for x in ledgers if x["is_cash_candidate"]], - "ledger_count": len(ledgers), + "ledger_count": len(ledgers) if ledgers else job["ledger_count"], + "company_name": job["company_name"], + "from_cache": bool(ledgers), "agent": self._agent_info(), } diff --git a/app/modules/documents/local_agent_runtime/erp_local_agent/dashboard.py b/app/modules/documents/local_agent_runtime/erp_local_agent/dashboard.py index d2948be..64fc745 100644 --- a/app/modules/documents/local_agent_runtime/erp_local_agent/dashboard.py +++ b/app/modules/documents/local_agent_runtime/erp_local_agent/dashboard.py @@ -171,7 +171,7 @@ function gb(n){return (n/1073741824).toFixed(1)+' GB'} function setProgress(title,pct,text,active=true){let b=document.getElementById('progressBox');b.style.display='block';document.getElementById('progressTitle').textContent=title;let p=Math.max(0,Math.min(100,Number(pct)||0));lastServerPercent=Math.max(lastServerPercent,p);document.getElementById('progressPct').textContent=Math.round(p)+'%';document.getElementById('progressFill').style.width=p+'%';document.getElementById('progressMsg').textContent=text||'';document.getElementById('progressFill').className='progressfill'+(active&&p<100?' pulse':'')} function applyProgress(u){let p=(u&&u.progress)||{};if(!p.operation)return;let title=p.operation==='check'?'Checking for Updates':p.operation==='download'?'Downloading Update':'Applying Update';setProgress(title,p.percent||0,p.message||p.phase||'',!['updated','downloaded','current','available','error'].includes(p.status));if(p.operation==='install'&&p.target_version){installTarget=p.target_version}} function setCashStep(id,state){let e=document.getElementById(id);e.className='cashstep'+(state?' '+state:'')} -async function refreshCashProgress(){try{const r=await fetch('/api/cash-progress',{cache:'no-store'});const d=await r.json();const j=d.job||{};if(!j.job_id){document.getElementById('cashJobSection').style.display='none';return}document.getElementById('cashJobSection').style.display='block';let p=Math.max(0,Math.min(100,Number(j.percent)||0));document.getElementById('cashJobPct').textContent=Math.round(p)+'%';document.getElementById('cashJobFill').style.width=p+'%';document.getElementById('cashJobStage').textContent=j.stage||j.status||'Working…';document.getElementById('cashJobSub').textContent=(j.company_name||'Tally company')+(j.cash_ledgers&&j.cash_ledgers.length?' • Cash ledger: '+j.cash_ledgers.join(', '):'');document.getElementById('cashDays').textContent=String(j.completed_days||0)+' / '+String(j.total_days||0);document.getElementById('cashVouchers').textContent=String(j.vouchers_cached||0);document.getElementById('cashRequests').textContent=String(j.tally_requests||0);document.getElementById('cashDate').textContent=j.current_date||'—';let st=String(j.stage||'').toLowerCase(),status=String(j.status||'').toLowerCase();setCashStep('cs1',(p>=10?'done':(p>0?'run':'')));setCashStep('cs2',(p>=20?'done':(st.includes('ledger')?'run':'')));setCashStep('cs3',(p>=93?'done':(p>=20?'run':'')));setCashStep('cs4',(status==='completed'?'done':(st.includes('analys')?'run':'')));setCashStep('cs5',(status==='completed'?'done':''));let er=document.getElementById('cashError');if(j.error){er.style.display='block';er.textContent=j.error}else{er.style.display='none'}}catch(e){}} +async function refreshCashProgress(){try{const r=await fetch('/api/cash-progress',{cache:'no-store'});const d=await r.json();const j=d.job||{};if(!j.job_id){document.getElementById('cashJobSection').style.display='none';return}document.getElementById('cashJobSection').style.display='block';let p=Math.max(0,Math.min(100,Number(j.percent)||0));document.getElementById('cashJobPct').textContent=Math.round(p)+'%';document.getElementById('cashJobFill').style.width=p+'%';document.getElementById('cashJobStage').textContent=j.stage||j.status||'Working…';let isLedger=String(j.job_kind||'')==='ledger_master';document.querySelector('#cashJobSection h2').textContent=isLedger?'Cash Payment — Tally Ledger Sync':'Cash Payment Compliance';document.getElementById('cashJobSub').textContent=(j.company_name||'Tally company')+(j.cash_ledgers&&j.cash_ledgers.length?' • Cash ledger: '+j.cash_ledgers.join(', '):'');if(isLedger){document.getElementById('cashDays').parentElement.querySelector('.label').textContent='Ledgers';document.getElementById('cashDays').textContent=String(j.ledger_count||0);document.getElementById('cashVouchers').parentElement.querySelector('.label').textContent='Status';document.getElementById('cashVouchers').textContent=j.status||'working';document.getElementById('cashRequests').parentElement.querySelector('.label').textContent='Progress';document.getElementById('cashRequests').textContent=Math.round(p)+'%';document.getElementById('cashDate').parentElement.querySelector('.label').textContent='Source';document.getElementById('cashDate').textContent='Tally ledger master';}else{document.getElementById('cashDays').parentElement.querySelector('.label').textContent='Days';document.getElementById('cashDays').textContent=String(j.completed_days||0)+' / '+String(j.total_days||0);document.getElementById('cashVouchers').parentElement.querySelector('.label').textContent='Cash vouchers';document.getElementById('cashVouchers').textContent=String(j.vouchers_cached||0);document.getElementById('cashRequests').parentElement.querySelector('.label').textContent='Tally requests';document.getElementById('cashRequests').textContent=String(j.tally_requests||0);document.getElementById('cashDate').parentElement.querySelector('.label').textContent='Current date';document.getElementById('cashDate').textContent=j.current_date||'—';}let st=String(j.stage||'').toLowerCase(),status=String(j.status||'').toLowerCase();if(isLedger){setCashStep('cs1',(p>=15?'done':(p>0?'run':'')));setCashStep('cs2',(status==='completed'?'done':'run'));setCashStep('cs3','');setCashStep('cs4','');setCashStep('cs5',(status==='completed'?'done':''));}else{setCashStep('cs1',(p>=10?'done':(p>0?'run':'')));setCashStep('cs2',(p>=20?'done':(st.includes('ledger')?'run':'')));setCashStep('cs3',(p>=93?'done':(p>=20?'run':'')));setCashStep('cs4',(status==='completed'?'done':(st.includes('analys')?'run':'')));setCashStep('cs5',(status==='completed'?'done':''));}let er=document.getElementById('cashError');if(j.error){er.style.display='block';er.textContent=j.error}else{er.style.display='none'}}catch(e){}} async function refresh(){try{const r=await fetch('/api/status',{cache:'no-store'});if(!r.ok)throw new Error('status unavailable');const s=await r.json();offlineTicks=0; let c=s.agent.connection_state||'unknown';let ce=document.getElementById('connection');ce.textContent=c.charAt(0).toUpperCase()+c.slice(1);ce.className='value '+(c==='connected'?'ok':'bad'); document.getElementById('heartbeat').textContent=s.agent.last_heartbeat_utc?'Last heartbeat: '+s.agent.last_heartbeat_utc:'No heartbeat recorded';