From 4739ad321e6a50f2e3b0bd1cc0c0b07a41898bc0 Mon Sep 17 00:00:00 2001 From: A R R R Associates Date: Thu, 3 Sep 2026 23:52:18 +0530 Subject: [PATCH] Add SQLite cached slow Tally extraction for cash payment review --- app/modules/accounting/cash_payment_ui.py | 135 +++++- .../accounting/cash_payment_review.html | 158 +++++-- app/modules/documents/agent_package.py | 2 +- .../erp_local_agent/__init__.py | 2 +- .../erp_local_agent/commands.py | 434 ++++++++++++++++++ .../erp_local_agent/tunnel.py | 7 +- 6 files changed, 682 insertions(+), 56 deletions(-) diff --git a/app/modules/accounting/cash_payment_ui.py b/app/modules/accounting/cash_payment_ui.py index d9a63c9..a751bfe 100644 --- a/app/modules/accounting/cash_payment_ui.py +++ b/app/modules/accounting/cash_payment_ui.py @@ -5,7 +5,7 @@ from decimal import Decimal, InvalidOperation from urllib.parse import quote from fastapi import APIRouter, Request -from fastapi.responses import RedirectResponse +from fastapi.responses import RedirectResponse, JSONResponse from sqlalchemy import select from app.core.db.common import CommonSessionLocal @@ -146,7 +146,8 @@ def cash_payment_review( tally_guid: str = "", date_from: str = "", date_to: str = "", - run: int = 0, + job_id: str = "", + analyze: int = 0, saved: int = 0, error: str = "", ): @@ -169,6 +170,7 @@ def cash_payment_review( end_text = date_to or today.isoformat() live_result = None review = None + cache_job = None command_error = error or "" if selected_client and node and online: @@ -183,28 +185,39 @@ def cash_payment_review( live_result = status_response.get("result") or {} else: command_error = str(status_response.get("error") or "Local Agent status failed.") - if run and tally_guid: - result = request_agent_command( + + if job_id: + status = request_agent_command( node.node_code, - "accounting_cash_payment_compliance", + "accounting_cash_payment_cache_status", { **_accounting_storage_payload(selected_client, _financial_year_for_date(date.fromisoformat(start_text))), - "tally_guid": str(tally_guid), - "date_from": start_text, - "date_to": end_text, + "job_id": str(job_id), + }, + timeout_seconds=20, + ) + if status.get("ok"): + cache_job = (status.get("result") or {}).get("job") + else: + command_error = str(status.get("error") or "Could not read cash-payment cache status.") + + if analyze and job_id and cache_job and cache_job.get("status") == "completed": + result = request_agent_command( + node.node_code, + "accounting_cash_payment_cache_analyze", + { + **_accounting_storage_payload(selected_client, _financial_year_for_date(date.fromisoformat(start_text))), + "job_id": str(job_id), "cash_limit": float(setting.cash_limit), "split_window_days": int(setting.split_window_days), "near_limit_percent": float(setting.near_limit_percent), - "tally_batch_days": 7, - "tally_pause_seconds": 1.5, - "tally_retry_pause_seconds": 3.0, }, - timeout_seconds=900, + timeout_seconds=120, ) if result.get("ok"): review = (result.get("result") or {}).get("cash_payment_review") else: - command_error = str(result.get("error") or "Cash payment review failed.") + command_error = str(result.get("error") or "Cash payment SQLite analysis failed.") except Exception as exc: command_error = str(exc) @@ -223,6 +236,8 @@ def cash_payment_review( date_to=end_text, setting=setting, review=review, + cache_job=cache_job, + active_job_id=str(job_id or ""), saved=bool(saved), command_error=command_error, allocation_plan=None, @@ -233,6 +248,96 @@ def cash_payment_review( db.close() +@router.post("/cache/start") +async def start_cash_payment_cache(request: Request): + form = await request.form() + validate_csrf(request, str(form.get("csrf_token") or "")) + db = CommonSessionLocal() + try: + user, response = _require_partner(request, db, "accounting.tally.view") + if response: + return JSONResponse({"ok": False, "error": "Access denied."}, status_code=403) + clients, scope = _visible_clients(db, request, user) + try: + client_id = int(form.get("client_id") or 0) + except Exception: + client_id = 0 + selected_client = next((row for row in clients if int(row.id) == client_id), None) + if not selected_client: + return JSONResponse({"ok": False, "error": "Select a valid client."}, status_code=400) + tally_guid = str(form.get("tally_guid") or "").strip() + date_from = str(form.get("date_from") or "").strip() + date_to = str(form.get("date_to") or "").strip() + try: + parsed_from = date.fromisoformat(date_from) + parsed_to = date.fromisoformat(date_to) + if parsed_to < parsed_from: + raise ValueError("To date cannot be before From date.") + except Exception as exc: + return JSONResponse({"ok": False, "error": str(exc)}, status_code=400) + if not tally_guid: + return JSONResponse({"ok": False, "error": "Select a mapped Tally company."}, status_code=400) + node = get_active_storage_node_for_branch(db, scope.tenant_id, scope.branch_id) + if not node or not _node_online(node): + return JSONResponse({"ok": False, "error": "ERP Local Agent is offline."}, status_code=409) + setting = _setting(db, _tenant_id(request, user)) + try: + result = request_agent_command( + node.node_code, + "accounting_cash_payment_cache_start", + { + **_accounting_storage_payload(selected_client, _financial_year_for_date(parsed_from)), + "tally_guid": tally_guid, + "date_from": date_from, + "date_to": date_to, + "cash_limit": float(setting.cash_limit), + "split_window_days": int(setting.split_window_days), + "near_limit_percent": float(setting.near_limit_percent), + "tally_pause_seconds": 3.0, + "requested_by_user_id": int(user.id), + }, + timeout_seconds=30, + ) + except Exception as exc: + return JSONResponse({"ok": False, "error": str(exc)}, status_code=502) + if not result.get("ok"): + return JSONResponse({"ok": False, "error": str(result.get("error") or "Could not start Tally extraction.")}, status_code=409) + return JSONResponse({"ok": True, **(result.get("result") or {})}) + finally: + db.close() + + +@router.get("/cache/progress") +def cash_payment_cache_progress(request: Request, client_id: int, job_id: str, date_from: str): + db = CommonSessionLocal() + try: + user, response = _require_partner(request, db, "accounting.tally.view") + if response: + return JSONResponse({"ok": False, "error": "Access denied."}, status_code=403) + clients, scope = _visible_clients(db, request, user) + selected_client = next((row for row in clients if int(row.id) == int(client_id)), None) + if not selected_client: + return JSONResponse({"ok": False, "error": "Client not found."}, status_code=404) + node = get_active_storage_node_for_branch(db, scope.tenant_id, scope.branch_id) + if not node or not _node_online(node): + return JSONResponse({"ok": False, "error": "ERP Local Agent is offline."}, status_code=409) + try: + fy = _financial_year_for_date(date.fromisoformat(date_from)) + result = request_agent_command( + node.node_code, + "accounting_cash_payment_cache_status", + {**_accounting_storage_payload(selected_client, fy), "job_id": str(job_id)}, + timeout_seconds=20, + ) + except Exception as exc: + return JSONResponse({"ok": False, "error": str(exc)}, status_code=502) + if not result.get("ok"): + return JSONResponse({"ok": False, "error": str(result.get("error") or "Could not read extraction progress.")}, status_code=409) + return JSONResponse({"ok": True, **(result.get("result") or {})}) + finally: + db.close() + + @router.get("/allocation") def cash_payment_allocation( request: Request, @@ -263,6 +368,8 @@ def cash_payment_allocation( date_to="", setting=setting, review=None, + cache_job=None, + active_job_id="", saved=False, command_error=error or "", allocation_plan=None, @@ -383,6 +490,8 @@ async def cash_payment_allocation_plan(request: Request): date_to="", setting=setting, review=None, + cache_job=None, + active_job_id="", saved=False, command_error="", allocation_plan=plan, diff --git a/app/modules/accounting/templates/accounting/cash_payment_review.html b/app/modules/accounting/templates/accounting/cash_payment_review.html index 2eedc8a..4d9a16d 100644 --- a/app/modules/accounting/templates/accounting/cash_payment_review.html +++ b/app/modules/accounting/templates/accounting/cash_payment_review.html @@ -22,6 +22,7 @@

Cash Payment Compliance Review

+
- -

Safe-paced mode reads Tally in monthly batches with a short pause between requests to reduce load on TallyPrime.

+

Transactions are copied very slowly into the Local Agent SQLite accounting cache one day at a time. Analysis starts only after TallyPrime has been released.

@@ -61,10 +61,24 @@ + {% if cache_job %} +
+
+
+

Local Tally SQLite Cache

+

Status: {{ cache_job.status }}{% if cache_job.stage %} · {{ cache_job.stage }}{% endif %}

+
+
{{ cache_job.completed_days }}/{{ cache_job.total_days }} day(s)
{{ cache_job.vouchers_cached }} voucher(s) cached
+
+
+ {% if cache_job.error %}

{{ cache_job.error }}

{% endif %} +
+ {% endif %} + {% if review %}
- {% if review.safe_paced_mode %} -
Completed in adaptive safe-paced mode using {{ review.tally_request_batches or 0 }} successful voucher batch(es) across {{ review.tally_request_count or 0 }} Tally request(s), with {{ review.tally_retry_count or 0 }} retry/retries and {{ review.tally_fallback_batches or 0 }} adaptive fallback(s). The agent pauses approximately {{ review.tally_pause_ms or 0 }} ms between Tally calls.
+ {% if review.sqlite_cache %} +
Analysis completed from the Local Agent SQLite cache. TallyPrime was used only for the slow background extraction and was released before this analysis ran. {{ review.tally_request_count or 0 }} Tally request(s), {{ review.tally_retry_count or 0 }} retry/retries, and approximately {{ review.tally_pause_ms or 0 }} ms cooling between successful day reads.
{% endif %}
{% for label, value in [('Vouchers reviewed', review.summary.vouchers_reviewed), ('Cash payments', review.summary.cash_payment_vouchers), ('Single-voucher exceptions', review.summary.single_voucher_exceptions), ('Same-day exceptions', review.summary.same_day_exceptions), ('Possible split patterns', review.summary.possible_split_patterns)] %} @@ -119,60 +133,126 @@
{% endif %}