From 6910b5b6b5f906b0bd5d72d9271e650d5adf89bb Mon Sep 17 00:00:00 2001 From: A R R R Associates Date: Sun, 23 Aug 2026 11:48:09 +0530 Subject: [PATCH] Fix live Purchase and Sales historical Tally sync --- .../accounting/historical_learning_ui.py | 41 ++++--- .../accounting/historical_sync_support.py | 114 ++++++++++++++++++ app/modules/accounting/sales_learning_ui.py | 45 +++---- .../accounting/historical_learning.html | 7 +- .../templates/accounting/sales_learning.html | 42 +++++-- app/modules/documents/agent_package.py | 2 +- .../erp_local_agent/__init__.py | 2 +- .../erp_local_agent/commands.py | 108 ++++++++++++++++- 8 files changed, 293 insertions(+), 68 deletions(-) create mode 100644 app/modules/accounting/historical_sync_support.py diff --git a/app/modules/accounting/historical_learning_ui.py b/app/modules/accounting/historical_learning_ui.py index b498d7a..402c48e 100644 --- a/app/modules/accounting/historical_learning_ui.py +++ b/app/modules/accounting/historical_learning_ui.py @@ -13,6 +13,7 @@ from app.core.templating import templates from app.modules.accounting.historical_learning_models import AccountingHistoricalLearningRun from app.modules.accounting.historical_learning_service import active_natures, classifiable_ledgers, evidence_rows, ingest_completed_run, ledger_mappings, party_suggestions, remove_ledger_mapping, save_ledger_mapping from app.modules.accounting.ui import _accounting_storage_payload, _find_visible_client, _require_partner, _visible_clients +from app.modules.accounting.historical_sync_support import eligible_tally_workstations, resolve_workstation_company, workstation_companies from app.modules.core.rbac.deps import get_user_permissions, get_user_roles from app.modules.core.rbac.permission_guard import require_permission from app.modules.documents.agent_jobs import enqueue_agent_job @@ -31,17 +32,6 @@ def _fy_dates(): return date(y,4,1).isoformat(), today.isoformat() -def _companies(workstation): - try: rows=json.loads(workstation.tally_companies_json or "[]") - except Exception: rows=[] - return rows if isinstance(rows,list) else [] - - -def _workstations(db, tenant_id: int, branch_id: int | None): - stmt=select(ERPWorkstationAgent).where(ERPWorkstationAgent.tenant_id==tenant_id, ERPWorkstationAgent.is_active.is_(True)) - if branch_id is not None: stmt=stmt.where(ERPWorkstationAgent.branch_id==branch_id) - return list(db.execute(stmt.order_by(ERPWorkstationAgent.last_seen_at_utc.desc())).scalars().all()) - def _sync_runs(db, tenant_id: int, client_id: int): runs=list(db.execute(select(AccountingHistoricalLearningRun).where(AccountingHistoricalLearningRun.tenant_id==tenant_id, AccountingHistoricalLearningRun.client_id==client_id).order_by(AccountingHistoricalLearningRun.id.desc()).limit(10)).scalars().all()) @@ -59,7 +49,7 @@ def historical_learning(request: Request, client_id: int | None=None, collected: clients,scope=_visible_clients(db,request,user) selected=next((c for c in clients if client_id and int(c.id)==int(client_id)),None) runs=[]; evid=[]; ledgers=[]; suggestions=[]; mappings=[] - workstations=_workstations(db, scope.tenant_id, scope.branch_id) + workstations=eligible_tally_workstations(db, scope.tenant_id, scope.branch_id) if selected: runs=_sync_runs(db,scope.tenant_id,selected.id) evid=evidence_rows(db,scope.tenant_id,selected.id) @@ -69,7 +59,7 @@ def historical_learning(request: Request, client_id: int | None=None, collected: df,dt=_fy_dates() return templates.TemplateResponse("modules/accounting/templates/accounting/historical_learning.html", { "request":request,"current_user":user,"current_user_roles":get_user_roles(db,user.id),"current_user_permissions":get_user_permissions(db,user.id),"csrf_token":get_or_create_csrf_token(request), - "title":"Historical Tally Learning","clients":clients,"selected_client":selected,"workstations":workstations,"workstation_companies":{w.id:_companies(w) for w in workstations}, + "title":"Historical Tally Learning","clients":clients,"selected_client":selected,"workstations":workstations,"workstation_companies":{w.id:workstation_companies(w) for w in workstations}, "runs":runs,"evidence":evid,"classifiable_ledgers":ledgers,"suggestions":suggestions,"mappings":mappings,"natures":active_natures(db,scope.tenant_id), "date_from":df,"date_to":dt,"collected":bool(collected),"mapped":bool(mapped),"error":error, }) @@ -84,14 +74,29 @@ def collect(request: Request, client_id: int=Form(...), workstation_id: int=Form if response:return response client,_,scope=_find_visible_client(db,request,user,client_id) if not client:return _denied() - workstation=db.get(ERPWorkstationAgent,workstation_id) - if not workstation or not workstation.is_active or workstation.tenant_id!=scope.tenant_id or (scope.branch_id is not None and workstation.branch_id!=scope.branch_id): return _denied() guid=str(tally_guid or "").strip() - if not guid: return RedirectResponse(url=f"/tools/accounting/historical-learning?client_id={client_id}&error=company_required",status_code=303) - payload=_accounting_storage_payload(client); payload.update({"tenant_id":scope.tenant_id,"requested_by_user_id":user.id,"tally_guid":guid,"date_from":date_from,"date_to":date_to,"voucher_scope":"purchase"}) + if not guid: + return RedirectResponse(url=f"/tools/accounting/historical-learning?client_id={client_id}&error=company_required",status_code=303) + workstation, company = resolve_workstation_company( + db, + workstation_id=workstation_id, + tenant_id=scope.tenant_id, + branch_id=scope.branch_id, + tally_guid=guid, + ) + payload=_accounting_storage_payload(client); payload.update({ + "tenant_id":scope.tenant_id, + "requested_by_user_id":user.id, + "tally_guid":guid, + "company_name":str(company.get("name") or company_name or "").strip(), + "date_from":date_from, + "date_to":date_to, + "voucher_scope":"purchase", + "refresh_transactions":True, + }) job=enqueue_agent_job(db,workstation_agent_id=workstation.id,action="accounting_historical_evidence",payload=payload,priority=6,max_attempts=2,created_by_user_id=user.id) db.flush() - run=AccountingHistoricalLearningRun(tenant_id=scope.tenant_id,client_id=client.id,workstation_agent_id=workstation.id,agent_job_id=job.id,tally_guid=guid,company_name=company_name or "",date_from=date_from or None,date_to=date_to or None,status="queued",requested_by_user_id=user.id) + run=AccountingHistoricalLearningRun(tenant_id=scope.tenant_id,client_id=client.id,workstation_agent_id=workstation.id,agent_job_id=job.id,tally_guid=guid,company_name=str(company.get("name") or company_name or ""),date_from=date_from or None,date_to=date_to or None,status="queued",requested_by_user_id=user.id) db.add(run); db.commit() return RedirectResponse(url=f"/tools/accounting/historical-learning?client_id={client_id}&collected=1",status_code=303) except Exception as exc: diff --git a/app/modules/accounting/historical_sync_support.py b/app/modules/accounting/historical_sync_support.py new file mode 100644 index 0000000..4411df6 --- /dev/null +++ b/app/modules/accounting/historical_sync_support.py @@ -0,0 +1,114 @@ +from __future__ import annotations + +import json +from datetime import datetime, timedelta, timezone + +from sqlalchemy import select + +from app.modules.documents.models import ERPWorkstationAgent + + +ONLINE_WINDOW = timedelta(minutes=5) + + +def workstation_companies(workstation) -> list[dict]: + try: + rows = json.loads(workstation.tally_companies_json or "[]") + except Exception: + rows = [] + if not isinstance(rows, list): + return [] + + result = [] + seen = set() + for row in rows: + if not isinstance(row, dict): + continue + guid = str(row.get("guid") or "").strip() + name = str(row.get("name") or "").strip() + if not guid or guid in seen: + continue + seen.add(guid) + result.append( + { + "guid": guid, + "name": name, + "gstin": str(row.get("gstin") or "").strip(), + "gstins": list(row.get("gstins") or []), + } + ) + return result + + +def workstation_is_recent(workstation) -> bool: + seen = getattr(workstation, "last_seen_at_utc", None) + if not seen: + return False + if seen.tzinfo is None: + seen = seen.replace(tzinfo=timezone.utc) + return datetime.now(timezone.utc) - seen <= ONLINE_WINDOW + + +def eligible_tally_workstations(db, tenant_id: int, branch_id: int | None = None): + stmt = select(ERPWorkstationAgent).where( + ERPWorkstationAgent.tenant_id == int(tenant_id), + ERPWorkstationAgent.is_active.is_(True), + ERPWorkstationAgent.tally_connected.is_(True), + ) + if branch_id is not None: + stmt = stmt.where(ERPWorkstationAgent.branch_id == int(branch_id)) + + rows = list( + db.execute( + stmt.order_by( + ERPWorkstationAgent.last_seen_at_utc.desc(), + ERPWorkstationAgent.machine_name.asc(), + ) + ).scalars().all() + ) + return [ + row + for row in rows + if workstation_is_recent(row) and workstation_companies(row) + ] + + +def resolve_workstation_company( + db, + *, + workstation_id: int, + tenant_id: int, + branch_id: int | None, + tally_guid: str, +): + workstation = db.get(ERPWorkstationAgent, int(workstation_id)) + if ( + not workstation + or int(workstation.tenant_id) != int(tenant_id) + or not workstation.is_active + or not workstation.tally_connected + or not workstation_is_recent(workstation) + or ( + branch_id is not None + and int(workstation.branch_id or 0) != int(branch_id) + ) + ): + raise ValueError( + "Selected ERP Local Agent workstation is offline, stale, or not connected to Tally." + ) + + guid = str(tally_guid or "").strip() + company = next( + ( + row + for row in workstation_companies(workstation) + if str(row.get("guid") or "").strip() == guid + ), + None, + ) + if not company: + raise ValueError( + "Selected Tally company is not currently advertised by this workstation. " + "Open the company in TallyPrime and allow the Local Agent to refresh." + ) + return workstation, company diff --git a/app/modules/accounting/sales_learning_ui.py b/app/modules/accounting/sales_learning_ui.py index 165a9c2..31425ee 100644 --- a/app/modules/accounting/sales_learning_ui.py +++ b/app/modules/accounting/sales_learning_ui.py @@ -11,7 +11,6 @@ from app.core.db.common import CommonSessionLocal from app.core.security.csrf import get_or_create_csrf_token, validate_csrf from app.core.templating import templates from app.modules.accounting.historical_learning_models import AccountingHistoricalLearningRun -from app.modules.accounting.ledger_learning_service import available_tally_guids from app.modules.accounting.sales_learning_models import AccountingSalesHistoricalRun from app.modules.accounting.sales_learning_service import ( historical_sales_rows, @@ -19,7 +18,8 @@ from app.modules.accounting.sales_learning_service import ( learning_summary, sales_mappings, ) -from app.modules.accounting.ui import _find_visible_client, _require_partner, _visible_clients +from app.modules.accounting.ui import _accounting_storage_payload, _find_visible_client, _require_partner, _visible_clients +from app.modules.accounting.historical_sync_support import eligible_tally_workstations, resolve_workstation_company, workstation_companies from app.modules.core.rbac.deps import get_user_permissions, get_user_roles from app.modules.documents.agent_jobs import enqueue_agent_job from app.modules.documents.models import ERPWorkstationAgent @@ -44,15 +44,6 @@ def _go(client_id=0, message="", error=""): ) -def _workstations(db, tenant_id: int): - return list(db.execute( - select(ERPWorkstationAgent).where( - ERPWorkstationAgent.tenant_id == int(tenant_id), - ERPWorkstationAgent.is_active.is_(True), - ERPWorkstationAgent.tally_connected.is_(True), - ).order_by(ERPWorkstationAgent.machine_name) - ).scalars().all()) - @router.get("") def page( @@ -76,7 +67,6 @@ def page( mappings = [] evidence = [] summary = {"mapping_count": 0, "historical_rows": 0, "review_confirmed": 0, "high_confidence": 0} - guids = [] workstations = [] runs = [] @@ -95,8 +85,7 @@ def page( mappings = sales_mappings(db, tenant_id=scope.tenant_id, client_id=selected.id) evidence = historical_sales_rows(db, tenant_id=scope.tenant_id, client_id=selected.id)[:200] summary = learning_summary(db, tenant_id=scope.tenant_id, client_id=selected.id) - guids = available_tally_guids(db, scope.tenant_id, selected.id) - workstations = _workstations(db, scope.tenant_id) + workstations = eligible_tally_workstations(db, scope.tenant_id, scope.branch_id) return templates.TemplateResponse( "modules/accounting/templates/accounting/sales_learning.html", @@ -112,8 +101,8 @@ def page( "mappings": mappings, "evidence": evidence, "summary": summary, - "tally_guids": guids, "workstations": workstations, + "workstation_companies": {w.id: workstation_companies(w) for w in workstations}, "runs": runs, "message": message, "error": error, @@ -144,18 +133,13 @@ def collect( if not client: return _go(error="Client is not visible.") - workstation = db.get(ERPWorkstationAgent, int(workstation_id)) - if ( - not workstation - or int(workstation.tenant_id) != int(scope.tenant_id) - or not workstation.is_active - or not workstation.tally_connected - ): - raise ValueError("Selected workstation is not available with Tally connected.") - - guid_options = dict(available_tally_guids(db, scope.tenant_id, client.id)) - if tally_guid not in guid_options: - raise ValueError("Selected Tally company is not mapped to this client.") + workstation, company = resolve_workstation_company( + db, + workstation_id=workstation_id, + tenant_id=scope.tenant_id, + branch_id=scope.branch_id, + tally_guid=tally_guid, + ) if not date_from: date_from = (date.today() - timedelta(days=730)).isoformat() @@ -163,12 +147,15 @@ def collect( date_to = date.today().isoformat() payload = { + **_accounting_storage_payload(client), "tenant_id": scope.tenant_id, - "client_id": client.id, + "requested_by_user_id": user.id, "tally_guid": tally_guid, + "company_name": str(company.get("name") or "").strip(), "date_from": date_from, "date_to": date_to, "voucher_scope": "sales", + "refresh_transactions": True, } job = enqueue_agent_job( db, @@ -184,7 +171,7 @@ def collect( tenant_id=scope.tenant_id, client_id=client.id, tally_guid=tally_guid, - company_name=guid_options.get(tally_guid, ""), + company_name=str(company.get("name") or ""), workstation_agent_id=workstation.id, agent_job_id=job.id, date_from=date_from, diff --git a/app/modules/accounting/templates/accounting/historical_learning.html b/app/modules/accounting/templates/accounting/historical_learning.html index 341e1fb..9a9c00f 100644 --- a/app/modules/accounting/templates/accounting/historical_learning.html +++ b/app/modules/accounting/templates/accounting/historical_learning.html @@ -2,7 +2,7 @@ {% block content %}
-

Tools · Accounting · Phase 5

Historical Tally Learning

Uses synced Tally purchase history as evidence. No voucher is created or altered in this phase.

+

Tools · Accounting · Phase 5

Historical Tally Learning

Refreshes the selected client's Purchase vouchers directly from the currently connected Tally company, then builds historical evidence. The operation is read-only; no voucher is created or altered.

Accounting TaxonomyBack to Tally
{% if collected %}
Historical evidence job queued. This page will ingest the result after the selected workstation completes it.
{% endif %} @@ -19,7 +19,10 @@ {% if selected_client %}

Collect Historical Purchase Evidence

-

Select the workstation that holds this client's synced .act data. The agent returns neutral ledger-history facts; classification stays on the ERP server.

+

Select the connected ERP Local Agent workstation and currently open mapped Tally company. The agent refreshes the selected period read-only before returning neutral ledger-history facts; classification stays on the ERP server.

+ {% if not workstations %} +
No recent ERP Local Agent workstation with Tally connected is advertising a company for this branch. Update/start the Local Agent, open the mapped company in TallyPrime, and refresh this page.
+ {% endif %}
diff --git a/app/modules/accounting/templates/accounting/sales_learning.html b/app/modules/accounting/templates/accounting/sales_learning.html index 8738cd3..dfd4fca 100644 --- a/app/modules/accounting/templates/accounting/sales_learning.html +++ b/app/modules/accounting/templates/accounting/sales_learning.html @@ -34,22 +34,25 @@

Collect Historical Sales Evidence from Tally

-

Read-only. Uses the existing durable Local Agent job queue and only Sales voucher history. It does not create or alter Tally vouchers.

+

Read-only. The selected workstation refreshes Sales vouchers from the currently open mapped Tally company for the chosen period, then stores neutral historical evidence. It does not create or alter Tally vouchers.

+ {% if not workstations %} +
No recent ERP Local Agent workstation with Tally connected is advertising a company for this branch. Update/start the Local Agent, open the mapped company in TallyPrime, and refresh this page.
+ {% endif %} - - + + @@ -89,4 +92,21 @@
{% endif %}
+ {% endblock %} diff --git a/app/modules/documents/agent_package.py b/app/modules/documents/agent_package.py index 7535c62..f1fc288 100644 --- a/app/modules/documents/agent_package.py +++ b/app/modules/documents/agent_package.py @@ -4,7 +4,7 @@ import io from pathlib import Path import zipfile -ERP_LOCAL_AGENT_VERSION = "1.15.0" +ERP_LOCAL_AGENT_VERSION = "1.15.1" ERP_LOCAL_AGENT_NAME = "ERP Local Agent" RUNTIME_ROOT = Path(__file__).resolve().parent / "local_agent_runtime" _DETERMINISTIC_ZIP_TIMESTAMP = (2026, 1, 1, 0, 0, 0) diff --git a/app/modules/documents/local_agent_runtime/erp_local_agent/__init__.py b/app/modules/documents/local_agent_runtime/erp_local_agent/__init__.py index 161fde1..479976a 100644 --- a/app/modules/documents/local_agent_runtime/erp_local_agent/__init__.py +++ b/app/modules/documents/local_agent_runtime/erp_local_agent/__init__.py @@ -1,2 +1,2 @@ -__version__ = "1.15.0" +__version__ = "1.15.1" AGENT_NAME = "ERP Local Agent" diff --git a/app/modules/documents/local_agent_runtime/erp_local_agent/commands.py b/app/modules/documents/local_agent_runtime/erp_local_agent/commands.py index 5e6ee66..b82ab09 100644 --- a/app/modules/documents/local_agent_runtime/erp_local_agent/commands.py +++ b/app/modules/documents/local_agent_runtime/erp_local_agent/commands.py @@ -216,14 +216,110 @@ class AgentCommandProcessor: return {"synced": True, "sync": sync, "accounting": self.store.snapshot(client_id), "tally": tally_status, "agent": self._agent_info()} def _historical_evidence(self, payload: dict[str, Any]) -> dict[str, Any]: + """Refresh Tally transactions read-only, then aggregate historical evidence. + + Earlier builds only aggregated transactions already present in the local + .act database. That made Purchase/Sales learning appear to "not sync" + unless the user had separately run transaction sync first. + """ + from datetime import date + client_id = int(payload.get("client_id")) - return self.store.historical_learning_evidence( - client_id, - tally_guid=str(payload.get("tally_guid") or ""), - date_from=str(payload.get("date_from") or ""), - date_to=str(payload.get("date_to") or ""), - voucher_scope=str(payload.get("voucher_scope") or "purchase"), + requested_guid = str(payload.get("tally_guid") or "").strip() + date_from = str(payload.get("date_from") or "").strip() + date_to = str(payload.get("date_to") or "").strip() + voucher_scope = str(payload.get("voucher_scope") or "purchase").strip().lower() + requested_by_user_id = payload.get("requested_by_user_id") + + if voucher_scope not in {"purchase", "sales", "all", "any"}: + raise ValueError("Historical evidence voucher scope must be purchase, sales or all.") + if not requested_guid: + raise ValueError("Select a currently open mapped Tally company.") + + try: + start = date.fromisoformat(date_from) + end = date.fromisoformat(date_to) + except Exception as exc: + raise ValueError("Select valid From Date and To Date for historical learning.") from exc + + if start > end: + raise ValueError("Historical learning From Date cannot be after To Date.") + if (end - start).days > 1826: + raise ValueError("Historical learning is limited to five years per collection run.") + + if not self.store.exists(client_id): + raise ValueError( + "Accounting storage is not initialized for this client. " + "Open Tally Accounting for the client and initialize/map the company first." + ) + + mapping = self.store.get_active_mapping_by_guid(client_id, requested_guid) + tally_status = self.tally.status() + if not tally_status.get("connected"): + raise ValueError(str(tally_status.get("error") or "TallyPrime is not connected.")) + + company = next( + ( + row + for row in (tally_status.get("companies") or []) + if str(row.get("guid") or "").strip() == requested_guid + ), + None, ) + if not company: + raise ValueError( + "The selected mapped Tally company is not currently open in TallyPrime. " + "Open it in Tally and retry." + ) + + company_name = str( + company.get("name") + or payload.get("company_name") + or mapping.get("company_name") + or "" + ).strip() + + # Read-only live refresh. No voucher is created or altered. + transactions = self.tally.fetch_accounting_transactions( + company_name, + date_from, + date_to, + ) + self.store.record_tally_status(client_id, tally_status) + sync = self.store.replace_transaction_snapshot( + client_id, + mapping={**mapping, "company_name": company_name}, + transactions=transactions, + requested_by_user_id=( + int(requested_by_user_id) + if requested_by_user_id not in (None, "") + else None + ), + ) + + evidence = self.store.historical_learning_evidence( + client_id, + tally_guid=requested_guid, + date_from=date_from, + date_to=date_to, + voucher_scope=voucher_scope, + ) + evidence["transaction_sync"] = sync + evidence["tally_company_name"] = company_name + evidence["refreshed_from_tally"] = True + + self.logger.info( + "Historical %s evidence refreshed from Tally client_id=%s company=%s " + "period=%s..%s vouchers=%s evidence_rows=%s", + voucher_scope, + client_id, + company_name, + date_from, + date_to, + (sync.get("counts") or {}).get("vouchers", sync.get("rows_processed")), + evidence.get("evidence_count"), + ) + return evidence def _resolve_open_company(self, payload: dict[str, Any]) -> tuple[dict[str, Any], str]: