From a9a5effcd0ac5cb47f708ecbaa6c480016d25b1e Mon Sep 17 00:00:00 2001 From: A R R R Associates Date: Sat, 22 Aug 2026 13:01:17 +0530 Subject: [PATCH] Add Phase 5 historical Tally learning foundation --- ...260822_historical_tally_learning_phase5.py | 47 +++++ app/core/startup.py | 2 + .../accounting/historical_learning_models.py | 74 ++++++++ .../accounting/historical_learning_service.py | 168 ++++++++++++++++++ .../accounting/historical_learning_ui.py | 125 +++++++++++++ .../accounting/historical_learning.html | 59 ++++++ .../templates/accounting/tally.html | 1 + app/modules/core/rbac/permissions_registry.py | 2 + app/modules/documents/agent_package.py | 4 +- .../erp_local_agent/__init__.py | 3 +- .../erp_local_agent/accounting_store.py | 48 +++++ .../erp_local_agent/commands.py | 13 ++ .../erp_local_agent/sync.py | 2 +- .../erp_local_agent/tunnel.py | 2 +- app/ui/app.py | 2 + 15 files changed, 546 insertions(+), 6 deletions(-) create mode 100644 alembic/versions/20260822_historical_tally_learning_phase5.py create mode 100644 app/modules/accounting/historical_learning_models.py create mode 100644 app/modules/accounting/historical_learning_service.py create mode 100644 app/modules/accounting/historical_learning_ui.py create mode 100644 app/modules/accounting/templates/accounting/historical_learning.html diff --git a/alembic/versions/20260822_historical_tally_learning_phase5.py b/alembic/versions/20260822_historical_tally_learning_phase5.py new file mode 100644 index 0000000..5ee0d49 --- /dev/null +++ b/alembic/versions/20260822_historical_tally_learning_phase5.py @@ -0,0 +1,47 @@ +"""Phase 5 historical Tally learning foundation. + +Revision ID: 20260822_historical_learning_p5 +Revises: 20260822_accounting_taxonomy_p4 +""" +from alembic import op +import sqlalchemy as sa + +revision = "20260822_historical_learning_p5" +down_revision = "20260822_accounting_taxonomy_p4" +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table("accounting_ledger_nature_mappings", + sa.Column("id",sa.Integer(),primary_key=True), sa.Column("tenant_id",sa.Integer(),sa.ForeignKey("tenants.id",ondelete="CASCADE"),nullable=False), + sa.Column("client_id",sa.Integer(),sa.ForeignKey("clients.id",ondelete="CASCADE"),nullable=False), sa.Column("tally_guid",sa.String(120),nullable=False), + sa.Column("company_name",sa.String(240),nullable=False,server_default=""), sa.Column("ledger_name",sa.String(240),nullable=False), sa.Column("parent_group",sa.String(240),nullable=False,server_default=""), + sa.Column("nature_id",sa.Integer(),sa.ForeignKey("accounting_natures.id",ondelete="RESTRICT"),nullable=False), sa.Column("source",sa.String(40),nullable=False,server_default="manual"), + sa.Column("confidence_percent",sa.Integer(),nullable=False,server_default="100"), sa.Column("confirmed_by_user_id",sa.Integer(),sa.ForeignKey("users.id",ondelete="SET NULL"),nullable=True), + sa.Column("confirmed_at_utc",sa.DateTime(timezone=True),nullable=False,server_default=sa.func.now()), sa.Column("created_at_utc",sa.DateTime(timezone=True),nullable=False,server_default=sa.func.now()), sa.Column("updated_at_utc",sa.DateTime(timezone=True),nullable=False,server_default=sa.func.now()), + sa.UniqueConstraint("tenant_id","client_id","tally_guid","ledger_name",name="uq_accounting_ledger_nature_mapping")) + for c in ["tenant_id","client_id","tally_guid","ledger_name","parent_group","nature_id","source"]: op.create_index(f"ix_accounting_ledger_nature_mappings_{c}","accounting_ledger_nature_mappings",[c]) + + op.create_table("accounting_historical_ledger_evidence", + sa.Column("id",sa.Integer(),primary_key=True), sa.Column("tenant_id",sa.Integer(),sa.ForeignKey("tenants.id",ondelete="CASCADE"),nullable=False), sa.Column("client_id",sa.Integer(),sa.ForeignKey("clients.id",ondelete="CASCADE"),nullable=False), + sa.Column("tally_guid",sa.String(120),nullable=False), sa.Column("company_name",sa.String(240),nullable=False,server_default=""), sa.Column("party_ledger_name",sa.String(240),nullable=False), sa.Column("party_parent_group",sa.String(240),nullable=False,server_default=""), + sa.Column("counter_ledger_name",sa.String(240),nullable=False), sa.Column("counter_parent_group",sa.String(240),nullable=False,server_default=""), sa.Column("voucher_type_name",sa.String(160),nullable=False,server_default="Purchase"), + sa.Column("voucher_count",sa.Integer(),nullable=False,server_default="0"), sa.Column("absolute_amount_total",sa.Float(),nullable=False,server_default="0"), sa.Column("first_voucher_date",sa.String(20),nullable=True), sa.Column("last_voucher_date",sa.String(20),nullable=True), + sa.Column("sample_narration",sa.Text(),nullable=True), sa.Column("refreshed_at_utc",sa.DateTime(timezone=True),nullable=False,server_default=sa.func.now()), + sa.UniqueConstraint("tenant_id","client_id","tally_guid","party_ledger_name","counter_ledger_name","voucher_type_name",name="uq_accounting_historical_ledger_evidence")) + for c in ["tenant_id","client_id","tally_guid","party_ledger_name","counter_ledger_name","counter_parent_group","voucher_type_name","refreshed_at_utc"]: op.create_index(f"ix_accounting_historical_ledger_evidence_{c}","accounting_historical_ledger_evidence",[c]) + + op.create_table("accounting_historical_learning_runs", + sa.Column("id",sa.Integer(),primary_key=True), sa.Column("tenant_id",sa.Integer(),sa.ForeignKey("tenants.id",ondelete="CASCADE"),nullable=False), sa.Column("client_id",sa.Integer(),sa.ForeignKey("clients.id",ondelete="CASCADE"),nullable=False), + sa.Column("workstation_agent_id",sa.Integer(),sa.ForeignKey("erp_workstation_agents.id",ondelete="SET NULL"),nullable=True), sa.Column("agent_job_id",sa.Integer(),sa.ForeignKey("erp_agent_jobs.id",ondelete="SET NULL"),nullable=True,unique=True), + sa.Column("tally_guid",sa.String(120),nullable=False), sa.Column("company_name",sa.String(240),nullable=False,server_default=""), sa.Column("date_from",sa.String(20),nullable=True), sa.Column("date_to",sa.String(20),nullable=True), + sa.Column("status",sa.String(30),nullable=False,server_default="queued"), sa.Column("evidence_rows",sa.Integer(),nullable=False,server_default="0"), sa.Column("error_message",sa.Text(),nullable=True), sa.Column("requested_by_user_id",sa.Integer(),sa.ForeignKey("users.id",ondelete="SET NULL"),nullable=True), + sa.Column("created_at_utc",sa.DateTime(timezone=True),nullable=False,server_default=sa.func.now()), sa.Column("completed_at_utc",sa.DateTime(timezone=True),nullable=True)) + for c in ["tenant_id","client_id","workstation_agent_id","agent_job_id","tally_guid","status","created_at_utc"]: op.create_index(f"ix_accounting_historical_learning_runs_{c}","accounting_historical_learning_runs",[c], unique=(c=="agent_job_id")) + + +def downgrade(): + op.drop_table("accounting_historical_learning_runs") + op.drop_table("accounting_historical_ledger_evidence") + op.drop_table("accounting_ledger_nature_mappings") diff --git a/app/core/startup.py b/app/core/startup.py index 03b01a9..1f8913c 100644 --- a/app/core/startup.py +++ b/app/core/startup.py @@ -342,6 +342,8 @@ ROLE_PERMISSION_MAP = { "accounting.tally.writeback", "accounting.taxonomy.view", "accounting.taxonomy.manage", + "accounting.learning.view", + "accounting.learning.manage", "accounting.act.initialize", "clients.view.own_only", "employees.dashboard.view", diff --git a/app/modules/accounting/historical_learning_models.py b/app/modules/accounting/historical_learning_models.py new file mode 100644 index 0000000..278105e --- /dev/null +++ b/app/modules/accounting/historical_learning_models.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +from datetime import datetime, timezone + +from sqlalchemy import Date, DateTime, Float, ForeignKey, Integer, String, Text, UniqueConstraint +from sqlalchemy.orm import Mapped, mapped_column + +from app.core.db.common import CommonBase + + +class AccountingLedgerNatureMapping(CommonBase): + __tablename__ = "accounting_ledger_nature_mappings" + __table_args__ = ( + UniqueConstraint("tenant_id", "client_id", "tally_guid", "ledger_name", name="uq_accounting_ledger_nature_mapping"), + ) + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + tenant_id: Mapped[int] = mapped_column(ForeignKey("tenants.id", ondelete="CASCADE"), nullable=False, index=True) + client_id: Mapped[int] = mapped_column(ForeignKey("clients.id", ondelete="CASCADE"), nullable=False, index=True) + tally_guid: Mapped[str] = mapped_column(String(120), nullable=False, index=True) + company_name: Mapped[str] = mapped_column(String(240), nullable=False, default="") + ledger_name: Mapped[str] = mapped_column(String(240), nullable=False, index=True) + parent_group: Mapped[str] = mapped_column(String(240), nullable=False, default="", index=True) + nature_id: Mapped[int] = mapped_column(ForeignKey("accounting_natures.id", ondelete="RESTRICT"), nullable=False, index=True) + source: Mapped[str] = mapped_column(String(40), nullable=False, default="manual", index=True) + confidence_percent: Mapped[int] = mapped_column(Integer, nullable=False, default=100) + confirmed_by_user_id: Mapped[int | None] = mapped_column(ForeignKey("users.id", ondelete="SET NULL"), nullable=True) + confirmed_at_utc: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc), nullable=False) + created_at_utc: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc), nullable=False) + updated_at_utc: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc), onupdate=lambda: datetime.now(timezone.utc), nullable=False) + + +class AccountingHistoricalLedgerEvidence(CommonBase): + __tablename__ = "accounting_historical_ledger_evidence" + __table_args__ = ( + UniqueConstraint("tenant_id", "client_id", "tally_guid", "party_ledger_name", "counter_ledger_name", "voucher_type_name", name="uq_accounting_historical_ledger_evidence"), + ) + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + tenant_id: Mapped[int] = mapped_column(ForeignKey("tenants.id", ondelete="CASCADE"), nullable=False, index=True) + client_id: Mapped[int] = mapped_column(ForeignKey("clients.id", ondelete="CASCADE"), nullable=False, index=True) + tally_guid: Mapped[str] = mapped_column(String(120), nullable=False, index=True) + company_name: Mapped[str] = mapped_column(String(240), nullable=False, default="") + party_ledger_name: Mapped[str] = mapped_column(String(240), nullable=False, index=True) + party_parent_group: Mapped[str] = mapped_column(String(240), nullable=False, default="") + counter_ledger_name: Mapped[str] = mapped_column(String(240), nullable=False, index=True) + counter_parent_group: Mapped[str] = mapped_column(String(240), nullable=False, default="", index=True) + voucher_type_name: Mapped[str] = mapped_column(String(160), nullable=False, default="Purchase", index=True) + voucher_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + absolute_amount_total: Mapped[float] = mapped_column(Float, nullable=False, default=0.0) + first_voucher_date: Mapped[str | None] = mapped_column(String(20), nullable=True) + last_voucher_date: Mapped[str | None] = mapped_column(String(20), nullable=True) + sample_narration: Mapped[str | None] = mapped_column(Text, nullable=True) + refreshed_at_utc: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc), nullable=False, index=True) + + +class AccountingHistoricalLearningRun(CommonBase): + __tablename__ = "accounting_historical_learning_runs" + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + tenant_id: Mapped[int] = mapped_column(ForeignKey("tenants.id", ondelete="CASCADE"), nullable=False, index=True) + client_id: Mapped[int] = mapped_column(ForeignKey("clients.id", ondelete="CASCADE"), nullable=False, index=True) + workstation_agent_id: Mapped[int] = mapped_column(ForeignKey("erp_workstation_agents.id", ondelete="SET NULL"), nullable=True, index=True) + agent_job_id: Mapped[int] = mapped_column(ForeignKey("erp_agent_jobs.id", ondelete="SET NULL"), nullable=True, unique=True, index=True) + tally_guid: Mapped[str] = mapped_column(String(120), nullable=False, index=True) + company_name: Mapped[str] = mapped_column(String(240), nullable=False, default="") + date_from: Mapped[str | None] = mapped_column(String(20), nullable=True) + date_to: Mapped[str | None] = mapped_column(String(20), nullable=True) + status: Mapped[str] = mapped_column(String(30), nullable=False, default="queued", index=True) + evidence_rows: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + error_message: Mapped[str | None] = mapped_column(Text, nullable=True) + requested_by_user_id: Mapped[int | None] = mapped_column(ForeignKey("users.id", ondelete="SET NULL"), nullable=True) + created_at_utc: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc), nullable=False, index=True) + completed_at_utc: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) diff --git a/app/modules/accounting/historical_learning_service.py b/app/modules/accounting/historical_learning_service.py new file mode 100644 index 0000000..3ba2fcc --- /dev/null +++ b/app/modules/accounting/historical_learning_service.py @@ -0,0 +1,168 @@ +from __future__ import annotations + +import json +from collections import defaultdict +from datetime import datetime, timezone + +from sqlalchemy import delete, select + +from app.modules.accounting.historical_learning_models import ( + AccountingHistoricalLedgerEvidence, AccountingHistoricalLearningRun, AccountingLedgerNatureMapping, +) +from app.modules.accounting.taxonomy_models import AccountingNature +from app.modules.documents.models import ERPAgentJob + + +NOISE_GROUPS = { + "bank accounts", "bank od a/c", "cash-in-hand", "cash in hand", "sundry creditors", "sundry debtors", + "duties & taxes", "duties and taxes", "loans (liability)", "secured loans", "unsecured loans", +} + + +def _utcnow(): + return datetime.now(timezone.utc) + + +def _json(value): + try: + return json.loads(value or "{}") + except Exception: + return {} + + +def active_natures(db, tenant_id: int): + return list(db.execute(select(AccountingNature).where( + AccountingNature.tenant_id == tenant_id, AccountingNature.is_active.is_(True), AccountingNature.is_posting_nature.is_(True) + ).order_by(AccountingNature.sort_order, AccountingNature.name)).scalars().all()) + + +def ledger_mappings(db, tenant_id: int, client_id: int, tally_guid: str = ""): + stmt = select(AccountingLedgerNatureMapping).where( + AccountingLedgerNatureMapping.tenant_id == tenant_id, AccountingLedgerNatureMapping.client_id == client_id + ) + if tally_guid: + stmt = stmt.where(AccountingLedgerNatureMapping.tally_guid == tally_guid) + return list(db.execute(stmt.order_by(AccountingLedgerNatureMapping.ledger_name)).scalars().all()) + + +def save_ledger_mapping(db, *, tenant_id: int, client_id: int, tally_guid: str, company_name: str, ledger_name: str, parent_group: str, nature_id: int, user_id: int): + nature = db.execute(select(AccountingNature).where( + AccountingNature.id == nature_id, AccountingNature.tenant_id == tenant_id, AccountingNature.is_active.is_(True), AccountingNature.is_posting_nature.is_(True) + )).scalar_one_or_none() + if not nature: + raise ValueError("Select an active posting accounting nature.") + row = db.execute(select(AccountingLedgerNatureMapping).where( + AccountingLedgerNatureMapping.tenant_id == tenant_id, AccountingLedgerNatureMapping.client_id == client_id, + AccountingLedgerNatureMapping.tally_guid == tally_guid, AccountingLedgerNatureMapping.ledger_name == ledger_name + )).scalar_one_or_none() + if not row: + row = AccountingLedgerNatureMapping(tenant_id=tenant_id, client_id=client_id, tally_guid=tally_guid, ledger_name=ledger_name, nature_id=nature.id) + row.company_name = company_name or row.company_name or "" + row.parent_group = parent_group or row.parent_group or "" + row.nature_id = nature.id + row.source = "manual" + row.confidence_percent = 100 + row.confirmed_by_user_id = user_id + row.confirmed_at_utc = _utcnow() + row.updated_at_utc = _utcnow() + db.add(row); db.commit(); db.refresh(row) + return row + + +def remove_ledger_mapping(db, *, tenant_id: int, client_id: int, mapping_id: int): + row = db.execute(select(AccountingLedgerNatureMapping).where( + AccountingLedgerNatureMapping.id == mapping_id, AccountingLedgerNatureMapping.tenant_id == tenant_id, AccountingLedgerNatureMapping.client_id == client_id + )).scalar_one_or_none() + if row: + db.delete(row); db.commit() + return row + + +def ingest_completed_run(db, run: AccountingHistoricalLearningRun): + if run.status == "completed" or not run.agent_job_id: + return run + job = db.get(ERPAgentJob, run.agent_job_id) + if not job: + run.status = "failed"; run.error_message = "Agent job no longer exists."; run.completed_at_utc = _utcnow(); db.commit(); return run + if job.status in {"queued", "claimed"}: + run.status = job.status; db.commit(); return run + if job.status != "succeeded": + run.status = "failed"; run.error_message = job.last_error or "Historical evidence collection failed."; run.completed_at_utc = _utcnow(); db.commit(); return run + result = _json(job.result_json) + rows = result.get("evidence") or [] + company_name = str(result.get("company_name") or run.company_name or "") + now = _utcnow() + # Snapshot semantics for this company: remove old aggregated rows then replace with current evidence. + db.execute(delete(AccountingHistoricalLedgerEvidence).where( + AccountingHistoricalLedgerEvidence.tenant_id == run.tenant_id, AccountingHistoricalLedgerEvidence.client_id == run.client_id, + AccountingHistoricalLedgerEvidence.tally_guid == run.tally_guid + )) + inserted = 0 + for item in rows: + party = str(item.get("party_ledger_name") or "").strip() + counter = str(item.get("counter_ledger_name") or "").strip() + if not party or not counter or party.casefold() == counter.casefold(): + continue + db.add(AccountingHistoricalLedgerEvidence( + tenant_id=run.tenant_id, client_id=run.client_id, tally_guid=run.tally_guid, company_name=company_name, + party_ledger_name=party, party_parent_group=str(item.get("party_parent_group") or ""), + counter_ledger_name=counter, counter_parent_group=str(item.get("counter_parent_group") or ""), + voucher_type_name=str(item.get("voucher_type_name") or "Purchase"), voucher_count=int(item.get("voucher_count") or 0), + absolute_amount_total=float(item.get("absolute_amount_total") or 0), first_voucher_date=str(item.get("first_voucher_date") or "") or None, + last_voucher_date=str(item.get("last_voucher_date") or "") or None, sample_narration=str(item.get("sample_narration") or "")[:2000] or None, refreshed_at_utc=now + )); inserted += 1 + run.company_name = company_name + run.status = "completed" + run.evidence_rows = inserted + run.error_message = None + run.completed_at_utc = now + db.commit(); db.refresh(run) + return run + + +def evidence_rows(db, tenant_id: int, client_id: int): + return list(db.execute(select(AccountingHistoricalLedgerEvidence).where( + AccountingHistoricalLedgerEvidence.tenant_id == tenant_id, AccountingHistoricalLedgerEvidence.client_id == client_id + ).order_by(AccountingHistoricalLedgerEvidence.voucher_count.desc(), AccountingHistoricalLedgerEvidence.party_ledger_name, AccountingHistoricalLedgerEvidence.counter_ledger_name)).scalars().all()) + + +def classifiable_ledgers(db, tenant_id: int, client_id: int): + rows = evidence_rows(db, tenant_id, client_id) + seen = {} + for r in rows: + group = (r.counter_parent_group or "").strip().casefold() + if group in NOISE_GROUPS: + continue + key = (r.tally_guid, r.counter_ledger_name.casefold()) + item = seen.setdefault(key, {"tally_guid": r.tally_guid, "company_name": r.company_name, "ledger_name": r.counter_ledger_name, "parent_group": r.counter_parent_group, "voucher_count": 0, "amount": 0.0}) + item["voucher_count"] += int(r.voucher_count or 0); item["amount"] += float(r.absolute_amount_total or 0) + mappings = {(m.tally_guid, m.ledger_name.casefold()): m for m in ledger_mappings(db, tenant_id, client_id)} + for key, item in seen.items(): item["mapping"] = mappings.get(key) + return sorted(seen.values(), key=lambda x: (-x["voucher_count"], x["ledger_name"].casefold())) + + +def party_suggestions(db, tenant_id: int, client_id: int): + rows = evidence_rows(db, tenant_id, client_id) + mappings = {(m.tally_guid, m.ledger_name.casefold()): m for m in ledger_mappings(db, tenant_id, client_id)} + natures = {n.id: n for n in db.execute(select(AccountingNature).where(AccountingNature.tenant_id == tenant_id)).scalars().all()} + by_party = defaultdict(lambda: defaultdict(lambda: {"count": 0, "amount": 0.0, "ledgers": set()})) + party_meta = {} + for r in rows: + m = mappings.get((r.tally_guid, r.counter_ledger_name.casefold())) + if not m: + continue + key=(r.tally_guid, r.party_ledger_name) + party_meta[key]=(r.company_name, r.party_parent_group) + bucket=by_party[key][m.nature_id] + bucket["count"] += int(r.voucher_count or 0); bucket["amount"] += float(r.absolute_amount_total or 0); bucket["ledgers"].add(r.counter_ledger_name) + out=[] + for key, nature_buckets in by_party.items(): + total=sum(v["count"] for v in nature_buckets.values()) + if total <= 0: continue + nature_id, best=max(nature_buckets.items(), key=lambda kv:(kv[1]["count"], kv[1]["amount"])) + nature=natures.get(nature_id) + if not nature: continue + confidence=round(best["count"]*100/total) + company, parent=party_meta[key] + out.append({"tally_guid": key[0], "company_name": company, "party_ledger_name": key[1], "party_parent_group": parent, "nature": nature, "voucher_count": best["count"], "mapped_voucher_count": total, "confidence": confidence, "amount": best["amount"], "ledgers": sorted(best["ledgers"])}) + return sorted(out, key=lambda x:(-x["confidence"], -x["voucher_count"], x["party_ledger_name"].casefold())) diff --git a/app/modules/accounting/historical_learning_ui.py b/app/modules/accounting/historical_learning_ui.py new file mode 100644 index 0000000..2367a57 --- /dev/null +++ b/app/modules/accounting/historical_learning_ui.py @@ -0,0 +1,125 @@ +from __future__ import annotations + +import json +from datetime import date, datetime, timezone + +from fastapi import APIRouter, Form, Request +from fastapi.responses import RedirectResponse +from sqlalchemy import select + +from app.core.db.common import CommonSessionLocal +from app.core.security import get_or_create_csrf_token, validate_csrf +from app.core.templating import templates +from app.modules.accounting.historical_learning_models import AccountingHistoricalLearningRun +from app.modules.accounting.historical_learning_service import active_natures, classifiable_ledgers, evidence_rows, ingest_completed_run, ledger_mappings, party_suggestions, remove_ledger_mapping, save_ledger_mapping +from app.modules.accounting.ui import _accounting_storage_payload, _find_visible_client, _require_partner, _visible_clients +from app.modules.core.rbac.deps import get_user_permissions, get_user_roles +from app.modules.core.rbac.permission_guard import require_permission +from app.modules.documents.agent_jobs import enqueue_agent_job +from app.modules.documents.models import ERPAgentJob, ERPWorkstationAgent + +router = APIRouter(prefix="/tools/accounting/historical-learning", tags=["accounting-historical-learning-ui"]) + + +def _denied(): + from app.core.http_responses import ui_access_denied + return ui_access_denied() + + +def _fy_dates(): + today=date.today(); y=today.year if today.month>=4 else today.year-1 + return date(y,4,1).isoformat(), today.isoformat() + + +def _companies(workstation): + try: rows=json.loads(workstation.tally_companies_json or "[]") + except Exception: rows=[] + return rows if isinstance(rows,list) else [] + + +def _workstations(db, tenant_id: int, branch_id: int | None): + stmt=select(ERPWorkstationAgent).where(ERPWorkstationAgent.tenant_id==tenant_id, ERPWorkstationAgent.is_active.is_(True)) + if branch_id is not None: stmt=stmt.where(ERPWorkstationAgent.branch_id==branch_id) + return list(db.execute(stmt.order_by(ERPWorkstationAgent.last_seen_at_utc.desc())).scalars().all()) + + +def _sync_runs(db, tenant_id: int, client_id: int): + runs=list(db.execute(select(AccountingHistoricalLearningRun).where(AccountingHistoricalLearningRun.tenant_id==tenant_id, AccountingHistoricalLearningRun.client_id==client_id).order_by(AccountingHistoricalLearningRun.id.desc()).limit(10)).scalars().all()) + for run in runs: + if run.status not in {"completed","failed"}: ingest_completed_run(db, run) + return runs + + +@router.get("") +def historical_learning(request: Request, client_id: int | None=None, collected: int=0, mapped: int=0, error: str=""): + db=CommonSessionLocal() + try: + user,response=_require_partner(request,db,"accounting.learning.view") + if response: return response + clients,scope=_visible_clients(db,request,user) + selected=next((c for c in clients if client_id and int(c.id)==int(client_id)),None) + runs=[]; evid=[]; ledgers=[]; suggestions=[]; mappings=[] + workstations=_workstations(db, scope.tenant_id, scope.branch_id) + if selected: + runs=_sync_runs(db,scope.tenant_id,selected.id) + evid=evidence_rows(db,scope.tenant_id,selected.id) + ledgers=classifiable_ledgers(db,scope.tenant_id,selected.id) + suggestions=party_suggestions(db,scope.tenant_id,selected.id) + mappings=ledger_mappings(db,scope.tenant_id,selected.id) + df,dt=_fy_dates() + return templates.TemplateResponse("modules/accounting/templates/accounting/historical_learning.html", { + "request":request,"current_user":user,"current_user_roles":get_user_roles(db,user.id),"current_user_permissions":get_user_permissions(db,user.id),"csrf_token":get_or_create_csrf_token(request), + "title":"Historical Tally Learning","clients":clients,"selected_client":selected,"workstations":workstations,"workstation_companies":{w.id:_companies(w) for w in workstations}, + "runs":runs,"evidence":evid,"classifiable_ledgers":ledgers,"suggestions":suggestions,"mappings":mappings,"natures":active_natures(db,scope.tenant_id), + "date_from":df,"date_to":dt,"collected":bool(collected),"mapped":bool(mapped),"error":error, + }) + finally: db.close() + + +@router.post("/collect") +def collect(request: Request, client_id: int=Form(...), workstation_id: int=Form(...), tally_guid: str=Form(...), company_name: str=Form(""), date_from: str=Form(""), date_to: str=Form(""), csrf_token: str=Form(...)): + validate_csrf(request,csrf_token); db=CommonSessionLocal() + try: + user,response=_require_partner(request,db,"accounting.learning.manage") + if response:return response + client,_,scope=_find_visible_client(db,request,user,client_id) + if not client:return _denied() + workstation=db.get(ERPWorkstationAgent,workstation_id) + if not workstation or not workstation.is_active or workstation.tenant_id!=scope.tenant_id or (scope.branch_id is not None and workstation.branch_id!=scope.branch_id): return _denied() + guid=str(tally_guid or "").strip() + if not guid: return RedirectResponse(url=f"/tools/accounting/historical-learning?client_id={client_id}&error=company_required",status_code=303) + payload=_accounting_storage_payload(client); payload.update({"tenant_id":scope.tenant_id,"requested_by_user_id":user.id,"tally_guid":guid,"date_from":date_from,"date_to":date_to,"voucher_scope":"purchase"}) + job=enqueue_agent_job(db,workstation_agent_id=workstation.id,action="accounting_historical_evidence",payload=payload,priority=6,max_attempts=2,created_by_user_id=user.id) + db.flush() + run=AccountingHistoricalLearningRun(tenant_id=scope.tenant_id,client_id=client.id,workstation_agent_id=workstation.id,agent_job_id=job.id,tally_guid=guid,company_name=company_name or "",date_from=date_from or None,date_to=date_to or None,status="queued",requested_by_user_id=user.id) + db.add(run); db.commit() + return RedirectResponse(url=f"/tools/accounting/historical-learning?client_id={client_id}&collected=1",status_code=303) + except Exception as exc: + db.rollback(); return RedirectResponse(url=f"/tools/accounting/historical-learning?client_id={client_id}&error={str(exc)[:120]}",status_code=303) + finally: db.close() + + +@router.post("/map-ledger") +def map_ledger(request: Request, client_id: int=Form(...), tally_guid: str=Form(...), company_name: str=Form(""), ledger_name: str=Form(...), parent_group: str=Form(""), nature_id: int=Form(...), csrf_token: str=Form(...)): + validate_csrf(request,csrf_token); db=CommonSessionLocal() + try: + user,response=_require_partner(request,db,"accounting.learning.manage") + if response:return response + client,_,scope=_find_visible_client(db,request,user,client_id) + if not client:return _denied() + save_ledger_mapping(db,tenant_id=scope.tenant_id,client_id=client.id,tally_guid=tally_guid,company_name=company_name,ledger_name=ledger_name,parent_group=parent_group,nature_id=nature_id,user_id=user.id) + return RedirectResponse(url=f"/tools/accounting/historical-learning?client_id={client_id}&mapped=1",status_code=303) + finally: db.close() + + +@router.post("/unmap-ledger") +def unmap_ledger(request: Request, client_id: int=Form(...), mapping_id: int=Form(...), csrf_token: str=Form(...)): + validate_csrf(request,csrf_token); db=CommonSessionLocal() + try: + user,response=_require_partner(request,db,"accounting.learning.manage") + if response:return response + client,_,scope=_find_visible_client(db,request,user,client_id) + if not client:return _denied() + remove_ledger_mapping(db,tenant_id=scope.tenant_id,client_id=client.id,mapping_id=mapping_id) + return RedirectResponse(url=f"/tools/accounting/historical-learning?client_id={client_id}",status_code=303) + finally: db.close() diff --git a/app/modules/accounting/templates/accounting/historical_learning.html b/app/modules/accounting/templates/accounting/historical_learning.html new file mode 100644 index 0000000..341e1fb --- /dev/null +++ b/app/modules/accounting/templates/accounting/historical_learning.html @@ -0,0 +1,59 @@ +{% extends "ui/templates/base/layout.html" %} +{% block content %} +
+
+

Tools · Accounting · Phase 5

Historical Tally Learning

Uses synced Tally purchase history as evidence. No voucher is created or altered in this phase.

+
Accounting TaxonomyBack to Tally
+
+ {% if collected %}
Historical evidence job queued. This page will ingest the result after the selected workstation completes it.
{% endif %} + {% if mapped %}
Ledger classification mapping saved.
{% endif %} + {% if error %}
{{ error }}
{% endif %} + +
+
+ + +
+
+ + {% if selected_client %} +
+

Collect Historical Purchase Evidence

+

Select the workstation that holds this client's synced .act data. The agent returns neutral ledger-history facts; classification stays on the ERP server.

+
+ + + + + +
+
+
+ +
+

Recent Learning Runs

+
{% for r in runs %}{% else %}{% endfor %}
CreatedCompanyStatusRowsError
{{ r.created_at_utc }}{{ r.company_name or r.tally_guid }}{{ r.status }}{{ r.evidence_rows }}{{ r.error_message or '' }}
No historical learning run yet.
+
+ +
+

Classify Historical Tally Ledgers

Map the client's actual Tally ledger names to the canonical Phase 4 accounting taxonomy. This mapping is client and Tally-company specific.

+
+ {% for row in classifiable_ledgers %}{% else %}{% endfor %} +
LedgerParentHistorical vouchersAmount evidenceAccounting nature
{{ row.ledger_name }}{{ row.parent_group }}{{ row.voucher_count }}{{ '%.2f'|format(row.amount) }}
Collect historical evidence first.
+
+ +
+

Supplier / Party Historical Suggestions

Confidence is the share of mapped historical purchase vouchers for that party that resolve to the leading accounting nature.

+
{% for s in suggestions %}{% else %}{% endfor %}
PartySuggested natureConfidenceEvidenceLedgers
{{ s.party_ledger_name }}{{ s.nature.name }}{{ s.confidence }}%{{ s.voucher_count }} / {{ s.mapped_voucher_count }} mapped vouchers{{ s.ledgers|join(', ') }}
Map historical Tally ledgers to accounting natures to generate party suggestions.
+
+ +
+

Raw Historical Evidence

+
{% for e in evidence[:300] %}{% else %}{% endfor %}
PartyCounter ledgerGroupVouchersFirstLast
{{ e.party_ledger_name }}{{ e.counter_ledger_name }}{{ e.counter_parent_group }}{{ e.voucher_count }}{{ e.first_voucher_date or '' }}{{ e.last_voucher_date or '' }}
No evidence collected.
+
+ {% endif %} +
+ +{% endblock %} diff --git a/app/modules/accounting/templates/accounting/tally.html b/app/modules/accounting/templates/accounting/tally.html index 790e6b5..294ed11 100644 --- a/app/modules/accounting/templates/accounting/tally.html +++ b/app/modules/accounting/templates/accounting/tally.html @@ -14,6 +14,7 @@
Accounting Taxonomy + {% if selected_client %}Historical Learning{% endif %} {% if selected_client %}Depreciation (IT){% endif %} Refresh Tally Companies
diff --git a/app/modules/core/rbac/permissions_registry.py b/app/modules/core/rbac/permissions_registry.py index 206ef2b..8ce5af9 100644 --- a/app/modules/core/rbac/permissions_registry.py +++ b/app/modules/core/rbac/permissions_registry.py @@ -156,6 +156,8 @@ PERMISSIONS = { "accounting.tally.writeback": "Approve and Post Accounting Entries to Tally", "accounting.taxonomy.view": "View Accounting Classification Taxonomy", "accounting.taxonomy.manage": "Activate or Deactivate Accounting Classification Natures", + "accounting.learning.view": "View Historical Tally Learning", + "accounting.learning.manage": "Collect Evidence and Map Historical Tally Ledgers", "accounting.act.initialize": "Initialize Client Accounting ACT Storage", "notice_cases.view": "View Notice and Case Management", diff --git a/app/modules/documents/agent_package.py b/app/modules/documents/agent_package.py index ec32202..d402977 100644 --- a/app/modules/documents/agent_package.py +++ b/app/modules/documents/agent_package.py @@ -4,7 +4,7 @@ import io from pathlib import Path import zipfile -ERP_LOCAL_AGENT_VERSION = "1.11.0" +ERP_LOCAL_AGENT_VERSION = "1.12.0" ERP_LOCAL_AGENT_NAME = "ERP Local Agent" RUNTIME_ROOT = Path(__file__).resolve().parent / "local_agent_runtime" _DETERMINISTIC_ZIP_TIMESTAMP = (2026, 1, 1, 0, 0, 0) @@ -53,7 +53,7 @@ def _build_zip(*, env_text: str | None, include_env: bool, include_admin_readme: text = ( f"ERP Local Agent {ERP_LOCAL_AGENT_VERSION}\n" "Existing storage, WebSocket tunnel, dashboard, Tally mapping and client .act functionality are preserved.\n" - "Phase 3 master synchronization is preserved. Phase 4 adds read-only date-scoped Tally voucher/transaction synchronization.\n" + "Existing master/transaction sync and depreciation are preserved. Phase 5 adds neutral historical-ledger evidence read capability for server-side learning.\n" "Dashboard: http://127.0.0.1:8788\n" ) _write_zip_bytes(dst, "README_ERP_LOCAL_AGENT.txt", text.encode("utf-8")) diff --git a/app/modules/documents/local_agent_runtime/erp_local_agent/__init__.py b/app/modules/documents/local_agent_runtime/erp_local_agent/__init__.py index 34d1752..b518f6e 100644 --- a/app/modules/documents/local_agent_runtime/erp_local_agent/__init__.py +++ b/app/modules/documents/local_agent_runtime/erp_local_agent/__init__.py @@ -1,2 +1 @@ -__version__ = "1.11.0" -AGENT_NAME = "ERP Local Agent" +__version__ = "1.12.0" diff --git a/app/modules/documents/local_agent_runtime/erp_local_agent/accounting_store.py b/app/modules/documents/local_agent_runtime/erp_local_agent/accounting_store.py index 02ae463..455358a 100644 --- a/app/modules/documents/local_agent_runtime/erp_local_agent/accounting_store.py +++ b/app/modules/documents/local_agent_runtime/erp_local_agent/accounting_store.py @@ -1077,6 +1077,54 @@ class LocalAccountingStore: ).fetchall() return [dict(row) for row in rows] + def historical_learning_evidence(self, client_id: int, *, tally_guid: str, date_from: str = "", date_to: str = "", voucher_scope: str = "purchase") -> dict: + """Return neutral aggregated historical ledger facts from the local .act store. + + No accounting classification is performed locally. The ERP server maps + these facts to its canonical taxonomy. + """ + if not self.exists(client_id): + raise ValueError("Accounting storage is not initialized for this client.") + guid = str(tally_guid or "").strip() + if not guid: + raise ValueError("Tally company GUID is required.") + mapping = self.get_active_mapping_by_guid(client_id, guid) + params: list = [guid] + where = ["v.tally_guid=?", "TRIM(COALESCE(v.party_ledger_name,''))<>''", "LOWER(TRIM(COALESCE(v.is_cancelled,''))) NOT IN ('yes','true','1')", "LOWER(TRIM(e.ledger_name))<>LOWER(TRIM(v.party_ledger_name))"] + if str(voucher_scope or "purchase").strip().lower() == "purchase": + where.append("LOWER(v.voucher_type_name) LIKE '%purchase%'") + if date_from: + where.append("v.voucher_date>=?"); params.append(str(date_from)) + if date_to: + where.append("v.voucher_date<=?"); params.append(str(date_to)) + sql = f""" + SELECT v.party_ledger_name AS party_ledger_name, + COALESCE(pl.parent,'') AS party_parent_group, + e.ledger_name AS counter_ledger_name, + COALESCE(cl.parent,'') AS counter_parent_group, + v.voucher_type_name AS voucher_type_name, + COUNT(DISTINCT v.id) AS voucher_count, + COALESCE(SUM(ABS(e.amount)),0) AS absolute_amount_total, + MIN(NULLIF(v.voucher_date,'')) AS first_voucher_date, + MAX(NULLIF(v.voucher_date,'')) AS last_voucher_date, + MIN(NULLIF(v.narration,'')) AS sample_narration + FROM tally_vouchers v + JOIN tally_voucher_ledger_entries e ON e.voucher_id=v.id + LEFT JOIN tally_ledgers pl ON pl.tally_guid=v.tally_guid AND LOWER(TRIM(pl.name))=LOWER(TRIM(v.party_ledger_name)) + LEFT JOIN tally_ledgers cl ON cl.tally_guid=v.tally_guid AND LOWER(TRIM(cl.name))=LOWER(TRIM(e.ledger_name)) + WHERE {' AND '.join(where)} + GROUP BY v.party_ledger_name, COALESCE(pl.parent,''), e.ledger_name, COALESCE(cl.parent,''), v.voucher_type_name + ORDER BY voucher_count DESC, v.party_ledger_name COLLATE NOCASE, e.ledger_name COLLATE NOCASE + LIMIT 5000 + """ + with self.connect(client_id) as db: + rows = [dict(r) for r in db.execute(sql, tuple(params)).fetchall()] + return { + "client_id": int(client_id), "tally_guid": guid, "company_name": str(mapping.get("company_name") or ""), + "date_from": str(date_from or ""), "date_to": str(date_to or ""), "voucher_scope": str(voucher_scope or "purchase"), + "evidence": rows, "evidence_count": len(rows), + } + def snapshot(self, client_id: int) -> dict: path = self.db_path(client_id) if not path.is_file(): diff --git a/app/modules/documents/local_agent_runtime/erp_local_agent/commands.py b/app/modules/documents/local_agent_runtime/erp_local_agent/commands.py index 1a88cd5..237116f 100644 --- a/app/modules/documents/local_agent_runtime/erp_local_agent/commands.py +++ b/app/modules/documents/local_agent_runtime/erp_local_agent/commands.py @@ -36,6 +36,8 @@ class AgentCommandProcessor: result = self._sync_masters(payload) elif action == "accounting_sync_transactions": result = self._sync_transactions(payload) + elif action == "accounting_historical_evidence": + result = self._historical_evidence(payload) elif action == "accounting_depreciation_preview": result = self._depreciation_preview(payload) elif action == "accounting_calculate_it_depreciation": @@ -67,6 +69,7 @@ class AgentCommandProcessor: "tally_capability": True, "accounting_act_capability": True, "tally_mapping_capability": True, "tally_master_sync_capability": True, "tally_transaction_sync_capability": True, + "historical_learning_read_capability": True, "it_depreciation_capability": True, "tally_writeback_capability": True, } @@ -201,6 +204,16 @@ class AgentCommandProcessor: self.logger.info("Tally transaction sync completed client_id=%s company=%s period=%s..%s rows=%s", client_id, company_name, date_from, date_to, sync.get("rows_processed")) return {"synced": True, "sync": sync, "accounting": self.store.snapshot(client_id), "tally": tally_status, "agent": self._agent_info()} + def _historical_evidence(self, payload: dict[str, Any]) -> dict[str, Any]: + client_id = int(payload.get("client_id")) + return self.store.historical_learning_evidence( + client_id, + tally_guid=str(payload.get("tally_guid") or ""), + date_from=str(payload.get("date_from") or ""), + date_to=str(payload.get("date_to") or ""), + voucher_scope=str(payload.get("voucher_scope") or "purchase"), + ) + def _depreciation_preview(self, payload: dict[str, Any]) -> dict[str, Any]: client_id = int(payload.get("client_id")) tally_guid = str(payload.get("tally_guid") or "").strip() diff --git a/app/modules/documents/local_agent_runtime/erp_local_agent/sync.py b/app/modules/documents/local_agent_runtime/erp_local_agent/sync.py index 06569fc..1501069 100644 --- a/app/modules/documents/local_agent_runtime/erp_local_agent/sync.py +++ b/app/modules/documents/local_agent_runtime/erp_local_agent/sync.py @@ -49,7 +49,7 @@ class StorageAgent: "free_bytes": free, "agent_version": __version__, "agent_name": "ERP Local Agent", - "capabilities": ["storage", "tally", "accounting_act", "tally_master_sync", "tally_transaction_sync", "local_dashboard", "manual_updates"], + "capabilities": ["storage", "tally", "accounting_act", "tally_master_sync", "tally_transaction_sync", "local_dashboard", "manual_updates", "historical_learning_read"], "agent_time_utc": datetime.now(timezone.utc).isoformat(), } self.client.heartbeat(payload) diff --git a/app/modules/documents/local_agent_runtime/erp_local_agent/tunnel.py b/app/modules/documents/local_agent_runtime/erp_local_agent/tunnel.py index c35a82b..30bbda4 100644 --- a/app/modules/documents/local_agent_runtime/erp_local_agent/tunnel.py +++ b/app/modules/documents/local_agent_runtime/erp_local_agent/tunnel.py @@ -190,7 +190,7 @@ class StorageAgentTunnel: def _capabilities(self) -> list[str]: return [ - "storage", "local_dashboard", "manual_updates", "accounting_act", "agent.jobs.v2", + "storage", "local_dashboard", "manual_updates", "historical_learning_read", "accounting_act", "agent.jobs.v2", "tally.status", "tally.company_identity", "tally.read_groups", "tally.read_ledgers", "tally.read_stock_items", "tally.read_vouchers", "tally.read_trial_balance", "tally.master_sync", "tally.transaction_sync", diff --git a/app/ui/app.py b/app/ui/app.py index 08d5d51..6014ccb 100644 --- a/app/ui/app.py +++ b/app/ui/app.py @@ -38,6 +38,7 @@ from app.modules.peer_review_export.ui import router as peer_review_export_route from app.modules.bank_statement_analyzer.ui import router as bank_statement_analyzer_router from app.modules.accounting.ui import router as accounting_ui_router from app.modules.accounting.taxonomy_ui import router as accounting_taxonomy_ui_router +from app.modules.accounting.historical_learning_ui import router as accounting_historical_learning_ui_router from app.modules.registrations.ui import router as registrations_ui_router from app.modules.credential_vault.ui import router as credential_vault_ui_router from app.modules.client_identity.ui import router as client_identity_ui_router @@ -62,6 +63,7 @@ def mount_ui(app: FastAPI) -> None: app.include_router(bank_statement_analyzer_router) app.include_router(accounting_ui_router) app.include_router(accounting_taxonomy_ui_router) + app.include_router(accounting_historical_learning_ui_router) app.include_router(work_tracker_ui_router) app.include_router(billing_ui_router) app.include_router(platform_billing_ui_router)