Add Phase 5 historical Tally learning foundation

app/core/startup.py

@@ -342,6 +342,8 @@ ROLE_PERMISSION_MAP = {
         "accounting.tally.writeback",
         "accounting.taxonomy.view",
         "accounting.taxonomy.manage",
+        "accounting.learning.view",
+        "accounting.learning.manage",
         "accounting.act.initialize",
         "clients.view.own_only",
         "employees.dashboard.view",

app/modules/accounting/historical_learning_models.py

@@ -0,0 +1,74 @@
+from __future__ import annotations
+
+from datetime import datetime, timezone
+
+from sqlalchemy import Date, DateTime, Float, ForeignKey, Integer, String, Text, UniqueConstraint
+from sqlalchemy.orm import Mapped, mapped_column
+
+from app.core.db.common import CommonBase
+
+
+class AccountingLedgerNatureMapping(CommonBase):
+    __tablename__ = "accounting_ledger_nature_mappings"
+    __table_args__ = (
+        UniqueConstraint("tenant_id", "client_id", "tally_guid", "ledger_name", name="uq_accounting_ledger_nature_mapping"),
+    )
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
+    tenant_id: Mapped[int] = mapped_column(ForeignKey("tenants.id", ondelete="CASCADE"), nullable=False, index=True)
+    client_id: Mapped[int] = mapped_column(ForeignKey("clients.id", ondelete="CASCADE"), nullable=False, index=True)
+    tally_guid: Mapped[str] = mapped_column(String(120), nullable=False, index=True)
+    company_name: Mapped[str] = mapped_column(String(240), nullable=False, default="")
+    ledger_name: Mapped[str] = mapped_column(String(240), nullable=False, index=True)
+    parent_group: Mapped[str] = mapped_column(String(240), nullable=False, default="", index=True)
+    nature_id: Mapped[int] = mapped_column(ForeignKey("accounting_natures.id", ondelete="RESTRICT"), nullable=False, index=True)
+    source: Mapped[str] = mapped_column(String(40), nullable=False, default="manual", index=True)
+    confidence_percent: Mapped[int] = mapped_column(Integer, nullable=False, default=100)
+    confirmed_by_user_id: Mapped[int | None] = mapped_column(ForeignKey("users.id", ondelete="SET NULL"), nullable=True)
+    confirmed_at_utc: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc), nullable=False)
+    created_at_utc: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc), nullable=False)
+    updated_at_utc: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc), onupdate=lambda: datetime.now(timezone.utc), nullable=False)
+
+
+class AccountingHistoricalLedgerEvidence(CommonBase):
+    __tablename__ = "accounting_historical_ledger_evidence"
+    __table_args__ = (
+        UniqueConstraint("tenant_id", "client_id", "tally_guid", "party_ledger_name", "counter_ledger_name", "voucher_type_name", name="uq_accounting_historical_ledger_evidence"),
+    )
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
+    tenant_id: Mapped[int] = mapped_column(ForeignKey("tenants.id", ondelete="CASCADE"), nullable=False, index=True)
+    client_id: Mapped[int] = mapped_column(ForeignKey("clients.id", ondelete="CASCADE"), nullable=False, index=True)
+    tally_guid: Mapped[str] = mapped_column(String(120), nullable=False, index=True)
+    company_name: Mapped[str] = mapped_column(String(240), nullable=False, default="")
+    party_ledger_name: Mapped[str] = mapped_column(String(240), nullable=False, index=True)
+    party_parent_group: Mapped[str] = mapped_column(String(240), nullable=False, default="")
+    counter_ledger_name: Mapped[str] = mapped_column(String(240), nullable=False, index=True)
+    counter_parent_group: Mapped[str] = mapped_column(String(240), nullable=False, default="", index=True)
+    voucher_type_name: Mapped[str] = mapped_column(String(160), nullable=False, default="Purchase", index=True)
+    voucher_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+    absolute_amount_total: Mapped[float] = mapped_column(Float, nullable=False, default=0.0)
+    first_voucher_date: Mapped[str | None] = mapped_column(String(20), nullable=True)
+    last_voucher_date: Mapped[str | None] = mapped_column(String(20), nullable=True)
+    sample_narration: Mapped[str | None] = mapped_column(Text, nullable=True)
+    refreshed_at_utc: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc), nullable=False, index=True)
+
+
+class AccountingHistoricalLearningRun(CommonBase):
+    __tablename__ = "accounting_historical_learning_runs"
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
+    tenant_id: Mapped[int] = mapped_column(ForeignKey("tenants.id", ondelete="CASCADE"), nullable=False, index=True)
+    client_id: Mapped[int] = mapped_column(ForeignKey("clients.id", ondelete="CASCADE"), nullable=False, index=True)
+    workstation_agent_id: Mapped[int] = mapped_column(ForeignKey("erp_workstation_agents.id", ondelete="SET NULL"), nullable=True, index=True)
+    agent_job_id: Mapped[int] = mapped_column(ForeignKey("erp_agent_jobs.id", ondelete="SET NULL"), nullable=True, unique=True, index=True)
+    tally_guid: Mapped[str] = mapped_column(String(120), nullable=False, index=True)
+    company_name: Mapped[str] = mapped_column(String(240), nullable=False, default="")
+    date_from: Mapped[str | None] = mapped_column(String(20), nullable=True)
+    date_to: Mapped[str | None] = mapped_column(String(20), nullable=True)
+    status: Mapped[str] = mapped_column(String(30), nullable=False, default="queued", index=True)
+    evidence_rows: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+    error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
+    requested_by_user_id: Mapped[int | None] = mapped_column(ForeignKey("users.id", ondelete="SET NULL"), nullable=True)
+    created_at_utc: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc), nullable=False, index=True)
+    completed_at_utc: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)

app/modules/accounting/historical_learning_service.py

@@ -0,0 +1,168 @@
+from __future__ import annotations
+
+import json
+from collections import defaultdict
+from datetime import datetime, timezone
+
+from sqlalchemy import delete, select
+
+from app.modules.accounting.historical_learning_models import (
+    AccountingHistoricalLedgerEvidence, AccountingHistoricalLearningRun, AccountingLedgerNatureMapping,
+)
+from app.modules.accounting.taxonomy_models import AccountingNature
+from app.modules.documents.models import ERPAgentJob
+
+
+NOISE_GROUPS = {
+    "bank accounts", "bank od a/c", "cash-in-hand", "cash in hand", "sundry creditors", "sundry debtors",
+    "duties & taxes", "duties and taxes", "loans (liability)", "secured loans", "unsecured loans",
+}
+
+
+def _utcnow():
+    return datetime.now(timezone.utc)
+
+
+def _json(value):
+    try:
+        return json.loads(value or "{}")
+    except Exception:
+        return {}
+
+
+def active_natures(db, tenant_id: int):
+    return list(db.execute(select(AccountingNature).where(
+        AccountingNature.tenant_id == tenant_id, AccountingNature.is_active.is_(True), AccountingNature.is_posting_nature.is_(True)
+    ).order_by(AccountingNature.sort_order, AccountingNature.name)).scalars().all())
+
+
+def ledger_mappings(db, tenant_id: int, client_id: int, tally_guid: str = ""):
+    stmt = select(AccountingLedgerNatureMapping).where(
+        AccountingLedgerNatureMapping.tenant_id == tenant_id, AccountingLedgerNatureMapping.client_id == client_id
+    )
+    if tally_guid:
+        stmt = stmt.where(AccountingLedgerNatureMapping.tally_guid == tally_guid)
+    return list(db.execute(stmt.order_by(AccountingLedgerNatureMapping.ledger_name)).scalars().all())
+
+
+def save_ledger_mapping(db, *, tenant_id: int, client_id: int, tally_guid: str, company_name: str, ledger_name: str, parent_group: str, nature_id: int, user_id: int):
+    nature = db.execute(select(AccountingNature).where(
+        AccountingNature.id == nature_id, AccountingNature.tenant_id == tenant_id, AccountingNature.is_active.is_(True), AccountingNature.is_posting_nature.is_(True)
+    )).scalar_one_or_none()
+    if not nature:
+        raise ValueError("Select an active posting accounting nature.")
+    row = db.execute(select(AccountingLedgerNatureMapping).where(
+        AccountingLedgerNatureMapping.tenant_id == tenant_id, AccountingLedgerNatureMapping.client_id == client_id,
+        AccountingLedgerNatureMapping.tally_guid == tally_guid, AccountingLedgerNatureMapping.ledger_name == ledger_name
+    )).scalar_one_or_none()
+    if not row:
+        row = AccountingLedgerNatureMapping(tenant_id=tenant_id, client_id=client_id, tally_guid=tally_guid, ledger_name=ledger_name, nature_id=nature.id)
+    row.company_name = company_name or row.company_name or ""
+    row.parent_group = parent_group or row.parent_group or ""
+    row.nature_id = nature.id
+    row.source = "manual"
+    row.confidence_percent = 100
+    row.confirmed_by_user_id = user_id
+    row.confirmed_at_utc = _utcnow()
+    row.updated_at_utc = _utcnow()
+    db.add(row); db.commit(); db.refresh(row)
+    return row
+
+
+def remove_ledger_mapping(db, *, tenant_id: int, client_id: int, mapping_id: int):
+    row = db.execute(select(AccountingLedgerNatureMapping).where(
+        AccountingLedgerNatureMapping.id == mapping_id, AccountingLedgerNatureMapping.tenant_id == tenant_id, AccountingLedgerNatureMapping.client_id == client_id
+    )).scalar_one_or_none()
+    if row:
+        db.delete(row); db.commit()
+    return row
+
+
+def ingest_completed_run(db, run: AccountingHistoricalLearningRun):
+    if run.status == "completed" or not run.agent_job_id:
+        return run
+    job = db.get(ERPAgentJob, run.agent_job_id)
+    if not job:
+        run.status = "failed"; run.error_message = "Agent job no longer exists."; run.completed_at_utc = _utcnow(); db.commit(); return run
+    if job.status in {"queued", "claimed"}:
+        run.status = job.status; db.commit(); return run
+    if job.status != "succeeded":
+        run.status = "failed"; run.error_message = job.last_error or "Historical evidence collection failed."; run.completed_at_utc = _utcnow(); db.commit(); return run
+    result = _json(job.result_json)
+    rows = result.get("evidence") or []
+    company_name = str(result.get("company_name") or run.company_name or "")
+    now = _utcnow()
+    # Snapshot semantics for this company: remove old aggregated rows then replace with current evidence.
+    db.execute(delete(AccountingHistoricalLedgerEvidence).where(
+        AccountingHistoricalLedgerEvidence.tenant_id == run.tenant_id, AccountingHistoricalLedgerEvidence.client_id == run.client_id,
+        AccountingHistoricalLedgerEvidence.tally_guid == run.tally_guid
+    ))
+    inserted = 0
+    for item in rows:
+        party = str(item.get("party_ledger_name") or "").strip()
+        counter = str(item.get("counter_ledger_name") or "").strip()
+        if not party or not counter or party.casefold() == counter.casefold():
+            continue
+        db.add(AccountingHistoricalLedgerEvidence(
+            tenant_id=run.tenant_id, client_id=run.client_id, tally_guid=run.tally_guid, company_name=company_name,
+            party_ledger_name=party, party_parent_group=str(item.get("party_parent_group") or ""),
+            counter_ledger_name=counter, counter_parent_group=str(item.get("counter_parent_group") or ""),
+            voucher_type_name=str(item.get("voucher_type_name") or "Purchase"), voucher_count=int(item.get("voucher_count") or 0),
+            absolute_amount_total=float(item.get("absolute_amount_total") or 0), first_voucher_date=str(item.get("first_voucher_date") or "") or None,
+            last_voucher_date=str(item.get("last_voucher_date") or "") or None, sample_narration=str(item.get("sample_narration") or "")[:2000] or None, refreshed_at_utc=now
+        )); inserted += 1
+    run.company_name = company_name
+    run.status = "completed"
+    run.evidence_rows = inserted
+    run.error_message = None
+    run.completed_at_utc = now
+    db.commit(); db.refresh(run)
+    return run
+
+
+def evidence_rows(db, tenant_id: int, client_id: int):
+    return list(db.execute(select(AccountingHistoricalLedgerEvidence).where(
+        AccountingHistoricalLedgerEvidence.tenant_id == tenant_id, AccountingHistoricalLedgerEvidence.client_id == client_id
+    ).order_by(AccountingHistoricalLedgerEvidence.voucher_count.desc(), AccountingHistoricalLedgerEvidence.party_ledger_name, AccountingHistoricalLedgerEvidence.counter_ledger_name)).scalars().all())
+
+
+def classifiable_ledgers(db, tenant_id: int, client_id: int):
+    rows = evidence_rows(db, tenant_id, client_id)
+    seen = {}
+    for r in rows:
+        group = (r.counter_parent_group or "").strip().casefold()
+        if group in NOISE_GROUPS:
+            continue
+        key = (r.tally_guid, r.counter_ledger_name.casefold())
+        item = seen.setdefault(key, {"tally_guid": r.tally_guid, "company_name": r.company_name, "ledger_name": r.counter_ledger_name, "parent_group": r.counter_parent_group, "voucher_count": 0, "amount": 0.0})
+        item["voucher_count"] += int(r.voucher_count or 0); item["amount"] += float(r.absolute_amount_total or 0)
+    mappings = {(m.tally_guid, m.ledger_name.casefold()): m for m in ledger_mappings(db, tenant_id, client_id)}
+    for key, item in seen.items(): item["mapping"] = mappings.get(key)
+    return sorted(seen.values(), key=lambda x: (-x["voucher_count"], x["ledger_name"].casefold()))
+
+
+def party_suggestions(db, tenant_id: int, client_id: int):
+    rows = evidence_rows(db, tenant_id, client_id)
+    mappings = {(m.tally_guid, m.ledger_name.casefold()): m for m in ledger_mappings(db, tenant_id, client_id)}
+    natures = {n.id: n for n in db.execute(select(AccountingNature).where(AccountingNature.tenant_id == tenant_id)).scalars().all()}
+    by_party = defaultdict(lambda: defaultdict(lambda: {"count": 0, "amount": 0.0, "ledgers": set()}))
+    party_meta = {}
+    for r in rows:
+        m = mappings.get((r.tally_guid, r.counter_ledger_name.casefold()))
+        if not m:
+            continue
+        key=(r.tally_guid, r.party_ledger_name)
+        party_meta[key]=(r.company_name, r.party_parent_group)
+        bucket=by_party[key][m.nature_id]
+        bucket["count"] += int(r.voucher_count or 0); bucket["amount"] += float(r.absolute_amount_total or 0); bucket["ledgers"].add(r.counter_ledger_name)
+    out=[]
+    for key, nature_buckets in by_party.items():
+        total=sum(v["count"] for v in nature_buckets.values())
+        if total <= 0: continue
+        nature_id, best=max(nature_buckets.items(), key=lambda kv:(kv[1]["count"], kv[1]["amount"]))
+        nature=natures.get(nature_id)
+        if not nature: continue
+        confidence=round(best["count"]*100/total)
+        company, parent=party_meta[key]
+        out.append({"tally_guid": key[0], "company_name": company, "party_ledger_name": key[1], "party_parent_group": parent, "nature": nature, "voucher_count": best["count"], "mapped_voucher_count": total, "confidence": confidence, "amount": best["amount"], "ledgers": sorted(best["ledgers"])})
+    return sorted(out, key=lambda x:(-x["confidence"], -x["voucher_count"], x["party_ledger_name"].casefold()))

app/modules/accounting/historical_learning_ui.py

@@ -0,0 +1,125 @@
+from __future__ import annotations
+
+import json
+from datetime import date, datetime, timezone
+
+from fastapi import APIRouter, Form, Request
+from fastapi.responses import RedirectResponse
+from sqlalchemy import select
+
+from app.core.db.common import CommonSessionLocal
+from app.core.security import get_or_create_csrf_token, validate_csrf
+from app.core.templating import templates
+from app.modules.accounting.historical_learning_models import AccountingHistoricalLearningRun
+from app.modules.accounting.historical_learning_service import active_natures, classifiable_ledgers, evidence_rows, ingest_completed_run, ledger_mappings, party_suggestions, remove_ledger_mapping, save_ledger_mapping
+from app.modules.accounting.ui import _accounting_storage_payload, _find_visible_client, _require_partner, _visible_clients
+from app.modules.core.rbac.deps import get_user_permissions, get_user_roles
+from app.modules.core.rbac.permission_guard import require_permission
+from app.modules.documents.agent_jobs import enqueue_agent_job
+from app.modules.documents.models import ERPAgentJob, ERPWorkstationAgent
+
+router = APIRouter(prefix="/tools/accounting/historical-learning", tags=["accounting-historical-learning-ui"])
+
+
+def _denied():
+    from app.core.http_responses import ui_access_denied
+    return ui_access_denied()
+
+
+def _fy_dates():
+    today=date.today(); y=today.year if today.month>=4 else today.year-1
+    return date(y,4,1).isoformat(), today.isoformat()
+
+
+def _companies(workstation):
+    try: rows=json.loads(workstation.tally_companies_json or "[]")
+    except Exception: rows=[]
+    return rows if isinstance(rows,list) else []
+
+
+def _workstations(db, tenant_id: int, branch_id: int | None):
+    stmt=select(ERPWorkstationAgent).where(ERPWorkstationAgent.tenant_id==tenant_id, ERPWorkstationAgent.is_active.is_(True))
+    if branch_id is not None: stmt=stmt.where(ERPWorkstationAgent.branch_id==branch_id)
+    return list(db.execute(stmt.order_by(ERPWorkstationAgent.last_seen_at_utc.desc())).scalars().all())
+
+
+def _sync_runs(db, tenant_id: int, client_id: int):
+    runs=list(db.execute(select(AccountingHistoricalLearningRun).where(AccountingHistoricalLearningRun.tenant_id==tenant_id, AccountingHistoricalLearningRun.client_id==client_id).order_by(AccountingHistoricalLearningRun.id.desc()).limit(10)).scalars().all())
+    for run in runs:
+        if run.status not in {"completed","failed"}: ingest_completed_run(db, run)
+    return runs
+
+
+@router.get("")
+def historical_learning(request: Request, client_id: int | None=None, collected: int=0, mapped: int=0, error: str=""):
+    db=CommonSessionLocal()
+    try:
+        user,response=_require_partner(request,db,"accounting.learning.view")
+        if response: return response
+        clients,scope=_visible_clients(db,request,user)
+        selected=next((c for c in clients if client_id and int(c.id)==int(client_id)),None)
+        runs=[]; evid=[]; ledgers=[]; suggestions=[]; mappings=[]
+        workstations=_workstations(db, scope.tenant_id, scope.branch_id)
+        if selected:
+            runs=_sync_runs(db,scope.tenant_id,selected.id)
+            evid=evidence_rows(db,scope.tenant_id,selected.id)
+            ledgers=classifiable_ledgers(db,scope.tenant_id,selected.id)
+            suggestions=party_suggestions(db,scope.tenant_id,selected.id)
+            mappings=ledger_mappings(db,scope.tenant_id,selected.id)
+        df,dt=_fy_dates()
+        return templates.TemplateResponse("modules/accounting/templates/accounting/historical_learning.html", {
+            "request":request,"current_user":user,"current_user_roles":get_user_roles(db,user.id),"current_user_permissions":get_user_permissions(db,user.id),"csrf_token":get_or_create_csrf_token(request),
+            "title":"Historical Tally Learning","clients":clients,"selected_client":selected,"workstations":workstations,"workstation_companies":{w.id:_companies(w) for w in workstations},
+            "runs":runs,"evidence":evid,"classifiable_ledgers":ledgers,"suggestions":suggestions,"mappings":mappings,"natures":active_natures(db,scope.tenant_id),
+            "date_from":df,"date_to":dt,"collected":bool(collected),"mapped":bool(mapped),"error":error,
+        })
+    finally: db.close()
+
+
+@router.post("/collect")
+def collect(request: Request, client_id: int=Form(...), workstation_id: int=Form(...), tally_guid: str=Form(...), company_name: str=Form(""), date_from: str=Form(""), date_to: str=Form(""), csrf_token: str=Form(...)):
+    validate_csrf(request,csrf_token); db=CommonSessionLocal()
+    try:
+        user,response=_require_partner(request,db,"accounting.learning.manage")
+        if response:return response
+        client,_,scope=_find_visible_client(db,request,user,client_id)
+        if not client:return _denied()
+        workstation=db.get(ERPWorkstationAgent,workstation_id)
+        if not workstation or not workstation.is_active or workstation.tenant_id!=scope.tenant_id or (scope.branch_id is not None and workstation.branch_id!=scope.branch_id): return _denied()
+        guid=str(tally_guid or "").strip()
+        if not guid: return RedirectResponse(url=f"/tools/accounting/historical-learning?client_id={client_id}&error=company_required",status_code=303)
+        payload=_accounting_storage_payload(client); payload.update({"tenant_id":scope.tenant_id,"requested_by_user_id":user.id,"tally_guid":guid,"date_from":date_from,"date_to":date_to,"voucher_scope":"purchase"})
+        job=enqueue_agent_job(db,workstation_agent_id=workstation.id,action="accounting_historical_evidence",payload=payload,priority=6,max_attempts=2,created_by_user_id=user.id)
+        db.flush()
+        run=AccountingHistoricalLearningRun(tenant_id=scope.tenant_id,client_id=client.id,workstation_agent_id=workstation.id,agent_job_id=job.id,tally_guid=guid,company_name=company_name or "",date_from=date_from or None,date_to=date_to or None,status="queued",requested_by_user_id=user.id)
+        db.add(run); db.commit()
+        return RedirectResponse(url=f"/tools/accounting/historical-learning?client_id={client_id}&collected=1",status_code=303)
+    except Exception as exc:
+        db.rollback(); return RedirectResponse(url=f"/tools/accounting/historical-learning?client_id={client_id}&error={str(exc)[:120]}",status_code=303)
+    finally: db.close()
+
+
+@router.post("/map-ledger")
+def map_ledger(request: Request, client_id: int=Form(...), tally_guid: str=Form(...), company_name: str=Form(""), ledger_name: str=Form(...), parent_group: str=Form(""), nature_id: int=Form(...), csrf_token: str=Form(...)):
+    validate_csrf(request,csrf_token); db=CommonSessionLocal()
+    try:
+        user,response=_require_partner(request,db,"accounting.learning.manage")
+        if response:return response
+        client,_,scope=_find_visible_client(db,request,user,client_id)
+        if not client:return _denied()
+        save_ledger_mapping(db,tenant_id=scope.tenant_id,client_id=client.id,tally_guid=tally_guid,company_name=company_name,ledger_name=ledger_name,parent_group=parent_group,nature_id=nature_id,user_id=user.id)
+        return RedirectResponse(url=f"/tools/accounting/historical-learning?client_id={client_id}&mapped=1",status_code=303)
+    finally: db.close()
+
+
+@router.post("/unmap-ledger")
+def unmap_ledger(request: Request, client_id: int=Form(...), mapping_id: int=Form(...), csrf_token: str=Form(...)):
+    validate_csrf(request,csrf_token); db=CommonSessionLocal()
+    try:
+        user,response=_require_partner(request,db,"accounting.learning.manage")
+        if response:return response
+        client,_,scope=_find_visible_client(db,request,user,client_id)
+        if not client:return _denied()
+        remove_ledger_mapping(db,tenant_id=scope.tenant_id,client_id=client.id,mapping_id=mapping_id)
+        return RedirectResponse(url=f"/tools/accounting/historical-learning?client_id={client_id}",status_code=303)
+    finally: db.close()

app/modules/accounting/templates/accounting/historical_learning.html

@@ -0,0 +1,59 @@
+{% extends "ui/templates/base/layout.html" %}
+{% block content %}
+<div class="space-y-6">
+  <div class="flex flex-wrap items-start justify-between gap-4">
+    <div><p class="text-xs font-semibold uppercase tracking-[0.16em] text-brand-600">Tools · Accounting · Phase 5</p><h1 class="mt-1 text-2xl font-semibold text-slate-900">Historical Tally Learning</h1><p class="mt-1 text-sm text-slate-500">Uses synced Tally purchase history as evidence. No voucher is created or altered in this phase.</p></div>
+    <div class="flex gap-2"><a href="/tools/accounting/taxonomy" class="rounded-xl border border-slate-300 bg-white px-4 py-2 text-sm font-semibold text-slate-700">Accounting Taxonomy</a><a href="/tools/tally{% if selected_client %}?client_id={{ selected_client.id }}{% endif %}" class="rounded-xl border border-slate-300 bg-white px-4 py-2 text-sm font-semibold text-slate-700">Back to Tally</a></div>
+  </div>
+  {% if collected %}<div class="rounded-xl border border-blue-200 bg-blue-50 px-4 py-3 text-sm text-blue-800">Historical evidence job queued. This page will ingest the result after the selected workstation completes it.</div>{% endif %}
+  {% if mapped %}<div class="rounded-xl border border-emerald-200 bg-emerald-50 px-4 py-3 text-sm text-emerald-800">Ledger classification mapping saved.</div>{% endif %}
+  {% if error %}<div class="rounded-xl border border-red-200 bg-red-50 px-4 py-3 text-sm text-red-800">{{ error }}</div>{% endif %}
+
+  <section class="rounded-2xl bg-white p-5 shadow-soft">
+    <form method="get" class="flex flex-wrap items-end gap-3">
+      <label class="min-w-[300px] flex-1 text-sm font-medium text-slate-700">Client<select name="client_id" class="mt-1 w-full rounded-xl border border-slate-300 px-3 py-2"><option value="">Select client</option>{% for c in clients %}<option value="{{ c.id }}" {% if selected_client and c.id==selected_client.id %}selected{% endif %}>{{ c.client_name }}</option>{% endfor %}</select></label>
+      <button class="rounded-xl bg-brand-600 px-4 py-2 text-sm font-semibold text-white">Open</button>
+    </form>
+  </section>
+
+  {% if selected_client %}
+  <section class="rounded-2xl bg-white p-5 shadow-soft">
+    <h2 class="text-lg font-semibold text-slate-900">Collect Historical Purchase Evidence</h2>
+    <p class="mt-1 text-sm text-slate-500">Select the workstation that holds this client's synced .act data. The agent returns neutral ledger-history facts; classification stays on the ERP server.</p>
+    <form method="post" action="/tools/accounting/historical-learning/collect" class="mt-4 grid gap-3 md:grid-cols-2 xl:grid-cols-5">
+      <input type="hidden" name="csrf_token" value="{{ csrf_token }}"><input type="hidden" name="client_id" value="{{ selected_client.id }}">
+      <label class="text-sm font-medium text-slate-700 xl:col-span-2">Workstation / Tally Company<select id="wsCompany" class="mt-1 w-full rounded-xl border border-slate-300 px-3 py-2" required><option value="">Select</option>{% for w in workstations %}{% for company in workstation_companies.get(w.id,[]) %}<option value="{{ w.id }}|{{ company.get('guid','') }}|{{ company.get('name','') }}">{{ w.machine_name }} · {{ company.get('name','') }}{% if not w.tally_connected %} (Tally not connected){% endif %}</option>{% endfor %}{% endfor %}</select></label>
+      <input type="hidden" name="workstation_id" id="workstationId"><input type="hidden" name="tally_guid" id="tallyGuid"><input type="hidden" name="company_name" id="companyName">
+      <label class="text-sm font-medium text-slate-700">From<input type="date" name="date_from" value="{{ date_from }}" class="mt-1 w-full rounded-xl border border-slate-300 px-3 py-2"></label>
+      <label class="text-sm font-medium text-slate-700">To<input type="date" name="date_to" value="{{ date_to }}" class="mt-1 w-full rounded-xl border border-slate-300 px-3 py-2"></label>
+      <div class="flex items-end"><button class="w-full rounded-xl bg-brand-600 px-4 py-2 text-sm font-semibold text-white">Collect Evidence</button></div>
+    </form>
+  </section>
+
+  <section class="rounded-2xl bg-white p-5 shadow-soft">
+    <h2 class="text-lg font-semibold text-slate-900">Recent Learning Runs</h2>
+    <div class="mt-3 overflow-x-auto"><table class="min-w-full text-sm"><thead class="text-left text-xs uppercase text-slate-500"><tr><th class="px-2 py-2">Created</th><th class="px-2 py-2">Company</th><th class="px-2 py-2">Status</th><th class="px-2 py-2">Rows</th><th class="px-2 py-2">Error</th></tr></thead><tbody>{% for r in runs %}<tr class="border-t"><td class="px-2 py-2">{{ r.created_at_utc }}</td><td class="px-2 py-2">{{ r.company_name or r.tally_guid }}</td><td class="px-2 py-2 font-medium">{{ r.status }}</td><td class="px-2 py-2">{{ r.evidence_rows }}</td><td class="px-2 py-2 text-red-700">{{ r.error_message or '' }}</td></tr>{% else %}<tr><td colspan="5" class="px-2 py-4 text-slate-500">No historical learning run yet.</td></tr>{% endfor %}</tbody></table></div>
+  </section>
+
+  <section class="rounded-2xl bg-white p-5 shadow-soft">
+    <h2 class="text-lg font-semibold text-slate-900">Classify Historical Tally Ledgers</h2><p class="mt-1 text-sm text-slate-500">Map the client's actual Tally ledger names to the canonical Phase 4 accounting taxonomy. This mapping is client and Tally-company specific.</p>
+    <div class="mt-3 overflow-x-auto"><table class="min-w-full text-sm"><thead class="text-left text-xs uppercase text-slate-500"><tr><th class="px-2 py-2">Ledger</th><th class="px-2 py-2">Parent</th><th class="px-2 py-2">Historical vouchers</th><th class="px-2 py-2">Amount evidence</th><th class="px-2 py-2">Accounting nature</th></tr></thead><tbody>
+    {% for row in classifiable_ledgers %}<tr class="border-t align-top"><td class="px-2 py-3 font-medium text-slate-900">{{ row.ledger_name }}</td><td class="px-2 py-3 text-slate-600">{{ row.parent_group }}</td><td class="px-2 py-3">{{ row.voucher_count }}</td><td class="px-2 py-3">{{ '%.2f'|format(row.amount) }}</td><td class="px-2 py-3"><form method="post" action="/tools/accounting/historical-learning/map-ledger" class="flex min-w-[360px] gap-2"><input type="hidden" name="csrf_token" value="{{ csrf_token }}"><input type="hidden" name="client_id" value="{{ selected_client.id }}"><input type="hidden" name="tally_guid" value="{{ row.tally_guid }}"><input type="hidden" name="company_name" value="{{ row.company_name }}"><input type="hidden" name="ledger_name" value="{{ row.ledger_name }}"><input type="hidden" name="parent_group" value="{{ row.parent_group }}"><select name="nature_id" class="min-w-[240px] rounded-lg border border-slate-300 px-2 py-1" required><option value="">Select nature</option>{% for n in natures %}<option value="{{ n.id }}" {% if row.mapping and row.mapping.nature_id==n.id %}selected{% endif %}>{{ n.name }}</option>{% endfor %}</select><button class="rounded-lg bg-slate-900 px-3 py-1 text-xs font-semibold text-white">Save</button></form></td></tr>{% else %}<tr><td colspan="5" class="px-2 py-4 text-slate-500">Collect historical evidence first.</td></tr>{% endfor %}
+    </tbody></table></div>
+  </section>
+
+  <section class="rounded-2xl bg-white p-5 shadow-soft">
+    <h2 class="text-lg font-semibold text-slate-900">Supplier / Party Historical Suggestions</h2><p class="mt-1 text-sm text-slate-500">Confidence is the share of mapped historical purchase vouchers for that party that resolve to the leading accounting nature.</p>
+    <div class="mt-3 overflow-x-auto"><table class="min-w-full text-sm"><thead class="text-left text-xs uppercase text-slate-500"><tr><th class="px-2 py-2">Party</th><th class="px-2 py-2">Suggested nature</th><th class="px-2 py-2">Confidence</th><th class="px-2 py-2">Evidence</th><th class="px-2 py-2">Ledgers</th></tr></thead><tbody>{% for s in suggestions %}<tr class="border-t"><td class="px-2 py-3 font-medium">{{ s.party_ledger_name }}</td><td class="px-2 py-3">{{ s.nature.name }}</td><td class="px-2 py-3"><span class="rounded-full px-2 py-1 text-xs font-semibold {% if s.confidence>=90 %}bg-emerald-100 text-emerald-800{% elif s.confidence>=70 %}bg-amber-100 text-amber-800{% else %}bg-slate-100 text-slate-700{% endif %}">{{ s.confidence }}%</span></td><td class="px-2 py-3">{{ s.voucher_count }} / {{ s.mapped_voucher_count }} mapped vouchers</td><td class="px-2 py-3 text-slate-600">{{ s.ledgers|join(', ') }}</td></tr>{% else %}<tr><td colspan="5" class="px-2 py-4 text-slate-500">Map historical Tally ledgers to accounting natures to generate party suggestions.</td></tr>{% endfor %}</tbody></table></div>
+  </section>
+
+  <section class="rounded-2xl bg-white p-5 shadow-soft">
+    <h2 class="text-lg font-semibold text-slate-900">Raw Historical Evidence</h2>
+    <div class="mt-3 overflow-x-auto"><table class="min-w-full text-xs"><thead class="text-left uppercase text-slate-500"><tr><th class="px-2 py-2">Party</th><th class="px-2 py-2">Counter ledger</th><th class="px-2 py-2">Group</th><th class="px-2 py-2">Vouchers</th><th class="px-2 py-2">First</th><th class="px-2 py-2">Last</th></tr></thead><tbody>{% for e in evidence[:300] %}<tr class="border-t"><td class="px-2 py-2">{{ e.party_ledger_name }}</td><td class="px-2 py-2">{{ e.counter_ledger_name }}</td><td class="px-2 py-2">{{ e.counter_parent_group }}</td><td class="px-2 py-2">{{ e.voucher_count }}</td><td class="px-2 py-2">{{ e.first_voucher_date or '' }}</td><td class="px-2 py-2">{{ e.last_voucher_date or '' }}</td></tr>{% else %}<tr><td colspan="6" class="px-2 py-4 text-slate-500">No evidence collected.</td></tr>{% endfor %}</tbody></table></div>
+  </section>
+  {% endif %}
+</div>
+<script>
+const s=document.getElementById('wsCompany'); if(s){s.addEventListener('change',()=>{const p=s.value.split('|');document.getElementById('workstationId').value=p[0]||'';document.getElementById('tallyGuid').value=p[1]||'';document.getElementById('companyName').value=p.slice(2).join('|')||'';});}
+</script>
+{% endblock %}

app/modules/accounting/templates/accounting/tally.html

@@ -14,6 +14,7 @@
     </div>
     <div class="flex flex-wrap gap-2">
       <a href="/tools/accounting/taxonomy" class="rounded-xl border border-slate-300 bg-white px-4 py-2 text-sm font-semibold text-slate-700 hover:bg-slate-50">Accounting Taxonomy</a>
+      {% if selected_client %}<a href="/tools/accounting/historical-learning?client_id={{ selected_client.id }}" class="rounded-xl border border-slate-300 bg-white px-4 py-2 text-sm font-semibold text-slate-700 hover:bg-slate-50">Historical Learning</a>{% endif %}
       {% if selected_client %}<a href="/tools/tally/depreciation?client_id={{ selected_client.id }}" class="rounded-xl border border-slate-300 bg-white px-4 py-2 text-sm font-semibold text-slate-700 hover:bg-slate-50">Depreciation (IT)</a>{% endif %}
       <a href="/tools/tally{% if selected_client %}?client_id={{ selected_client.id }}&refresh=1{% else %}?refresh=1{% endif %}" class="rounded-xl bg-brand-600 px-4 py-2 text-sm font-semibold text-white shadow-soft hover:bg-brand-700">Refresh Tally Companies</a>
     </div>

app/modules/core/rbac/permissions_registry.py

@@ -156,6 +156,8 @@ PERMISSIONS = {
     "accounting.tally.writeback": "Approve and Post Accounting Entries to Tally",
     "accounting.taxonomy.view": "View Accounting Classification Taxonomy",
     "accounting.taxonomy.manage": "Activate or Deactivate Accounting Classification Natures",
+    "accounting.learning.view": "View Historical Tally Learning",
+    "accounting.learning.manage": "Collect Evidence and Map Historical Tally Ledgers",
     "accounting.act.initialize": "Initialize Client Accounting ACT Storage",
 
     "notice_cases.view": "View Notice and Case Management",

app/modules/documents/agent_package.py

@@ -4,7 +4,7 @@ import io
 from pathlib import Path
 import zipfile
 
-ERP_LOCAL_AGENT_VERSION = "1.11.0"
+ERP_LOCAL_AGENT_VERSION = "1.12.0"
 ERP_LOCAL_AGENT_NAME = "ERP Local Agent"
 RUNTIME_ROOT = Path(__file__).resolve().parent / "local_agent_runtime"
 _DETERMINISTIC_ZIP_TIMESTAMP = (2026, 1, 1, 0, 0, 0)
@@ -53,7 +53,7 @@ def _build_zip(*, env_text: str | None, include_env: bool, include_admin_readme:
             text = (
                 f"ERP Local Agent {ERP_LOCAL_AGENT_VERSION}\n"
                 "Existing storage, WebSocket tunnel, dashboard, Tally mapping and client .act functionality are preserved.\n"
-                "Phase 3 master synchronization is preserved. Phase 4 adds read-only date-scoped Tally voucher/transaction synchronization.\n"
+                "Existing master/transaction sync and depreciation are preserved. Phase 5 adds neutral historical-ledger evidence read capability for server-side learning.\n"
                 "Dashboard: http://127.0.0.1:8788\n"
             )
             _write_zip_bytes(dst, "README_ERP_LOCAL_AGENT.txt", text.encode("utf-8"))

app/modules/documents/local_agent_runtime/erp_local_agent/__init__.py

@@ -1,2 +1 @@
-__version__ = "1.11.0"
-AGENT_NAME = "ERP Local Agent"
+__version__ = "1.12.0"

app/modules/documents/local_agent_runtime/erp_local_agent/accounting_store.py

@@ -1077,6 +1077,54 @@ class LocalAccountingStore:
             ).fetchall()
         return [dict(row) for row in rows]
 
+    def historical_learning_evidence(self, client_id: int, *, tally_guid: str, date_from: str = "", date_to: str = "", voucher_scope: str = "purchase") -> dict:
+        """Return neutral aggregated historical ledger facts from the local .act store.
+
+        No accounting classification is performed locally.  The ERP server maps
+        these facts to its canonical taxonomy.
+        """
+        if not self.exists(client_id):
+            raise ValueError("Accounting storage is not initialized for this client.")
+        guid = str(tally_guid or "").strip()
+        if not guid:
+            raise ValueError("Tally company GUID is required.")
+        mapping = self.get_active_mapping_by_guid(client_id, guid)
+        params: list = [guid]
+        where = ["v.tally_guid=?", "TRIM(COALESCE(v.party_ledger_name,''))<>''", "LOWER(TRIM(COALESCE(v.is_cancelled,''))) NOT IN ('yes','true','1')", "LOWER(TRIM(e.ledger_name))<>LOWER(TRIM(v.party_ledger_name))"]
+        if str(voucher_scope or "purchase").strip().lower() == "purchase":
+            where.append("LOWER(v.voucher_type_name) LIKE '%purchase%'")
+        if date_from:
+            where.append("v.voucher_date>=?"); params.append(str(date_from))
+        if date_to:
+            where.append("v.voucher_date<=?"); params.append(str(date_to))
+        sql = f"""
+            SELECT v.party_ledger_name AS party_ledger_name,
+                   COALESCE(pl.parent,'') AS party_parent_group,
+                   e.ledger_name AS counter_ledger_name,
+                   COALESCE(cl.parent,'') AS counter_parent_group,
+                   v.voucher_type_name AS voucher_type_name,
+                   COUNT(DISTINCT v.id) AS voucher_count,
+                   COALESCE(SUM(ABS(e.amount)),0) AS absolute_amount_total,
+                   MIN(NULLIF(v.voucher_date,'')) AS first_voucher_date,
+                   MAX(NULLIF(v.voucher_date,'')) AS last_voucher_date,
+                   MIN(NULLIF(v.narration,'')) AS sample_narration
+              FROM tally_vouchers v
+              JOIN tally_voucher_ledger_entries e ON e.voucher_id=v.id
+         LEFT JOIN tally_ledgers pl ON pl.tally_guid=v.tally_guid AND LOWER(TRIM(pl.name))=LOWER(TRIM(v.party_ledger_name))
+         LEFT JOIN tally_ledgers cl ON cl.tally_guid=v.tally_guid AND LOWER(TRIM(cl.name))=LOWER(TRIM(e.ledger_name))
+             WHERE {' AND '.join(where)}
+          GROUP BY v.party_ledger_name, COALESCE(pl.parent,''), e.ledger_name, COALESCE(cl.parent,''), v.voucher_type_name
+          ORDER BY voucher_count DESC, v.party_ledger_name COLLATE NOCASE, e.ledger_name COLLATE NOCASE
+             LIMIT 5000
+        """
+        with self.connect(client_id) as db:
+            rows = [dict(r) for r in db.execute(sql, tuple(params)).fetchall()]
+        return {
+            "client_id": int(client_id), "tally_guid": guid, "company_name": str(mapping.get("company_name") or ""),
+            "date_from": str(date_from or ""), "date_to": str(date_to or ""), "voucher_scope": str(voucher_scope or "purchase"),
+            "evidence": rows, "evidence_count": len(rows),
+        }
+
     def snapshot(self, client_id: int) -> dict:
         path = self.db_path(client_id)
         if not path.is_file():

app/modules/documents/local_agent_runtime/erp_local_agent/commands.py

@@ -36,6 +36,8 @@ class AgentCommandProcessor:
                 result = self._sync_masters(payload)
             elif action == "accounting_sync_transactions":
                 result = self._sync_transactions(payload)
+            elif action == "accounting_historical_evidence":
+                result = self._historical_evidence(payload)
             elif action == "accounting_depreciation_preview":
                 result = self._depreciation_preview(payload)
             elif action == "accounting_calculate_it_depreciation":
@@ -67,6 +69,7 @@ class AgentCommandProcessor:
             "tally_capability": True, "accounting_act_capability": True,
             "tally_mapping_capability": True, "tally_master_sync_capability": True,
             "tally_transaction_sync_capability": True,
+            "historical_learning_read_capability": True,
             "it_depreciation_capability": True,
             "tally_writeback_capability": True,
         }
@@ -201,6 +204,16 @@ class AgentCommandProcessor:
         self.logger.info("Tally transaction sync completed client_id=%s company=%s period=%s..%s rows=%s", client_id, company_name, date_from, date_to, sync.get("rows_processed"))
         return {"synced": True, "sync": sync, "accounting": self.store.snapshot(client_id), "tally": tally_status, "agent": self._agent_info()}
 
+    def _historical_evidence(self, payload: dict[str, Any]) -> dict[str, Any]:
+        client_id = int(payload.get("client_id"))
+        return self.store.historical_learning_evidence(
+            client_id,
+            tally_guid=str(payload.get("tally_guid") or ""),
+            date_from=str(payload.get("date_from") or ""),
+            date_to=str(payload.get("date_to") or ""),
+            voucher_scope=str(payload.get("voucher_scope") or "purchase"),
+        )
+
     def _depreciation_preview(self, payload: dict[str, Any]) -> dict[str, Any]:
         client_id = int(payload.get("client_id"))
         tally_guid = str(payload.get("tally_guid") or "").strip()

app/modules/documents/local_agent_runtime/erp_local_agent/sync.py

@@ -49,7 +49,7 @@ class StorageAgent:
                 "free_bytes": free,
                 "agent_version": __version__,
                 "agent_name": "ERP Local Agent",
-                "capabilities": ["storage", "tally", "accounting_act", "tally_master_sync", "tally_transaction_sync", "local_dashboard", "manual_updates"],
+                "capabilities": ["storage", "tally", "accounting_act", "tally_master_sync", "tally_transaction_sync", "local_dashboard", "manual_updates", "historical_learning_read"],
                 "agent_time_utc": datetime.now(timezone.utc).isoformat(),
             }
             self.client.heartbeat(payload)

app/modules/documents/local_agent_runtime/erp_local_agent/tunnel.py

@@ -190,7 +190,7 @@ class StorageAgentTunnel:
 
     def _capabilities(self) -> list[str]:
         return [
-            "storage", "local_dashboard", "manual_updates", "accounting_act", "agent.jobs.v2",
+            "storage", "local_dashboard", "manual_updates", "historical_learning_read", "accounting_act", "agent.jobs.v2",
             "tally.status", "tally.company_identity", "tally.read_groups",
             "tally.read_ledgers", "tally.read_stock_items", "tally.read_vouchers",
             "tally.read_trial_balance", "tally.master_sync", "tally.transaction_sync",

app/ui/app.py

@@ -38,6 +38,7 @@ from app.modules.peer_review_export.ui import router as peer_review_export_route
 from app.modules.bank_statement_analyzer.ui import router as bank_statement_analyzer_router
 from app.modules.accounting.ui import router as accounting_ui_router
 from app.modules.accounting.taxonomy_ui import router as accounting_taxonomy_ui_router
+from app.modules.accounting.historical_learning_ui import router as accounting_historical_learning_ui_router
 from app.modules.registrations.ui import router as registrations_ui_router
 from app.modules.credential_vault.ui import router as credential_vault_ui_router
 from app.modules.client_identity.ui import router as client_identity_ui_router
@@ -62,6 +63,7 @@ def mount_ui(app: FastAPI) -> None:
     app.include_router(bank_statement_analyzer_router)
     app.include_router(accounting_ui_router)
     app.include_router(accounting_taxonomy_ui_router)
+    app.include_router(accounting_historical_learning_ui_router)
     app.include_router(work_tracker_ui_router)
     app.include_router(billing_ui_router)
     app.include_router(platform_billing_ui_router)