Add adaptive paced Tally extraction for cash payment review

app/modules/accounting/cash_payment_ui.py

@@ -195,8 +195,11 @@ def cash_payment_review(
                             "cash_limit": float(setting.cash_limit),
                             "split_window_days": int(setting.split_window_days),
                             "near_limit_percent": float(setting.near_limit_percent),
+                            "tally_batch_days": 7,
+                            "tally_pause_seconds": 1.5,
+                            "tally_retry_pause_seconds": 3.0,
                         },
-                        timeout_seconds=300,
+                        timeout_seconds=900,
                     )
                     if result.get("ok"):
                         review = (result.get("result") or {}).get("cash_payment_review")

app/modules/accounting/templates/accounting/cash_payment_review.html

@@ -64,7 +64,7 @@
   {% if review %}
   <section class="space-y-4">
     {% if review.safe_paced_mode %}
-    <div class="rounded-2xl border border-sky-200 bg-sky-50 px-4 py-3 text-sm text-sky-800">Completed in safe-paced mode using {{ review.tally_request_batches or 0 }} Tally voucher batch(es), with approximately {{ review.tally_pause_ms or 0 }} ms pause between batches.</div>
+    <div class="rounded-2xl border border-sky-200 bg-sky-50 px-4 py-3 text-sm text-sky-800">Completed in adaptive safe-paced mode using {{ review.tally_request_batches or 0 }} successful voucher batch(es) across {{ review.tally_request_count or 0 }} Tally request(s), with {{ review.tally_retry_count or 0 }} retry/retries and {{ review.tally_fallback_batches or 0 }} adaptive fallback(s). The agent pauses approximately {{ review.tally_pause_ms or 0 }} ms between Tally calls.</div>
     {% endif %}
     <div class="grid gap-4 md:grid-cols-2 xl:grid-cols-5">
       {% for label, value in [('Vouchers reviewed', review.summary.vouchers_reviewed), ('Cash payments', review.summary.cash_payment_vouchers), ('Single-voucher exceptions', review.summary.single_voucher_exceptions), ('Same-day exceptions', review.summary.same_day_exceptions), ('Possible split patterns', review.summary.possible_split_patterns)] %}
@@ -133,7 +133,7 @@
     <div class="mt-4 grid grid-cols-4 gap-2 text-center text-[11px] font-semibold uppercase tracking-wide text-slate-400">
       <span>Agent</span><span>Tally</span><span>Vouchers</span><span>Analysis</span>
     </div>
-    <p class="mt-4 text-xs text-slate-500">Tally is deliberately read in smaller monthly batches with pauses between requests. Please keep TallyPrime and the ERP Local Agent open until this finishes.</p>
+    <p class="mt-4 text-xs text-slate-500">Tally is deliberately read in 7-day batches with pauses between requests. If a batch times out, the agent automatically retries and reduces it to 3-day and then 1-day batches. Please keep TallyPrime and the ERP Local Agent open until this finishes.</p>
   </div>
 </div>
 <script>
@@ -149,8 +149,9 @@
     [8, 'Sending the review request to the ERP Local Agent…'],
     [18, 'Connecting to TallyPrime on the workstation…'],
     [32, 'Reading cash and ledger masters…'],
-    [48, 'Reading vouchers in safe monthly batches…'],
+    [42, 'Reading vouchers in safe 7-day batches…'],
-    [64, 'Continuing voucher scan with controlled pauses…'],
+    [58, 'Continuing voucher extraction with 1.5 second pauses…'],
+    [70, 'Large batches automatically fall back to 3-day or 1-day reads if needed…'],
     [78, 'Checking same-day cash payment aggregation…'],
     [88, 'Checking possible split-payment patterns…'],
     [94, 'Preparing the compliance report…']
@@ -169,7 +170,7 @@
       bar.style.width = stage[0] + '%';
       pct.textContent = stage[0] + '%';
       text.textContent = stage[1];
-      if (i < stages.length) window.setTimeout(advance, i < 4 ? 1300 : 2200);
+      if (i < stages.length) window.setTimeout(advance, i < 4 ? 1800 : 4500);
     };
     advance();
   });

app/modules/documents/agent_package.py

@@ -4,7 +4,7 @@ import io
 from pathlib import Path
 import zipfile
 
-ERP_LOCAL_AGENT_VERSION = "1.22.6"
+ERP_LOCAL_AGENT_VERSION = "1.22.7"
 ERP_LOCAL_AGENT_NAME = "ERP Local Agent"
 RUNTIME_ROOT = Path(__file__).resolve().parent / "local_agent_runtime"
 _DETERMINISTIC_ZIP_TIMESTAMP = (2026, 1, 1, 0, 0, 0)

app/modules/documents/local_agent_runtime/erp_local_agent/__init__.py

@@ -1,2 +1,2 @@
-__version__ = "1.22.6"
+__version__ = "1.22.7"
 AGENT_NAME = "ERP Local Agent"

app/modules/documents/local_agent_runtime/erp_local_agent/commands.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 from datetime import datetime, timezone, date as _dt_date, timedelta as _timedelta
 import re
 import time
+import threading
 from typing import Any
 
 from . import __version__
@@ -11,6 +12,9 @@ from .tally import TallyLiveConnector
 from .native_voucher_engine import NativeVoucherEngine
 
 
+_CASH_TALLY_EXTRACTION_LOCK = threading.Lock()
+
+
 class AgentCommandProcessor:
     def __init__(self, config, logger):
         self.config = config
@@ -662,23 +666,6 @@ class AgentCommandProcessor:
         if not 1 <= near_limit_percent <= 100:
             raise ValueError("Near-limit percentage must be between 1 and 100.")
 
-        # TallyPrime is intentionally queried in a paced manner. A full-year XML
-        # voucher export in one request can make desktop Tally unresponsive on
-        # larger books. Load masters once, pause briefly, then read vouchers in
-        # month-sized batches with a small gap between calls.
-        pace_seconds = 0.75
-        ledger_masters = self.tally.export_master_collection(company_name, "ledgers")
-        time.sleep(pace_seconds)
-        cash_ledgers = set()
-        for ledger in ledger_masters:
-            name = str(ledger.get("name") or "").strip()
-            parent = str(ledger.get("parent") or "").strip().casefold().replace("-", " ")
-            reserved = str(ledger.get("reserved_name") or "").strip().casefold()
-            if name and (parent in {"cash in hand", "cash-in-hand"} or reserved == "cash" or name.casefold() == "cash"):
-                cash_ledgers.add(name.casefold())
-        if not cash_ledgers:
-            cash_ledgers.add("cash")
-
         try:
             scan_from = _dt_date.fromisoformat(date_from)
             scan_to = _dt_date.fromisoformat(date_to)
@@ -687,151 +674,254 @@ class AgentCommandProcessor:
         if scan_to < scan_from:
             raise ValueError("Cash payment review To date cannot be before From date.")
 
-        vouchers = []
+        # TallyPrime is single-desktop software and large XML exports can make it
-        batch_count = 0
+        # appear frozen. Keep only one heavy compliance extraction active on a
-        cursor = scan_from
+        # workstation and deliberately pace every XML request.
-        while cursor <= scan_to:
+        if not _CASH_TALLY_EXTRACTION_LOCK.acquire(blocking=False):
-            next_month = (cursor.replace(day=28) + _timedelta(days=4)).replace(day=1)
+            raise ValueError("Another Cash Payment Tally extraction is already running on this workstation. Please wait for it to finish and try again.")
-            batch_to = min(scan_to, next_month - _timedelta(days=1))
-            batch = self.tally.export_vouchers(company_name, cursor.isoformat(), batch_to.isoformat())
-            vouchers.extend(batch or [])
-            batch_count += 1
-            cursor = batch_to + _timedelta(days=1)
-            if cursor <= scan_to:
-                time.sleep(pace_seconds)
 
-        payments = []
+        pace_seconds = max(0.5, min(float(payload.get("tally_pause_seconds") or 1.5), 10.0))
-        for voucher in vouchers:
+        retry_pause_seconds = max(1.0, min(float(payload.get("tally_retry_pause_seconds") or 3.0), 30.0))
-            if str(voucher.get("is_cancelled") or "").strip().lower() in {"yes", "true", "1"}:
+        base_batch_days = max(1, min(int(payload.get("tally_batch_days") or 7), 31))
-                continue
+        request_count = 0
-            if str(voucher.get("is_optional") or "").strip().lower() in {"yes", "true", "1"}:
+        successful_batches = 0
-                continue
+        fallback_batches = 0
-            entries = list(voucher.get("ledger_entries") or [])
+        retry_count = 0
-            cash_credit = []
+        batch_log: list[dict[str, Any]] = []
-            for entry in entries:
-                lname = str(entry.get("ledger_name") or "").strip()
-                if lname.casefold() not in cash_ledgers:
-                    continue
-                amount = float(entry.get("amount") or 0)
-                deemed = str(entry.get("is_deemed_positive") or "").strip().lower()
-                if amount > 0 or deemed == "no":
-                    cash_credit.append(abs(amount))
-            cash_amount = round(sum(cash_credit), 2)
-            if cash_amount <= 0:
-                continue
 
-            non_cash = [entry for entry in entries if str(entry.get("ledger_name") or "").strip().casefold() not in cash_ledgers]
+        def paced_call(callable_, *, label: str):
-            party = str(voucher.get("party_ledger_name") or "").strip()
+            nonlocal request_count, retry_count
-            if not party or party.casefold() in cash_ledgers:
+            last_exc = None
-                candidates = sorted(non_cash, key=lambda row: abs(float(row.get("amount") or 0)), reverse=True)
+            for attempt in range(1, 3):
-                party = str(candidates[0].get("ledger_name") or "").strip() if candidates else "Unidentified counter-ledger"
+                try:
-            payments.append({
+                    request_count += 1
-                "date": str(voucher.get("date") or voucher.get("effective_date") or ""),
+                    result = callable_()
-                "party": party or "Unidentified counter-ledger",
+                    time.sleep(pace_seconds)
-                "amount": cash_amount,
+                    return result
-                "voucher_number": str(voucher.get("voucher_number") or ""),
+                except Exception as exc:
-                "voucher_type": str(voucher.get("voucher_type_name") or ""),
+                    last_exc = exc
-                "reference": str(voucher.get("reference") or ""),
+                    if attempt < 2:
-                "narration": str(voucher.get("narration") or ""),
+                        retry_count += 1
-            })
+                        self.logger.warning("Tally request retry label=%s attempt=%s error=%s", label, attempt, exc)
+                        time.sleep(retry_pause_seconds)
+            raise last_exc  # type: ignore[misc]
 
-        by_party_date = defaultdict(list)
+        def fetch_voucher_range(start_day: _dt_date, end_day: _dt_date, preferred_days: int) -> list[dict[str, Any]]:
-        for row in payments:
+            nonlocal successful_batches, fallback_batches
-            by_party_date[(row["party"].casefold(), row["date"])].append(row)
+            span = (end_day - start_day).days + 1
-
+            label = f"cash vouchers {start_day.isoformat()} to {end_day.isoformat()}"
-        exceptions = []
+            try:
-        seen = set()
+                rows = paced_call(
-        single_count = 0
+                    lambda: self.tally.export_vouchers(company_name, start_day.isoformat(), end_day.isoformat()),
-        same_day_count = 0
+                    label=label,
-        for row in payments:
+                ) or []
-            if row["amount"] > cash_limit + 0.009:
+                successful_batches += 1
-                single_count += 1
+                batch_log.append({
-                key = (row["party"].casefold(), row["date"], "single", row["voucher_number"])
+                    "date_from": start_day.isoformat(),
-                if key not in seen:
+                    "date_to": end_day.isoformat(),
-                    seen.add(key)
+                    "days": span,
-                    exceptions.append({
+                    "vouchers": len(rows),
-                        "date": row["date"], "party": row["party"], "amount": row["amount"],
+                    "status": "ok",
-                        "reason": "Single cash-payment voucher exceeds configured limit",
+                })
-                        "voucher_numbers": [row["voucher_number"] or "-"],
+                return list(rows)
-                    })
+            except Exception as exc:
-        for (_party_key, paid_on), rows in by_party_date.items():
+                # Adaptive fallback: 7-day request -> approx 3-day chunks -> 1-day.
-            total = round(sum(row["amount"] for row in rows), 2)
+                if span <= 1:
-            if total > cash_limit + 0.009 and len(rows) > 1:
+                    batch_log.append({
-                same_day_count += 1
+                        "date_from": start_day.isoformat(),
-                key = (_party_key, paid_on, "aggregate")
+                        "date_to": end_day.isoformat(),
-                if key not in seen:
+                        "days": 1,
-                    seen.add(key)
+                        "vouchers": 0,
-                    exceptions.append({
+                        "status": "failed",
-                        "date": paid_on, "party": rows[0]["party"], "amount": total,
+                        "error": str(exc),
-                        "reason": "Same-day aggregate cash payments to the same party exceed configured limit",
-                        "voucher_numbers": [row["voucher_number"] or "-" for row in rows],
                     })
+                    raise ValueError(
+                        f"TallyPrime timed out while reading vouchers for {start_day.isoformat()}. "
+                        "The agent already retried the request slowly. Keep TallyPrime open and try this period again. "
+                        f"Details: {exc}"
+                    ) from exc
 
-        daily_by_party = defaultdict(lambda: defaultdict(float))
+                fallback_batches += 1
-        for row in payments:
+                if span > 3:
-            daily_by_party[row["party"].casefold()][row["date"]] += row["amount"]
+                    chunk_days = 3
-        near_floor = cash_limit * near_limit_percent / 100.0
+                else:
-        split_patterns = []
+                    chunk_days = 1
-        for party_key, day_map in daily_by_party.items():
+                self.logger.warning(
-            day_rows = sorted((_date.fromisoformat(day), round(amount, 2)) for day, amount in day_map.items() if day)
+                    "Reducing Tally cash extraction batch %s..%s from %s day(s) to %s day chunk(s) after error: %s",
-            for start_idx in range(len(day_rows)):
+                    start_day, end_day, span, chunk_days, exc,
-                window = []
+                )
-                for idx in range(start_idx, len(day_rows)):
+                output: list[dict[str, Any]] = []
-                    day_value, amount = day_rows[idx]
+                cursor = start_day
-                    if (day_value - day_rows[start_idx][0]).days >= split_window_days:
+                while cursor <= end_day:
-                        break
+                    child_end = min(end_day, cursor + _timedelta(days=chunk_days - 1))
-                    window.append((day_value, amount))
+                    output.extend(fetch_voucher_range(cursor, child_end, chunk_days))
-                if len(window) < 2:
+                    cursor = child_end + _timedelta(days=1)
+                return output
+
+        try:
+            # Masters are also paced and retried; the previous implementation
+            # could fail before voucher extraction even began on large books.
+            ledger_masters = paced_call(
+                lambda: self.tally.export_master_collection(company_name, "ledgers"),
+                label="cash ledger masters",
+            ) or []
+            cash_ledgers = set()
+            for ledger in ledger_masters:
+                name = str(ledger.get("name") or "").strip()
+                parent = str(ledger.get("parent") or "").strip().casefold().replace("-", " ")
+                reserved = str(ledger.get("reserved_name") or "").strip().casefold()
+                if name and (parent in {"cash in hand", "cash-in-hand"} or reserved == "cash" or name.casefold() == "cash"):
+                    cash_ledgers.add(name.casefold())
+            if not cash_ledgers:
+                cash_ledgers.add("cash")
+
+            vouchers: list[dict[str, Any]] = []
+            cursor = scan_from
+            while cursor <= scan_to:
+                batch_to = min(scan_to, cursor + _timedelta(days=base_batch_days - 1))
+                vouchers.extend(fetch_voucher_range(cursor, batch_to, base_batch_days))
+                cursor = batch_to + _timedelta(days=1)
+
+            payments = []
+            for voucher in vouchers:
+                if str(voucher.get("is_cancelled") or "").strip().lower() in {"yes", "true", "1"}:
                     continue
-                if any(amount > cash_limit + 0.009 for _, amount in window):
+                if str(voucher.get("is_optional") or "").strip().lower() in {"yes", "true", "1"}:
                     continue
-                total = round(sum(amount for _, amount in window), 2)
+                entries = list(voucher.get("ledger_entries") or [])
-                if total <= cash_limit + 0.009:
+                cash_credit = []
+                for entry in entries:
+                    lname = str(entry.get("ledger_name") or "").strip()
+                    if lname.casefold() not in cash_ledgers:
+                        continue
+                    amount = float(entry.get("amount") or 0)
+                    deemed = str(entry.get("is_deemed_positive") or "").strip().lower()
+                    if amount > 0 or deemed == "no":
+                        cash_credit.append(abs(amount))
+                cash_amount = round(sum(cash_credit), 2)
+                if cash_amount <= 0:
                     continue
-                near_days = sum(1 for _, amount in window if amount >= near_floor)
+
-                if near_days < 2:
+                non_cash = [entry for entry in entries if str(entry.get("ledger_name") or "").strip().casefold() not in cash_ledgers]
-                    continue
+                party = str(voucher.get("party_ledger_name") or "").strip()
-                party_name = next((row["party"] for row in payments if row["party"].casefold() == party_key), party_key)
+                if not party or party.casefold() in cash_ledgers:
-                signature = (party_key, window[0][0].isoformat(), window[-1][0].isoformat())
+                    candidates = sorted(non_cash, key=lambda row: abs(float(row.get("amount") or 0)), reverse=True)
-                if any((r["party"].casefold(), r["date_from"], r["date_to"]) == signature for r in split_patterns):
+                    party = str(candidates[0].get("ledger_name") or "").strip() if candidates else "Unidentified counter-ledger"
-                    continue
+                payments.append({
-                split_patterns.append({
+                    "date": str(voucher.get("date") or voucher.get("effective_date") or ""),
-                    "party": party_name,
+                    "party": party or "Unidentified counter-ledger",
-                    "date_from": window[0][0].isoformat(),
+                    "amount": cash_amount,
-                    "date_to": window[-1][0].isoformat(),
+                    "voucher_number": str(voucher.get("voucher_number") or ""),
-                    "total_amount": total,
+                    "voucher_type": str(voucher.get("voucher_type_name") or ""),
-                    "days": [{"date": d.isoformat(), "amount": a} for d, a in window],
+                    "reference": str(voucher.get("reference") or ""),
-                    "review_only": True,
+                    "narration": str(voucher.get("narration") or ""),
                 })
 
-        exceptions.sort(key=lambda row: (row["date"], row["party"].casefold()))
+            by_party_date = defaultdict(list)
-        split_patterns.sort(key=lambda row: (row["date_from"], row["party"].casefold()))
+            for row in payments:
-        return {
+                by_party_date[(row["party"].casefold(), row["date"])].append(row)
-            "cash_payment_review": {
+
-                "company_name": company_name,
+            exceptions = []
-                "company_guid": str(company.get("guid") or ""),
+            seen = set()
-                "date_from": date_from,
+            single_count = 0
-                "date_to": date_to,
+            same_day_count = 0
-                "cash_limit": cash_limit,
+            for row in payments:
-                "cash_ledgers": sorted(cash_ledgers),
+                if row["amount"] > cash_limit + 0.009:
-                "summary": {
+                    single_count += 1
-                    "vouchers_reviewed": len(vouchers),
+                    key = (row["party"].casefold(), row["date"], "single", row["voucher_number"])
-                    "cash_payment_vouchers": len(payments),
+                    if key not in seen:
-                    "single_voucher_exceptions": single_count,
+                        seen.add(key)
-                    "same_day_exceptions": same_day_count,
+                        exceptions.append({
-                    "possible_split_patterns": len(split_patterns),
+                            "date": row["date"], "party": row["party"], "amount": row["amount"],
+                            "reason": "Single cash-payment voucher exceeds configured limit",
+                            "voucher_numbers": [row["voucher_number"] or "-"],
+                        })
+            for (_party_key, paid_on), rows in by_party_date.items():
+                total = round(sum(row["amount"] for row in rows), 2)
+                if total > cash_limit + 0.009 and len(rows) > 1:
+                    same_day_count += 1
+                    key = (_party_key, paid_on, "aggregate")
+                    if key not in seen:
+                        seen.add(key)
+                        exceptions.append({
+                            "date": paid_on, "party": rows[0]["party"], "amount": total,
+                            "reason": "Same-day aggregate cash payments to the same party exceed configured limit",
+                            "voucher_numbers": [row["voucher_number"] or "-" for row in rows],
+                        })
+
+            daily_by_party = defaultdict(lambda: defaultdict(float))
+            for row in payments:
+                daily_by_party[row["party"].casefold()][row["date"]] += row["amount"]
+            near_floor = cash_limit * near_limit_percent / 100.0
+            split_patterns = []
+            for party_key, day_map in daily_by_party.items():
+                day_rows = sorted((_date.fromisoformat(day), round(amount, 2)) for day, amount in day_map.items() if day)
+                for start_idx in range(len(day_rows)):
+                    window = []
+                    for idx in range(start_idx, len(day_rows)):
+                        day_value, amount = day_rows[idx]
+                        if (day_value - day_rows[start_idx][0]).days >= split_window_days:
+                            break
+                        window.append((day_value, amount))
+                    if len(window) < 2:
+                        continue
+                    if any(amount > cash_limit + 0.009 for _, amount in window):
+                        continue
+                    total = round(sum(amount for _, amount in window), 2)
+                    if total <= cash_limit + 0.009:
+                        continue
+                    near_days = sum(1 for _, amount in window if amount >= near_floor)
+                    if near_days < 2:
+                        continue
+                    party_name = next((row["party"] for row in payments if row["party"].casefold() == party_key), party_key)
+                    signature = (party_key, window[0][0].isoformat(), window[-1][0].isoformat())
+                    if any((r["party"].casefold(), r["date_from"], r["date_to"]) == signature for r in split_patterns):
+                        continue
+                    split_patterns.append({
+                        "party": party_name,
+                        "date_from": window[0][0].isoformat(),
+                        "date_to": window[-1][0].isoformat(),
+                        "total_amount": total,
+                        "days": [{"date": d.isoformat(), "amount": a} for d, a in window],
+                        "review_only": True,
+                    })
+
+            exceptions.sort(key=lambda row: (row["date"], row["party"].casefold()))
+            split_patterns.sort(key=lambda row: (row["date_from"], row["party"].casefold()))
+            return {
+                "cash_payment_review": {
+                    "company_name": company_name,
+                    "company_guid": str(company.get("guid") or ""),
+                    "date_from": date_from,
+                    "date_to": date_to,
+                    "cash_limit": cash_limit,
+                    "cash_ledgers": sorted(cash_ledgers),
+                    "summary": {
+                        "vouchers_reviewed": len(vouchers),
+                        "cash_payment_vouchers": len(payments),
+                        "single_voucher_exceptions": single_count,
+                        "same_day_exceptions": same_day_count,
+                        "possible_split_patterns": len(split_patterns),
+                    },
+                    "exceptions": exceptions,
+                    "possible_split_payments": split_patterns,
+                    "cash_payments": payments,
+                    "read_only": True,
+                    "safe_paced_mode": True,
+                    "adaptive_extraction": True,
+                    "tally_request_batches": successful_batches,
+                    "tally_request_count": request_count,
+                    "tally_retry_count": retry_count,
+                    "tally_fallback_batches": fallback_batches,
+                    "tally_batch_days": base_batch_days,
+                    "tally_pause_ms": int(pace_seconds * 1000),
+                    "tally_batch_log": batch_log,
+                    "review_note": "Possible split-payment patterns are review indicators only and do not determine tax treatment.",
                 },
-                "exceptions": exceptions,
+                "agent": self._agent_info(),
-                "possible_split_payments": split_patterns,
+            }
-                "cash_payments": payments,
+        finally:
-                "read_only": True,
+            _CASH_TALLY_EXTRACTION_LOCK.release()
-                "safe_paced_mode": True,
-                "tally_request_batches": batch_count,
-                "tally_pause_ms": int(pace_seconds * 1000),
-                "review_note": "Possible split-payment patterns are review indicators only and do not determine tax treatment.",
-            },
-            "agent": self._agent_info(),
-        }
 
     def _tds_compliance(self, payload: dict[str, Any]) -> dict[str, Any]:
         from collections import defaultdict