Add adaptive paced Tally extraction for cash payment review

app/modules/accounting/cash_payment_ui.py

@@ -195,8 +195,11 @@ def cash_payment_review(
                             "cash_limit": float(setting.cash_limit),
                             "split_window_days": int(setting.split_window_days),
                             "near_limit_percent": float(setting.near_limit_percent),
+                            "tally_batch_days": 7,
+                            "tally_pause_seconds": 1.5,
+                            "tally_retry_pause_seconds": 3.0,
                         },
-                        timeout_seconds=300,
+                        timeout_seconds=900,
                     )
                     if result.get("ok"):
                         review = (result.get("result") or {}).get("cash_payment_review")

app/modules/accounting/templates/accounting/cash_payment_review.html

@@ -64,7 +64,7 @@
   {% if review %}
   <section class="space-y-4">
     {% if review.safe_paced_mode %}
-    <div class="rounded-2xl border border-sky-200 bg-sky-50 px-4 py-3 text-sm text-sky-800">Completed in safe-paced mode using {{ review.tally_request_batches or 0 }} Tally voucher batch(es), with approximately {{ review.tally_pause_ms or 0 }} ms pause between batches.</div>
+    <div class="rounded-2xl border border-sky-200 bg-sky-50 px-4 py-3 text-sm text-sky-800">Completed in adaptive safe-paced mode using {{ review.tally_request_batches or 0 }} successful voucher batch(es) across {{ review.tally_request_count or 0 }} Tally request(s), with {{ review.tally_retry_count or 0 }} retry/retries and {{ review.tally_fallback_batches or 0 }} adaptive fallback(s). The agent pauses approximately {{ review.tally_pause_ms or 0 }} ms between Tally calls.</div>
     {% endif %}
     <div class="grid gap-4 md:grid-cols-2 xl:grid-cols-5">
       {% for label, value in [('Vouchers reviewed', review.summary.vouchers_reviewed), ('Cash payments', review.summary.cash_payment_vouchers), ('Single-voucher exceptions', review.summary.single_voucher_exceptions), ('Same-day exceptions', review.summary.same_day_exceptions), ('Possible split patterns', review.summary.possible_split_patterns)] %}
@@ -133,7 +133,7 @@
     <div class="mt-4 grid grid-cols-4 gap-2 text-center text-[11px] font-semibold uppercase tracking-wide text-slate-400">
       <span>Agent</span><span>Tally</span><span>Vouchers</span><span>Analysis</span>
     </div>
-    <p class="mt-4 text-xs text-slate-500">Tally is deliberately read in smaller monthly batches with pauses between requests. Please keep TallyPrime and the ERP Local Agent open until this finishes.</p>
+    <p class="mt-4 text-xs text-slate-500">Tally is deliberately read in 7-day batches with pauses between requests. If a batch times out, the agent automatically retries and reduces it to 3-day and then 1-day batches. Please keep TallyPrime and the ERP Local Agent open until this finishes.</p>
   </div>
 </div>
 <script>
@@ -149,8 +149,9 @@
     [8, 'Sending the review request to the ERP Local Agent…'],
     [18, 'Connecting to TallyPrime on the workstation…'],
     [32, 'Reading cash and ledger masters…'],
-    [48, 'Reading vouchers in safe monthly batches…'],
+    [42, 'Reading vouchers in safe 7-day batches…'],
-    [64, 'Continuing voucher scan with controlled pauses…'],
+    [58, 'Continuing voucher extraction with 1.5 second pauses…'],
+    [70, 'Large batches automatically fall back to 3-day or 1-day reads if needed…'],
     [78, 'Checking same-day cash payment aggregation…'],
     [88, 'Checking possible split-payment patterns…'],
     [94, 'Preparing the compliance report…']
@@ -169,7 +170,7 @@
       bar.style.width = stage[0] + '%';
       pct.textContent = stage[0] + '%';
       text.textContent = stage[1];
-      if (i < stages.length) window.setTimeout(advance, i < 4 ? 1300 : 2200);
+      if (i < stages.length) window.setTimeout(advance, i < 4 ? 1800 : 4500);
     };
     advance();
   });

app/modules/documents/agent_package.py

@@ -4,7 +4,7 @@ import io
 from pathlib import Path
 import zipfile
 
-ERP_LOCAL_AGENT_VERSION = "1.22.6"
+ERP_LOCAL_AGENT_VERSION = "1.22.7"
 ERP_LOCAL_AGENT_NAME = "ERP Local Agent"
 RUNTIME_ROOT = Path(__file__).resolve().parent / "local_agent_runtime"
 _DETERMINISTIC_ZIP_TIMESTAMP = (2026, 1, 1, 0, 0, 0)

app/modules/documents/local_agent_runtime/erp_local_agent/__init__.py

@@ -1,2 +1,2 @@
-__version__ = "1.22.6"
+__version__ = "1.22.7"
 AGENT_NAME = "ERP Local Agent"

app/modules/documents/local_agent_runtime/erp_local_agent/commands.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 from datetime import datetime, timezone, date as _dt_date, timedelta as _timedelta
 import re
 import time
+import threading
 from typing import Any
 
 from . import __version__
@@ -11,6 +12,9 @@ from .tally import TallyLiveConnector
 from .native_voucher_engine import NativeVoucherEngine
 
 
+_CASH_TALLY_EXTRACTION_LOCK = threading.Lock()
+
+
 class AgentCommandProcessor:
     def __init__(self, config, logger):
         self.config = config
@@ -662,13 +666,105 @@ class AgentCommandProcessor:
         if not 1 <= near_limit_percent <= 100:
             raise ValueError("Near-limit percentage must be between 1 and 100.")
 
-        # TallyPrime is intentionally queried in a paced manner. A full-year XML
+        try:
-        # voucher export in one request can make desktop Tally unresponsive on
+            scan_from = _dt_date.fromisoformat(date_from)
-        # larger books. Load masters once, pause briefly, then read vouchers in
+            scan_to = _dt_date.fromisoformat(date_to)
-        # month-sized batches with a small gap between calls.
+        except ValueError as exc:
-        pace_seconds = 0.75
+            raise ValueError("Cash payment review dates must be valid YYYY-MM-DD dates.") from exc
-        ledger_masters = self.tally.export_master_collection(company_name, "ledgers")
+        if scan_to < scan_from:
+            raise ValueError("Cash payment review To date cannot be before From date.")
+
+        # TallyPrime is single-desktop software and large XML exports can make it
+        # appear frozen. Keep only one heavy compliance extraction active on a
+        # workstation and deliberately pace every XML request.
+        if not _CASH_TALLY_EXTRACTION_LOCK.acquire(blocking=False):
+            raise ValueError("Another Cash Payment Tally extraction is already running on this workstation. Please wait for it to finish and try again.")
+
+        pace_seconds = max(0.5, min(float(payload.get("tally_pause_seconds") or 1.5), 10.0))
+        retry_pause_seconds = max(1.0, min(float(payload.get("tally_retry_pause_seconds") or 3.0), 30.0))
+        base_batch_days = max(1, min(int(payload.get("tally_batch_days") or 7), 31))
+        request_count = 0
+        successful_batches = 0
+        fallback_batches = 0
+        retry_count = 0
+        batch_log: list[dict[str, Any]] = []
+
+        def paced_call(callable_, *, label: str):
+            nonlocal request_count, retry_count
+            last_exc = None
+            for attempt in range(1, 3):
+                try:
+                    request_count += 1
+                    result = callable_()
                     time.sleep(pace_seconds)
+                    return result
+                except Exception as exc:
+                    last_exc = exc
+                    if attempt < 2:
+                        retry_count += 1
+                        self.logger.warning("Tally request retry label=%s attempt=%s error=%s", label, attempt, exc)
+                        time.sleep(retry_pause_seconds)
+            raise last_exc  # type: ignore[misc]
+
+        def fetch_voucher_range(start_day: _dt_date, end_day: _dt_date, preferred_days: int) -> list[dict[str, Any]]:
+            nonlocal successful_batches, fallback_batches
+            span = (end_day - start_day).days + 1
+            label = f"cash vouchers {start_day.isoformat()} to {end_day.isoformat()}"
+            try:
+                rows = paced_call(
+                    lambda: self.tally.export_vouchers(company_name, start_day.isoformat(), end_day.isoformat()),
+                    label=label,
+                ) or []
+                successful_batches += 1
+                batch_log.append({
+                    "date_from": start_day.isoformat(),
+                    "date_to": end_day.isoformat(),
+                    "days": span,
+                    "vouchers": len(rows),
+                    "status": "ok",
+                })
+                return list(rows)
+            except Exception as exc:
+                # Adaptive fallback: 7-day request -> approx 3-day chunks -> 1-day.
+                if span <= 1:
+                    batch_log.append({
+                        "date_from": start_day.isoformat(),
+                        "date_to": end_day.isoformat(),
+                        "days": 1,
+                        "vouchers": 0,
+                        "status": "failed",
+                        "error": str(exc),
+                    })
+                    raise ValueError(
+                        f"TallyPrime timed out while reading vouchers for {start_day.isoformat()}. "
+                        "The agent already retried the request slowly. Keep TallyPrime open and try this period again. "
+                        f"Details: {exc}"
+                    ) from exc
+
+                fallback_batches += 1
+                if span > 3:
+                    chunk_days = 3
+                else:
+                    chunk_days = 1
+                self.logger.warning(
+                    "Reducing Tally cash extraction batch %s..%s from %s day(s) to %s day chunk(s) after error: %s",
+                    start_day, end_day, span, chunk_days, exc,
+                )
+                output: list[dict[str, Any]] = []
+                cursor = start_day
+                while cursor <= end_day:
+                    child_end = min(end_day, cursor + _timedelta(days=chunk_days - 1))
+                    output.extend(fetch_voucher_range(cursor, child_end, chunk_days))
+                    cursor = child_end + _timedelta(days=1)
+                return output
+
+        try:
+            # Masters are also paced and retried; the previous implementation
+            # could fail before voucher extraction even began on large books.
+            ledger_masters = paced_call(
+                lambda: self.tally.export_master_collection(company_name, "ledgers"),
+                label="cash ledger masters",
+            ) or []
             cash_ledgers = set()
             for ledger in ledger_masters:
                 name = str(ledger.get("name") or "").strip()
@@ -679,26 +775,12 @@ class AgentCommandProcessor:
             if not cash_ledgers:
                 cash_ledgers.add("cash")
 
-        try:
+            vouchers: list[dict[str, Any]] = []
-            scan_from = _dt_date.fromisoformat(date_from)
-            scan_to = _dt_date.fromisoformat(date_to)
-        except ValueError as exc:
-            raise ValueError("Cash payment review dates must be valid YYYY-MM-DD dates.") from exc
-        if scan_to < scan_from:
-            raise ValueError("Cash payment review To date cannot be before From date.")
-
-        vouchers = []
-        batch_count = 0
             cursor = scan_from
             while cursor <= scan_to:
-            next_month = (cursor.replace(day=28) + _timedelta(days=4)).replace(day=1)
+                batch_to = min(scan_to, cursor + _timedelta(days=base_batch_days - 1))
-            batch_to = min(scan_to, next_month - _timedelta(days=1))
+                vouchers.extend(fetch_voucher_range(cursor, batch_to, base_batch_days))
-            batch = self.tally.export_vouchers(company_name, cursor.isoformat(), batch_to.isoformat())
-            vouchers.extend(batch or [])
-            batch_count += 1
                 cursor = batch_to + _timedelta(days=1)
-            if cursor <= scan_to:
-                time.sleep(pace_seconds)
 
             payments = []
             for voucher in vouchers:
@@ -826,12 +908,20 @@ class AgentCommandProcessor:
                     "cash_payments": payments,
                     "read_only": True,
                     "safe_paced_mode": True,
-                "tally_request_batches": batch_count,
+                    "adaptive_extraction": True,
+                    "tally_request_batches": successful_batches,
+                    "tally_request_count": request_count,
+                    "tally_retry_count": retry_count,
+                    "tally_fallback_batches": fallback_batches,
+                    "tally_batch_days": base_batch_days,
                     "tally_pause_ms": int(pace_seconds * 1000),
+                    "tally_batch_log": batch_log,
                     "review_note": "Possible split-payment patterns are review indicators only and do not determine tax treatment.",
                 },
                 "agent": self._agent_info(),
             }
+        finally:
+            _CASH_TALLY_EXTRACTION_LOCK.release()
 
     def _tds_compliance(self, payload: dict[str, Any]) -> dict[str, Any]:
         from collections import defaultdict