Rakseeibra/arrr-erp
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add adaptive paced Tally extraction for cash payment review

Browse Source
This commit is contained in:
A R R R Associates
2026-09-03 23:37:14 +05:30
parent 3a4370d35e
commit 247c545d20
5 changed files with 253 additions and 159 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/modules/accounting/cash_payment_ui.py
+4 -1
View File
@@ -195,8 +195,11 @@ def cash_payment_review(
"cash_limit": float(setting.cash_limit),
"split_window_days": int(setting.split_window_days),
"near_limit_percent": float(setting.near_limit_percent),
"tally_batch_days": 7,
"tally_pause_seconds": 1.5,
"tally_retry_pause_seconds": 3.0,
},
timeout_seconds=300,
timeout_seconds=900,
)
if result.get("ok"):
review = (result.get("result") or {}).get("cash_payment_review")
app/modules/accounting/templates/accounting/cash_payment_review.html
+6 -5
View File
@@ -64,7 +64,7 @@
{% if review %}
<section class="space-y-4">
{% if review.safe_paced_mode %}
<div class="rounded-2xl border border-sky-200 bg-sky-50 px-4 py-3 text-sm text-sky-800">Completed in safe-paced mode using {{ review.tally_request_batches or 0 }} Tally voucher batch(es), with approximately {{ review.tally_pause_ms or 0 }} ms pause between batches.</div>
<div class="rounded-2xl border border-sky-200 bg-sky-50 px-4 py-3 text-sm text-sky-800">Completed in adaptive safe-paced mode using {{ review.tally_request_batches or 0 }} successful voucher batch(es) across {{ review.tally_request_count or 0 }} Tally request(s), with {{ review.tally_retry_count or 0 }} retry/retries and {{ review.tally_fallback_batches or 0 }} adaptive fallback(s). The agent pauses approximately {{ review.tally_pause_ms or 0 }} ms between Tally calls.</div>
{% endif %}
<div class="grid gap-4 md:grid-cols-2 xl:grid-cols-5">
{% for label, value in [('Vouchers reviewed', review.summary.vouchers_reviewed), ('Cash payments', review.summary.cash_payment_vouchers), ('Single-voucher exceptions', review.summary.single_voucher_exceptions), ('Same-day exceptions', review.summary.same_day_exceptions), ('Possible split patterns', review.summary.possible_split_patterns)] %}
@@ -133,7 +133,7 @@
<div class="mt-4 grid grid-cols-4 gap-2 text-center text-[11px] font-semibold uppercase tracking-wide text-slate-400">
<span>Agent</span><span>Tally</span><span>Vouchers</span><span>Analysis</span>
</div>
<p class="mt-4 text-xs text-slate-500">Tally is deliberately read in smaller monthly batches with pauses between requests. Please keep TallyPrime and the ERP Local Agent open until this finishes.</p>
<p class="mt-4 text-xs text-slate-500">Tally is deliberately read in 7-day batches with pauses between requests. If a batch times out, the agent automatically retries and reduces it to 3-day and then 1-day batches. Please keep TallyPrime and the ERP Local Agent open until this finishes.</p>
</div>
</div>
<script>
@@ -149,8 +149,9 @@
[8, 'Sending the review request to the ERP Local Agent…'],
[18, 'Connecting to TallyPrime on the workstation…'],
[32, 'Reading cash and ledger masters…'],
[48, 'Reading vouchers in safe monthly batches…'],
[64, 'Continuing voucher scan with controlled pauses…'],
[42, 'Reading vouchers in safe 7-day batches…'],
[58, 'Continuing voucher extraction with 1.5 second pauses…'],
[70, 'Large batches automatically fall back to 3-day or 1-day reads if needed…'],
[78, 'Checking same-day cash payment aggregation…'],
[88, 'Checking possible split-payment patterns…'],
[94, 'Preparing the compliance report…']
@@ -169,7 +170,7 @@
bar.style.width = stage[0] + '%';
pct.textContent = stage[0] + '%';
text.textContent = stage[1];
if (i < stages.length) window.setTimeout(advance, i < 4 ? 1300 : 2200);
if (i < stages.length) window.setTimeout(advance, i < 4 ? 1800 : 4500);
};
advance();
});
app/modules/documents/agent_package.py
+1 -1
View File
@@ -4,7 +4,7 @@ import io
from pathlib import Path
import zipfile
ERP_LOCAL_AGENT_VERSION = "1.22.6"
ERP_LOCAL_AGENT_VERSION = "1.22.7"
ERP_LOCAL_AGENT_NAME = "ERP Local Agent"
RUNTIME_ROOT = Path(__file__).resolve().parent / "local_agent_runtime"
_DETERMINISTIC_ZIP_TIMESTAMP = (2026, 1, 1, 0, 0, 0)
app/modules/documents/local_agent_runtime/erp_local_agent/__init__.py
+1 -1
View File
@@ -1,2 +1,2 @@
__version__ = "1.22.6"
__version__ = "1.22.7"
AGENT_NAME = "ERP Local Agent"
app/modules/documents/local_agent_runtime/erp_local_agent/commands.py
+241 -151
View File
@@ -3,6 +3,7 @@ from __future__ import annotations
from datetime import datetime, timezone, date as _dt_date, timedelta as _timedelta
import re
import time
import threading
from typing import Any
from . import __version__
@@ -11,6 +12,9 @@ from .tally import TallyLiveConnector
from .native_voucher_engine import NativeVoucherEngine
_CASH_TALLY_EXTRACTION_LOCK = threading.Lock()
class AgentCommandProcessor:
def __init__(self, config, logger):
self.config = config
@@ -662,23 +666,6 @@ class AgentCommandProcessor:
if not 1 <= near_limit_percent <= 100:
raise ValueError("Near-limit percentage must be between 1 and 100.")
# TallyPrime is intentionally queried in a paced manner. A full-year XML
# voucher export in one request can make desktop Tally unresponsive on
# larger books. Load masters once, pause briefly, then read vouchers in
# month-sized batches with a small gap between calls.
pace_seconds = 0.75
ledger_masters = self.tally.export_master_collection(company_name, "ledgers")
time.sleep(pace_seconds)
cash_ledgers = set()
for ledger in ledger_masters:
name = str(ledger.get("name") or "").strip()
parent = str(ledger.get("parent") or "").strip().casefold().replace("-", " ")
reserved = str(ledger.get("reserved_name") or "").strip().casefold()
if name and (parent in {"cash in hand", "cash-in-hand"} or reserved == "cash" or name.casefold() == "cash"):
cash_ledgers.add(name.casefold())
if not cash_ledgers:
cash_ledgers.add("cash")
try:
scan_from = _dt_date.fromisoformat(date_from)
scan_to = _dt_date.fromisoformat(date_to)
@@ -687,151 +674,254 @@ class AgentCommandProcessor:
if scan_to < scan_from:
raise ValueError("Cash payment review To date cannot be before From date.")
vouchers = []
batch_count = 0
cursor = scan_from
while cursor <= scan_to:
next_month = (cursor.replace(day=28) + _timedelta(days=4)).replace(day=1)
batch_to = min(scan_to, next_month - _timedelta(days=1))
batch = self.tally.export_vouchers(company_name, cursor.isoformat(), batch_to.isoformat())
vouchers.extend(batch or [])
batch_count += 1
cursor = batch_to + _timedelta(days=1)
if cursor <= scan_to:
time.sleep(pace_seconds)
# TallyPrime is single-desktop software and large XML exports can make it
# appear frozen. Keep only one heavy compliance extraction active on a
# workstation and deliberately pace every XML request.
if not _CASH_TALLY_EXTRACTION_LOCK.acquire(blocking=False):
raise ValueError("Another Cash Payment Tally extraction is already running on this workstation. Please wait for it to finish and try again.")
payments = []
for voucher in vouchers:
if str(voucher.get("is_cancelled") or "").strip().lower() in {"yes", "true", "1"}:
continue
if str(voucher.get("is_optional") or "").strip().lower() in {"yes", "true", "1"}:
continue
entries = list(voucher.get("ledger_entries") or [])
cash_credit = []
for entry in entries:
lname = str(entry.get("ledger_name") or "").strip()
if lname.casefold() not in cash_ledgers:
continue
amount = float(entry.get("amount") or 0)
deemed = str(entry.get("is_deemed_positive") or "").strip().lower()
if amount > 0 or deemed == "no":
cash_credit.append(abs(amount))
cash_amount = round(sum(cash_credit), 2)
if cash_amount <= 0:
continue
pace_seconds = max(0.5, min(float(payload.get("tally_pause_seconds") or 1.5), 10.0))
retry_pause_seconds = max(1.0, min(float(payload.get("tally_retry_pause_seconds") or 3.0), 30.0))
base_batch_days = max(1, min(int(payload.get("tally_batch_days") or 7), 31))
request_count = 0
successful_batches = 0
fallback_batches = 0
retry_count = 0
batch_log: list[dict[str, Any]] = []
non_cash = [entry for entry in entries if str(entry.get("ledger_name") or "").strip().casefold() not in cash_ledgers]
party = str(voucher.get("party_ledger_name") or "").strip()
if not party or party.casefold() in cash_ledgers:
candidates = sorted(non_cash, key=lambda row: abs(float(row.get("amount") or 0)), reverse=True)
party = str(candidates[0].get("ledger_name") or "").strip() if candidates else "Unidentified counter-ledger"
payments.append({
"date": str(voucher.get("date") or voucher.get("effective_date") or ""),
"party": party or "Unidentified counter-ledger",
"amount": cash_amount,
"voucher_number": str(voucher.get("voucher_number") or ""),
"voucher_type": str(voucher.get("voucher_type_name") or ""),
"reference": str(voucher.get("reference") or ""),
"narration": str(voucher.get("narration") or ""),
})
def paced_call(callable_, *, label: str):
nonlocal request_count, retry_count
last_exc = None
for attempt in range(1, 3):
try:
request_count += 1
result = callable_()
time.sleep(pace_seconds)
return result
except Exception as exc:
last_exc = exc
if attempt < 2:
retry_count += 1
self.logger.warning("Tally request retry label=%s attempt=%s error=%s", label, attempt, exc)
time.sleep(retry_pause_seconds)
raise last_exc # type: ignore[misc]
by_party_date = defaultdict(list)
for row in payments:
by_party_date[(row["party"].casefold(), row["date"])].append(row)
exceptions = []
seen = set()
single_count = 0
same_day_count = 0
for row in payments:
if row["amount"] > cash_limit + 0.009:
single_count += 1
key = (row["party"].casefold(), row["date"], "single", row["voucher_number"])
if key not in seen:
seen.add(key)
exceptions.append({
"date": row["date"], "party": row["party"], "amount": row["amount"],
"reason": "Single cash-payment voucher exceeds configured limit",
"voucher_numbers": [row["voucher_number"] or "-"],
})
for (_party_key, paid_on), rows in by_party_date.items():
total = round(sum(row["amount"] for row in rows), 2)
if total > cash_limit + 0.009 and len(rows) > 1:
same_day_count += 1
key = (_party_key, paid_on, "aggregate")
if key not in seen:
seen.add(key)
exceptions.append({
"date": paid_on, "party": rows[0]["party"], "amount": total,
"reason": "Same-day aggregate cash payments to the same party exceed configured limit",
"voucher_numbers": [row["voucher_number"] or "-" for row in rows],
def fetch_voucher_range(start_day: _dt_date, end_day: _dt_date, preferred_days: int) -> list[dict[str, Any]]:
nonlocal successful_batches, fallback_batches
span = (end_day - start_day).days + 1
label = f"cash vouchers {start_day.isoformat()} to {end_day.isoformat()}"
try:
rows = paced_call(
lambda: self.tally.export_vouchers(company_name, start_day.isoformat(), end_day.isoformat()),
label=label,
) or []
successful_batches += 1
batch_log.append({
"date_from": start_day.isoformat(),
"date_to": end_day.isoformat(),
"days": span,
"vouchers": len(rows),
"status": "ok",
})
return list(rows)
except Exception as exc:
# Adaptive fallback: 7-day request -> approx 3-day chunks -> 1-day.
if span <= 1:
batch_log.append({
"date_from": start_day.isoformat(),
"date_to": end_day.isoformat(),
"days": 1,
"vouchers": 0,
"status": "failed",
"error": str(exc),
})
raise ValueError(
f"TallyPrime timed out while reading vouchers for {start_day.isoformat()}. "
"The agent already retried the request slowly. Keep TallyPrime open and try this period again. "
f"Details: {exc}"
) from exc
daily_by_party = defaultdict(lambda: defaultdict(float))
for row in payments:
daily_by_party[row["party"].casefold()][row["date"]] += row["amount"]
near_floor = cash_limit * near_limit_percent / 100.0
split_patterns = []
for party_key, day_map in daily_by_party.items():
day_rows = sorted((_date.fromisoformat(day), round(amount, 2)) for day, amount in day_map.items() if day)
for start_idx in range(len(day_rows)):
window = []
for idx in range(start_idx, len(day_rows)):
day_value, amount = day_rows[idx]
if (day_value - day_rows[start_idx][0]).days >= split_window_days:
break
window.append((day_value, amount))
if len(window) < 2:
fallback_batches += 1
if span > 3:
chunk_days = 3
else:
chunk_days = 1
self.logger.warning(
"Reducing Tally cash extraction batch %s..%s from %s day(s) to %s day chunk(s) after error: %s",
start_day, end_day, span, chunk_days, exc,
)
output: list[dict[str, Any]] = []
cursor = start_day
while cursor <= end_day:
child_end = min(end_day, cursor + _timedelta(days=chunk_days - 1))
output.extend(fetch_voucher_range(cursor, child_end, chunk_days))
cursor = child_end + _timedelta(days=1)
return output
try:
# Masters are also paced and retried; the previous implementation
# could fail before voucher extraction even began on large books.
ledger_masters = paced_call(
lambda: self.tally.export_master_collection(company_name, "ledgers"),
label="cash ledger masters",
) or []
cash_ledgers = set()
for ledger in ledger_masters:
name = str(ledger.get("name") or "").strip()
parent = str(ledger.get("parent") or "").strip().casefold().replace("-", " ")
reserved = str(ledger.get("reserved_name") or "").strip().casefold()
if name and (parent in {"cash in hand", "cash-in-hand"} or reserved == "cash" or name.casefold() == "cash"):
cash_ledgers.add(name.casefold())
if not cash_ledgers:
cash_ledgers.add("cash")
vouchers: list[dict[str, Any]] = []
cursor = scan_from
while cursor <= scan_to:
batch_to = min(scan_to, cursor + _timedelta(days=base_batch_days - 1))
vouchers.extend(fetch_voucher_range(cursor, batch_to, base_batch_days))
cursor = batch_to + _timedelta(days=1)
payments = []
for voucher in vouchers:
if str(voucher.get("is_cancelled") or "").strip().lower() in {"yes", "true", "1"}:
continue
if any(amount > cash_limit + 0.009 for _, amount in window):
if str(voucher.get("is_optional") or "").strip().lower() in {"yes", "true", "1"}:
continue
total = round(sum(amount for _, amount in window), 2)
if total <= cash_limit + 0.009:
entries = list(voucher.get("ledger_entries") or [])
cash_credit = []
for entry in entries:
lname = str(entry.get("ledger_name") or "").strip()
if lname.casefold() not in cash_ledgers:
continue
amount = float(entry.get("amount") or 0)
deemed = str(entry.get("is_deemed_positive") or "").strip().lower()
if amount > 0 or deemed == "no":
cash_credit.append(abs(amount))
cash_amount = round(sum(cash_credit), 2)
if cash_amount <= 0:
continue
near_days = sum(1 for _, amount in window if amount >= near_floor)
if near_days < 2:
continue
party_name = next((row["party"] for row in payments if row["party"].casefold() == party_key), party_key)
signature = (party_key, window[0][0].isoformat(), window[-1][0].isoformat())
if any((r["party"].casefold(), r["date_from"], r["date_to"]) == signature for r in split_patterns):
continue
split_patterns.append({
"party": party_name,
"date_from": window[0][0].isoformat(),
"date_to": window[-1][0].isoformat(),
"total_amount": total,
"days": [{"date": d.isoformat(), "amount": a} for d, a in window],
"review_only": True,
non_cash = [entry for entry in entries if str(entry.get("ledger_name") or "").strip().casefold() not in cash_ledgers]
party = str(voucher.get("party_ledger_name") or "").strip()
if not party or party.casefold() in cash_ledgers:
candidates = sorted(non_cash, key=lambda row: abs(float(row.get("amount") or 0)), reverse=True)
party = str(candidates[0].get("ledger_name") or "").strip() if candidates else "Unidentified counter-ledger"
payments.append({
"date": str(voucher.get("date") or voucher.get("effective_date") or ""),
"party": party or "Unidentified counter-ledger",
"amount": cash_amount,
"voucher_number": str(voucher.get("voucher_number") or ""),
"voucher_type": str(voucher.get("voucher_type_name") or ""),
"reference": str(voucher.get("reference") or ""),
"narration": str(voucher.get("narration") or ""),
})
exceptions.sort(key=lambda row: (row["date"], row["party"].casefold()))
split_patterns.sort(key=lambda row: (row["date_from"], row["party"].casefold()))
return {
"cash_payment_review": {
"company_name": company_name,
"company_guid": str(company.get("guid") or ""),
"date_from": date_from,
"date_to": date_to,
"cash_limit": cash_limit,
"cash_ledgers": sorted(cash_ledgers),
"summary": {
"vouchers_reviewed": len(vouchers),
"cash_payment_vouchers": len(payments),
"single_voucher_exceptions": single_count,
"same_day_exceptions": same_day_count,
"possible_split_patterns": len(split_patterns),
by_party_date = defaultdict(list)
for row in payments:
by_party_date[(row["party"].casefold(), row["date"])].append(row)
exceptions = []
seen = set()
single_count = 0
same_day_count = 0
for row in payments:
if row["amount"] > cash_limit + 0.009:
single_count += 1
key = (row["party"].casefold(), row["date"], "single", row["voucher_number"])
if key not in seen:
seen.add(key)
exceptions.append({
"date": row["date"], "party": row["party"], "amount": row["amount"],
"reason": "Single cash-payment voucher exceeds configured limit",
"voucher_numbers": [row["voucher_number"] or "-"],
})
for (_party_key, paid_on), rows in by_party_date.items():
total = round(sum(row["amount"] for row in rows), 2)
if total > cash_limit + 0.009 and len(rows) > 1:
same_day_count += 1
key = (_party_key, paid_on, "aggregate")
if key not in seen:
seen.add(key)
exceptions.append({
"date": paid_on, "party": rows[0]["party"], "amount": total,
"reason": "Same-day aggregate cash payments to the same party exceed configured limit",
"voucher_numbers": [row["voucher_number"] or "-" for row in rows],
})
daily_by_party = defaultdict(lambda: defaultdict(float))
for row in payments:
daily_by_party[row["party"].casefold()][row["date"]] += row["amount"]
near_floor = cash_limit * near_limit_percent / 100.0
split_patterns = []
for party_key, day_map in daily_by_party.items():
day_rows = sorted((_date.fromisoformat(day), round(amount, 2)) for day, amount in day_map.items() if day)
for start_idx in range(len(day_rows)):
window = []
for idx in range(start_idx, len(day_rows)):
day_value, amount = day_rows[idx]
if (day_value - day_rows[start_idx][0]).days >= split_window_days:
break
window.append((day_value, amount))
if len(window) < 2:
continue
if any(amount > cash_limit + 0.009 for _, amount in window):
continue
total = round(sum(amount for _, amount in window), 2)
if total <= cash_limit + 0.009:
continue
near_days = sum(1 for _, amount in window if amount >= near_floor)
if near_days < 2:
continue
party_name = next((row["party"] for row in payments if row["party"].casefold() == party_key), party_key)
signature = (party_key, window[0][0].isoformat(), window[-1][0].isoformat())
if any((r["party"].casefold(), r["date_from"], r["date_to"]) == signature for r in split_patterns):
continue
split_patterns.append({
"party": party_name,
"date_from": window[0][0].isoformat(),
"date_to": window[-1][0].isoformat(),
"total_amount": total,
"days": [{"date": d.isoformat(), "amount": a} for d, a in window],
"review_only": True,
})
exceptions.sort(key=lambda row: (row["date"], row["party"].casefold()))
split_patterns.sort(key=lambda row: (row["date_from"], row["party"].casefold()))
return {
"cash_payment_review": {
"company_name": company_name,
"company_guid": str(company.get("guid") or ""),
"date_from": date_from,
"date_to": date_to,
"cash_limit": cash_limit,
"cash_ledgers": sorted(cash_ledgers),
"summary": {
"vouchers_reviewed": len(vouchers),
"cash_payment_vouchers": len(payments),
"single_voucher_exceptions": single_count,
"same_day_exceptions": same_day_count,
"possible_split_patterns": len(split_patterns),
},
"exceptions": exceptions,
"possible_split_payments": split_patterns,
"cash_payments": payments,
"read_only": True,
"safe_paced_mode": True,
"adaptive_extraction": True,
"tally_request_batches": successful_batches,
"tally_request_count": request_count,
"tally_retry_count": retry_count,
"tally_fallback_batches": fallback_batches,
"tally_batch_days": base_batch_days,
"tally_pause_ms": int(pace_seconds * 1000),
"tally_batch_log": batch_log,
"review_note": "Possible split-payment patterns are review indicators only and do not determine tax treatment.",
},
"exceptions": exceptions,
"possible_split_payments": split_patterns,
"cash_payments": payments,
"read_only": True,
"safe_paced_mode": True,
"tally_request_batches": batch_count,
"tally_pause_ms": int(pace_seconds * 1000),
"review_note": "Possible split-payment patterns are review indicators only and do not determine tax treatment.",
},
"agent": self._agent_info(),
}
"agent": self._agent_info(),
}
finally:
_CASH_TALLY_EXTRACTION_LOCK.release()
def _tds_compliance(self, payload: dict[str, Any]) -> dict[str, Any]:
from collections import defaultdict