Add SQLite cached slow Tally extraction for cash payment review

app/modules/accounting/cash_payment_ui.py

@@ -5,7 +5,7 @@ from decimal import Decimal, InvalidOperation
 from urllib.parse import quote
 
 from fastapi import APIRouter, Request
-from fastapi.responses import RedirectResponse
+from fastapi.responses import RedirectResponse, JSONResponse
 from sqlalchemy import select
 
 from app.core.db.common import CommonSessionLocal
@@ -146,7 +146,8 @@ def cash_payment_review(
     tally_guid: str = "",
     date_from: str = "",
     date_to: str = "",
-    run: int = 0,
+    job_id: str = "",
+    analyze: int = 0,
     saved: int = 0,
     error: str = "",
 ):
@@ -169,6 +170,7 @@ def cash_payment_review(
         end_text = date_to or today.isoformat()
         live_result = None
         review = None
+        cache_job = None
         command_error = error or ""
 
         if selected_client and node and online:
@@ -183,28 +185,39 @@ def cash_payment_review(
                     live_result = status_response.get("result") or {}
                 else:
                     command_error = str(status_response.get("error") or "Local Agent status failed.")
-                if run and tally_guid:
-                    result = request_agent_command(
+
+                if job_id:
+                    status = request_agent_command(
                         node.node_code,
-                        "accounting_cash_payment_compliance",
+                        "accounting_cash_payment_cache_status",
                         {
                             **_accounting_storage_payload(selected_client, _financial_year_for_date(date.fromisoformat(start_text))),
-                            "tally_guid": str(tally_guid),
-                            "date_from": start_text,
-                            "date_to": end_text,
+                            "job_id": str(job_id),
+                        },
+                        timeout_seconds=20,
+                    )
+                    if status.get("ok"):
+                        cache_job = (status.get("result") or {}).get("job")
+                    else:
+                        command_error = str(status.get("error") or "Could not read cash-payment cache status.")
+
+                if analyze and job_id and cache_job and cache_job.get("status") == "completed":
+                    result = request_agent_command(
+                        node.node_code,
+                        "accounting_cash_payment_cache_analyze",
+                        {
+                            **_accounting_storage_payload(selected_client, _financial_year_for_date(date.fromisoformat(start_text))),
+                            "job_id": str(job_id),
                             "cash_limit": float(setting.cash_limit),
                             "split_window_days": int(setting.split_window_days),
                             "near_limit_percent": float(setting.near_limit_percent),
-                            "tally_batch_days": 7,
-                            "tally_pause_seconds": 1.5,
-                            "tally_retry_pause_seconds": 3.0,
                         },
-                        timeout_seconds=900,
+                        timeout_seconds=120,
                     )
                     if result.get("ok"):
                         review = (result.get("result") or {}).get("cash_payment_review")
                     else:
-                        command_error = str(result.get("error") or "Cash payment review failed.")
+                        command_error = str(result.get("error") or "Cash payment SQLite analysis failed.")
             except Exception as exc:
                 command_error = str(exc)
 
@@ -223,6 +236,8 @@ def cash_payment_review(
             date_to=end_text,
             setting=setting,
             review=review,
+            cache_job=cache_job,
+            active_job_id=str(job_id or ""),
             saved=bool(saved),
             command_error=command_error,
             allocation_plan=None,
@@ -233,6 +248,96 @@ def cash_payment_review(
         db.close()
 
 
+@router.post("/cache/start")
+async def start_cash_payment_cache(request: Request):
+    form = await request.form()
+    validate_csrf(request, str(form.get("csrf_token") or ""))
+    db = CommonSessionLocal()
+    try:
+        user, response = _require_partner(request, db, "accounting.tally.view")
+        if response:
+            return JSONResponse({"ok": False, "error": "Access denied."}, status_code=403)
+        clients, scope = _visible_clients(db, request, user)
+        try:
+            client_id = int(form.get("client_id") or 0)
+        except Exception:
+            client_id = 0
+        selected_client = next((row for row in clients if int(row.id) == client_id), None)
+        if not selected_client:
+            return JSONResponse({"ok": False, "error": "Select a valid client."}, status_code=400)
+        tally_guid = str(form.get("tally_guid") or "").strip()
+        date_from = str(form.get("date_from") or "").strip()
+        date_to = str(form.get("date_to") or "").strip()
+        try:
+            parsed_from = date.fromisoformat(date_from)
+            parsed_to = date.fromisoformat(date_to)
+            if parsed_to < parsed_from:
+                raise ValueError("To date cannot be before From date.")
+        except Exception as exc:
+            return JSONResponse({"ok": False, "error": str(exc)}, status_code=400)
+        if not tally_guid:
+            return JSONResponse({"ok": False, "error": "Select a mapped Tally company."}, status_code=400)
+        node = get_active_storage_node_for_branch(db, scope.tenant_id, scope.branch_id)
+        if not node or not _node_online(node):
+            return JSONResponse({"ok": False, "error": "ERP Local Agent is offline."}, status_code=409)
+        setting = _setting(db, _tenant_id(request, user))
+        try:
+            result = request_agent_command(
+                node.node_code,
+                "accounting_cash_payment_cache_start",
+                {
+                    **_accounting_storage_payload(selected_client, _financial_year_for_date(parsed_from)),
+                    "tally_guid": tally_guid,
+                    "date_from": date_from,
+                    "date_to": date_to,
+                    "cash_limit": float(setting.cash_limit),
+                    "split_window_days": int(setting.split_window_days),
+                    "near_limit_percent": float(setting.near_limit_percent),
+                    "tally_pause_seconds": 3.0,
+                    "requested_by_user_id": int(user.id),
+                },
+                timeout_seconds=30,
+            )
+        except Exception as exc:
+            return JSONResponse({"ok": False, "error": str(exc)}, status_code=502)
+        if not result.get("ok"):
+            return JSONResponse({"ok": False, "error": str(result.get("error") or "Could not start Tally extraction.")}, status_code=409)
+        return JSONResponse({"ok": True, **(result.get("result") or {})})
+    finally:
+        db.close()
+
+
+@router.get("/cache/progress")
+def cash_payment_cache_progress(request: Request, client_id: int, job_id: str, date_from: str):
+    db = CommonSessionLocal()
+    try:
+        user, response = _require_partner(request, db, "accounting.tally.view")
+        if response:
+            return JSONResponse({"ok": False, "error": "Access denied."}, status_code=403)
+        clients, scope = _visible_clients(db, request, user)
+        selected_client = next((row for row in clients if int(row.id) == int(client_id)), None)
+        if not selected_client:
+            return JSONResponse({"ok": False, "error": "Client not found."}, status_code=404)
+        node = get_active_storage_node_for_branch(db, scope.tenant_id, scope.branch_id)
+        if not node or not _node_online(node):
+            return JSONResponse({"ok": False, "error": "ERP Local Agent is offline."}, status_code=409)
+        try:
+            fy = _financial_year_for_date(date.fromisoformat(date_from))
+            result = request_agent_command(
+                node.node_code,
+                "accounting_cash_payment_cache_status",
+                {**_accounting_storage_payload(selected_client, fy), "job_id": str(job_id)},
+                timeout_seconds=20,
+            )
+        except Exception as exc:
+            return JSONResponse({"ok": False, "error": str(exc)}, status_code=502)
+        if not result.get("ok"):
+            return JSONResponse({"ok": False, "error": str(result.get("error") or "Could not read extraction progress.")}, status_code=409)
+        return JSONResponse({"ok": True, **(result.get("result") or {})})
+    finally:
+        db.close()
+
+
 @router.get("/allocation")
 def cash_payment_allocation(
     request: Request,
@@ -263,6 +368,8 @@ def cash_payment_allocation(
             date_to="",
             setting=setting,
             review=None,
+            cache_job=None,
+            active_job_id="",
             saved=False,
             command_error=error or "",
             allocation_plan=None,
@@ -383,6 +490,8 @@ async def cash_payment_allocation_plan(request: Request):
             date_to="",
             setting=setting,
             review=None,
+            cache_job=None,
+            active_job_id="",
             saved=False,
             command_error="",
             allocation_plan=plan,

app/modules/accounting/templates/accounting/cash_payment_review.html

@@ -22,6 +22,7 @@
     <div class="rounded-2xl bg-white p-5 shadow-soft lg:col-span-2">
       <h2 class="text-lg font-semibold text-slate-900">Cash Payment Compliance Review</h2>
       <form id="cash-review-form" method="get" action="/tools/accounting/cash-payments" class="mt-4 grid gap-4 md:grid-cols-2 xl:grid-cols-4">
+        <input type="hidden" id="cash-csrf-token" value="{{ csrf_token }}">
         <div>
           <label class="text-xs font-semibold uppercase text-slate-500">Client</label>
           <select name="client_id" class="mt-1 w-full rounded-xl border-slate-300 text-sm" required onchange="this.form.submit()">
@@ -42,8 +43,7 @@
         </div>
         <div><label class="text-xs font-semibold uppercase text-slate-500">From</label><input type="date" name="date_from" value="{{ date_from }}" class="mt-1 w-full rounded-xl border-slate-300 text-sm" required></div>
         <div><label class="text-xs font-semibold uppercase text-slate-500">To</label><input type="date" name="date_to" value="{{ date_to }}" class="mt-1 w-full rounded-xl border-slate-300 text-sm" required></div>
-        <input type="hidden" name="run" value="1">
-        <div class="md:col-span-2 xl:col-span-4"><button id="cash-review-button" class="rounded-xl bg-brand-600 px-4 py-2 text-sm font-semibold text-white" {% if not selected_client %}disabled{% endif %}>Run Tally Review</button><p class="mt-2 text-xs text-slate-500">Safe-paced mode reads Tally in monthly batches with a short pause between requests to reduce load on TallyPrime.</p></div>
+        <div class="md:col-span-2 xl:col-span-4"><button type="button" id="cash-review-button" class="rounded-xl bg-brand-600 px-4 py-2 text-sm font-semibold text-white" {% if not selected_client %}disabled{% endif %}>Extract / Refresh &amp; Analyze</button><p class="mt-2 text-xs text-slate-500">Transactions are copied very slowly into the Local Agent SQLite accounting cache one day at a time. Analysis starts only after TallyPrime has been released.</p></div>
       </form>
     </div>
 
@@ -61,10 +61,24 @@
     </div>
   </section>
 
+  {% if cache_job %}
+  <section class="rounded-2xl border border-sky-200 bg-sky-50 p-5">
+    <div class="flex flex-wrap items-start justify-between gap-3">
+      <div>
+        <h3 class="font-semibold text-sky-950">Local Tally SQLite Cache</h3>
+        <p class="mt-1 text-sm text-sky-800">Status: <strong>{{ cache_job.status }}</strong>{% if cache_job.stage %} · {{ cache_job.stage }}{% endif %}</p>
+      </div>
+      <div class="text-right text-sm text-sky-900"><div>{{ cache_job.completed_days }}/{{ cache_job.total_days }} day(s)</div><div>{{ cache_job.vouchers_cached }} voucher(s) cached</div></div>
+    </div>
+    <div class="mt-3 h-2 overflow-hidden rounded-full bg-sky-100"><div class="h-full bg-sky-600" style="width:{{ cache_job.percent or 0 }}%"></div></div>
+    {% if cache_job.error %}<p class="mt-3 text-sm font-medium text-amber-900">{{ cache_job.error }}</p>{% endif %}
+  </section>
+  {% endif %}
+
   {% if review %}
   <section class="space-y-4">
-    {% if review.safe_paced_mode %}
-    <div class="rounded-2xl border border-sky-200 bg-sky-50 px-4 py-3 text-sm text-sky-800">Completed in adaptive safe-paced mode using {{ review.tally_request_batches or 0 }} successful voucher batch(es) across {{ review.tally_request_count or 0 }} Tally request(s), with {{ review.tally_retry_count or 0 }} retry/retries and {{ review.tally_fallback_batches or 0 }} adaptive fallback(s). The agent pauses approximately {{ review.tally_pause_ms or 0 }} ms between Tally calls.</div>
+    {% if review.sqlite_cache %}
+    <div class="rounded-2xl border border-emerald-200 bg-emerald-50 px-4 py-3 text-sm text-emerald-800">Analysis completed from the Local Agent SQLite cache. TallyPrime was used only for the slow background extraction and was released before this analysis ran. {{ review.tally_request_count or 0 }} Tally request(s), {{ review.tally_retry_count or 0 }} retry/retries, and approximately {{ review.tally_pause_ms or 0 }} ms cooling between successful day reads.</div>
     {% endif %}
     <div class="grid gap-4 md:grid-cols-2 xl:grid-cols-5">
       {% for label, value in [('Vouchers reviewed', review.summary.vouchers_reviewed), ('Cash payments', review.summary.cash_payment_vouchers), ('Single-voucher exceptions', review.summary.single_voucher_exceptions), ('Same-day exceptions', review.summary.same_day_exceptions), ('Possible split patterns', review.summary.possible_split_patterns)] %}
@@ -119,60 +133,126 @@
   </section>
   {% endif %}
 <div id="cash-review-progress" class="fixed inset-0 z-[100] hidden items-center justify-center bg-slate-950/40 p-4 backdrop-blur-sm">
-  <div class="w-full max-w-xl rounded-2xl bg-white p-6 shadow-2xl">
+  <div class="w-full max-w-2xl rounded-2xl bg-white p-6 shadow-2xl">
     <div class="flex items-start justify-between gap-4">
       <div>
-        <h3 class="text-lg font-semibold text-slate-900">Cash Payment Analysis in Progress</h3>
-        <p id="cash-progress-text" class="mt-1 text-sm text-slate-600">Sending the review request to the ERP Local Agent…</p>
+        <h3 class="text-lg font-semibold text-slate-900">Slow Tally Extraction to Local SQLite</h3>
+        <p id="cash-progress-text" class="mt-1 text-sm text-slate-600">Submitting a background extraction job to the ERP Local Agent…</p>
       </div>
-      <div id="cash-progress-percent" class="text-lg font-semibold text-brand-700">5%</div>
+      <div id="cash-progress-percent" class="text-lg font-semibold text-brand-700">0%</div>
     </div>
     <div class="mt-5 h-3 overflow-hidden rounded-full bg-slate-200">
-      <div id="cash-progress-bar" class="h-full rounded-full bg-brand-600 transition-all duration-700" style="width:5%"></div>
+      <div id="cash-progress-bar" class="h-full rounded-full bg-brand-600 transition-all duration-500" style="width:0%"></div>
     </div>
-    <div class="mt-4 grid grid-cols-4 gap-2 text-center text-[11px] font-semibold uppercase tracking-wide text-slate-400">
-      <span>Agent</span><span>Tally</span><span>Vouchers</span><span>Analysis</span>
+    <div class="mt-4 grid grid-cols-2 gap-3 text-sm md:grid-cols-4">
+      <div class="rounded-xl bg-slate-50 p-3"><div class="text-xs uppercase text-slate-500">Days</div><div id="cash-progress-days" class="mt-1 font-semibold">0 / 0</div></div>
+      <div class="rounded-xl bg-slate-50 p-3"><div class="text-xs uppercase text-slate-500">Vouchers cached</div><div id="cash-progress-vouchers" class="mt-1 font-semibold">0</div></div>
+      <div class="rounded-xl bg-slate-50 p-3"><div class="text-xs uppercase text-slate-500">Tally requests</div><div id="cash-progress-requests" class="mt-1 font-semibold">0</div></div>
+      <div class="rounded-xl bg-slate-50 p-3"><div class="text-xs uppercase text-slate-500">Current date</div><div id="cash-progress-date" class="mt-1 font-semibold">—</div></div>
     </div>
-    <p class="mt-4 text-xs text-slate-500">Tally is deliberately read in 7-day batches with pauses between requests. If a batch times out, the agent automatically retries and reduces it to 3-day and then 1-day batches. Please keep TallyPrime and the ERP Local Agent open until this finishes.</p>
+    <p class="mt-4 text-xs text-slate-500">Only one day is requested from TallyPrime at a time. The agent waits about 3 seconds after every successful day. If Tally responds slowly, it cools for 20 seconds and retries only once. The tunnel heartbeat stays responsive because progress is read from SQLite, not from Tally.</p>
+    <div id="cash-progress-error" class="mt-4 hidden rounded-xl border border-amber-200 bg-amber-50 p-3 text-sm text-amber-900"></div>
+    <div class="mt-4 flex justify-end"><button type="button" id="cash-progress-close" class="hidden rounded-xl border border-slate-300 bg-white px-4 py-2 text-sm font-semibold text-slate-700">Close</button></div>
   </div>
 </div>
 <script>
 (function () {
   const form = document.getElementById('cash-review-form');
-  if (!form) return;
+  const button = document.getElementById('cash-review-button');
+  if (!form || !button) return;
   const overlay = document.getElementById('cash-review-progress');
   const bar = document.getElementById('cash-progress-bar');
   const pct = document.getElementById('cash-progress-percent');
   const text = document.getElementById('cash-progress-text');
-  const button = document.getElementById('cash-review-button');
-  const stages = [
-    [8, 'Sending the review request to the ERP Local Agent…'],
-    [18, 'Connecting to TallyPrime on the workstation…'],
-    [32, 'Reading cash and ledger masters…'],
-    [42, 'Reading vouchers in safe 7-day batches…'],
-    [58, 'Continuing voucher extraction with 1.5 second pauses…'],
-    [70, 'Large batches automatically fall back to 3-day or 1-day reads if needed…'],
-    [78, 'Checking same-day cash payment aggregation…'],
-    [88, 'Checking possible split-payment patterns…'],
-    [94, 'Preparing the compliance report…']
-  ];
-  form.addEventListener('submit', function (event) {
-    const submitter = event.submitter;
-    // Client dropdown auto-submit should remain quick and should not show a review progress dialog.
-    if (!submitter || submitter.id !== 'cash-review-button') return;
+  const days = document.getElementById('cash-progress-days');
+  const vouchers = document.getElementById('cash-progress-vouchers');
+  const requests = document.getElementById('cash-progress-requests');
+  const currentDate = document.getElementById('cash-progress-date');
+  const errorBox = document.getElementById('cash-progress-error');
+  const closeButton = document.getElementById('cash-progress-close');
+  let pollTimer = null;
+
+  function show() {
     overlay.classList.remove('hidden');
     overlay.classList.add('flex');
+    errorBox.classList.add('hidden');
+    closeButton.classList.add('hidden');
+  }
+  function stopPolling() {
+    if (pollTimer) window.clearTimeout(pollTimer);
+    pollTimer = null;
+  }
+  function fail(message) {
+    stopPolling();
+    button.disabled = false;
+    text.textContent = 'Extraction paused safely.';
+    errorBox.textContent = message || 'The extraction did not complete.';
+    errorBox.classList.remove('hidden');
+    closeButton.classList.remove('hidden');
+  }
+  function update(job) {
+    const value = Math.max(0, Math.min(100, Number(job.percent || 0)));
+    bar.style.width = value + '%';
+    pct.textContent = value + '%';
+    text.textContent = job.stage || job.status || 'Working…';
+    days.textContent = String(job.completed_days || 0) + ' / ' + String(job.total_days || 0);
+    vouchers.textContent = String(job.vouchers_cached || 0);
+    requests.textContent = String(job.tally_requests || 0);
+    currentDate.textContent = job.current_date || '—';
+  }
+  async function poll(jobId, clientId, dateFrom, tallyGuid, dateTo) {
+    try {
+      const url = new URL('/tools/accounting/cash-payments/cache/progress', window.location.origin);
+      url.searchParams.set('client_id', clientId);
+      url.searchParams.set('job_id', jobId);
+      url.searchParams.set('date_from', dateFrom);
+      const response = await fetch(url.toString(), {headers: {'Accept': 'application/json'}});
+      const data = await response.json();
+      if (!response.ok || !data.ok) throw new Error(data.error || 'Could not read extraction progress.');
+      const job = data.job || {};
+      update(job);
+      if (job.status === 'completed') {
+        text.textContent = 'SQLite extraction complete. Running cash-payment analysis locally…';
+        bar.style.width = '100%'; pct.textContent = '100%';
+        const target = new URL('/tools/accounting/cash-payments', window.location.origin);
+        target.searchParams.set('client_id', clientId);
+        target.searchParams.set('tally_guid', tallyGuid);
+        target.searchParams.set('date_from', dateFrom);
+        target.searchParams.set('date_to', dateTo);
+        target.searchParams.set('job_id', jobId);
+        target.searchParams.set('analyze', '1');
+        window.location.assign(target.toString());
+        return;
+      }
+      if (job.status === 'paused' || job.status === 'failed') {
+        fail(job.error || 'Tally extraction paused. Restart/verify TallyPrime and click Extract / Refresh & Analyze again to resume.');
+        return;
+      }
+      pollTimer = window.setTimeout(() => poll(jobId, clientId, dateFrom, tallyGuid, dateTo), 2000);
+    } catch (err) {
+      fail(err && err.message ? err.message : String(err));
+    }
+  }
+
+  button.addEventListener('click', async function () {
+    if (!form.reportValidity()) return;
+    const fd = new FormData(form);
+    fd.set('csrf_token', document.getElementById('cash-csrf-token').value);
     button.disabled = true;
-    let i = 0;
-    const advance = function () {
-      if (i >= stages.length) return;
-      const stage = stages[i++];
-      bar.style.width = stage[0] + '%';
-      pct.textContent = stage[0] + '%';
-      text.textContent = stage[1];
-      if (i < stages.length) window.setTimeout(advance, i < 4 ? 1800 : 4500);
-    };
-    advance();
+    show();
+    try {
+      const response = await fetch('/tools/accounting/cash-payments/cache/start', {method: 'POST', body: fd, headers: {'Accept': 'application/json'}});
+      const data = await response.json();
+      if (!response.ok || !data.ok) throw new Error(data.error || 'Could not start extraction.');
+      const job = data.job || {};
+      update(job);
+      poll(job.job_id, fd.get('client_id'), fd.get('date_from'), fd.get('tally_guid'), fd.get('date_to'));
+    } catch (err) {
+      fail(err && err.message ? err.message : String(err));
+    }
+  });
+  closeButton.addEventListener('click', function () {
+    overlay.classList.add('hidden'); overlay.classList.remove('flex');
   });
 })();
 </script>