Add lightweight GST operator agent for portal automation

app/modules/accounting/gst_operator_agent_runtime/README.txt

@@ -0,0 +1,18 @@
+ARRR GST Operator Agent 1.0.0
+
+Purpose
+- Runs only on the ERP operator workstation.
+- Listens on 127.0.0.1:8791 only.
+- Opens installed Chrome/Edge visibly.
+- Receives only a short-lived ERP job token from the ERP page.
+- Redeems GST username/password directly from ERP Credential Vault over HTTPS.
+- Autofills username/password; CAPTCHA/OTP remain manual.
+- Downloads selected GST return data and uploads it back to ERP for transfer to the configured Storage Agent.
+- Does not provide Tally, Accounting Mirror or client storage services.
+
+Install
+Right-click PowerShell and run:
+  powershell.exe -NoProfile -ExecutionPolicy Bypass -File .\install_gst_operator_agent.ps1
+
+Requirement
+Python 3.11 or later and installed Google Chrome or Microsoft Edge.

app/modules/accounting/gst_operator_agent_runtime/gst_operator_agent.py

@@ -0,0 +1,370 @@
+from __future__ import annotations
+
+import json
+import re
+import shutil
+import threading
+import time
+import zipfile
+from datetime import datetime, timezone
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from pathlib import Path
+from urllib.parse import parse_qs, urlsplit
+
+import requests
+
+VERSION = "1.0.0"
+PORT = 8791
+ROOT = Path(__file__).resolve().parent
+DATA = ROOT / "data"
+CONFIG_PATH = ROOT / "config.json"
+GST_LOGIN_URL = "https://services.gst.gov.in/services/login"
+GSTR1_URL = "https://return.gst.gov.in/returns/auth/api/offline/download/generate?flag=0&rtn_prd={period}&rtn_typ=GSTR1"
+GSTR2A_URL = "https://return.gst.gov.in/returns/auth/api/offline/download/generate?flag=0&rtn_prd={period}&rtn_typ=GSTR2A"
+GSTR1_DOWNLOAD_URL = "https://return.gst.gov.in/returns/auth/api/offline/download/url?rtn_prd={period}&rtn_typ=GSTR1&file_num={file_num}"
+GSTR2B_URL = "https://gstr2b.gst.gov.in/gstr2b/auth/api/gstr2b/getjson?rtnprd={period}"
+GSTR3B_SUMMARY_URL = "https://return.gst.gov.in/returns/auth/api/gstr3b/summary?rtn_prd={period}"
+GSTR3B_URL = "https://return.gst.gov.in/returns/auth/api/gstr3b/taxpayble?rtn_prd={period}"
+THREADS: dict[str, threading.Thread] = {}
+
+
+def now() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+def read_config() -> dict:
+    try:
+        return json.loads(CONFIG_PATH.read_text(encoding="utf-8"))
+    except Exception:
+        return {"erp_base_url": "https://office.arrrassociates.com", "port": PORT}
+
+
+def safe(value: str, fallback: str = "item") -> str:
+    value = re.sub(r"[^A-Za-z0-9._ -]+", "_", str(value or "").strip()).strip(" ._")
+    return value or fallback
+
+
+def write_json(path: Path, payload) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    tmp = path.with_suffix(path.suffix + ".tmp")
+    tmp.write_text(json.dumps(payload, ensure_ascii=False, indent=2, default=str), encoding="utf-8")
+    tmp.replace(path)
+
+
+def read_json(path: Path) -> dict:
+    try:
+        value = json.loads(path.read_text(encoding="utf-8"))
+        return value if isinstance(value, dict) else {}
+    except Exception:
+        return {}
+
+
+def job_path(job_id: str) -> Path:
+    return DATA / f"gst_job_{safe(job_id)}.json"
+
+
+def walk(obj):
+    if isinstance(obj, dict):
+        yield obj
+        for value in obj.values():
+            yield from walk(value)
+    elif isinstance(obj, list):
+        for value in obj:
+            yield from walk(value)
+
+
+def extract_file_num(text: str) -> str:
+    try:
+        data = json.loads(text or "{}")
+    except Exception:
+        return "1"
+    for obj in walk(data):
+        if isinstance(obj, dict):
+            for key in ("file_num", "fileNum", "filenum", "file_number", "fileNumber"):
+                if obj.get(key) not in (None, ""):
+                    return str(obj.get(key))
+    return "1"
+
+
+def extract_payload(text: str) -> str:
+    try:
+        data = json.loads(text or "{}")
+    except Exception:
+        return text or ""
+    if isinstance(data, dict):
+        for key in ("data", "payload", "json", "response"):
+            value = data.get(key)
+            if isinstance(value, (dict, list)):
+                return json.dumps(value, ensure_ascii=False)
+            if isinstance(value, str) and value.strip().startswith(("{", "[")):
+                return value
+    return text or ""
+
+
+def api_text(context, url: str) -> str:
+    response = context.request.get(url, timeout=90000)
+    if not response.ok:
+        raise RuntimeError(f"GST portal request failed ({response.status}) for {url.split('?')[0]}")
+    return response.text()
+
+
+def download_period(context, work_root: Path, period: str, return_types: list[str], progress) -> list[dict]:
+    raw_dir = work_root / period / "raw"
+    raw_dir.mkdir(parents=True, exist_ok=True)
+    selected = {str(x or "").upper() for x in return_types}
+    downloaded: list[dict] = []
+
+    if "GSTR1" in selected:
+        progress(stage=f"Downloading GSTR-1 {period}", message=f"Reading GSTR-1 for {period}.")
+        generated = api_text(context, GSTR1_URL.format(period=period))
+        (raw_dir / f"{period}_GSTR1_GENERATE.json").write_text(generated, encoding="utf-8")
+        file_num = extract_file_num(generated)
+        content = extract_payload(api_text(context, GSTR1_DOWNLOAD_URL.format(period=period, file_num=file_num)))
+        path = raw_dir / f"{period}_GSTR1.json"
+        path.write_text(content, encoding="utf-8")
+        downloaded.append({"return_type": "GSTR1", "path": str(path.relative_to(work_root)), "bytes": path.stat().st_size})
+
+    if "GSTR2B" in selected:
+        progress(stage=f"Downloading GSTR-2B {period}", message=f"Reading GSTR-2B for {period}.")
+        content = api_text(context, GSTR2B_URL.format(period=period))
+        path = raw_dir / f"{period}_GSTR2B.json"
+        path.write_text(content, encoding="utf-8")
+        downloaded.append({"return_type": "GSTR2B", "path": str(path.relative_to(work_root)), "bytes": path.stat().st_size})
+
+    if "GSTR3B" in selected:
+        progress(stage=f"Downloading GSTR-3B {period}", message=f"Reading GSTR-3B for {period}.")
+        summary = api_text(context, GSTR3B_SUMMARY_URL.format(period=period))
+        payable = api_text(context, GSTR3B_URL.format(period=period))
+        (raw_dir / f"{period}_GSTR3B_SUMMARY.json").write_text(summary, encoding="utf-8")
+        (raw_dir / f"{period}_GSTR3B_TAXPAYBLE.json").write_text(payable, encoding="utf-8")
+        try:
+            combined = json.loads(summary or "{}")
+        except Exception:
+            combined = {"summary_raw": summary}
+        if not isinstance(combined, dict):
+            combined = {"summary": combined}
+        try:
+            combined["taxpayble"] = json.loads(payable or "{}")
+        except Exception:
+            combined["taxpayble"] = {"raw": payable}
+        path = raw_dir / f"{period}_GSTR3B.json"
+        path.write_text(json.dumps(combined, ensure_ascii=False, indent=2), encoding="utf-8")
+        downloaded.append({"return_type": "GSTR3B", "path": str(path.relative_to(work_root)), "bytes": path.stat().st_size})
+
+    if "GSTR2A" in selected:
+        progress(stage=f"Requesting GSTR-2A {period}", message=f"Requesting GSTR-2A for {period}.")
+        content = api_text(context, GSTR2A_URL.format(period=period))
+        path = raw_dir / f"{period}_GSTR2A.json"
+        path.write_text(content, encoding="utf-8")
+        downloaded.append({"return_type": "GSTR2A", "path": str(path.relative_to(work_root)), "bytes": path.stat().st_size})
+
+    write_json(work_root / period / "download_manifest.json", {
+        "period": period, "downloaded_at_utc": now(), "source": "gst_lightweight_operator_agent", "downloaded": downloaded
+    })
+    return downloaded
+
+
+def login_complete(page) -> bool:
+    url = (page.url or "").lower()
+    if "/dashboard" in url or "/returns" in url:
+        return True
+    try:
+        return bool(page.locator("text=Search Taxpayer").count() and not page.locator("#username").count())
+    except Exception:
+        return False
+
+
+def browser_worker(payload: dict, token: str, path: Path) -> None:
+    job_id = str(payload.get("jti") or "")
+    work_root = DATA / "work" / safe(job_id)
+    package_path = DATA / f"gst_{safe(job_id)}.zip"
+    try:
+        shutil.rmtree(work_root, ignore_errors=True)
+        work_root.mkdir(parents=True, exist_ok=True)
+
+        def progress(**updates):
+            current = read_json(path)
+            current.update(updates)
+            current["updated_at_utc"] = now()
+            write_json(path, current)
+
+        progress(status="running", percent=5, stage="Opening GST Login", message="Opening GST Portal in visible Chrome/Edge on this workstation.")
+        from playwright.sync_api import sync_playwright
+        with sync_playwright() as pw:
+            context = None
+            errors = []
+            profile_root = DATA / "browser_profile"
+            profile_root.mkdir(parents=True, exist_ok=True)
+            for channel in ("chrome", "msedge"):
+                try:
+                    context = pw.chromium.launch_persistent_context(
+                        user_data_dir=str(profile_root / channel), channel=channel, headless=False,
+                        no_viewport=True, accept_downloads=True,
+                        args=["--start-maximized", "--no-first-run", "--disable-blink-features=AutomationControlled"],
+                    )
+                    break
+                except Exception as exc:
+                    errors.append(f"{channel}: {exc}")
+            if context is None:
+                raise RuntimeError("Could not open installed Chrome or Edge. " + " | ".join(errors))
+            page = context.pages[0] if context.pages else context.new_page()
+            page.goto(GST_LOGIN_URL, wait_until="domcontentloaded", timeout=90000)
+            page.locator("#username").wait_for(state="visible", timeout=30000)
+            page.fill("#username", str(payload.get("username") or ""))
+            # GST portal has used user_pass for the password field; fall back to generic password locator.
+            password_filled = False
+            for selector in ("#user_pass", "input[type=password]"):
+                try:
+                    locator = page.locator(selector).first
+                    if locator.count():
+                        locator.fill(str(payload.get("password") or ""))
+                        password_filled = True
+                        break
+                except Exception:
+                    pass
+            if not password_filled:
+                raise RuntimeError("GST password field could not be located. The portal login page may have changed.")
+            progress(percent=10, stage="Waiting for CAPTCHA / OTP", message="GST username/password filled. Complete CAPTCHA/OTP in the visible GST browser.")
+            deadline = time.time() + int(payload.get("login_timeout_seconds") or 900)
+            while time.time() < deadline:
+                if login_complete(page):
+                    break
+                time.sleep(2)
+            else:
+                raise RuntimeError("GST login was not completed within 15 minutes.")
+
+            periods = [str(p) for p in payload.get("periods") or []]
+            return_types = [str(r) for r in payload.get("return_types") or []]
+            all_downloaded = []
+            for index, period in enumerate(periods, 1):
+                base_pct = 10 + int((index - 1) * 75 / max(1, len(periods)))
+                progress(percent=base_pct, period=period, period_index=index, period_total=len(periods), stage=f"Period {index}/{len(periods)}", message=f"Downloading selected GST returns for {period}.")
+                all_downloaded.extend(download_period(context, work_root, period, return_types, progress))
+            context.close()
+
+        if package_path.exists():
+            package_path.unlink()
+        with zipfile.ZipFile(package_path, "w", zipfile.ZIP_DEFLATED, compresslevel=6) as archive:
+            for item in sorted(work_root.rglob("*")):
+                if item.is_file():
+                    archive.write(item, item.relative_to(work_root).as_posix())
+
+        progress(percent=90, stage="Transferring to Client Storage", message="Uploading GST return package to ERP for transfer to the configured Storage Agent.")
+        with package_path.open("rb") as handle:
+            response = requests.post(
+                str(payload.get("upload_url") or ""), data={"token": token},
+                files={"package": (package_path.name, handle, "application/zip")}, timeout=240,
+            )
+        try:
+            body = response.json()
+        except Exception:
+            body = {"ok": False, "error": response.text[:1000]}
+        if not response.ok or not body.get("ok"):
+            raise RuntimeError(body.get("error") or f"ERP storage transfer failed with HTTP {response.status_code}.")
+        progress(status="completed", percent=100, stage="Completed", message="GST returns downloaded and stored in the configured client local storage.", result={"stored": body.get("stored") or {}}, finished_at_utc=now())
+    except Exception as exc:
+        progress = read_json(path)
+        progress.update({"status": "failed", "percent": 100, "stage": "Failed", "message": str(exc), "error": str(exc), "finished_at_utc": now(), "updated_at_utc": now()})
+        write_json(path, progress)
+    finally:
+        shutil.rmtree(work_root, ignore_errors=True)
+        try:
+            package_path.unlink(missing_ok=True)
+        except Exception:
+            pass
+
+
+def start_job(token: str) -> dict:
+    cfg = read_config()
+    erp = str(cfg.get("erp_base_url") or "").rstrip("/")
+    if not token:
+        raise ValueError("GST operator token is required.")
+    response = requests.post(erp + "/tools/accounting/gst-reconciliation/operator/redeem", json={"token": token}, timeout=30)
+    try:
+        body = response.json()
+    except Exception:
+        body = {"ok": False, "error": response.text[:1000]}
+    if not response.ok or not body.get("ok"):
+        raise RuntimeError(body.get("error") or "ERP could not authorize this GST download.")
+    payload = dict(body.get("payload") or {})
+    job_id = str(payload.get("jti") or "")
+    if not job_id:
+        raise RuntimeError("ERP did not return a GST job id.")
+    path = job_path(job_id)
+    current = read_json(path)
+    if current.get("status") in {"queued", "running"}:
+        return current
+    initial = {
+        "status": "queued", "percent": 1, "stage": "Queued", "message": "GST browser job queued.",
+        "job_id": job_id, "periods": payload.get("periods") or [], "return_types": payload.get("return_types") or [],
+        "started_at_utc": now(), "updated_at_utc": now(),
+    }
+    write_json(path, initial)
+    thread = threading.Thread(target=browser_worker, args=(payload, token, path), daemon=True, name=f"gst-{job_id}")
+    THREADS[job_id] = thread
+    thread.start()
+    return initial
+
+
+class Handler(BaseHTTPRequestHandler):
+    def log_message(self, fmt, *args):
+        return
+
+    def cors(self):
+        cfg = read_config()
+        origin = self.headers.get("Origin") or ""
+        allowed = str(cfg.get("erp_base_url") or "").rstrip("/")
+        if origin.rstrip("/") == allowed:
+            self.send_header("Access-Control-Allow-Origin", origin)
+            self.send_header("Vary", "Origin")
+        if (self.headers.get("Access-Control-Request-Private-Network") or "").lower() == "true":
+            self.send_header("Access-Control-Allow-Private-Network", "true")
+
+    def reply(self, payload, status=200):
+        data = json.dumps(payload, ensure_ascii=False, default=str).encode("utf-8")
+        self.send_response(status)
+        self.send_header("Content-Type", "application/json; charset=utf-8")
+        self.send_header("Cache-Control", "no-store")
+        self.cors()
+        self.send_header("Content-Length", str(len(data)))
+        self.end_headers()
+        self.wfile.write(data)
+
+    def do_OPTIONS(self):
+        self.send_response(204)
+        self.cors()
+        self.send_header("Access-Control-Allow-Methods", "GET, POST, OPTIONS")
+        self.send_header("Access-Control-Allow-Headers", "Content-Type")
+        self.end_headers()
+
+    def do_GET(self):
+        path = urlsplit(self.path)
+        if path.path == "/api/status":
+            return self.reply({"ok": True, "name": "ARRR GST Operator Agent", "version": VERSION, "port": PORT})
+        if path.path == "/api/gst/status":
+            job_id = str((parse_qs(path.query).get("job_id") or [""])[0])
+            return self.reply({"ok": True, "job": read_json(job_path(job_id))})
+        return self.reply({"ok": False, "error": "Not found"}, 404)
+
+    def do_POST(self):
+        try:
+            if urlsplit(self.path).path != "/api/gst/start":
+                return self.reply({"ok": False, "error": "Not found"}, 404)
+            length = int(self.headers.get("Content-Length") or 0)
+            body = json.loads((self.rfile.read(length) if length else b"{}").decode("utf-8"))
+            job = start_job(str(body.get("token") or ""))
+            return self.reply({"ok": True, "job": job})
+        except Exception as exc:
+            return self.reply({"ok": False, "error": str(exc)}, 400)
+
+
+def main():
+    DATA.mkdir(parents=True, exist_ok=True)
+    cfg = read_config()
+    port = int(cfg.get("port") or PORT)
+    server = ThreadingHTTPServer(("127.0.0.1", port), Handler)
+    server.serve_forever()
+
+
+if __name__ == "__main__":
+    main()

app/modules/accounting/gst_operator_agent_runtime/install_gst_operator_agent.ps1

@@ -0,0 +1,57 @@
+param(
+    [string]$ErpBaseUrl = "https://office.arrrassociates.com"
+)
+$ErrorActionPreference = "Stop"
+$Source = Split-Path -Parent $MyInvocation.MyCommand.Path
+$InstallRoot = Join-Path $env:LOCALAPPDATA "ARRR\GSTOperatorAgent"
+$Venv = Join-Path $InstallRoot ".venv"
+$Startup = Join-Path $env:APPDATA "Microsoft\Windows\Start Menu\Programs\Startup\ARRR GST Operator Agent.cmd"
+
+Write-Host "ARRR GST Operator Agent 1.0.0" -ForegroundColor Cyan
+Write-Host "Install root: $InstallRoot"
+
+$Python = $null
+if (Get-Command py -ErrorAction SilentlyContinue) { $Python = @("py", "-3") }
+elseif (Get-Command python -ErrorAction SilentlyContinue) { $Python = @("python") }
+else { throw "Python 3 is required on this workstation. Install Python 3.11+ and run this installer again." }
+
+New-Item -ItemType Directory -Path $InstallRoot -Force | Out-Null
+Copy-Item (Join-Path $Source "gst_operator_agent.py") (Join-Path $InstallRoot "gst_operator_agent.py") -Force
+Copy-Item (Join-Path $Source "requirements.txt") (Join-Path $InstallRoot "requirements.txt") -Force
+
+$config = @{ erp_base_url = $ErpBaseUrl.TrimEnd('/'); port = 8791 } | ConvertTo-Json
+[System.IO.File]::WriteAllText((Join-Path $InstallRoot "config.json"), $config, [System.Text.UTF8Encoding]::new($false))
+
+if (-not (Test-Path (Join-Path $Venv "Scripts\python.exe"))) {
+    if ($Python.Count -eq 2) { & $Python[0] $Python[1] -m venv $Venv }
+    else { & $Python[0] -m venv $Venv }
+    if ($LASTEXITCODE -ne 0) { throw "Could not create GST Operator Agent virtual environment." }
+}
+
+$VenvPython = Join-Path $Venv "Scripts\python.exe"
+& $VenvPython -m pip install --disable-pip-version-check --upgrade pip
+if ($LASTEXITCODE -ne 0) { throw "pip upgrade failed." }
+& $VenvPython -m pip install --disable-pip-version-check -r (Join-Path $InstallRoot "requirements.txt")
+if ($LASTEXITCODE -ne 0) { throw "GST Operator Agent dependencies could not be installed." }
+
+$Pythonw = Join-Path $Venv "Scripts\pythonw.exe"
+$Cmd = "@echo off`r`nstart `"`" `"$Pythonw`" `"$InstallRoot\gst_operator_agent.py`"`r`n"
+[System.IO.File]::WriteAllText($Startup, $Cmd, [System.Text.ASCIIEncoding]::new())
+
+# Stop an older copy bound to the operator port, if present.
+try {
+    $Connections = Get-NetTCPConnection -LocalPort 8791 -State Listen -ErrorAction SilentlyContinue
+    foreach ($Connection in $Connections) {
+        if ($Connection.OwningProcess) { Stop-Process -Id $Connection.OwningProcess -Force -ErrorAction SilentlyContinue }
+    }
+} catch {}
+
+Start-Process -FilePath $Pythonw -ArgumentList @((Join-Path $InstallRoot "gst_operator_agent.py")) -WorkingDirectory $InstallRoot
+Start-Sleep -Seconds 2
+try {
+    $Status = Invoke-RestMethod -Uri "http://127.0.0.1:8791/api/status" -TimeoutSec 5
+    Write-Host "Agent status: $($Status.name) v$($Status.version) - Online" -ForegroundColor Green
+} catch {
+    throw "GST Operator Agent was installed but did not answer on http://127.0.0.1:8791. $($_.Exception.Message)"
+}
+Write-Host "Installed successfully. It will start automatically when this Windows user logs in." -ForegroundColor Green

app/modules/accounting/gst_operator_agent_runtime/requirements.txt

@@ -0,0 +1,2 @@
+requests==2.32.3
+playwright==1.55.0

app/modules/accounting/gst_operator_agent_runtime/uninstall_gst_operator_agent.ps1

@@ -0,0 +1,11 @@
+$ErrorActionPreference = "Stop"
+$InstallRoot = Join-Path $env:LOCALAPPDATA "ARRR\GSTOperatorAgent"
+$Startup = Join-Path $env:APPDATA "Microsoft\Windows\Start Menu\Programs\Startup\ARRR GST Operator Agent.cmd"
+try {
+    Get-NetTCPConnection -LocalPort 8791 -State Listen -ErrorAction SilentlyContinue | ForEach-Object {
+        if ($_.OwningProcess) { Stop-Process -Id $_.OwningProcess -Force -ErrorAction SilentlyContinue }
+    }
+} catch {}
+Remove-Item $Startup -Force -ErrorAction SilentlyContinue
+Remove-Item $InstallRoot -Recurse -Force -ErrorAction SilentlyContinue
+Write-Host "ARRR GST Operator Agent removed." -ForegroundColor Green

app/modules/accounting/gst_reconciliation_ui.py

@@ -13,7 +13,7 @@ from urllib.parse import urlencode
 
 import jwt
 from fastapi import APIRouter, File, Form, Request, UploadFile
-from fastapi.responses import FileResponse, JSONResponse, RedirectResponse
+from fastapi.responses import FileResponse, JSONResponse, RedirectResponse, StreamingResponse
 from sqlalchemy import select
 
 from app.core.db.common import CommonSessionLocal
@@ -144,9 +144,12 @@ def extract_gstr3b_itc(data) -> dict[str, float]:
     return totals
 
 router = APIRouter(prefix="/tools/accounting/gst-reconciliation", tags=["accounting-gst-reconciliation-ui"])
-_TOKEN_PURPOSE = "gst_operator_browser_v1"
+_TOKEN_PURPOSE = "gst_lightweight_operator_v2"
-_TOKEN_MINUTES = 30
+_TOKEN_MINUTES = 15
 _UPLOAD_ROOT = Path(tempfile.gettempdir()) / "audit_firm_gst_operator_uploads"
+_OPERATOR_AGENT_VERSION = "1.0.0"
+_OPERATOR_AGENT_PORT = 8791
+_OPERATOR_RUNTIME_ROOT = Path(__file__).resolve().parent / "gst_operator_agent_runtime"
 
 
 def _fy_bounds(fy: str) -> tuple[date, date]:
@@ -410,12 +413,35 @@ def page(request: Request, client_id: int | None = None, registration_id: int |
         return templates.TemplateResponse("modules/accounting/templates/accounting/gst_reconciliation.html",{
             "request":request,"current_user":user,"current_user_roles":get_user_roles(db,user.id),"current_user_permissions":get_user_permissions(db,user.id),"csrf_token":get_or_create_csrf_token(request),
             "clients":clients,"selected_client":selected,"registrations":registrations,"selected_registration":selected_reg,"credentials":credentials,"node":node,"node_online":_node_online(node) if node else False,
-            "period":period,"financial_year":financial_year,"download_mode":download_mode,"operator_job":operator_job,"gst_login_url":GST_LOGIN_URL,"message":message,"error":error,"title":"GST Return Reconciliation",
+            "period":period,"financial_year":financial_year,"download_mode":download_mode,"operator_job":operator_job,"gst_login_url":GST_LOGIN_URL,
+            "operator_agent_version":_OPERATOR_AGENT_VERSION,"operator_agent_port":_OPERATOR_AGENT_PORT,
+            "message":message,"error":error,"title":"GST Return Reconciliation",
         })
     finally:
         db.close()
 
 
+@router.get("/operator-agent/download")
+def download_operator_agent(request: Request):
+    db = CommonSessionLocal()
+    try:
+        user, response = _require_partner(request, db, "accounting.learning.view")
+        if response:
+            return response
+        if not _OPERATOR_RUNTIME_ROOT.is_dir():
+            return JSONResponse({"ok": False, "error": "GST Operator Agent runtime is missing from this ERP build."}, status_code=404)
+        memory = io.BytesIO()
+        with zipfile.ZipFile(memory, "w", zipfile.ZIP_DEFLATED, compresslevel=6) as archive:
+            for path in sorted(_OPERATOR_RUNTIME_ROOT.rglob("*")):
+                if path.is_file() and "__pycache__" not in path.parts:
+                    archive.write(path, Path("ARRR_GST_Operator_Agent") / path.relative_to(_OPERATOR_RUNTIME_ROOT))
+        memory.seek(0)
+        headers = {"Content-Disposition": f'attachment; filename="ARRR_GST_Operator_Agent_{_OPERATOR_AGENT_VERSION}.zip"'}
+        return StreamingResponse(memory, media_type="application/zip", headers=headers)
+    finally:
+        db.close()
+
+
 @router.post("/download/start")
 def start_download(
     request: Request,
@@ -459,7 +485,7 @@ def start_download(
         log_access(db,request,user,cred,"use_for_gst_download",reason=f"GST returns {financial_year}: {','.join(return_types)}",fields="username,secret",success=True)
         db.commit()
         request.session["gst_operator_job"]={"token":token,"job_id":jti,"periods":periods,"return_types":return_types}
-        return _redirect(client_id,registration_id=registration_id,period=period,financial_year=financial_year,download_mode=download_mode,message="GST download prepared. The ERP will now ask the Local Agent on this computer to open the visible GST browser; CAPTCHA/OTP will appear here. Completed files will be transferred to the configured client local storage.")
+        return _redirect(client_id,registration_id=registration_id,period=period,financial_year=financial_year,download_mode=download_mode,message="GST download prepared. The lightweight GST Operator Agent on this computer will open the visible GST browser and autofill the selected Credential Vault login. Complete CAPTCHA/OTP there; downloaded return data will then be transferred to the configured client storage.")
     except Exception as exc:
         db.rollback(); return _redirect(client_id,registration_id=registration_id,period=period,financial_year=financial_year,download_mode=download_mode,error=str(exc))
     finally:
@@ -482,7 +508,7 @@ async def operator_redeem(request: Request):
                 raise ValueError("GST username/password is missing in Credential Vault.")
         finally:
             db.close()
-        return JSONResponse({"ok":True,"payload":{**{k:v for k,v in data.items() if k not in {"iat","exp","purpose"}},"username":username,"password":password,"upload_url":str(get_settings().ERP_PUBLIC_BASE_URL).rstrip("/")+"/tools/accounting/gst-reconciliation/operator/upload","login_timeout_seconds":900}})
+        return JSONResponse({"ok":True,"payload":{**{k:v for k,v in data.items() if k not in {"iat","exp","purpose"}},"username":username,"password":password,"upload_url":str(get_settings().ERP_PUBLIC_BASE_URL).rstrip("/")+"/tools/accounting/gst-reconciliation/operator/upload","login_timeout_seconds":900,"operator_agent_min_version":_OPERATOR_AGENT_VERSION}})
     except Exception as exc:
         return JSONResponse({"ok":False,"error":str(exc)},status_code=400)
 

app/modules/accounting/templates/accounting/gst_reconciliation.html

@@ -2,40 +2,90 @@
 {% block content %}
 <div class="mx-auto max-w-7xl space-y-5 p-4">
   <div class="flex items-center justify-between gap-3">
-    <div><h1 class="text-2xl font-bold">GST Return Reconciliation</h1><p class="text-sm text-slate-600">Open GST Portal directly in your browser, import downloaded return files into the configured client local storage, and reconcile against Accounting Mirror.</p></div>
+    <div>
+      <h1 class="text-2xl font-bold">GST Return Reconciliation</h1>
+      <p class="text-sm text-slate-600">Download GST returns through the lightweight operator agent, store them in the configured client local storage, and reconcile against Accounting Mirror.</p>
+    </div>
     <a href="/tools/tally{% if selected_client %}?client_id={{ selected_client.id }}{% endif %}" class="rounded-lg border px-3 py-2 text-sm">Back to Accounting</a>
   </div>
+
   {% if message %}<div class="rounded-lg border border-emerald-200 bg-emerald-50 p-3 text-emerald-800">{{ message }}</div>{% endif %}
   {% if error %}<div class="rounded-lg border border-red-200 bg-red-50 p-3 text-red-800">{{ error }}</div>{% endif %}
 
   <form method="get" class="grid gap-3 rounded-xl border bg-white p-4 md:grid-cols-5">
-    <label class="text-sm">Client<select name="client_id" class="mt-1 w-full rounded border p-2" onchange="this.form.submit()"><option value="">Select client</option>{% for c in clients %}<option value="{{ c.id }}" {% if selected_client and c.id==selected_client.id %}selected{% endif %}>{{ c.client_name }}</option>{% endfor %}</select></label>
+    <label class="text-sm">Client
-    <label class="text-sm">GSTIN<select name="registration_id" class="mt-1 w-full rounded border p-2" onchange="this.form.submit()"><option value="">Select GSTIN</option>{% for r,t in registrations %}<option value="{{ r.id }}" {% if selected_registration and r.id==selected_registration.id %}selected{% endif %}>{{ r.registration_number }}{% if r.trade_name %} — {{ r.trade_name }}{% endif %}</option>{% endfor %}</select></label>
+      <select name="client_id" class="mt-1 w-full rounded border p-2" onchange="this.form.submit()">
+        <option value="">Select client</option>
+        {% for c in clients %}<option value="{{ c.id }}" {% if selected_client and c.id==selected_client.id %}selected{% endif %}>{{ c.client_name }}</option>{% endfor %}
+      </select>
+    </label>
+    <label class="text-sm">GSTIN
+      <select name="registration_id" class="mt-1 w-full rounded border p-2" onchange="this.form.submit()">
+        <option value="">Select GSTIN</option>
+        {% for r,t in registrations %}<option value="{{ r.id }}" {% if selected_registration and r.id==selected_registration.id %}selected{% endif %}>{{ r.registration_number }}{% if r.trade_name %} — {{ r.trade_name }}{% endif %}</option>{% endfor %}
+      </select>
+    </label>
     <label class="text-sm">Financial Year<input name="financial_year" value="{{ financial_year }}" pattern="[0-9]{4}-[0-9]{2}" placeholder="2025-26" class="mt-1 w-full rounded border p-2"></label>
-    <label class="text-sm">Mode<select name="download_mode" class="mt-1 w-full rounded border p-2"><option value="single" {% if download_mode=='single' %}selected{% endif %}>Single Month</option><option value="full_fy" {% if download_mode=='full_fy' %}selected{% endif %}>Full Financial Year</option></select></label>
+    <label class="text-sm">Mode
+      <select name="download_mode" class="mt-1 w-full rounded border p-2">
+        <option value="single" {% if download_mode=='single' %}selected{% endif %}>Single Month</option>
+        <option value="full_fy" {% if download_mode=='full_fy' %}selected{% endif %}>Full Financial Year</option>
+      </select>
+    </label>
     <div class="flex items-end"><button class="w-full rounded bg-slate-800 px-3 py-2 text-white">Load</button></div>
-    <label class="text-sm md:col-span-2">Single Month Period (MMYYYY)<input name="period" value="{{ period }}" pattern="[0-9]{6}" placeholder="042025" class="mt-1 w-full rounded border p-2"><span class="text-xs text-slate-500">Used only when Mode = Single Month.</span></label>
+    <label class="text-sm md:col-span-2">Single Month Period (MMYYYY)
+      <input name="period" value="{{ period }}" pattern="[0-9]{6}" placeholder="042025" class="mt-1 w-full rounded border p-2">
+      <span class="text-xs text-slate-500">Used only when Mode = Single Month.</span>
+    </label>
   </form>
 
   {% if selected_client and selected_registration %}
   <div class="grid gap-4 lg:grid-cols-2">
-    <div class="rounded-xl border bg-white p-4 space-y-4">
+    <form method="post" action="/tools/accounting/gst-reconciliation/download/start" class="rounded-xl border bg-white p-4 space-y-4">
+      <input type="hidden" name="csrf_token" value="{{ csrf_token }}">
+      <input type="hidden" name="client_id" value="{{ selected_client.id }}">
+      <input type="hidden" name="registration_id" value="{{ selected_registration.id }}">
+      <input type="hidden" name="period" value="{{ period }}">
+      <input type="hidden" name="financial_year" value="{{ financial_year }}">
+      <input type="hidden" name="download_mode" value="{{ download_mode }}">
       <div>
-        <h2 class="font-semibold">1. Download from GST Portal</h2>
+        <h2 class="font-semibold">1. Automatic GST Download</h2>
-        <p class="mt-1 text-sm text-slate-600">Open the official GST Portal directly in this browser. No ERP Local Agent is required on this computer. Complete login, CAPTCHA and OTP normally, then download the required return JSON/ZIP files.</p>
+        <p class="mt-1 text-sm text-slate-600">The lightweight GST Operator Agent runs only on this workstation. It opens visible Chrome/Edge, autofills the selected Credential Vault username/password, and waits for you to complete CAPTCHA/OTP. It does not run Tally or hold the client storage.</p>
       </div>
+
+      <div class="flex flex-wrap items-center gap-2 rounded-lg border bg-slate-50 p-3 text-sm">
+        <span>GST Operator Agent:</span>
+        <span id="gst-agent-status" class="font-semibold text-slate-600">Checking…</span>
+        <span class="text-slate-400">v{{ operator_agent_version }}</span>
+        <a href="/tools/accounting/gst-reconciliation/operator-agent/download" class="ml-auto rounded border bg-white px-3 py-1.5 text-xs font-medium">Download / Install Agent</a>
+      </div>
+
       <label class="text-sm">Credential Vault Entry
-        <select class="mt-1 w-full rounded border p-2" disabled>
+        <select name="credential_id" required class="mt-1 w-full rounded border p-2" {% if not credentials %}disabled{% endif %}>
-          {% if credentials %}{% for c in credentials %}<option {% if credentials|length==1 %}selected{% endif %}>{{ c.title }}{% if c.reference_number %} — {{ c.reference_number }}{% endif %}</option>{% endfor %}{% else %}<option>No GST Portal credential found</option>{% endif %}
+          {% if credentials %}
+            {% for c in credentials %}<option value="{{ c.id }}" {% if credentials|length==1 %}selected{% endif %}>{{ c.title }}{% if c.reference_number %} — {{ c.reference_number }}{% endif %}</option>{% endfor %}
+          {% else %}
+            <option>No GST Portal credential found</option>
+          {% endif %}
         </select>
       </label>
-      <p class="text-xs text-slate-500">Credential Vault remains linked to this GSTIN for reference and future API-based workflows. The browser login is manual because the GST Portal is on a different website and ERP cannot read or autofill that page across browser security boundaries.</p>
+      {% if credentials|length==1 %}<p class="text-xs text-slate-500">The only eligible GST Portal credential has been selected automatically.</p>{% endif %}
-      <a href="{{ gst_login_url }}" target="_blank" rel="noopener noreferrer" class="inline-flex rounded bg-indigo-600 px-4 py-2 text-white">Open GST Portal</a>
+
-      <div class="rounded-lg border bg-slate-50 p-3 text-xs text-slate-600">
+      <div class="rounded-lg border p-3">
-        <b>For Full Financial Year:</b> download the selected GSTR-1, GSTR-2B and GSTR-3B JSON/ZIP files for all required months. You can select multiple files below in one import.
+        <div class="mb-2 text-sm font-medium">Returns to download</div>
+        <div class="grid grid-cols-2 gap-3 text-sm">
+          <label><input type="checkbox" name="gstr1" value="1" checked class="mr-2">GSTR-1</label>
+          <label><input type="checkbox" name="gstr2b" value="1" checked class="mr-2">GSTR-2B</label>
+          <label><input type="checkbox" name="gstr3b" value="1" checked class="mr-2">GSTR-3B</label>
+          <label><input type="checkbox" name="gstr2a" value="1" class="mr-2">GSTR-2A</label>
         </div>
       </div>
 
+      <button {% if not node_online or not credentials %}disabled{% endif %} class="rounded bg-indigo-600 px-4 py-2 text-white disabled:opacity-50">Start GST Download</button>
+      <div class="text-xs">Configured Storage Agent: <b>{{ 'Online' if node_online else 'Offline' }}</b></div>
+      <p class="text-xs text-slate-500">For Full Financial Year, the job processes April through March and reports progress month by month. The password is redeemed directly by the localhost agent and is never placed in this page's HTML or URL.</p>
+    </form>
+
     <form method="post" action="/tools/accounting/gst-reconciliation/import" enctype="multipart/form-data" class="rounded-xl border bg-white p-4 space-y-3">
       <input type="hidden" name="csrf_token" value="{{ csrf_token }}">
       <input type="hidden" name="client_id" value="{{ selected_client.id }}">
@@ -44,23 +94,28 @@
       <input type="hidden" name="financial_year" value="{{ financial_year }}">
       <input type="hidden" name="download_mode" value="{{ download_mode }}">
       <div>
-        <h2 class="font-semibold">2. Import Downloaded Returns to Client Storage</h2>
+        <h2 class="font-semibold">2. Manual Import Fallback</h2>
-        <p class="mt-1 text-sm text-slate-600">Select the JSON or ZIP files downloaded from GST Portal. ERP identifies each return period, normalizes the data, and transfers it to the configured Local Storage Agent under the client's FY/GST/GSTIN/period directory.</p>
+        <p class="mt-1 text-sm text-slate-600">If a GST portal download cannot be automated, keep the workflow moving by importing the portal JSON/ZIP files. They use the same client GST storage hierarchy and the same reconciliation engine.</p>
       </div>
       <label class="block text-sm">GSTR-1 files<input type="file" name="gstr1_files" accept=".json,.zip,application/json,application/zip" multiple class="mt-1 block w-full rounded border p-2"></label>
       <label class="block text-sm">GSTR-2B files<input type="file" name="gstr2b_files" accept=".json,.zip,application/json,application/zip" multiple class="mt-1 block w-full rounded border p-2"></label>
       <label class="block text-sm">GSTR-3B files<input type="file" name="gstr3b_files" accept=".json,.zip,application/json,application/zip" multiple class="mt-1 block w-full rounded border p-2"></label>
       <label class="block text-sm">GSTR-2A files <span class="text-slate-400">(optional)</span><input type="file" name="gstr2a_files" accept=".json,.zip,application/json,application/zip" multiple class="mt-1 block w-full rounded border p-2"></label>
       <button {% if not node_online %}disabled{% endif %} class="rounded bg-sky-700 px-4 py-2 text-white disabled:opacity-50">Import & Store GST Returns</button>
-      <div class="text-xs">Configured Storage Agent: <b>{{ 'Online' if node_online else 'Offline' }}</b></div>
-      <div class="text-xs text-slate-500">No Local Agent is required on the computer where this browser is open.</div>
     </form>
   </div>
 
+  <div id="gst-job-panel" class="hidden rounded-xl border bg-white p-4 space-y-2">
+    <div class="flex items-center justify-between"><h2 class="font-semibold">Interactive GST Download</h2><span id="gst-job-percent" class="text-sm font-medium">0%</span></div>
+    <div class="h-2 overflow-hidden rounded bg-slate-200"><div id="gst-job-bar" class="h-full bg-indigo-600" style="width:0%"></div></div>
+    <div id="gst-job-stage" class="text-sm font-medium"></div>
+    <div id="gst-job-message" class="text-sm text-slate-600"></div>
+  </div>
+
   <form method="post" action="/tools/accounting/gst-reconciliation/analyze" class="rounded-xl border bg-white p-4 space-y-3">
     <input type="hidden" name="csrf_token" value="{{ csrf_token }}"><input type="hidden" name="client_id" value="{{ selected_client.id }}"><input type="hidden" name="registration_id" value="{{ selected_registration.id }}"><input type="hidden" name="period" value="{{ period }}"><input type="hidden" name="financial_year" value="{{ financial_year }}">
     <h2 class="font-semibold">3. Reconcile Stored Data</h2>
-    <p class="text-sm text-slate-600">Sales: Accounting Mirror vs GSTR-1. Purchases: Accounting Mirror vs GSTR-2B. ITC: GSTR-2B vs GSTR-3B. Analysis reads only return data already stored in the client's local-storage GST directory.</p>
+    <p class="text-sm text-slate-600">Sales: Accounting Mirror vs GSTR-1. Purchases: Accounting Mirror vs GSTR-2B. ITC: GSTR-2B vs GSTR-3B. Analysis reads only stored client data, so download and reconciliation remain independent.</p>
     <div class="flex flex-wrap items-end gap-3">
       <label class="text-sm min-w-64">Analysis scope<select name="analyze_mode" class="mt-1 w-full rounded border p-2"><option value="single">Single Month</option><option value="full_fy" {% if download_mode=='full_fy' %}selected{% endif %}>Full Financial Year</option></select></label>
       <button {% if not node_online %}disabled{% endif %} class="rounded bg-emerald-600 px-4 py-2 text-white disabled:opacity-50">Run Reconciliation</button>
@@ -78,5 +133,80 @@
   {% endif %}
 </div>
 
+<script>
+(() => {
+  const port = {{ operator_agent_port|int }};
+  const base = `http://127.0.0.1:${port}`;
+  const statusEl = document.getElementById('gst-agent-status');
+  const panel = document.getElementById('gst-job-panel');
+  const bar = document.getElementById('gst-job-bar');
+  const pct = document.getElementById('gst-job-percent');
+  const stage = document.getElementById('gst-job-stage');
+  const msg = document.getElementById('gst-job-message');
 
+  async function agentStatus() {
+    if (!statusEl) return false;
+    try {
+      const r = await fetch(base + '/api/status', {cache:'no-store'});
+      const b = await r.json();
+      if (!r.ok || !b.ok) throw new Error(b.error || 'Agent unavailable');
+      statusEl.textContent = `Online · v${b.version}`;
+      statusEl.className = 'font-semibold text-emerald-700';
+      return true;
+    } catch (e) {
+      statusEl.textContent = 'Not installed / not running';
+      statusEl.className = 'font-semibold text-red-600';
+      return false;
+    }
+  }
+
+  function renderJob(job) {
+    panel?.classList.remove('hidden');
+    const total = Number(job.period_total || (job.periods || []).length || 1);
+    const index = Number(job.period_index || 0);
+    let percent = Number(job.percent || 0);
+    if (!percent && total) percent = Math.min(95, Math.round((index / total) * 100));
+    if (job.status === 'completed') percent = 100;
+    if (bar) bar.style.width = `${percent}%`;
+    if (pct) pct.textContent = `${percent}%`;
+    if (stage) stage.textContent = job.stage || job.status || '';
+    if (msg) msg.textContent = job.message || job.error || '';
+  }
+
+  async function poll(jobId) {
+    try {
+      const r = await fetch(base + '/api/gst/status?job_id=' + encodeURIComponent(jobId), {cache:'no-store'});
+      const b = await r.json();
+      if (b.job) renderJob(b.job);
+      if (b.job && !['completed','failed'].includes(b.job.status)) setTimeout(() => poll(jobId), 1800);
+    } catch (e) {
+      if (msg) msg.textContent = 'Could not read GST Operator Agent progress: ' + e.message;
+    }
+  }
+
+  agentStatus();
+
+  {% if operator_job %}
+  const prepared = {{ operator_job|tojson }};
+  (async () => {
+    panel?.classList.remove('hidden');
+    if (stage) stage.textContent = 'Connecting to GST Operator Agent';
+    if (msg) msg.textContent = 'Starting visible GST login on this workstation…';
+    try {
+      const online = await agentStatus();
+      if (!online) throw new Error('Install/start the GST Operator Agent using the button above, then retry Start GST Download.');
+      const r = await fetch(base + '/api/gst/start', {
+        method:'POST', headers:{'Content-Type':'application/json'}, body:JSON.stringify({token:prepared.token})
+      });
+      const b = await r.json();
+      if (!r.ok || !b.ok) throw new Error(b.error || 'Agent could not start GST browser.');
+      renderJob(b.job || {status:'queued',stage:'Queued',message:'GST browser job queued.'});
+      poll(prepared.job_id);
+    } catch (e) {
+      renderJob({status:'failed',percent:100,stage:'Could not start GST browser',message:e.message});
+    }
+  })();
+  {% endif %}
+})();
+</script>
 {% endblock %}