From 782feaeeea53bc0392ee70ab42608f111db626d4 Mon Sep 17 00:00:00 2001 From: A R R R Associates Date: Sat, 12 Sep 2026 09:57:32 +0530 Subject: [PATCH] Verify GST return downloads before completion --- .../gst_operator_agent_runtime/README.txt | 20 +- .../gst_operator_agent.py | 277 +++++++++++++----- .../install_gst_operator_agent.ps1 | 2 +- .../accounting/gst_reconciliation_ui.py | 2 +- 4 files changed, 226 insertions(+), 75 deletions(-) diff --git a/app/modules/accounting/gst_operator_agent_runtime/README.txt b/app/modules/accounting/gst_operator_agent_runtime/README.txt index 6264f2f..2760417 100644 --- a/app/modules/accounting/gst_operator_agent_runtime/README.txt +++ b/app/modules/accounting/gst_operator_agent_runtime/README.txt @@ -1,4 +1,4 @@ -ARRR GST Operator Agent 1.2.0 +ARRR GST Operator Agent 1.4.0 Purpose ------- @@ -23,3 +23,21 @@ The custom URL contains only a short-lived signed job token. GST username/passwo Uninstall --------- Run uninstall_gst_operator_agent.ps1. It removes the protocol registration and also cleans legacy v1.0.x localhost listener/startup/certificate state. + + +V1.4.0 VERIFIED POST-DASHBOARD DOWNLOAD SEQUENCE +- After CAPTCHA/OTP login, do not deep-link to return pages. +- Close optional Aadhaar/E-KYC reminder. +- From the authenticated GST Welcome page click Return Dashboard exactly as a user would. +- Wait for return.gst.gov.in /returns/auth/ session to load. +- Only then call GSTR-1/GSTR-2B/GSTR-3B APIs using the authenticated browser context. +- Manual JSON/ZIP import remains available in ERP as fallback. + + +V1.4.0 POST-DASHBOARD REPAIR +- Does not mark a job complete merely because Return Dashboard opened. +- Requires an actual saved JSON result for every selected return and period. +- GSTR-2B now enters its module only after the natural Return Dashboard session is established, then calls the GSTR-2B API same-origin. +- Does not call GST logout after completion. +- On failure the browser remains visible for 45 seconds with the exact failure message. +- Upload to ERP/client storage happens only after all expected return files are present. diff --git a/app/modules/accounting/gst_operator_agent_runtime/gst_operator_agent.py b/app/modules/accounting/gst_operator_agent_runtime/gst_operator_agent.py index be94117..f28e8ed 100644 --- a/app/modules/accounting/gst_operator_agent_runtime/gst_operator_agent.py +++ b/app/modules/accounting/gst_operator_agent_runtime/gst_operator_agent.py @@ -13,7 +13,7 @@ from urllib.parse import parse_qs, urlsplit import requests -VERSION = "1.3.0" +VERSION = "1.4.0" ROOT = Path(__file__).resolve().parent DATA = ROOT / "data" CONFIG_PATH = ROOT / "config.json" @@ -450,36 +450,120 @@ def prepare_portal_context(page, return_types: list[str], progress) -> None: verify_return_dashboard_session(page) +def validate_gst_json(text: str, label: str, *, allow_empty: bool = False) -> object: + """Validate that GST returned JSON rather than an empty/session shell response.""" + raw = (text or "").strip() + if not raw: + raise RuntimeError(f"{label}: GST returned an empty response.") + try: + data = json.loads(raw) + except Exception as exc: + raise RuntimeError(f"{label}: GST response was not valid JSON: {raw[:300]}") from exc + if not allow_empty and data in ({}, [], None, ""): + raise RuntimeError(f"{label}: GST returned empty JSON. The module/session is not ready.") + return data + + +def return_to_return_dashboard(page, progress) -> None: + """Return to the already-established GST Return Dashboard without logging out.""" + if "return.gst.gov.in" in (page.url or "").lower() and "/returns/auth/dashboard" in (page.url or "").lower(): + verify_return_dashboard_session(page) + return + progress(stage="Returning to Return Dashboard", message="Returning to GST Return Dashboard before the next return download.") + for _ in range(15): + try: + page.go_back(wait_until="domcontentloaded", timeout=20000) + except Exception as exc: + _log(f"Return Dashboard Back warning: {exc}") + page.wait_for_timeout(1000) + url = (page.url or "").lower() + if "return.gst.gov.in" in url and "/returns/auth/dashboard" in url and not is_access_denied_page(page): + show_status_overlay(page, "GST Return Dashboard ready for the next return.", "#15803d") + return + if is_logged_in_services_page(page): + open_return_dashboard_naturally(page, progress) + return + raise RuntimeError(f"Could not return to GST Return Dashboard. Current page: {page.url}") + + +def open_gstr2b_module_from_established_session(page, progress) -> None: + """Enter GSTR-2B only after Return Dashboard has created the return-domain session. + + The failed v9 path called the GSTR-2B API cross-domain directly from the Return + Dashboard. This version first establishes the normal Return Dashboard session, + then opens the GSTR-2B module page and performs the JSON call same-origin from + that page, matching the browser-session pattern used by the working GST tools. + """ + verify_return_dashboard_session(page) + progress(stage="Opening GSTR-2B Module", message="Return Dashboard is ready. Opening the GST GSTR-2B module before requesting JSON.") + show_status_overlay(page, "Return Dashboard ready. Opening GSTR-2B module...", "#0f766e") + try: + page.goto(GSTR2B_PAGE_URL, wait_until="domcontentloaded", timeout=90000) + except Exception as exc: + raise RuntimeError(f"Could not enter the GSTR-2B module from the established Return Dashboard session: {exc}") from exc + page.wait_for_timeout(2500) + if is_access_denied_page(page): + raise RuntimeError( + "GST returned Access Denied while transferring from Return Dashboard to GSTR-2B. " + "The browser has been kept open temporarily for inspection." + ) + url = (page.url or "").lower() + if "gstr2b.gst.gov.in" not in url: + raise RuntimeError(f"GST did not enter the GSTR-2B module. Current page: {page.url}") + show_status_overlay(page, "GSTR-2B module ready. Downloading JSON...", "#15803d") + + +def keep_browser_visible(page, message: str, color: str, milliseconds: int) -> None: + """Keep the visible GST browser available long enough to see success/failure.""" + try: + page.bring_to_front() + show_status_overlay(page, message, color) + page.wait_for_timeout(milliseconds) + except Exception: + pass + + def download_period(page, work_root: Path, period: str, return_types: list[str], progress) -> list[dict]: raw_dir = work_root / period / "raw" raw_dir.mkdir(parents=True, exist_ok=True) selected = {str(x or "").upper() for x in return_types} downloaded: list[dict] = [] + # GSTR-1, GSTR-3B and GSTR-2A use the return.gst.gov.in session. + # Always confirm/restore Return Dashboard before each of those calls. if "GSTR1" in selected: + return_to_return_dashboard(page, progress) progress(stage=f"Downloading GSTR-1 {period}", message=f"Generating and downloading GSTR-1 for {period}.") - verify_return_dashboard_session(page) generated = page_fetch_text(page, GSTR1_URL.format(period=period), referer=RETURN_DASHBOARD_URL) + validate_gst_json(generated, f"GSTR-1 generate {period}") (raw_dir / f"{period}_GSTR1_GENERATE.json").write_text(generated, encoding="utf-8") file_num = extract_file_num(generated) content = extract_payload(page_fetch_text(page, GSTR1_DOWNLOAD_URL.format(period=period, file_num=file_num))) + validate_gst_json(content, f"GSTR-1 download {period}") path = raw_dir / f"{period}_GSTR1.json" path.write_text(content, encoding="utf-8") downloaded.append({"return_type": "GSTR1", "path": str(path.relative_to(work_root)), "bytes": path.stat().st_size}) if "GSTR2B" in selected: - progress(stage=f"Downloading GSTR-2B {period}", message=f"Downloading GSTR-2B JSON for {period}.") - verify_return_dashboard_session(page) - content = page_fetch_text(page, GSTR2B_URL.format(period=period), referer="https://gstr2b.gst.gov.in") + # Important: do not call GSTR-2B cross-domain directly from Return Dashboard. + # Establish Return Dashboard first, enter the GSTR-2B module, then call its API + # from the GSTR-2B page so the module/session cookies and origin are correct. + return_to_return_dashboard(page, progress) + open_gstr2b_module_from_established_session(page, progress) + progress(stage=f"Downloading GSTR-2B {period}", message=f"Downloading GSTR-2B JSON for {period} from the established GSTR-2B session.") + content = page_fetch_text(page, GSTR2B_URL.format(period=period), referer=GSTR2B_PAGE_URL) + validate_gst_json(content, f"GSTR-2B {period}") path = raw_dir / f"{period}_GSTR2B.json" path.write_text(content, encoding="utf-8") downloaded.append({"return_type": "GSTR2B", "path": str(path.relative_to(work_root)), "bytes": path.stat().st_size}) if "GSTR3B" in selected: + return_to_return_dashboard(page, progress) progress(stage=f"Downloading GSTR-3B {period}", message=f"Downloading GSTR-3B summary and tax payable for {period}.") - verify_return_dashboard_session(page) summary = page_fetch_text(page, GSTR3B_SUMMARY_URL.format(period=period), referer=RETURN_DASHBOARD_URL) payable = page_fetch_text(page, GSTR3B_URL.format(period=period), referer=RETURN_DASHBOARD_URL) + validate_gst_json(summary, f"GSTR-3B summary {period}", allow_empty=True) + validate_gst_json(payable, f"GSTR-3B tax payable {period}", allow_empty=True) (raw_dir / f"{period}_GSTR3B_SUMMARY.json").write_text(summary, encoding="utf-8") (raw_dir / f"{period}_GSTR3B_TAXPAYBLE.json").write_text(payable, encoding="utf-8") try: @@ -497,24 +581,34 @@ def download_period(page, work_root: Path, period: str, return_types: list[str], downloaded.append({"return_type": "GSTR3B", "path": str(path.relative_to(work_root)), "bytes": path.stat().st_size}) if "GSTR2A" in selected: + return_to_return_dashboard(page, progress) progress(stage=f"Requesting GSTR-2A {period}", message=f"Requesting GSTR-2A offline return for {period}.") - verify_return_dashboard_session(page) content = page_fetch_text(page, GSTR2A_URL.format(period=period), referer=RETURN_DASHBOARD_URL) + validate_gst_json(content, f"GSTR-2A request {period}") path = raw_dir / f"{period}_GSTR2A.json" path.write_text(content, encoding="utf-8") downloaded.append({"return_type": "GSTR2A", "path": str(path.relative_to(work_root)), "bytes": path.stat().st_size}) + expected = len(selected) + if len(downloaded) != expected: + got = ", ".join(x.get("return_type", "?") for x in downloaded) or "none" + raise RuntimeError( + f"GST period {period} did not complete every selected return. Expected {expected}, downloaded {len(downloaded)} ({got})." + ) + write_json( work_root / period / "download_manifest.json", - {"period": period, "downloaded_at_utc": now(), "source": "arrr_gst_operator_agent_1.3.0", "downloaded": downloaded}, + {"period": period, "downloaded_at_utc": now(), "source": "arrr_gst_operator_agent_1.4.0", "downloaded": downloaded}, ) return downloaded - def browser_worker(payload: dict, token: str, path: Path) -> None: job_id = str(payload.get("jti") or "") work_root = DATA / "work" / safe(job_id) package_path = DATA / f"gst_{safe(job_id)}.zip" + pw = None + context = None + page = None def progress(**updates): current = read_json(path) @@ -528,68 +622,76 @@ def browser_worker(payload: dict, token: str, path: Path) -> None: progress(status="running", percent=5, stage="Opening GST Login", message="Opening GST Portal in visible Chrome/Edge.") from playwright.sync_api import sync_playwright - with sync_playwright() as pw: - context = None - errors = [] - profile_root = DATA / "browser_profile" - profile_root.mkdir(parents=True, exist_ok=True) - for channel in ("chrome", "msedge"): - try: - context = pw.chromium.launch_persistent_context( - user_data_dir=str(profile_root / f"{channel}_{safe(job_id)}"), - channel=channel, - headless=False, - no_viewport=True, - accept_downloads=True, - args=["--start-maximized", "--no-first-run", "--disable-blink-features=AutomationControlled"], - timeout=45000, - ) - break - except Exception as exc: - errors.append(f"{channel}: {exc}") - if context is None: - raise RuntimeError("Could not open installed Chrome or Edge. " + " | ".join(errors)) - - page = context.pages[0] if context.pages else context.new_page() - open_visible_page(page, GST_LOGIN_URL, "GST Login", 1000) - page.locator("#username").wait_for(state="visible", timeout=30000) - page.fill("#username", str(payload.get("username") or "")) - password_filled = False - for selector in ("#user_pass", "input[type=password]"): - try: - locator = page.locator(selector).first - if locator.count(): - locator.fill(str(payload.get("password") or "")) - password_filled = True - break - except Exception: - pass - if not password_filled: - raise RuntimeError("GST password field could not be located. The GST login page may have changed.") - - wait_for_login(page, progress, int(payload.get("login_timeout_seconds") or 900)) - periods = [str(p) for p in payload.get("periods") or []] - return_types = [str(r) for r in payload.get("return_types") or []] - prepare_portal_context(page, return_types, progress) - all_downloaded = [] - for index, period in enumerate(periods, 1): - pct = 15 + int((index - 1) * 70 / max(1, len(periods))) - progress( - percent=pct, - period=period, - period_index=index, - period_total=len(periods), - stage=f"Period {index}/{len(periods)}", - message=f"Downloading selected GST returns for {period}.", + pw = sync_playwright().start() + errors = [] + profile_root = DATA / "browser_profile" + profile_root.mkdir(parents=True, exist_ok=True) + for channel in ("chrome", "msedge"): + try: + context = pw.chromium.launch_persistent_context( + user_data_dir=str(profile_root / f"{channel}_{safe(job_id)}"), + channel=channel, + headless=False, + no_viewport=True, + accept_downloads=True, + args=["--start-maximized", "--no-first-run", "--disable-blink-features=AutomationControlled"], + timeout=45000, ) - all_downloaded.extend(download_period(page, work_root, period, return_types, progress)) + break + except Exception as exc: + errors.append(f"{channel}: {exc}") + if context is None: + raise RuntimeError("Could not open installed Chrome or Edge. " + " | ".join(errors)) - show_status_overlay( - page, - "Selected GST return downloads are complete. Transferring them to ARRR client storage.", - "#065f46", + page = context.pages[0] if context.pages else context.new_page() + open_visible_page(page, GST_LOGIN_URL, "GST Login", 1000) + page.locator("#username").wait_for(state="visible", timeout=30000) + page.fill("#username", str(payload.get("username") or "")) + password_filled = False + for selector in ("#user_pass", "input[type=password]"): + try: + locator = page.locator(selector).first + if locator.count(): + locator.fill(str(payload.get("password") or "")) + password_filled = True + break + except Exception: + pass + if not password_filled: + raise RuntimeError("GST password field could not be located. The GST login page may have changed.") + + wait_for_login(page, progress, int(payload.get("login_timeout_seconds") or 900)) + periods = [str(p) for p in payload.get("periods") or []] + return_types = [str(r) for r in payload.get("return_types") or []] + if not periods: + raise RuntimeError("GST job does not contain any return period.") + if not return_types: + raise RuntimeError("GST job does not contain any selected return type.") + + prepare_portal_context(page, return_types, progress) + all_downloaded = [] + for index, period in enumerate(periods, 1): + pct = 15 + int((index - 1) * 70 / max(1, len(periods))) + progress( + percent=pct, + period=period, + period_index=index, + period_total=len(periods), + stage=f"Period {index}/{len(periods)}", + message=f"Downloading selected GST returns for {period}.", + ) + period_downloaded = download_period(page, work_root, period, return_types, progress) + all_downloaded.extend(period_downloaded) + _log(f"GST period {period} downloaded files: {period_downloaded}") + + expected_total = len(periods) * len({str(r or "").upper() for r in return_types}) + if not all_downloaded: + raise RuntimeError("No GST return file was downloaded. The job will not be marked completed.") + if len(all_downloaded) != expected_total: + raise RuntimeError( + f"GST download is incomplete. Expected {expected_total} return files, received {len(all_downloaded)}. " + "The job will not be marked completed." ) - context.close() write_json( work_root / "job_manifest.json", @@ -601,6 +703,7 @@ def browser_worker(payload: dict, token: str, path: Path) -> None: "return_types": payload.get("return_types") or [], "downloaded": all_downloaded, "completed_at_utc": now(), + "agent_version": VERSION, }, ) if package_path.exists(): @@ -613,8 +716,15 @@ def browser_worker(payload: dict, token: str, path: Path) -> None: progress( percent=90, stage="Transferring to Client Storage", - message="Uploading GST return package to ERP for transfer to configured Storage Agent.", + message="GST JSON download completed. Uploading the verified package to ERP/client storage. GST remains logged in.", ) + keep_browser_visible( + page, + "GST JSON download completed. Transferring verified files to ARRR ERP/client storage. GST remains logged in.", + "#065f46", + 1500, + ) + upload_url = str(payload.get("upload_url") or "").strip() if not upload_url.startswith("https://"): raise RuntimeError(f"ERP upload URL is invalid: {upload_url or '(empty)'}") @@ -631,14 +741,21 @@ def browser_worker(payload: dict, token: str, path: Path) -> None: body = {"ok": False, "error": response.text[:1000]} if not response.ok or not body.get("ok"): raise RuntimeError(body.get("error") or f"ERP storage transfer failed with HTTP {response.status_code}.") + progress( status="completed", percent=100, stage="Completed", - message="GST returns downloaded and stored in configured client local storage.", - result={"stored": body.get("stored") or {}}, + message="GST returns downloaded and stored in configured client local storage. GST portal was not logged out automatically.", + result={"stored": body.get("stored") or {}, "downloaded": all_downloaded}, finished_at_utc=now(), ) + keep_browser_visible( + page, + "ARRR GST download completed successfully and client storage confirmed. No automatic GST logout was performed. This browser will close in 12 seconds.", + "#15803d", + 12000, + ) except Exception as exc: current = read_json(path) current.update( @@ -654,14 +771,30 @@ def browser_worker(payload: dict, token: str, path: Path) -> None: ) write_json(path, current) _log(traceback.format_exc()) + if page is not None: + keep_browser_visible( + page, + "GST download FAILED: " + str(exc)[:650] + "\n\nThe browser is being kept open for 45 seconds so the failed GST page can be inspected. No automatic logout was performed.", + "#b91c1c", + 45000, + ) finally: + try: + if context is not None: + context.close() + except Exception: + pass + try: + if pw is not None: + pw.stop() + except Exception: + pass shutil.rmtree(work_root, ignore_errors=True) try: package_path.unlink(missing_ok=True) except Exception: pass - def redeem_job(token: str) -> dict: cfg = read_config() erp = str(cfg.get("erp_base_url") or "").rstrip("/") diff --git a/app/modules/accounting/gst_operator_agent_runtime/install_gst_operator_agent.ps1 b/app/modules/accounting/gst_operator_agent_runtime/install_gst_operator_agent.ps1 index 4d1b861..ae00751 100644 --- a/app/modules/accounting/gst_operator_agent_runtime/install_gst_operator_agent.ps1 +++ b/app/modules/accounting/gst_operator_agent_runtime/install_gst_operator_agent.ps1 @@ -9,7 +9,7 @@ $Startup = Join-Path $env:APPDATA "Microsoft\Windows\Start Menu\Programs\Startup $ThumbprintFile = Join-Path $InstallRoot "cert_thumbprint.txt" $ProtocolKey = "HKCU:\Software\Classes\arrrgst" -Write-Host "ARRR GST Operator Agent 1.2.0 - clean installation" -ForegroundColor Cyan +Write-Host "ARRR GST Operator Agent 1.4.0 - clean installation" -ForegroundColor Cyan Write-Host "Install root: $InstallRoot" # Clean legacy v1.0.x listener/startup/certificate/protocol state first. diff --git a/app/modules/accounting/gst_reconciliation_ui.py b/app/modules/accounting/gst_reconciliation_ui.py index 6c22b2f..bb4f453 100644 --- a/app/modules/accounting/gst_reconciliation_ui.py +++ b/app/modules/accounting/gst_reconciliation_ui.py @@ -147,7 +147,7 @@ router = APIRouter(prefix="/tools/accounting/gst-reconciliation", tags=["account _TOKEN_PURPOSE = "gst_lightweight_operator_v3" _TOKEN_MINUTES = 15 _UPLOAD_ROOT = Path(tempfile.gettempdir()) / "audit_firm_gst_operator_uploads" -_OPERATOR_AGENT_VERSION = "1.3.0" +_OPERATOR_AGENT_VERSION = "1.4.0" _OPERATOR_RUNTIME_ROOT = Path(__file__).resolve().parent / "gst_operator_agent_runtime" _OPERATOR_PACKAGE_FILES = ("gst_operator_agent.py", "requirements.txt", "README.txt", "install_gst_operator_agent.ps1", "uninstall_gst_operator_agent.ps1")