Verify GST return downloads before completion

2026-09-12 09:57:32 +05:30
parent fdc7580b50
commit 782feaeeea
4 changed files with 226 additions and 75 deletions
@@ -1,4 +1,4 @@
-ARRR GST Operator Agent 1.2.0
+ARRR GST Operator Agent 1.4.0

 Purpose
 -------
@@ -23,3 +23,21 @@ The custom URL contains only a short-lived signed job token. GST username/passwo
 Uninstall
 ---------
 Run uninstall_gst_operator_agent.ps1. It removes the protocol registration and also cleans legacy v1.0.x localhost listener/startup/certificate state.
+
+
+V1.4.0 VERIFIED POST-DASHBOARD DOWNLOAD SEQUENCE
+- After CAPTCHA/OTP login, do not deep-link to return pages.
+- Close optional Aadhaar/E-KYC reminder.
+- From the authenticated GST Welcome page click Return Dashboard exactly as a user would.
+- Wait for return.gst.gov.in /returns/auth/ session to load.
+- Only then call GSTR-1/GSTR-2B/GSTR-3B APIs using the authenticated browser context.
+- Manual JSON/ZIP import remains available in ERP as fallback.
+
+
+V1.4.0 POST-DASHBOARD REPAIR
+- Does not mark a job complete merely because Return Dashboard opened.
+- Requires an actual saved JSON result for every selected return and period.
+- GSTR-2B now enters its module only after the natural Return Dashboard session is established, then calls the GSTR-2B API same-origin.
+- Does not call GST logout after completion.
+- On failure the browser remains visible for 45 seconds with the exact failure message.
+- Upload to ERP/client storage happens only after all expected return files are present.
@@ -13,7 +13,7 @@ from urllib.parse import parse_qs, urlsplit

 import requests

-VERSION = "1.3.0"
+VERSION = "1.4.0"
 ROOT = Path(__file__).resolve().parent
 DATA = ROOT / "data"
 CONFIG_PATH = ROOT / "config.json"
@@ -450,36 +450,120 @@ def prepare_portal_context(page, return_types: list[str], progress) -> None:
    verify_return_dashboard_session(page)


+def validate_gst_json(text: str, label: str, *, allow_empty: bool = False) -> object:
+    """Validate that GST returned JSON rather than an empty/session shell response."""
+    raw = (text or "").strip()
+    if not raw:
+        raise RuntimeError(f"{label}: GST returned an empty response.")
+    try:
+        data = json.loads(raw)
+    except Exception as exc:
+        raise RuntimeError(f"{label}: GST response was not valid JSON: {raw[:300]}") from exc
+    if not allow_empty and data in ({}, [], None, ""):
+        raise RuntimeError(f"{label}: GST returned empty JSON. The module/session is not ready.")
+    return data
+
+
+def return_to_return_dashboard(page, progress) -> None:
+    """Return to the already-established GST Return Dashboard without logging out."""
+    if "return.gst.gov.in" in (page.url or "").lower() and "/returns/auth/dashboard" in (page.url or "").lower():
+        verify_return_dashboard_session(page)
+        return
+    progress(stage="Returning to Return Dashboard", message="Returning to GST Return Dashboard before the next return download.")
+    for _ in range(15):
+        try:
+            page.go_back(wait_until="domcontentloaded", timeout=20000)
+        except Exception as exc:
+            _log(f"Return Dashboard Back warning: {exc}")
+        page.wait_for_timeout(1000)
+        url = (page.url or "").lower()
+        if "return.gst.gov.in" in url and "/returns/auth/dashboard" in url and not is_access_denied_page(page):
+            show_status_overlay(page, "GST Return Dashboard ready for the next return.", "#15803d")
+            return
+        if is_logged_in_services_page(page):
+            open_return_dashboard_naturally(page, progress)
+            return
+    raise RuntimeError(f"Could not return to GST Return Dashboard. Current page: {page.url}")
+
+
+def open_gstr2b_module_from_established_session(page, progress) -> None:
+    """Enter GSTR-2B only after Return Dashboard has created the return-domain session.
+
+    The failed v9 path called the GSTR-2B API cross-domain directly from the Return
+    Dashboard. This version first establishes the normal Return Dashboard session,
+    then opens the GSTR-2B module page and performs the JSON call same-origin from
+    that page, matching the browser-session pattern used by the working GST tools.
+    """
+    verify_return_dashboard_session(page)
+    progress(stage="Opening GSTR-2B Module", message="Return Dashboard is ready. Opening the GST GSTR-2B module before requesting JSON.")
+    show_status_overlay(page, "Return Dashboard ready. Opening GSTR-2B module...", "#0f766e")
+    try:
+        page.goto(GSTR2B_PAGE_URL, wait_until="domcontentloaded", timeout=90000)
+    except Exception as exc:
+        raise RuntimeError(f"Could not enter the GSTR-2B module from the established Return Dashboard session: {exc}") from exc
+    page.wait_for_timeout(2500)
+    if is_access_denied_page(page):
+        raise RuntimeError(
+            "GST returned Access Denied while transferring from Return Dashboard to GSTR-2B. "
+            "The browser has been kept open temporarily for inspection."
+        )
+    url = (page.url or "").lower()
+    if "gstr2b.gst.gov.in" not in url:
+        raise RuntimeError(f"GST did not enter the GSTR-2B module. Current page: {page.url}")
+    show_status_overlay(page, "GSTR-2B module ready. Downloading JSON...", "#15803d")
+
+
+def keep_browser_visible(page, message: str, color: str, milliseconds: int) -> None:
+    """Keep the visible GST browser available long enough to see success/failure."""
+    try:
+        page.bring_to_front()
+        show_status_overlay(page, message, color)
+        page.wait_for_timeout(milliseconds)
+    except Exception:
+        pass
+
+
 def download_period(page, work_root: Path, period: str, return_types: list[str], progress) -> list[dict]:
    raw_dir = work_root / period / "raw"
    raw_dir.mkdir(parents=True, exist_ok=True)
    selected = {str(x or "").upper() for x in return_types}
    downloaded: list[dict] = []

+    # GSTR-1, GSTR-3B and GSTR-2A use the return.gst.gov.in session.
+    # Always confirm/restore Return Dashboard before each of those calls.
    if "GSTR1" in selected:
+        return_to_return_dashboard(page, progress)
        progress(stage=f"Downloading GSTR-1 {period}", message=f"Generating and downloading GSTR-1 for {period}.")
-        verify_return_dashboard_session(page)
        generated = page_fetch_text(page, GSTR1_URL.format(period=period), referer=RETURN_DASHBOARD_URL)
+        validate_gst_json(generated, f"GSTR-1 generate {period}")
        (raw_dir / f"{period}_GSTR1_GENERATE.json").write_text(generated, encoding="utf-8")
        file_num = extract_file_num(generated)
        content = extract_payload(page_fetch_text(page, GSTR1_DOWNLOAD_URL.format(period=period, file_num=file_num)))
+        validate_gst_json(content, f"GSTR-1 download {period}")
        path = raw_dir / f"{period}_GSTR1.json"
        path.write_text(content, encoding="utf-8")
        downloaded.append({"return_type": "GSTR1", "path": str(path.relative_to(work_root)), "bytes": path.stat().st_size})

    if "GSTR2B" in selected:
-        progress(stage=f"Downloading GSTR-2B {period}", message=f"Downloading GSTR-2B JSON for {period}.")
-        verify_return_dashboard_session(page)
-        content = page_fetch_text(page, GSTR2B_URL.format(period=period), referer="https://gstr2b.gst.gov.in")
+        # Important: do not call GSTR-2B cross-domain directly from Return Dashboard.
+        # Establish Return Dashboard first, enter the GSTR-2B module, then call its API
+        # from the GSTR-2B page so the module/session cookies and origin are correct.
+        return_to_return_dashboard(page, progress)
+        open_gstr2b_module_from_established_session(page, progress)
+        progress(stage=f"Downloading GSTR-2B {period}", message=f"Downloading GSTR-2B JSON for {period} from the established GSTR-2B session.")
+        content = page_fetch_text(page, GSTR2B_URL.format(period=period), referer=GSTR2B_PAGE_URL)
+        validate_gst_json(content, f"GSTR-2B {period}")
        path = raw_dir / f"{period}_GSTR2B.json"
        path.write_text(content, encoding="utf-8")
        downloaded.append({"return_type": "GSTR2B", "path": str(path.relative_to(work_root)), "bytes": path.stat().st_size})

    if "GSTR3B" in selected:
+        return_to_return_dashboard(page, progress)
        progress(stage=f"Downloading GSTR-3B {period}", message=f"Downloading GSTR-3B summary and tax payable for {period}.")
-        verify_return_dashboard_session(page)
        summary = page_fetch_text(page, GSTR3B_SUMMARY_URL.format(period=period), referer=RETURN_DASHBOARD_URL)
        payable = page_fetch_text(page, GSTR3B_URL.format(period=period), referer=RETURN_DASHBOARD_URL)
+        validate_gst_json(summary, f"GSTR-3B summary {period}", allow_empty=True)
+        validate_gst_json(payable, f"GSTR-3B tax payable {period}", allow_empty=True)
        (raw_dir / f"{period}_GSTR3B_SUMMARY.json").write_text(summary, encoding="utf-8")
        (raw_dir / f"{period}_GSTR3B_TAXPAYBLE.json").write_text(payable, encoding="utf-8")
        try:
@@ -497,24 +581,34 @@ def download_period(page, work_root: Path, period: str, return_types: list[str],
        downloaded.append({"return_type": "GSTR3B", "path": str(path.relative_to(work_root)), "bytes": path.stat().st_size})

    if "GSTR2A" in selected:
+        return_to_return_dashboard(page, progress)
        progress(stage=f"Requesting GSTR-2A {period}", message=f"Requesting GSTR-2A offline return for {period}.")
-        verify_return_dashboard_session(page)
        content = page_fetch_text(page, GSTR2A_URL.format(period=period), referer=RETURN_DASHBOARD_URL)
+        validate_gst_json(content, f"GSTR-2A request {period}")
        path = raw_dir / f"{period}_GSTR2A.json"
        path.write_text(content, encoding="utf-8")
        downloaded.append({"return_type": "GSTR2A", "path": str(path.relative_to(work_root)), "bytes": path.stat().st_size})

+    expected = len(selected)
+    if len(downloaded) != expected:
+        got = ", ".join(x.get("return_type", "?") for x in downloaded) or "none"
+        raise RuntimeError(
+            f"GST period {period} did not complete every selected return. Expected {expected}, downloaded {len(downloaded)} ({got})."
+        )
+
    write_json(
        work_root / period / "download_manifest.json",
-        {"period": period, "downloaded_at_utc": now(), "source": "arrr_gst_operator_agent_1.3.0", "downloaded": downloaded},
+        {"period": period, "downloaded_at_utc": now(), "source": "arrr_gst_operator_agent_1.4.0", "downloaded": downloaded},
    )
    return downloaded

-
 def browser_worker(payload: dict, token: str, path: Path) -> None:
    job_id = str(payload.get("jti") or "")
    work_root = DATA / "work" / safe(job_id)
    package_path = DATA / f"gst_{safe(job_id)}.zip"
+    pw = None
+    context = None
+    page = None

    def progress(**updates):
        current = read_json(path)
@@ -528,68 +622,76 @@ def browser_worker(payload: dict, token: str, path: Path) -> None:
        progress(status="running", percent=5, stage="Opening GST Login", message="Opening GST Portal in visible Chrome/Edge.")
        from playwright.sync_api import sync_playwright

-        with sync_playwright() as pw:
-            context = None
-            errors = []
-            profile_root = DATA / "browser_profile"
-            profile_root.mkdir(parents=True, exist_ok=True)
-            for channel in ("chrome", "msedge"):
-                try:
-                    context = pw.chromium.launch_persistent_context(
-                        user_data_dir=str(profile_root / f"{channel}_{safe(job_id)}"),
-                        channel=channel,
-                        headless=False,
-                        no_viewport=True,
-                        accept_downloads=True,
-                        args=["--start-maximized", "--no-first-run", "--disable-blink-features=AutomationControlled"],
-                        timeout=45000,
-                    )
-                    break
-                except Exception as exc:
-                    errors.append(f"{channel}: {exc}")
-            if context is None:
-                raise RuntimeError("Could not open installed Chrome or Edge. " + " | ".join(errors))
-
-            page = context.pages[0] if context.pages else context.new_page()
-            open_visible_page(page, GST_LOGIN_URL, "GST Login", 1000)
-            page.locator("#username").wait_for(state="visible", timeout=30000)
-            page.fill("#username", str(payload.get("username") or ""))
-            password_filled = False
-            for selector in ("#user_pass", "input[type=password]"):
-                try:
-                    locator = page.locator(selector).first
-                    if locator.count():
-                        locator.fill(str(payload.get("password") or ""))
-                        password_filled = True
-                        break
-                except Exception:
-                    pass
-            if not password_filled:
-                raise RuntimeError("GST password field could not be located. The GST login page may have changed.")
-
-            wait_for_login(page, progress, int(payload.get("login_timeout_seconds") or 900))
-            periods = [str(p) for p in payload.get("periods") or []]
-            return_types = [str(r) for r in payload.get("return_types") or []]
-            prepare_portal_context(page, return_types, progress)
-            all_downloaded = []
-            for index, period in enumerate(periods, 1):
-                pct = 15 + int((index - 1) * 70 / max(1, len(periods)))
-                progress(
-                    percent=pct,
-                    period=period,
-                    period_index=index,
-                    period_total=len(periods),
-                    stage=f"Period {index}/{len(periods)}",
-                    message=f"Downloading selected GST returns for {period}.",
+        pw = sync_playwright().start()
+        errors = []
+        profile_root = DATA / "browser_profile"
+        profile_root.mkdir(parents=True, exist_ok=True)
+        for channel in ("chrome", "msedge"):
+            try:
+                context = pw.chromium.launch_persistent_context(
+                    user_data_dir=str(profile_root / f"{channel}_{safe(job_id)}"),
+                    channel=channel,
+                    headless=False,
+                    no_viewport=True,
+                    accept_downloads=True,
+                    args=["--start-maximized", "--no-first-run", "--disable-blink-features=AutomationControlled"],
+                    timeout=45000,
                )
-                all_downloaded.extend(download_period(page, work_root, period, return_types, progress))
+                break
+            except Exception as exc:
+                errors.append(f"{channel}: {exc}")
+        if context is None:
+            raise RuntimeError("Could not open installed Chrome or Edge. " + " | ".join(errors))

-            show_status_overlay(
-                page,
-                "Selected GST return downloads are complete. Transferring them to ARRR client storage.",
-                "#065f46",
+        page = context.pages[0] if context.pages else context.new_page()
+        open_visible_page(page, GST_LOGIN_URL, "GST Login", 1000)
+        page.locator("#username").wait_for(state="visible", timeout=30000)
+        page.fill("#username", str(payload.get("username") or ""))
+        password_filled = False
+        for selector in ("#user_pass", "input[type=password]"):
+            try:
+                locator = page.locator(selector).first
+                if locator.count():
+                    locator.fill(str(payload.get("password") or ""))
+                    password_filled = True
+                    break
+            except Exception:
+                pass
+        if not password_filled:
+            raise RuntimeError("GST password field could not be located. The GST login page may have changed.")
+
+        wait_for_login(page, progress, int(payload.get("login_timeout_seconds") or 900))
+        periods = [str(p) for p in payload.get("periods") or []]
+        return_types = [str(r) for r in payload.get("return_types") or []]
+        if not periods:
+            raise RuntimeError("GST job does not contain any return period.")
+        if not return_types:
+            raise RuntimeError("GST job does not contain any selected return type.")
+
+        prepare_portal_context(page, return_types, progress)
+        all_downloaded = []
+        for index, period in enumerate(periods, 1):
+            pct = 15 + int((index - 1) * 70 / max(1, len(periods)))
+            progress(
+                percent=pct,
+                period=period,
+                period_index=index,
+                period_total=len(periods),
+                stage=f"Period {index}/{len(periods)}",
+                message=f"Downloading selected GST returns for {period}.",
+            )
+            period_downloaded = download_period(page, work_root, period, return_types, progress)
+            all_downloaded.extend(period_downloaded)
+            _log(f"GST period {period} downloaded files: {period_downloaded}")
+
+        expected_total = len(periods) * len({str(r or "").upper() for r in return_types})
+        if not all_downloaded:
+            raise RuntimeError("No GST return file was downloaded. The job will not be marked completed.")
+        if len(all_downloaded) != expected_total:
+            raise RuntimeError(
+                f"GST download is incomplete. Expected {expected_total} return files, received {len(all_downloaded)}. "
+                "The job will not be marked completed."
            )
-            context.close()

        write_json(
            work_root / "job_manifest.json",
@@ -601,6 +703,7 @@ def browser_worker(payload: dict, token: str, path: Path) -> None:
                "return_types": payload.get("return_types") or [],
                "downloaded": all_downloaded,
                "completed_at_utc": now(),
+                "agent_version": VERSION,
            },
        )
        if package_path.exists():
@@ -613,8 +716,15 @@ def browser_worker(payload: dict, token: str, path: Path) -> None:
        progress(
            percent=90,
            stage="Transferring to Client Storage",
-            message="Uploading GST return package to ERP for transfer to configured Storage Agent.",
+            message="GST JSON download completed. Uploading the verified package to ERP/client storage. GST remains logged in.",
        )
+        keep_browser_visible(
+            page,
+            "GST JSON download completed. Transferring verified files to ARRR ERP/client storage. GST remains logged in.",
+            "#065f46",
+            1500,
+        )
+
        upload_url = str(payload.get("upload_url") or "").strip()
        if not upload_url.startswith("https://"):
            raise RuntimeError(f"ERP upload URL is invalid: {upload_url or '(empty)'}")
@@ -631,14 +741,21 @@ def browser_worker(payload: dict, token: str, path: Path) -> None:
            body = {"ok": False, "error": response.text[:1000]}
        if not response.ok or not body.get("ok"):
            raise RuntimeError(body.get("error") or f"ERP storage transfer failed with HTTP {response.status_code}.")
+
        progress(
            status="completed",
            percent=100,
            stage="Completed",
-            message="GST returns downloaded and stored in configured client local storage.",
-            result={"stored": body.get("stored") or {}},
+            message="GST returns downloaded and stored in configured client local storage. GST portal was not logged out automatically.",
+            result={"stored": body.get("stored") or {}, "downloaded": all_downloaded},
            finished_at_utc=now(),
        )
+        keep_browser_visible(
+            page,
+            "ARRR GST download completed successfully and client storage confirmed. No automatic GST logout was performed. This browser will close in 12 seconds.",
+            "#15803d",
+            12000,
+        )
    except Exception as exc:
        current = read_json(path)
        current.update(
@@ -654,14 +771,30 @@ def browser_worker(payload: dict, token: str, path: Path) -> None:
        )
        write_json(path, current)
        _log(traceback.format_exc())
+        if page is not None:
+            keep_browser_visible(
+                page,
+                "GST download FAILED: " + str(exc)[:650] + "\n\nThe browser is being kept open for 45 seconds so the failed GST page can be inspected. No automatic logout was performed.",
+                "#b91c1c",
+                45000,
+            )
    finally:
+        try:
+            if context is not None:
+                context.close()
+        except Exception:
+            pass
+        try:
+            if pw is not None:
+                pw.stop()
+        except Exception:
+            pass
        shutil.rmtree(work_root, ignore_errors=True)
        try:
            package_path.unlink(missing_ok=True)
        except Exception:
            pass

-
 def redeem_job(token: str) -> dict:
    cfg = read_config()
    erp = str(cfg.get("erp_base_url") or "").rstrip("/")
@@ -9,7 +9,7 @@ $Startup = Join-Path $env:APPDATA "Microsoft\Windows\Start Menu\Programs\Startup
 $ThumbprintFile = Join-Path $InstallRoot "cert_thumbprint.txt"
 $ProtocolKey = "HKCU:\Software\Classes\arrrgst"

-Write-Host "ARRR GST Operator Agent 1.2.0 - clean installation" -ForegroundColor Cyan
+Write-Host "ARRR GST Operator Agent 1.4.0 - clean installation" -ForegroundColor Cyan
 Write-Host "Install root: $InstallRoot"

 # Clean legacy v1.0.x listener/startup/certificate/protocol state first.
@@ -147,7 +147,7 @@ router = APIRouter(prefix="/tools/accounting/gst-reconciliation", tags=["account
 _TOKEN_PURPOSE = "gst_lightweight_operator_v3"
 _TOKEN_MINUTES = 15
 _UPLOAD_ROOT = Path(tempfile.gettempdir()) / "audit_firm_gst_operator_uploads"
-_OPERATOR_AGENT_VERSION = "1.3.0"
+_OPERATOR_AGENT_VERSION = "1.4.0"
 _OPERATOR_RUNTIME_ROOT = Path(__file__).resolve().parent / "gst_operator_agent_runtime"
 _OPERATOR_PACKAGE_FILES = ("gst_operator_agent.py", "requirements.txt", "README.txt", "install_gst_operator_agent.ps1", "uninstall_gst_operator_agent.ps1")