Use natural GST return dashboard navigation in operator agent

app/modules/accounting/gst_operator_agent_runtime/gst_operator_agent.py

@@ -13,7 +13,7 @@ from urllib.parse import parse_qs, urlsplit
 
 import requests
 
-VERSION = "1.2.0"
+VERSION = "1.3.0"
 ROOT = Path(__file__).resolve().parent
 DATA = ROOT / "data"
 CONFIG_PATH = ROOT / "config.json"
@@ -26,6 +26,7 @@ GSTR1_URL = "https://return.gst.gov.in/returns/auth/api/offline/download/generat
 GSTR2A_URL = "https://return.gst.gov.in/returns/auth/api/offline/download/generate?flag=0&rtn_prd={period}&rtn_typ=GSTR2A"
 GSTR1_DOWNLOAD_URL = "https://return.gst.gov.in/returns/auth/api/offline/download/url?rtn_prd={period}&rtn_typ=GSTR1&file_num={file_num}"
 GSTR2B_URL = "https://gstr2b.gst.gov.in/gstr2b/auth/api/gstr2b/getjson?rtnprd={period}"
+ROLESTATUS_URL = "https://return.gst.gov.in/returns/auth/api/rolestatus?rtn_prd={period}"
 GSTR3B_SUMMARY_URL = "https://return.gst.gov.in/returns/auth/api/gstr3b/summary?rtn_prd={period}"
 GSTR3B_URL = "https://return.gst.gov.in/returns/auth/api/gstr3b/taxpayble?rtn_prd={period}"
 
@@ -265,45 +266,188 @@ def open_visible_page(page, url: str, label: str = "GST page", wait_ms: int = 18
         raise RuntimeError(f"GST portal returned Access Denied / Session Expired while opening {label}.")
 
 
-def ensure_origin_page(page, url: str) -> None:
+def close_aadhaar_popup_if_visible(page) -> None:
-    origin = _origin_for_url(url)
+    """Close the optional GST Aadhaar / E-KYC reminder without disturbing login state."""
-    if (page.url or "").startswith(origin):
+    for selector in (
-        return
+        "a.btn.btn-primary:has-text('Remind me later')",
-    if origin == "https://gstr2b.gst.gov.in":
+        "text=Remind me later",
-        open_visible_page(page, GSTR2B_PAGE_URL, "GSTR-2B download page", 2500)
+    ):
-    elif origin == "https://return.gst.gov.in":
+        try:
-        open_visible_page(page, RETURN_DASHBOARD_URL, "GST Return Dashboard", 2500)
+            loc = page.locator(selector).first
-    elif origin == "https://services.gst.gov.in":
+            if loc.is_visible(timeout=2500):
-        open_visible_page(page, SERVICES_DASHBOARD_URL, "GST services dashboard", 1800)
+                loc.click(timeout=10000)
+                page.wait_for_timeout(1200)
+                _log("Closed GST Aadhaar/E-KYC reminder popup.")
+                return
+        except Exception:
+            pass
 
 
-def page_fetch_text(page, url: str, method: str = "GET", payload=None) -> str:
+def is_logged_in_services_page(page) -> bool:
-    ensure_origin_page(page, url)
+    url = (page.url or "").lower()
-    result = page.evaluate(
+    return (
-        """async ({url,method,payload})=>{const headers={Accept:'application/json, text/plain, */*'};const opt={method,credentials:'include',headers};if(payload!==null&&payload!==undefined){headers['Content-Type']='application/json;charset=UTF-8';opt.body=JSON.stringify(payload);}const r=await fetch(url,opt);return {status:r.status,ctype:r.headers.get('content-type')||'',text:await r.text(),url:r.url};}""",
+        "services.gst.gov.in/services/auth/fowelcome" in url
-        {"url": url, "method": method, "payload": payload},
+        or "services.gst.gov.in/services/auth/dashboard" in url
+        or ("services.gst.gov.in" in url and "/services/auth/" in url)
     )
-    status = int(result.get("status") or 0)
+
-    text = result.get("text") or ""
+
-    ctype = (result.get("ctype") or "").lower()
+def return_to_services_welcome(page, progress, timeout_seconds: int = 60) -> None:
+    """Return through browser history to the logged-in GST services/welcome page.
+
+    The V16 trial established that GST subdomain sessions are most reliable when
+    navigation starts from the authenticated services page rather than by typing
+    a return/payment URL directly.
+    """
+    if is_logged_in_services_page(page):
+        close_aadhaar_popup_if_visible(page)
+        return
+    progress(stage="Returning to GST Welcome", message="Returning to the authenticated GST welcome page before the next portal flow.")
+    started = time.time()
+    for _ in range(15):
+        if is_logged_in_services_page(page):
+            close_aadhaar_popup_if_visible(page)
+            return
+        try:
+            page.go_back(wait_until="domcontentloaded", timeout=20000)
+        except Exception as exc:
+            _log(f"GST browser Back warning: {exc}")
+        page.wait_for_timeout(1200)
+        if time.time() - started > timeout_seconds:
+            break
+    if not is_logged_in_services_page(page):
+        raise RuntimeError(
+            "Could not return to the authenticated GST Welcome page. Keep the GST browser open, "
+            "return to the logged-in Home/Welcome page and start the download again."
+        )
+    close_aadhaar_popup_if_visible(page)
+
+
+def open_return_dashboard_naturally(page, progress) -> None:
+    """Reach Return Dashboard using the same working V16 human-navigation sequence.
+
+    Never deep-link from a fresh login. GST creates/refreshes the return-domain
+    session when the operator clicks the Return Dashboard control on the logged-in
+    services page. The OctaGST extension likewise expects return.gst.gov.in to be
+    the active dashboard before it calls return APIs.
+    """
+    if "return.gst.gov.in/returns/auth/dashboard" in (page.url or "").lower():
+        return
+    return_to_services_welcome(page, progress)
+    close_aadhaar_popup_if_visible(page)
+    progress(stage="Opening Return Dashboard", message="Opening Return Dashboard through the GST Welcome page (natural portal navigation).")
+    show_status_overlay(page, "Opening GST Return Dashboard through the portal...", "#0f766e")
+    clicked = False
+    errors = []
+    candidates = [
+        lambda: page.get_by_role("button", name="Return Dashboard"),
+        lambda: page.locator("button:has-text('Return Dashboard')").first,
+        lambda: page.locator("a:has-text('Return Dashboard')").first,
+        lambda: page.get_by_text("Return Dashboard", exact=True).first,
+    ]
+    for factory in candidates:
+        try:
+            loc = factory()
+            if loc.is_visible(timeout=4000):
+                loc.click(timeout=15000)
+                clicked = True
+                break
+        except Exception as exc:
+            errors.append(str(exc))
+    if not clicked:
+        raise RuntimeError(
+            "GST Return Dashboard control was not found on the logged-in Welcome page. "
+            "The portal layout may have changed. " + (" | ".join(errors[-2:]) if errors else "")
+        )
+    try:
+        page.wait_for_url("**/returns/auth/dashboard**", timeout=60000)
+    except Exception:
+        # Some portal builds change query/hash while remaining on the same dashboard host.
+        deadline = time.time() + 45
+        while time.time() < deadline:
+            if "return.gst.gov.in" in (page.url or "").lower() and "/returns/auth/" in (page.url or "").lower():
+                break
+            page.wait_for_timeout(1000)
+    try:
+        page.wait_for_load_state("domcontentloaded", timeout=15000)
+    except Exception:
+        pass
+    page.wait_for_timeout(1800)
+    if is_access_denied_page(page):
+        raise RuntimeError(
+            "GST returned Access Denied while entering Return Dashboard through the normal portal flow. "
+            "Logout other GST sessions, login again and retry."
+        )
+    if "return.gst.gov.in" not in (page.url or "").lower():
+        raise RuntimeError(f"GST did not enter the Return Dashboard. Current page: {page.url}")
+    show_status_overlay(page, "GST Return Dashboard ready. Downloading selected return data...", "#15803d")
+    progress(stage="Return Dashboard Ready", message="GST Return Dashboard opened through the normal portal sequence.")
+
+
+def _same_origin(page, url: str) -> bool:
+    try:
+        return urlsplit(page.url).scheme == urlsplit(url).scheme and urlsplit(page.url).netloc == urlsplit(url).netloc
+    except Exception:
+        return False
+
+
+def page_fetch_text(page, url: str, method: str = "GET", payload=None, referer: str | None = None) -> str:
+    """Fetch GST JSON only after Return Dashboard has been established naturally.
+
+    Same-origin return.gst.gov.in calls execute in the visible page just like the
+    OctaGST content-script flow. Cross-subdomain calls (notably GSTR-2B) use the
+    Playwright browser context request client, which shares this browser context's
+    GST cookies, and carries the referer expected by the GST module.
+    """
+    if is_access_denied_page(page):
+        raise RuntimeError("GST session is on an Access Denied / Session Expired page.")
+    if _same_origin(page, url):
+        result = page.evaluate(
+            """async ({url,method,payload})=>{const headers={Accept:'application/json, text/plain, */*'};const opt={method,credentials:'include',headers};if(payload!==null&&payload!==undefined){headers['Content-Type']='application/json;charset=UTF-8';opt.body=JSON.stringify(payload);}const r=await fetch(url,opt);return {status:r.status,ctype:r.headers.get('content-type')||'',text:await r.text(),url:r.url};}""",
+            {"url": url, "method": method, "payload": payload},
+        )
+        status = int(result.get("status") or 0)
+        text = result.get("text") or ""
+        ctype = (result.get("ctype") or "").lower()
+    else:
+        headers = {
+            "Accept": "application/json, text/plain, */*",
+            "Referer": referer or _origin_for_url(url),
+        }
+        try:
+            if method.upper() == "POST":
+                resp = page.context.request.post(url, data=payload if payload is not None else {}, headers=headers, timeout=60000)
+            else:
+                resp = page.context.request.get(url, headers=headers, timeout=60000)
+            status = int(resp.status or 0)
+            text = resp.text() or ""
+            ctype = (resp.headers.get("content-type") or "").lower()
+        except Exception as exc:
+            raise RuntimeError(f"GST cross-module request failed for {url}: {exc}") from exc
     sample = text.lstrip().lower()
     if status >= 400:
         raise RuntimeError(f"GST API HTTP {status}: {text[:500]}")
     if ("text/html" in ctype or sample.startswith(("<!doctype", "<html"))) and not sample.startswith(("{", "[")):
         raise RuntimeError(
-            "GST returned an HTML/login page instead of return JSON. The portal session/module is not ready."
+            "GST returned an HTML/login page instead of return JSON. The return-domain session is not ready or has expired."
         )
     return text
 
 
-def prepare_portal_context(page, return_types: list[str]) -> None:
+def verify_return_dashboard_session(page) -> None:
-    selected = {str(x or "").upper() for x in return_types}
+    """Confirm the OctaGST prerequisite: authenticated return.gst.gov.in dashboard."""
-    if "GSTR1" in selected or "GSTR2A" in selected:
+    url = (page.url or "").lower()
-        open_visible_page(page, OFFLINE_DOWNLOAD_PAGE_URL, "GST offline return download page", 2500)
+    if "return.gst.gov.in" not in url or "/returns/auth/" not in url:
-    if "GSTR2B" in selected:
+        raise RuntimeError("GST Return Dashboard is not active. The agent will not deep-link to a return page.")
-        open_visible_page(page, GSTR2B_PAGE_URL, "GSTR-2B download page", 2500)
+    if is_access_denied_page(page):
-    if "GSTR3B" in selected:
+        raise RuntimeError("GST Return Dashboard session is Access Denied / expired.")
-        open_visible_page(page, RETURN_DASHBOARD_URL, "GST Return Dashboard", 2500)
+
+
+def prepare_portal_context(page, return_types: list[str], progress) -> None:
+    # The extension supplied by the user explicitly requires Return Dashboard to
+    # already be open. Keep that one established session for all return API calls.
+    open_return_dashboard_naturally(page, progress)
+    verify_return_dashboard_session(page)
 
 
 def download_period(page, work_root: Path, period: str, return_types: list[str], progress) -> list[dict]:
@@ -314,8 +458,8 @@ def download_period(page, work_root: Path, period: str, return_types: list[str],
 
     if "GSTR1" in selected:
         progress(stage=f"Downloading GSTR-1 {period}", message=f"Generating and downloading GSTR-1 for {period}.")
-        open_visible_page(page, OFFLINE_DOWNLOAD_PAGE_URL, "GSTR-1 offline download page", 1800)
+        verify_return_dashboard_session(page)
-        generated = page_fetch_text(page, GSTR1_URL.format(period=period))
+        generated = page_fetch_text(page, GSTR1_URL.format(period=period), referer=RETURN_DASHBOARD_URL)
         (raw_dir / f"{period}_GSTR1_GENERATE.json").write_text(generated, encoding="utf-8")
         file_num = extract_file_num(generated)
         content = extract_payload(page_fetch_text(page, GSTR1_DOWNLOAD_URL.format(period=period, file_num=file_num)))
@@ -325,17 +469,17 @@ def download_period(page, work_root: Path, period: str, return_types: list[str],
 
     if "GSTR2B" in selected:
         progress(stage=f"Downloading GSTR-2B {period}", message=f"Downloading GSTR-2B JSON for {period}.")
-        open_visible_page(page, GSTR2B_PAGE_URL, "GSTR-2B download page", 1800)
+        verify_return_dashboard_session(page)
-        content = page_fetch_text(page, GSTR2B_URL.format(period=period))
+        content = page_fetch_text(page, GSTR2B_URL.format(period=period), referer="https://gstr2b.gst.gov.in")
         path = raw_dir / f"{period}_GSTR2B.json"
         path.write_text(content, encoding="utf-8")
         downloaded.append({"return_type": "GSTR2B", "path": str(path.relative_to(work_root)), "bytes": path.stat().st_size})
 
     if "GSTR3B" in selected:
         progress(stage=f"Downloading GSTR-3B {period}", message=f"Downloading GSTR-3B summary and tax payable for {period}.")
-        open_visible_page(page, RETURN_DASHBOARD_URL, "GST Return Dashboard", 1800)
+        verify_return_dashboard_session(page)
-        summary = page_fetch_text(page, GSTR3B_SUMMARY_URL.format(period=period))
+        summary = page_fetch_text(page, GSTR3B_SUMMARY_URL.format(period=period), referer=RETURN_DASHBOARD_URL)
-        payable = page_fetch_text(page, GSTR3B_URL.format(period=period))
+        payable = page_fetch_text(page, GSTR3B_URL.format(period=period), referer=RETURN_DASHBOARD_URL)
         (raw_dir / f"{period}_GSTR3B_SUMMARY.json").write_text(summary, encoding="utf-8")
         (raw_dir / f"{period}_GSTR3B_TAXPAYBLE.json").write_text(payable, encoding="utf-8")
         try:
@@ -354,15 +498,15 @@ def download_period(page, work_root: Path, period: str, return_types: list[str],
 
     if "GSTR2A" in selected:
         progress(stage=f"Requesting GSTR-2A {period}", message=f"Requesting GSTR-2A offline return for {period}.")
-        open_visible_page(page, OFFLINE_DOWNLOAD_PAGE_URL, "GSTR-2A offline download page", 1800)
+        verify_return_dashboard_session(page)
-        content = page_fetch_text(page, GSTR2A_URL.format(period=period))
+        content = page_fetch_text(page, GSTR2A_URL.format(period=period), referer=RETURN_DASHBOARD_URL)
         path = raw_dir / f"{period}_GSTR2A.json"
         path.write_text(content, encoding="utf-8")
         downloaded.append({"return_type": "GSTR2A", "path": str(path.relative_to(work_root)), "bytes": path.stat().st_size})
 
     write_json(
         work_root / period / "download_manifest.json",
-        {"period": period, "downloaded_at_utc": now(), "source": "arrr_gst_operator_agent_1.2.0", "downloaded": downloaded},
+        {"period": period, "downloaded_at_utc": now(), "source": "arrr_gst_operator_agent_1.3.0", "downloaded": downloaded},
     )
     return downloaded
 
@@ -426,7 +570,7 @@ def browser_worker(payload: dict, token: str, path: Path) -> None:
             wait_for_login(page, progress, int(payload.get("login_timeout_seconds") or 900))
             periods = [str(p) for p in payload.get("periods") or []]
             return_types = [str(r) for r in payload.get("return_types") or []]
-            prepare_portal_context(page, return_types)
+            prepare_portal_context(page, return_types, progress)
             all_downloaded = []
             for index, period in enumerate(periods, 1):
                 pct = 15 + int((index - 1) * 70 / max(1, len(periods)))

app/modules/accounting/gst_reconciliation_ui.py

@@ -147,7 +147,7 @@ router = APIRouter(prefix="/tools/accounting/gst-reconciliation", tags=["account
 _TOKEN_PURPOSE = "gst_lightweight_operator_v3"
 _TOKEN_MINUTES = 15
 _UPLOAD_ROOT = Path(tempfile.gettempdir()) / "audit_firm_gst_operator_uploads"
-_OPERATOR_AGENT_VERSION = "1.2.0"
+_OPERATOR_AGENT_VERSION = "1.3.0"
 _OPERATOR_RUNTIME_ROOT = Path(__file__).resolve().parent / "gst_operator_agent_runtime"
 _OPERATOR_PACKAGE_FILES = ("gst_operator_agent.py", "requirements.txt", "README.txt", "install_gst_operator_agent.ps1", "uninstall_gst_operator_agent.ps1")
 
@@ -496,7 +496,7 @@ def start_download(
         log_access(db,request,user,cred,"use_for_gst_download",reason=f"GST returns {financial_year}: {','.join(return_types)}",fields="username,secret",success=True)
         db.commit()
         request.session["gst_operator_job"]={"job_id":jti,"periods":periods,"return_types":return_types,"launch_url":"arrrgst://start?"+urlencode({"token":token})}
-        return _redirect(client_id,registration_id=registration_id,period=period,financial_year=financial_year,download_mode=download_mode,message="GST download prepared. Windows will launch the lightweight GST Operator Agent through the ARRR GST protocol. The agent will open the visible GST browser and autofill the selected Credential Vault login. Complete CAPTCHA/OTP there; downloaded return data will then be transferred to the configured client storage.")
+        return _redirect(client_id,registration_id=registration_id,period=period,financial_year=financial_year,download_mode=download_mode,message="GST download prepared. Windows will launch the lightweight GST Operator Agent through the ARRR GST protocol. The agent will open the visible GST browser and autofill the selected Credential Vault login. Complete CAPTCHA/OTP there; the agent will enter Return Dashboard through the normal GST portal sequence before downloading return data and transferring it to configured client storage.")
     except Exception as exc:
         db.rollback(); return _redirect(client_id,registration_id=registration_id,period=period,financial_year=financial_year,download_mode=download_mode,error=str(exc))
     finally: