Use natural GST return dashboard navigation in operator agent

2026-09-11 22:45:31 +05:30
parent 756daf67da
commit fdc7580b50
2 changed files with 185 additions and 41 deletions
@@ -13,7 +13,7 @@ from urllib.parse import parse_qs, urlsplit

 import requests

-VERSION = "1.2.0"
+VERSION = "1.3.0"
 ROOT = Path(__file__).resolve().parent
 DATA = ROOT / "data"
 CONFIG_PATH = ROOT / "config.json"
@@ -26,6 +26,7 @@ GSTR1_URL = "https://return.gst.gov.in/returns/auth/api/offline/download/generat
 GSTR2A_URL = "https://return.gst.gov.in/returns/auth/api/offline/download/generate?flag=0&rtn_prd={period}&rtn_typ=GSTR2A"
 GSTR1_DOWNLOAD_URL = "https://return.gst.gov.in/returns/auth/api/offline/download/url?rtn_prd={period}&rtn_typ=GSTR1&file_num={file_num}"
 GSTR2B_URL = "https://gstr2b.gst.gov.in/gstr2b/auth/api/gstr2b/getjson?rtnprd={period}"
+ROLESTATUS_URL = "https://return.gst.gov.in/returns/auth/api/rolestatus?rtn_prd={period}"
 GSTR3B_SUMMARY_URL = "https://return.gst.gov.in/returns/auth/api/gstr3b/summary?rtn_prd={period}"
 GSTR3B_URL = "https://return.gst.gov.in/returns/auth/api/gstr3b/taxpayble?rtn_prd={period}"

@@ -265,45 +266,188 @@ def open_visible_page(page, url: str, label: str = "GST page", wait_ms: int = 18
        raise RuntimeError(f"GST portal returned Access Denied / Session Expired while opening {label}.")


-def ensure_origin_page(page, url: str) -> None:
-    origin = _origin_for_url(url)
-    if (page.url or "").startswith(origin):
-        return
-    if origin == "https://gstr2b.gst.gov.in":
-        open_visible_page(page, GSTR2B_PAGE_URL, "GSTR-2B download page", 2500)
-    elif origin == "https://return.gst.gov.in":
-        open_visible_page(page, RETURN_DASHBOARD_URL, "GST Return Dashboard", 2500)
-    elif origin == "https://services.gst.gov.in":
-        open_visible_page(page, SERVICES_DASHBOARD_URL, "GST services dashboard", 1800)
+def close_aadhaar_popup_if_visible(page) -> None:
+    """Close the optional GST Aadhaar / E-KYC reminder without disturbing login state."""
+    for selector in (
+        "a.btn.btn-primary:has-text('Remind me later')",
+        "text=Remind me later",
+    ):
+        try:
+            loc = page.locator(selector).first
+            if loc.is_visible(timeout=2500):
+                loc.click(timeout=10000)
+                page.wait_for_timeout(1200)
+                _log("Closed GST Aadhaar/E-KYC reminder popup.")
+                return
+        except Exception:
+            pass


-def page_fetch_text(page, url: str, method: str = "GET", payload=None) -> str:
-    ensure_origin_page(page, url)
-    result = page.evaluate(
-        """async ({url,method,payload})=>{const headers={Accept:'application/json, text/plain, */*'};const opt={method,credentials:'include',headers};if(payload!==null&&payload!==undefined){headers['Content-Type']='application/json;charset=UTF-8';opt.body=JSON.stringify(payload);}const r=await fetch(url,opt);return {status:r.status,ctype:r.headers.get('content-type')||'',text:await r.text(),url:r.url};}""",
-        {"url": url, "method": method, "payload": payload},
+def is_logged_in_services_page(page) -> bool:
+    url = (page.url or "").lower()
+    return (
+        "services.gst.gov.in/services/auth/fowelcome" in url
+        or "services.gst.gov.in/services/auth/dashboard" in url
+        or ("services.gst.gov.in" in url and "/services/auth/" in url)
    )
-    status = int(result.get("status") or 0)
-    text = result.get("text") or ""
-    ctype = (result.get("ctype") or "").lower()
+
+
+def return_to_services_welcome(page, progress, timeout_seconds: int = 60) -> None:
+    """Return through browser history to the logged-in GST services/welcome page.
+
+    The V16 trial established that GST subdomain sessions are most reliable when
+    navigation starts from the authenticated services page rather than by typing
+    a return/payment URL directly.
+    """
+    if is_logged_in_services_page(page):
+        close_aadhaar_popup_if_visible(page)
+        return
+    progress(stage="Returning to GST Welcome", message="Returning to the authenticated GST welcome page before the next portal flow.")
+    started = time.time()
+    for _ in range(15):
+        if is_logged_in_services_page(page):
+            close_aadhaar_popup_if_visible(page)
+            return
+        try:
+            page.go_back(wait_until="domcontentloaded", timeout=20000)
+        except Exception as exc:
+            _log(f"GST browser Back warning: {exc}")
+        page.wait_for_timeout(1200)
+        if time.time() - started > timeout_seconds:
+            break
+    if not is_logged_in_services_page(page):
+        raise RuntimeError(
+            "Could not return to the authenticated GST Welcome page. Keep the GST browser open, "
+            "return to the logged-in Home/Welcome page and start the download again."
+        )
+    close_aadhaar_popup_if_visible(page)
+
+
+def open_return_dashboard_naturally(page, progress) -> None:
+    """Reach Return Dashboard using the same working V16 human-navigation sequence.
+
+    Never deep-link from a fresh login. GST creates/refreshes the return-domain
+    session when the operator clicks the Return Dashboard control on the logged-in
+    services page. The OctaGST extension likewise expects return.gst.gov.in to be
+    the active dashboard before it calls return APIs.
+    """
+    if "return.gst.gov.in/returns/auth/dashboard" in (page.url or "").lower():
+        return
+    return_to_services_welcome(page, progress)
+    close_aadhaar_popup_if_visible(page)
+    progress(stage="Opening Return Dashboard", message="Opening Return Dashboard through the GST Welcome page (natural portal navigation).")
+    show_status_overlay(page, "Opening GST Return Dashboard through the portal...", "#0f766e")
+    clicked = False
+    errors = []
+    candidates = [
+        lambda: page.get_by_role("button", name="Return Dashboard"),
+        lambda: page.locator("button:has-text('Return Dashboard')").first,
+        lambda: page.locator("a:has-text('Return Dashboard')").first,
+        lambda: page.get_by_text("Return Dashboard", exact=True).first,
+    ]
+    for factory in candidates:
+        try:
+            loc = factory()
+            if loc.is_visible(timeout=4000):
+                loc.click(timeout=15000)
+                clicked = True
+                break
+        except Exception as exc:
+            errors.append(str(exc))
+    if not clicked:
+        raise RuntimeError(
+            "GST Return Dashboard control was not found on the logged-in Welcome page. "
+            "The portal layout may have changed. " + (" | ".join(errors[-2:]) if errors else "")
+        )
+    try:
+        page.wait_for_url("**/returns/auth/dashboard**", timeout=60000)
+    except Exception:
+        # Some portal builds change query/hash while remaining on the same dashboard host.
+        deadline = time.time() + 45
+        while time.time() < deadline:
+            if "return.gst.gov.in" in (page.url or "").lower() and "/returns/auth/" in (page.url or "").lower():
+                break
+            page.wait_for_timeout(1000)
+    try:
+        page.wait_for_load_state("domcontentloaded", timeout=15000)
+    except Exception:
+        pass
+    page.wait_for_timeout(1800)
+    if is_access_denied_page(page):
+        raise RuntimeError(
+            "GST returned Access Denied while entering Return Dashboard through the normal portal flow. "
+            "Logout other GST sessions, login again and retry."
+        )
+    if "return.gst.gov.in" not in (page.url or "").lower():
+        raise RuntimeError(f"GST did not enter the Return Dashboard. Current page: {page.url}")
+    show_status_overlay(page, "GST Return Dashboard ready. Downloading selected return data...", "#15803d")
+    progress(stage="Return Dashboard Ready", message="GST Return Dashboard opened through the normal portal sequence.")
+
+
+def _same_origin(page, url: str) -> bool:
+    try:
+        return urlsplit(page.url).scheme == urlsplit(url).scheme and urlsplit(page.url).netloc == urlsplit(url).netloc
+    except Exception:
+        return False
+
+
+def page_fetch_text(page, url: str, method: str = "GET", payload=None, referer: str | None = None) -> str:
+    """Fetch GST JSON only after Return Dashboard has been established naturally.
+
+    Same-origin return.gst.gov.in calls execute in the visible page just like the
+    OctaGST content-script flow. Cross-subdomain calls (notably GSTR-2B) use the
+    Playwright browser context request client, which shares this browser context's
+    GST cookies, and carries the referer expected by the GST module.
+    """
+    if is_access_denied_page(page):
+        raise RuntimeError("GST session is on an Access Denied / Session Expired page.")
+    if _same_origin(page, url):
+        result = page.evaluate(
+            """async ({url,method,payload})=>{const headers={Accept:'application/json, text/plain, */*'};const opt={method,credentials:'include',headers};if(payload!==null&&payload!==undefined){headers['Content-Type']='application/json;charset=UTF-8';opt.body=JSON.stringify(payload);}const r=await fetch(url,opt);return {status:r.status,ctype:r.headers.get('content-type')||'',text:await r.text(),url:r.url};}""",
+            {"url": url, "method": method, "payload": payload},
+        )
+        status = int(result.get("status") or 0)
+        text = result.get("text") or ""
+        ctype = (result.get("ctype") or "").lower()
+    else:
+        headers = {
+            "Accept": "application/json, text/plain, */*",
+            "Referer": referer or _origin_for_url(url),
+        }
+        try:
+            if method.upper() == "POST":
+                resp = page.context.request.post(url, data=payload if payload is not None else {}, headers=headers, timeout=60000)
+            else:
+                resp = page.context.request.get(url, headers=headers, timeout=60000)
+            status = int(resp.status or 0)
+            text = resp.text() or ""
+            ctype = (resp.headers.get("content-type") or "").lower()
+        except Exception as exc:
+            raise RuntimeError(f"GST cross-module request failed for {url}: {exc}") from exc
    sample = text.lstrip().lower()
    if status >= 400:
        raise RuntimeError(f"GST API HTTP {status}: {text[:500]}")
    if ("text/html" in ctype or sample.startswith(("<!doctype", "<html"))) and not sample.startswith(("{", "[")):
        raise RuntimeError(
-            "GST returned an HTML/login page instead of return JSON. The portal session/module is not ready."
+            "GST returned an HTML/login page instead of return JSON. The return-domain session is not ready or has expired."
        )
    return text


-def prepare_portal_context(page, return_types: list[str]) -> None:
-    selected = {str(x or "").upper() for x in return_types}
-    if "GSTR1" in selected or "GSTR2A" in selected:
-        open_visible_page(page, OFFLINE_DOWNLOAD_PAGE_URL, "GST offline return download page", 2500)
-    if "GSTR2B" in selected:
-        open_visible_page(page, GSTR2B_PAGE_URL, "GSTR-2B download page", 2500)
-    if "GSTR3B" in selected:
-        open_visible_page(page, RETURN_DASHBOARD_URL, "GST Return Dashboard", 2500)
+def verify_return_dashboard_session(page) -> None:
+    """Confirm the OctaGST prerequisite: authenticated return.gst.gov.in dashboard."""
+    url = (page.url or "").lower()
+    if "return.gst.gov.in" not in url or "/returns/auth/" not in url:
+        raise RuntimeError("GST Return Dashboard is not active. The agent will not deep-link to a return page.")
+    if is_access_denied_page(page):
+        raise RuntimeError("GST Return Dashboard session is Access Denied / expired.")
+
+
+def prepare_portal_context(page, return_types: list[str], progress) -> None:
+    # The extension supplied by the user explicitly requires Return Dashboard to
+    # already be open. Keep that one established session for all return API calls.
+    open_return_dashboard_naturally(page, progress)
+    verify_return_dashboard_session(page)


 def download_period(page, work_root: Path, period: str, return_types: list[str], progress) -> list[dict]:
@@ -314,8 +458,8 @@ def download_period(page, work_root: Path, period: str, return_types: list[str],

    if "GSTR1" in selected:
        progress(stage=f"Downloading GSTR-1 {period}", message=f"Generating and downloading GSTR-1 for {period}.")
-        open_visible_page(page, OFFLINE_DOWNLOAD_PAGE_URL, "GSTR-1 offline download page", 1800)
-        generated = page_fetch_text(page, GSTR1_URL.format(period=period))
+        verify_return_dashboard_session(page)
+        generated = page_fetch_text(page, GSTR1_URL.format(period=period), referer=RETURN_DASHBOARD_URL)
        (raw_dir / f"{period}_GSTR1_GENERATE.json").write_text(generated, encoding="utf-8")
        file_num = extract_file_num(generated)
        content = extract_payload(page_fetch_text(page, GSTR1_DOWNLOAD_URL.format(period=period, file_num=file_num)))
@@ -325,17 +469,17 @@ def download_period(page, work_root: Path, period: str, return_types: list[str],

    if "GSTR2B" in selected:
        progress(stage=f"Downloading GSTR-2B {period}", message=f"Downloading GSTR-2B JSON for {period}.")
-        open_visible_page(page, GSTR2B_PAGE_URL, "GSTR-2B download page", 1800)
-        content = page_fetch_text(page, GSTR2B_URL.format(period=period))
+        verify_return_dashboard_session(page)
+        content = page_fetch_text(page, GSTR2B_URL.format(period=period), referer="https://gstr2b.gst.gov.in")
        path = raw_dir / f"{period}_GSTR2B.json"
        path.write_text(content, encoding="utf-8")
        downloaded.append({"return_type": "GSTR2B", "path": str(path.relative_to(work_root)), "bytes": path.stat().st_size})

    if "GSTR3B" in selected:
        progress(stage=f"Downloading GSTR-3B {period}", message=f"Downloading GSTR-3B summary and tax payable for {period}.")
-        open_visible_page(page, RETURN_DASHBOARD_URL, "GST Return Dashboard", 1800)
-        summary = page_fetch_text(page, GSTR3B_SUMMARY_URL.format(period=period))
-        payable = page_fetch_text(page, GSTR3B_URL.format(period=period))
+        verify_return_dashboard_session(page)
+        summary = page_fetch_text(page, GSTR3B_SUMMARY_URL.format(period=period), referer=RETURN_DASHBOARD_URL)
+        payable = page_fetch_text(page, GSTR3B_URL.format(period=period), referer=RETURN_DASHBOARD_URL)
        (raw_dir / f"{period}_GSTR3B_SUMMARY.json").write_text(summary, encoding="utf-8")
        (raw_dir / f"{period}_GSTR3B_TAXPAYBLE.json").write_text(payable, encoding="utf-8")
        try:
@@ -354,15 +498,15 @@ def download_period(page, work_root: Path, period: str, return_types: list[str],

    if "GSTR2A" in selected:
        progress(stage=f"Requesting GSTR-2A {period}", message=f"Requesting GSTR-2A offline return for {period}.")
-        open_visible_page(page, OFFLINE_DOWNLOAD_PAGE_URL, "GSTR-2A offline download page", 1800)
-        content = page_fetch_text(page, GSTR2A_URL.format(period=period))
+        verify_return_dashboard_session(page)
+        content = page_fetch_text(page, GSTR2A_URL.format(period=period), referer=RETURN_DASHBOARD_URL)
        path = raw_dir / f"{period}_GSTR2A.json"
        path.write_text(content, encoding="utf-8")
        downloaded.append({"return_type": "GSTR2A", "path": str(path.relative_to(work_root)), "bytes": path.stat().st_size})

    write_json(
        work_root / period / "download_manifest.json",
-        {"period": period, "downloaded_at_utc": now(), "source": "arrr_gst_operator_agent_1.2.0", "downloaded": downloaded},
+        {"period": period, "downloaded_at_utc": now(), "source": "arrr_gst_operator_agent_1.3.0", "downloaded": downloaded},
    )
    return downloaded

@@ -426,7 +570,7 @@ def browser_worker(payload: dict, token: str, path: Path) -> None:
            wait_for_login(page, progress, int(payload.get("login_timeout_seconds") or 900))
            periods = [str(p) for p in payload.get("periods") or []]
            return_types = [str(r) for r in payload.get("return_types") or []]
-            prepare_portal_context(page, return_types)
+            prepare_portal_context(page, return_types, progress)
            all_downloaded = []
            for index, period in enumerate(periods, 1):
                pct = 15 + int((index - 1) * 70 / max(1, len(periods)))