Use natural monthly GST return navigation and pending download polling

app/modules/accounting/gst_operator_agent_runtime/README.txt

@@ -1,4 +1,4 @@
-ARRR GST Operator Agent 1.5.0
+ARRR GST Operator Agent 1.5.1
 
 Purpose
 -------
@@ -25,7 +25,7 @@ Uninstall
 Run uninstall_gst_operator_agent.ps1. It removes the protocol registration and also cleans legacy v1.0.x localhost listener/startup/certificate state.
 
 
-V1.5.0 OCTAGST-STYLE RETURN-DASHBOARD EXTRACTION
+V1.5.1 OCTAGST-STYLE RETURN-DASHBOARD EXTRACTION
 - Preserves the v1.4.0 Credential Vault login, manual CAPTCHA/OTP, natural Welcome -> Return Dashboard flow, ERP upload and failure diagnostics.
 - GSTR-2B no longer transfers the browser to gstr2b.gst.gov.in.
 - GSTR-2B is requested from the authenticated return.gst.gov.in Return Dashboard using the GST offline-download API pattern used by the supplied OctaGST extension.
@@ -34,3 +34,12 @@ V1.5.0 OCTAGST-STYLE RETURN-DASHBOARD EXTRACTION
 - No automatic GST logout is performed.
 - A job is completed only after every selected period/return has a saved result and ERP/client-storage upload is confirmed.
 - Manual JSON/ZIP import remains available in ERP as fallback.
+
+
+V1.5.1 NATURAL MONTHLY RETURN FLOW + GSTR-2B PENDING POLL
+--------------------------------------------------------
+- Restores robust Aadhaar/E-KYC Remind me later / Later dismissal.
+- GSTR-2B generation-accepted responses are PENDING, not failures; polls up to 25 minutes.
+- GSTR-1: selects FY/Quarter/Month on Return Dashboard, clicks Search, opens the actual GSTR-1 card, then performs offline generation/download from the initialized page context.
+- GSTR-3B: selects FY/Quarter/Month, clicks Search, opens the actual GSTR-3B card, then reads summary/tax-payable from the initialized return page context.
+- Existing Credential Vault, custom protocol, login/CAPTCHA, manual fallback, ERP upload and Storage Agent flow are unchanged.

app/modules/accounting/gst_operator_agent_runtime/gst_operator_agent.py

@@ -14,7 +14,7 @@ from urllib.parse import parse_qs, urljoin, urlsplit
 
 import requests
 
-VERSION = "1.5.0"
+VERSION = "1.5.1"
 ROOT = Path(__file__).resolve().parent
 DATA = ROOT / "data"
 CONFIG_PATH = ROOT / "config.json"
@@ -269,20 +269,35 @@ def open_visible_page(page, url: str, label: str = "GST page", wait_ms: int = 18
 
 
 def close_aadhaar_popup_if_visible(page) -> None:
-    """Close the optional GST Aadhaar / E-KYC reminder without disturbing login state."""
-    for selector in (
+    """Close the optional GST Aadhaar / E-KYC reminder using the proven V16 flow."""
+    candidates = (
         "a.btn.btn-primary:has-text('Remind me later')",
+        "button:has-text('Remind me later')",
+        "a:has-text('Remind me later')",
         "text=Remind me later",
-    ):
+        "button:has-text('LATER')",
+        "a:has-text('LATER')",
+        "text=LATER",
+    )
+    for _ in range(8):
+        for selector in candidates:
+            try:
+                loc = page.locator(selector).first
+                if loc.is_visible(timeout=1200):
+                    loc.scroll_into_view_if_needed(timeout=3000)
+                    loc.click(timeout=10000, force=True)
+                    page.wait_for_timeout(1400)
+                    _log("Closed GST Aadhaar/E-KYC reminder popup using Later/Remind me later.")
+                    return
+            except Exception:
+                pass
         try:
-            loc = page.locator(selector).first
-            if loc.is_visible(timeout=2500):
-                loc.click(timeout=10000)
-                page.wait_for_timeout(1200)
-                _log("Closed GST Aadhaar/E-KYC reminder popup.")
+            body = (page.locator("body").inner_text(timeout=1000) or "").lower()
+            if "aadhaar" not in body and "e-kyc" not in body and "remind me later" not in body:
                 return
         except Exception:
             pass
+        page.wait_for_timeout(700)
 
 
 def is_logged_in_services_page(page) -> bool:
@@ -593,6 +608,242 @@ def _store_gst_download_blob(raw_dir: Path, period: str, return_type: str, index
     return stored
 
 
+
+def _period_ui_parts(period: str) -> tuple[str, str, str]:
+    value = str(period or "").strip()
+    if not re.fullmatch(r"\d{6}", value):
+        raise ValueError(f"Invalid GST return period: {period!r}")
+    month = int(value[:2])
+    year = int(value[2:])
+    fy_start = year if month >= 4 else year - 1
+    fy = f"{fy_start}-{str(fy_start + 1)[-2:]}"
+    month_name = datetime(year, month, 1).strftime("%B")
+    quarter_no = 1 if month in (4, 5, 6) else 2 if month in (7, 8, 9) else 3 if month in (10, 11, 12) else 4
+    return fy, f"Quarter {quarter_no}", month_name
+
+
+def _set_dashboard_select_by_text(page, preferred_text: str, *, contains: bool = False) -> bool:
+    """Select a visible Return Dashboard dropdown option by its displayed text."""
+    target = preferred_text.strip().lower()
+    selects = page.locator("select:visible")
+    try:
+        count = selects.count()
+    except Exception:
+        count = 0
+    for i in range(count):
+        select = selects.nth(i)
+        try:
+            options = select.locator("option").all_text_contents()
+        except Exception:
+            continue
+        for label in options:
+            norm = str(label or "").strip().lower()
+            match = target in norm if contains else norm == target
+            if not match:
+                continue
+            try:
+                select.select_option(label=label)
+                page.wait_for_timeout(400)
+                return True
+            except Exception:
+                try:
+                    page.evaluate(
+                        """({el,text})=>{const opt=[...el.options].find(o=>o.textContent.trim().toLowerCase()===text.toLowerCase()||o.textContent.trim().toLowerCase().includes(text.toLowerCase()));if(!opt)return false;el.value=opt.value;el.dispatchEvent(new Event('input',{bubbles:true}));el.dispatchEvent(new Event('change',{bubbles:true}));return true;}""",
+                        {"el": select.element_handle(), "text": preferred_text},
+                    )
+                    page.wait_for_timeout(400)
+                    return True
+                except Exception:
+                    pass
+    return False
+
+
+def configure_return_dashboard_period(page, period: str, progress) -> None:
+    return_to_return_dashboard(page, progress)
+    verify_return_dashboard_session(page)
+    fy, quarter, month_name = _period_ui_parts(period)
+    progress(stage=f"Selecting GST period {period}", message=f"Selecting {month_name} {period[2:]} on the GST Return Dashboard.")
+    show_status_overlay(page, f"Selecting {month_name} {period[2:]} on GST Return Dashboard...", "#0f766e")
+    if not _set_dashboard_select_by_text(page, fy):
+        raise RuntimeError(f"GST Return Dashboard Financial Year {fy} option was not found.")
+    if not _set_dashboard_select_by_text(page, quarter, contains=True):
+        raise RuntimeError(f"GST Return Dashboard {quarter} option was not found.")
+    if not _set_dashboard_select_by_text(page, month_name, contains=True):
+        raise RuntimeError(f"GST Return Dashboard month {month_name} option was not found.")
+    clicked = False
+    for selector in ("button:has-text('SEARCH')", "input[type='submit'][value*='SEARCH' i]", "text=SEARCH"):
+        try:
+            loc = page.locator(selector).first
+            if loc.is_visible(timeout=2500):
+                loc.click(timeout=10000)
+                clicked = True
+                break
+        except Exception:
+            pass
+    if not clicked:
+        raise RuntimeError("GST Return Dashboard Search button was not found.")
+    page.wait_for_timeout(2600)
+    if is_access_denied_page(page):
+        raise RuntimeError("GST returned Access Denied after Return Dashboard period Search.")
+
+
+def open_return_card_naturally(page, period: str, return_label: str, progress) -> None:
+    """Open GSTR-1/GSTR-3B from the searched Return Dashboard card like a user."""
+    configure_return_dashboard_period(page, period, progress)
+    label_variants = [return_label, return_label.replace("-", ""), return_label.replace("GSTR", "GSTR ")]
+    card = None
+    for label in label_variants:
+        try:
+            hit = page.get_by_text(re.compile(rf"^\s*{re.escape(label)}\s*$", re.I)).first
+            if not hit.is_visible(timeout=1800):
+                continue
+            for xpath in (
+                "xpath=ancestor::tr[1]",
+                "xpath=ancestor::div[contains(@class,'card')][1]",
+                "xpath=ancestor::div[.//button or .//a][1]",
+            ):
+                try:
+                    candidate = hit.locator(xpath)
+                    if candidate.count() and candidate.first.is_visible(timeout=1200):
+                        card = candidate.first
+                        break
+                except Exception:
+                    pass
+            if card is not None:
+                break
+        except Exception:
+            pass
+    if card is None:
+        raise RuntimeError(f"{return_label} card was not found for period {period} after GST Search.")
+
+    actions = re.compile(r"PREPARE\s+ONLINE|VIEW|VIEW\s*/\s*DOWNLOAD|DOWNLOAD", re.I)
+    controls = card.locator("button:visible, a:visible")
+    chosen = None
+    try:
+        count = controls.count()
+    except Exception:
+        count = 0
+    for i in range(count):
+        control = controls.nth(i)
+        try:
+            txt = (control.inner_text(timeout=1000) or "").strip()
+        except Exception:
+            txt = ""
+        if actions.search(txt):
+            chosen = control
+            break
+    if chosen is None:
+        raise RuntimeError(f"{return_label} action button (Prepare Online/View/Download) was not found for period {period}.")
+
+    before = page.url
+    progress(stage=f"Opening {return_label} {period}", message=f"Opening {return_label} {period} through the GST portal card.")
+    show_status_overlay(page, f"Opening {return_label} {period} through GST portal...", "#0f766e")
+    chosen.click(timeout=15000)
+    try:
+        page.wait_for_load_state("domcontentloaded", timeout=30000)
+    except Exception:
+        pass
+    page.wait_for_timeout(2200)
+    if is_access_denied_page(page):
+        raise RuntimeError(f"GST returned Access Denied while opening {return_label} {period} naturally.")
+    if page.url == before:
+        # Angular routes can keep the same document URL while rendering the module.
+        _log(f"{return_label} card action remained on the same URL; continuing with the initialized page context.")
+
+
+def _poll_generated_download_urls(page, period: str, return_type: str, initial_payload, progress, *, wait_seconds: int = 1500, interval_seconds: int = 60) -> tuple[list[str], object]:
+    """Wait for GST offline generation instead of treating 'wait 20 min' as failure."""
+    payload = initial_payload
+    urls = _extract_download_urls(payload)
+    file_num = "1"
+    try:
+        file_num = extract_file_num(json.dumps(payload, ensure_ascii=False)) or "1"
+    except Exception:
+        file_num = "1"
+    if urls:
+        return urls, payload
+
+    deadline = time.time() + max(60, int(wait_seconds))
+    download_url = f"https://return.gst.gov.in/returns/auth/api/offline/download/url?rtn_prd={period}&rtn_typ={return_type}&file_num={file_num}"
+    last_message = _gst_response_message(payload)
+    while time.time() < deadline:
+        remaining = max(0, int(deadline - time.time()))
+        progress(
+            stage=f"Waiting for {return_type} {period}",
+            message=f"GST is preparing {return_type} {period}. Checking again in {interval_seconds} seconds; up to {remaining // 60 + 1} minute(s) remain.",
+        )
+        show_status_overlay(page, f"{return_type} {period}: GST accepted generation. Waiting for the file; next check in {interval_seconds} seconds...", "#b45309")
+        page.wait_for_timeout(interval_seconds * 1000)
+        try:
+            response_text = page_fetch_text(page, download_url, referer=RETURN_DASHBOARD_URL)
+            candidate = validate_gst_json(response_text, f"{return_type} download status {period}", allow_empty=True)
+            candidate_urls = _extract_download_urls(candidate)
+            if candidate_urls:
+                return candidate_urls, candidate
+            message = _gst_response_message(candidate)
+            if message:
+                last_message = message
+        except Exception as exc:
+            _log(f"{return_type} download-status poll warning for {period}: {exc}")
+        # Re-read generation metadata occasionally because GST variants sometimes
+        # expose the final URL there instead of the /url endpoint.
+        try:
+            generate_url = GSTR2B_OFFLINE_URL.format(period=period) if return_type == "GSTR2B" else GSTR1_URL.format(period=period)
+            response_text = page_fetch_text(page, generate_url, referer=RETURN_DASHBOARD_URL)
+            candidate = validate_gst_json(response_text, f"{return_type} generation status {period}", allow_empty=True)
+            candidate_urls = _extract_download_urls(candidate)
+            if candidate_urls:
+                return candidate_urls, candidate
+            message = _gst_response_message(candidate)
+            if message:
+                last_message = message
+        except Exception as exc:
+            _log(f"{return_type} generation-status poll warning for {period}: {exc}")
+
+    suffix = f" Last GST message: {last_message}" if last_message else ""
+    raise RuntimeError(f"{return_type} {period}: GST did not make the generated file available within 25 minutes.{suffix}")
+
+
+def download_gstr1_natural_ui(page, raw_dir: Path, period: str, progress) -> dict:
+    open_return_card_naturally(page, period, "GSTR-1", progress)
+    response_text = page_fetch_text(page, GSTR1_URL.format(period=period), referer=page.url or RETURN_DASHBOARD_URL)
+    response_data = validate_gst_json(response_text, f"GSTR-1 generate {period}", allow_empty=True)
+    (raw_dir / f"{period}_GSTR1_GENERATE.json").write_text(json.dumps(response_data, ensure_ascii=False, indent=2), encoding="utf-8")
+    urls, final_meta = _poll_generated_download_urls(page, period, "GSTR1", response_data, progress)
+    files: list[Path] = []
+    for index, url in enumerate(urls, 1):
+        body, ctype = fetch_binary(page, url, referer=page.url or RETURN_DASHBOARD_URL)
+        files.extend(_store_gst_download_blob(raw_dir, period, "GSTR1", index, body, ctype))
+    json_files = [p for p in files if p.suffix.lower() == ".json"]
+    primary = json_files[0] if json_files else (files[0] if files else None)
+    if primary is None:
+        raise RuntimeError(f"GSTR-1 {period}: GST download completed but no file was saved.")
+    return {"return_type": "GSTR1", "path": str(primary.relative_to(raw_dir.parent.parent)), "bytes": sum(p.stat().st_size for p in files if p.exists()), "files": [str(p.relative_to(raw_dir.parent.parent)) for p in files], "source": "natural_return_card_then_offline_download"}
+
+
+def download_gstr3b_natural_ui(page, raw_dir: Path, period: str, progress) -> dict:
+    open_return_card_naturally(page, period, "GSTR-3B", progress)
+    progress(stage=f"Reading GSTR-3B {period}", message=f"Reading GSTR-3B {period} after the GST portal initialized that return page.")
+    summary = page_fetch_text(page, GSTR3B_SUMMARY_URL.format(period=period), referer=page.url or RETURN_DASHBOARD_URL)
+    payable = page_fetch_text(page, GSTR3B_URL.format(period=period), referer=page.url or RETURN_DASHBOARD_URL)
+    validate_gst_json(summary, f"GSTR-3B summary {period}", allow_empty=True)
+    validate_gst_json(payable, f"GSTR-3B tax payable {period}", allow_empty=True)
+    (raw_dir / f"{period}_GSTR3B_SUMMARY.json").write_text(summary, encoding="utf-8")
+    (raw_dir / f"{period}_GSTR3B_TAXPAYBLE.json").write_text(payable, encoding="utf-8")
+    try:
+        combined = json.loads(summary or "{}")
+    except Exception:
+        combined = {"summary_raw": summary}
+    if not isinstance(combined, dict):
+        combined = {"summary": combined}
+    try:
+        combined["taxpayble"] = json.loads(payable or "{}")
+    except Exception:
+        combined["taxpayble"] = {"raw": payable}
+    path = raw_dir / f"{period}_GSTR3B.json"
+    path.write_text(json.dumps(combined, ensure_ascii=False, indent=2), encoding="utf-8")
+    return {"return_type": "GSTR3B", "path": str(path.relative_to(raw_dir.parent.parent)), "bytes": path.stat().st_size, "source": "natural_return_card_then_gstr3b_page_api"}
+
 def download_gstr2b_octagst_style(page, raw_dir: Path, period: str, progress) -> dict:
     """Download GSTR-2B from Return Dashboard without entering gstr2b.gst.gov.in.
 
@@ -617,10 +868,11 @@ def download_gstr2b_octagst_style(page, raw_dir: Path, period: str, progress) ->
     urls = _extract_download_urls(response_data)
     if not urls:
         message = _gst_response_message(response_data)
-        suffix = f" GST message: {message}" if message else ""
-        raise RuntimeError(
-            f"GSTR-2B {period}: GST did not return a downloadable file URL from the Return Dashboard API.{suffix}"
+        _log(f"GSTR-2B {period} generation pending. GST message: {message}")
+        urls, response_data = _poll_generated_download_urls(
+            page, period, "GSTR2B", response_data, progress, wait_seconds=1500, interval_seconds=60
         )
+        metadata_path.write_text(json.dumps(response_data, ensure_ascii=False, indent=2), encoding="utf-8")
 
     files: list[Path] = []
     for index, url in enumerate(urls, 1):
@@ -661,50 +913,18 @@ def download_period(page, work_root: Path, period: str, return_types: list[str],
     selected = {str(x or "").upper() for x in return_types}
     downloaded: list[dict] = []
 
-    # GSTR-1, GSTR-3B and GSTR-2A use the return.gst.gov.in session.
-    # Always confirm/restore Return Dashboard before each of those calls.
+    # GSTR-1 and GSTR-3B are opened through the actual Return Dashboard card
+    # for the selected month before reading their return data. GSTR-2B retains
+    # the working Return Dashboard offline-generation flow with pending polling.
     if "GSTR1" in selected:
-        return_to_return_dashboard(page, progress)
-        progress(stage=f"Downloading GSTR-1 {period}", message=f"Generating and downloading GSTR-1 for {period}.")
-        generated = page_fetch_text(page, GSTR1_URL.format(period=period), referer=RETURN_DASHBOARD_URL)
-        validate_gst_json(generated, f"GSTR-1 generate {period}")
-        (raw_dir / f"{period}_GSTR1_GENERATE.json").write_text(generated, encoding="utf-8")
-        file_num = extract_file_num(generated)
-        content = extract_payload(page_fetch_text(page, GSTR1_DOWNLOAD_URL.format(period=period, file_num=file_num)))
-        validate_gst_json(content, f"GSTR-1 download {period}")
-        path = raw_dir / f"{period}_GSTR1.json"
-        path.write_text(content, encoding="utf-8")
-        downloaded.append({"return_type": "GSTR1", "path": str(path.relative_to(work_root)), "bytes": path.stat().st_size})
+        downloaded.append(download_gstr1_natural_ui(page, raw_dir, period, progress))
 
     if "GSTR2B" in selected:
-        # OctaGST-style path: remain on the authenticated Return Dashboard and use
-        # the return-domain offline-download API.  Do not transfer the browser to
-        # gstr2b.gst.gov.in, which is the step GST rejected with Access Denied.
         return_to_return_dashboard(page, progress)
         downloaded.append(download_gstr2b_octagst_style(page, raw_dir, period, progress))
 
     if "GSTR3B" in selected:
-        return_to_return_dashboard(page, progress)
-        progress(stage=f"Downloading GSTR-3B {period}", message=f"Downloading GSTR-3B summary and tax payable for {period}.")
-        summary = page_fetch_text(page, GSTR3B_SUMMARY_URL.format(period=period), referer=RETURN_DASHBOARD_URL)
-        payable = page_fetch_text(page, GSTR3B_URL.format(period=period), referer=RETURN_DASHBOARD_URL)
-        validate_gst_json(summary, f"GSTR-3B summary {period}", allow_empty=True)
-        validate_gst_json(payable, f"GSTR-3B tax payable {period}", allow_empty=True)
-        (raw_dir / f"{period}_GSTR3B_SUMMARY.json").write_text(summary, encoding="utf-8")
-        (raw_dir / f"{period}_GSTR3B_TAXPAYBLE.json").write_text(payable, encoding="utf-8")
-        try:
-            combined = json.loads(summary or "{}")
-        except Exception:
-            combined = {"summary_raw": summary}
-        if not isinstance(combined, dict):
-            combined = {"summary": combined}
-        try:
-            combined["taxpayble"] = json.loads(payable or "{}")
-        except Exception:
-            combined["taxpayble"] = {"raw": payable}
-        path = raw_dir / f"{period}_GSTR3B.json"
-        path.write_text(json.dumps(combined, ensure_ascii=False, indent=2), encoding="utf-8")
-        downloaded.append({"return_type": "GSTR3B", "path": str(path.relative_to(work_root)), "bytes": path.stat().st_size})
+        downloaded.append(download_gstr3b_natural_ui(page, raw_dir, period, progress))
 
     if "GSTR2A" in selected:
         return_to_return_dashboard(page, progress)
@@ -724,7 +944,7 @@ def download_period(page, work_root: Path, period: str, return_types: list[str],
 
     write_json(
         work_root / period / "download_manifest.json",
-        {"period": period, "downloaded_at_utc": now(), "source": "arrr_gst_operator_agent_1.5.0", "downloaded": downloaded},
+        {"period": period, "downloaded_at_utc": now(), "source": "arrr_gst_operator_agent_1.5.1", "downloaded": downloaded},
     )
     return downloaded
 

app/modules/accounting/gst_operator_agent_runtime/install_gst_operator_agent.ps1

@@ -9,7 +9,7 @@ $Startup = Join-Path $env:APPDATA "Microsoft\Windows\Start Menu\Programs\Startup
 $ThumbprintFile = Join-Path $InstallRoot "cert_thumbprint.txt"
 $ProtocolKey = "HKCU:\Software\Classes\arrrgst"
 
-Write-Host "ARRR GST Operator Agent 1.5.0 - clean installation" -ForegroundColor Cyan
+Write-Host "ARRR GST Operator Agent 1.5.1 - clean installation" -ForegroundColor Cyan
 Write-Host "Install root: $InstallRoot"
 
 # Clean legacy v1.0.x listener/startup/certificate/protocol state first.

app/modules/accounting/gst_reconciliation_ui.py

@@ -147,7 +147,7 @@ router = APIRouter(prefix="/tools/accounting/gst-reconciliation", tags=["account
 _TOKEN_PURPOSE = "gst_lightweight_operator_v3"
 _TOKEN_MINUTES = 15
 _UPLOAD_ROOT = Path(tempfile.gettempdir()) / "audit_firm_gst_operator_uploads"
-_OPERATOR_AGENT_VERSION = "1.5.0"
+_OPERATOR_AGENT_VERSION = "1.5.1"
 _OPERATOR_RUNTIME_ROOT = Path(__file__).resolve().parent / "gst_operator_agent_runtime"
 _OPERATOR_PACKAGE_FILES = ("gst_operator_agent.py", "requirements.txt", "README.txt", "install_gst_operator_agent.ps1", "uninstall_gst_operator_agent.ps1")