From 4d0c53b9ba5d90723f649bb7603d648094db0f80 Mon Sep 17 00:00:00 2001 From: A R R R Associates Date: Sat, 12 Sep 2026 12:35:44 +0530 Subject: [PATCH] Use authenticated GST dashboard XHR for return downloads --- .../gst_operator_agent_runtime/README.txt | 25 +-- .../gst_operator_agent.py | 196 ++++++++++++++---- .../install_gst_operator_agent.ps1 | 2 +- .../accounting/gst_reconciliation_ui.py | 2 +- 4 files changed, 172 insertions(+), 53 deletions(-) diff --git a/app/modules/accounting/gst_operator_agent_runtime/README.txt b/app/modules/accounting/gst_operator_agent_runtime/README.txt index 2760417..15eaccb 100644 --- a/app/modules/accounting/gst_operator_agent_runtime/README.txt +++ b/app/modules/accounting/gst_operator_agent_runtime/README.txt @@ -1,4 +1,4 @@ -ARRR GST Operator Agent 1.4.0 +ARRR GST Operator Agent 1.5.0 Purpose ------- @@ -25,19 +25,12 @@ Uninstall Run uninstall_gst_operator_agent.ps1. It removes the protocol registration and also cleans legacy v1.0.x localhost listener/startup/certificate state. -V1.4.0 VERIFIED POST-DASHBOARD DOWNLOAD SEQUENCE -- After CAPTCHA/OTP login, do not deep-link to return pages. -- Close optional Aadhaar/E-KYC reminder. -- From the authenticated GST Welcome page click Return Dashboard exactly as a user would. -- Wait for return.gst.gov.in /returns/auth/ session to load. -- Only then call GSTR-1/GSTR-2B/GSTR-3B APIs using the authenticated browser context. +V1.5.0 OCTAGST-STYLE RETURN-DASHBOARD EXTRACTION +- Preserves the v1.4.0 Credential Vault login, manual CAPTCHA/OTP, natural Welcome -> Return Dashboard flow, ERP upload and failure diagnostics. +- GSTR-2B no longer transfers the browser to gstr2b.gst.gov.in. +- GSTR-2B is requested from the authenticated return.gst.gov.in Return Dashboard using the GST offline-download API pattern used by the supplied OctaGST extension. +- The agent follows GST-provided download URLs, saves returned ZIP/JSON files, and safely extracts JSON members locally. +- Same-origin GSTR-1 and GSTR-3B API calls remain unchanged. +- No automatic GST logout is performed. +- A job is completed only after every selected period/return has a saved result and ERP/client-storage upload is confirmed. - Manual JSON/ZIP import remains available in ERP as fallback. - - -V1.4.0 POST-DASHBOARD REPAIR -- Does not mark a job complete merely because Return Dashboard opened. -- Requires an actual saved JSON result for every selected return and period. -- GSTR-2B now enters its module only after the natural Return Dashboard session is established, then calls the GSTR-2B API same-origin. -- Does not call GST logout after completion. -- On failure the browser remains visible for 45 seconds with the exact failure message. -- Upload to ERP/client storage happens only after all expected return files are present. diff --git a/app/modules/accounting/gst_operator_agent_runtime/gst_operator_agent.py b/app/modules/accounting/gst_operator_agent_runtime/gst_operator_agent.py index f28e8ed..b28be86 100644 --- a/app/modules/accounting/gst_operator_agent_runtime/gst_operator_agent.py +++ b/app/modules/accounting/gst_operator_agent_runtime/gst_operator_agent.py @@ -1,5 +1,6 @@ from __future__ import annotations +import io import json import re import shutil @@ -9,11 +10,11 @@ import traceback import zipfile from datetime import datetime, timezone from pathlib import Path -from urllib.parse import parse_qs, urlsplit +from urllib.parse import parse_qs, urljoin, urlsplit import requests -VERSION = "1.4.0" +VERSION = "1.5.0" ROOT = Path(__file__).resolve().parent DATA = ROOT / "data" CONFIG_PATH = ROOT / "config.json" @@ -26,6 +27,7 @@ GSTR1_URL = "https://return.gst.gov.in/returns/auth/api/offline/download/generat GSTR2A_URL = "https://return.gst.gov.in/returns/auth/api/offline/download/generate?flag=0&rtn_prd={period}&rtn_typ=GSTR2A" GSTR1_DOWNLOAD_URL = "https://return.gst.gov.in/returns/auth/api/offline/download/url?rtn_prd={period}&rtn_typ=GSTR1&file_num={file_num}" GSTR2B_URL = "https://gstr2b.gst.gov.in/gstr2b/auth/api/gstr2b/getjson?rtnprd={period}" +GSTR2B_OFFLINE_URL = "https://return.gst.gov.in/returns/auth/api/offline/download/generate?flag=0&rtn_prd={period}&rtn_typ=GSTR2B" ROLESTATUS_URL = "https://return.gst.gov.in/returns/auth/api/rolestatus?rtn_prd={period}" GSTR3B_SUMMARY_URL = "https://return.gst.gov.in/returns/auth/api/gstr3b/summary?rtn_prd={period}" GSTR3B_URL = "https://return.gst.gov.in/returns/auth/api/gstr3b/taxpayble?rtn_prd={period}" @@ -486,32 +488,162 @@ def return_to_return_dashboard(page, progress) -> None: raise RuntimeError(f"Could not return to GST Return Dashboard. Current page: {page.url}") -def open_gstr2b_module_from_established_session(page, progress) -> None: - """Enter GSTR-2B only after Return Dashboard has created the return-domain session. +def _extract_download_urls(payload) -> list[str]: + """Extract GST offline-download URLs from the return API response. - The failed v9 path called the GSTR-2B API cross-domain directly from the Return - Dashboard. This version first establishes the normal Return Dashboard session, - then opens the GSTR-2B module page and performs the JSON call same-origin from - that page, matching the browser-session pattern used by the working GST tools. + OctaGST uses the return.gst.gov.in offline download API from the already + authenticated Return Dashboard and then downloads the URLs returned in + data.url. Keep this parser tolerant of GST response-shape changes. + """ + found: list[str] = [] + + def visit(value): + if isinstance(value, dict): + for key, item in value.items(): + if str(key).lower() in {"url", "urls", "downloadurl", "download_url"}: + if isinstance(item, str) and item.strip(): + found.append(item.strip()) + elif isinstance(item, list): + for x in item: + if isinstance(x, str) and x.strip(): + found.append(x.strip()) + visit(item) + elif isinstance(value, list): + for item in value: + visit(item) + + visit(payload) + result = [] + seen = set() + for item in found: + absolute = urljoin(RETURN_DASHBOARD_URL, item) + if absolute not in seen: + seen.add(absolute) + result.append(absolute) + return result + + +def _gst_response_message(payload) -> str: + """Return the most useful short GST message for operator diagnostics.""" + candidates = [] + for obj in walk(payload): + if not isinstance(obj, dict): + continue + for key in ("message", "msg", "error", "status_desc", "statusDesc", "desc", "description"): + value = obj.get(key) + if isinstance(value, str) and value.strip(): + candidates.append(value.strip()) + return candidates[0][:500] if candidates else "" + + +def fetch_binary(page, url: str, *, referer: str = RETURN_DASHBOARD_URL) -> tuple[bytes, str]: + """Download a GST-generated file using the authenticated browser context.""" + headers = {"Accept": "*/*", "Referer": referer} + try: + response = page.context.request.get(url, headers=headers, timeout=120000) + except Exception as exc: + raise RuntimeError(f"GST generated-file download failed: {exc}") from exc + status = int(response.status or 0) + body = response.body() or b"" + ctype = (response.headers.get("content-type") or "").lower() + if status >= 400: + sample = body[:500].decode("utf-8", errors="replace") + raise RuntimeError(f"GST generated-file download HTTP {status}: {sample}") + if not body: + raise RuntimeError("GST generated-file download returned an empty file.") + return body, ctype + + +def _store_gst_download_blob(raw_dir: Path, period: str, return_type: str, index: int, body: bytes, ctype: str) -> list[Path]: + """Persist a GST JSON/ZIP response and safely extract JSON members from ZIPs.""" + stored: list[Path] = [] + is_zip = body[:4] == b"PK\x03\x04" or "zip" in (ctype or "") + if is_zip: + zip_path = raw_dir / f"{period}_{return_type}_{index}.zip" + zip_path.write_bytes(body) + stored.append(zip_path) + try: + with zipfile.ZipFile(io.BytesIO(body)) as archive: + for member in archive.infolist(): + if member.is_dir() or not member.filename.lower().endswith(".json"): + continue + data = archive.read(member) + if not data: + continue + name = safe(Path(member.filename).name, f"{period}_{return_type}_{index}.json") + target = raw_dir / name + if target.exists(): + target = raw_dir / f"{target.stem}_{index}{target.suffix}" + target.write_bytes(data) + stored.append(target) + except zipfile.BadZipFile as exc: + raise RuntimeError("GST returned a file marked as ZIP but it could not be opened.") from exc + return stored + + text = body.decode("utf-8", errors="replace").strip() + try: + parsed = json.loads(text) + except Exception as exc: + raise RuntimeError(f"GST generated file was neither ZIP nor valid JSON: {text[:300]}") from exc + if parsed in ({}, [], None, ""): + raise RuntimeError("GST generated JSON file was empty.") + json_path = raw_dir / f"{period}_{return_type}_{index}.json" + json_path.write_text(json.dumps(parsed, ensure_ascii=False, indent=2), encoding="utf-8") + stored.append(json_path) + return stored + + +def download_gstr2b_octagst_style(page, raw_dir: Path, period: str, progress) -> dict: + """Download GSTR-2B from Return Dashboard without entering gstr2b.gst.gov.in. + + The supplied OctaGST extension performs return retrieval from the authenticated + Return Dashboard by XHR against return.gst.gov.in and then follows the + generated download URLs. Its GSTR-2B configuration uses apiCode GSTR2B with + the same offline-download endpoint. This avoids the cross-subdomain transfer + that GST is currently rejecting with Access Denied. """ verify_return_dashboard_session(page) - progress(stage="Opening GSTR-2B Module", message="Return Dashboard is ready. Opening the GST GSTR-2B module before requesting JSON.") - show_status_overlay(page, "Return Dashboard ready. Opening GSTR-2B module...", "#0f766e") - try: - page.goto(GSTR2B_PAGE_URL, wait_until="domcontentloaded", timeout=90000) - except Exception as exc: - raise RuntimeError(f"Could not enter the GSTR-2B module from the established Return Dashboard session: {exc}") from exc - page.wait_for_timeout(2500) - if is_access_denied_page(page): - raise RuntimeError( - "GST returned Access Denied while transferring from Return Dashboard to GSTR-2B. " - "The browser has been kept open temporarily for inspection." - ) - url = (page.url or "").lower() - if "gstr2b.gst.gov.in" not in url: - raise RuntimeError(f"GST did not enter the GSTR-2B module. Current page: {page.url}") - show_status_overlay(page, "GSTR-2B module ready. Downloading JSON...", "#15803d") + progress( + stage=f"Downloading GSTR-2B {period}", + message=f"Requesting GSTR-2B {period} directly from the authenticated Return Dashboard session.", + ) + show_status_overlay(page, f"GSTR-2B {period}: requesting return file from Return Dashboard...", "#0f766e") + response_text = page_fetch_text(page, GSTR2B_OFFLINE_URL.format(period=period), referer=RETURN_DASHBOARD_URL) + response_data = validate_gst_json(response_text, f"GSTR-2B file request {period}") + metadata_path = raw_dir / f"{period}_GSTR2B_DOWNLOAD_META.json" + metadata_path.write_text(json.dumps(response_data, ensure_ascii=False, indent=2), encoding="utf-8") + + urls = _extract_download_urls(response_data) + if not urls: + message = _gst_response_message(response_data) + suffix = f" GST message: {message}" if message else "" + raise RuntimeError( + f"GSTR-2B {period}: GST did not return a downloadable file URL from the Return Dashboard API.{suffix}" + ) + + files: list[Path] = [] + for index, url in enumerate(urls, 1): + progress( + stage=f"Downloading GSTR-2B {period}", + message=f"Downloading GSTR-2B file {index}/{len(urls)} for {period}.", + ) + body, ctype = fetch_binary(page, url, referer=RETURN_DASHBOARD_URL) + files.extend(_store_gst_download_blob(raw_dir, period, "GSTR2B", index, body, ctype)) + + json_files = [p for p in files if p.suffix.lower() == ".json"] + primary = json_files[0] if json_files else (files[0] if files else None) + if primary is None: + raise RuntimeError(f"GSTR-2B {period}: GST download completed but no file was saved.") + + show_status_overlay(page, f"GSTR-2B {period} downloaded successfully from Return Dashboard.", "#15803d") + return { + "return_type": "GSTR2B", + "path": str(primary.relative_to(raw_dir.parent.parent)), + "bytes": sum(p.stat().st_size for p in files if p.exists()), + "files": [str(p.relative_to(raw_dir.parent.parent)) for p in files], + "source": "return_dashboard_offline_download", + } def keep_browser_visible(page, message: str, color: str, milliseconds: int) -> None: """Keep the visible GST browser available long enough to see success/failure.""" @@ -545,17 +677,11 @@ def download_period(page, work_root: Path, period: str, return_types: list[str], downloaded.append({"return_type": "GSTR1", "path": str(path.relative_to(work_root)), "bytes": path.stat().st_size}) if "GSTR2B" in selected: - # Important: do not call GSTR-2B cross-domain directly from Return Dashboard. - # Establish Return Dashboard first, enter the GSTR-2B module, then call its API - # from the GSTR-2B page so the module/session cookies and origin are correct. + # OctaGST-style path: remain on the authenticated Return Dashboard and use + # the return-domain offline-download API. Do not transfer the browser to + # gstr2b.gst.gov.in, which is the step GST rejected with Access Denied. return_to_return_dashboard(page, progress) - open_gstr2b_module_from_established_session(page, progress) - progress(stage=f"Downloading GSTR-2B {period}", message=f"Downloading GSTR-2B JSON for {period} from the established GSTR-2B session.") - content = page_fetch_text(page, GSTR2B_URL.format(period=period), referer=GSTR2B_PAGE_URL) - validate_gst_json(content, f"GSTR-2B {period}") - path = raw_dir / f"{period}_GSTR2B.json" - path.write_text(content, encoding="utf-8") - downloaded.append({"return_type": "GSTR2B", "path": str(path.relative_to(work_root)), "bytes": path.stat().st_size}) + downloaded.append(download_gstr2b_octagst_style(page, raw_dir, period, progress)) if "GSTR3B" in selected: return_to_return_dashboard(page, progress) @@ -598,7 +724,7 @@ def download_period(page, work_root: Path, period: str, return_types: list[str], write_json( work_root / period / "download_manifest.json", - {"period": period, "downloaded_at_utc": now(), "source": "arrr_gst_operator_agent_1.4.0", "downloaded": downloaded}, + {"period": period, "downloaded_at_utc": now(), "source": "arrr_gst_operator_agent_1.5.0", "downloaded": downloaded}, ) return downloaded diff --git a/app/modules/accounting/gst_operator_agent_runtime/install_gst_operator_agent.ps1 b/app/modules/accounting/gst_operator_agent_runtime/install_gst_operator_agent.ps1 index ae00751..1dfba4a 100644 --- a/app/modules/accounting/gst_operator_agent_runtime/install_gst_operator_agent.ps1 +++ b/app/modules/accounting/gst_operator_agent_runtime/install_gst_operator_agent.ps1 @@ -9,7 +9,7 @@ $Startup = Join-Path $env:APPDATA "Microsoft\Windows\Start Menu\Programs\Startup $ThumbprintFile = Join-Path $InstallRoot "cert_thumbprint.txt" $ProtocolKey = "HKCU:\Software\Classes\arrrgst" -Write-Host "ARRR GST Operator Agent 1.4.0 - clean installation" -ForegroundColor Cyan +Write-Host "ARRR GST Operator Agent 1.5.0 - clean installation" -ForegroundColor Cyan Write-Host "Install root: $InstallRoot" # Clean legacy v1.0.x listener/startup/certificate/protocol state first. diff --git a/app/modules/accounting/gst_reconciliation_ui.py b/app/modules/accounting/gst_reconciliation_ui.py index bb4f453..4ffc70a 100644 --- a/app/modules/accounting/gst_reconciliation_ui.py +++ b/app/modules/accounting/gst_reconciliation_ui.py @@ -147,7 +147,7 @@ router = APIRouter(prefix="/tools/accounting/gst-reconciliation", tags=["account _TOKEN_PURPOSE = "gst_lightweight_operator_v3" _TOKEN_MINUTES = 15 _UPLOAD_ROOT = Path(tempfile.gettempdir()) / "audit_firm_gst_operator_uploads" -_OPERATOR_AGENT_VERSION = "1.4.0" +_OPERATOR_AGENT_VERSION = "1.5.0" _OPERATOR_RUNTIME_ROOT = Path(__file__).resolve().parent / "gst_operator_agent_runtime" _OPERATOR_PACKAGE_FILES = ("gst_operator_agent.py", "requirements.txt", "README.txt", "install_gst_operator_agent.ps1", "uninstall_gst_operator_agent.ps1")