diff --git a/app/modules/accounting/gst_operator_agent_runtime/README.txt b/app/modules/accounting/gst_operator_agent_runtime/README.txt index 15eaccb..f894485 100644 --- a/app/modules/accounting/gst_operator_agent_runtime/README.txt +++ b/app/modules/accounting/gst_operator_agent_runtime/README.txt @@ -1,4 +1,4 @@ -ARRR GST Operator Agent 1.5.0 +ARRR GST Operator Agent 1.5.1 Purpose ------- @@ -25,7 +25,7 @@ Uninstall Run uninstall_gst_operator_agent.ps1. It removes the protocol registration and also cleans legacy v1.0.x localhost listener/startup/certificate state. -V1.5.0 OCTAGST-STYLE RETURN-DASHBOARD EXTRACTION +V1.5.1 OCTAGST-STYLE RETURN-DASHBOARD EXTRACTION - Preserves the v1.4.0 Credential Vault login, manual CAPTCHA/OTP, natural Welcome -> Return Dashboard flow, ERP upload and failure diagnostics. - GSTR-2B no longer transfers the browser to gstr2b.gst.gov.in. - GSTR-2B is requested from the authenticated return.gst.gov.in Return Dashboard using the GST offline-download API pattern used by the supplied OctaGST extension. @@ -34,3 +34,12 @@ V1.5.0 OCTAGST-STYLE RETURN-DASHBOARD EXTRACTION - No automatic GST logout is performed. - A job is completed only after every selected period/return has a saved result and ERP/client-storage upload is confirmed. - Manual JSON/ZIP import remains available in ERP as fallback. + + +V1.5.1 NATURAL MONTHLY RETURN FLOW + GSTR-2B PENDING POLL +-------------------------------------------------------- +- Restores robust Aadhaar/E-KYC Remind me later / Later dismissal. +- GSTR-2B generation-accepted responses are PENDING, not failures; polls up to 25 minutes. +- GSTR-1: selects FY/Quarter/Month on Return Dashboard, clicks Search, opens the actual GSTR-1 card, then performs offline generation/download from the initialized page context. +- GSTR-3B: selects FY/Quarter/Month, clicks Search, opens the actual GSTR-3B card, then reads summary/tax-payable from the initialized return page context. +- Existing Credential Vault, custom protocol, login/CAPTCHA, manual fallback, ERP upload and Storage Agent flow are unchanged. diff --git a/app/modules/accounting/gst_operator_agent_runtime/gst_operator_agent.py b/app/modules/accounting/gst_operator_agent_runtime/gst_operator_agent.py index b28be86..db992c3 100644 --- a/app/modules/accounting/gst_operator_agent_runtime/gst_operator_agent.py +++ b/app/modules/accounting/gst_operator_agent_runtime/gst_operator_agent.py @@ -14,7 +14,7 @@ from urllib.parse import parse_qs, urljoin, urlsplit import requests -VERSION = "1.5.0" +VERSION = "1.5.1" ROOT = Path(__file__).resolve().parent DATA = ROOT / "data" CONFIG_PATH = ROOT / "config.json" @@ -269,20 +269,35 @@ def open_visible_page(page, url: str, label: str = "GST page", wait_ms: int = 18 def close_aadhaar_popup_if_visible(page) -> None: - """Close the optional GST Aadhaar / E-KYC reminder without disturbing login state.""" - for selector in ( + """Close the optional GST Aadhaar / E-KYC reminder using the proven V16 flow.""" + candidates = ( "a.btn.btn-primary:has-text('Remind me later')", + "button:has-text('Remind me later')", + "a:has-text('Remind me later')", "text=Remind me later", - ): + "button:has-text('LATER')", + "a:has-text('LATER')", + "text=LATER", + ) + for _ in range(8): + for selector in candidates: + try: + loc = page.locator(selector).first + if loc.is_visible(timeout=1200): + loc.scroll_into_view_if_needed(timeout=3000) + loc.click(timeout=10000, force=True) + page.wait_for_timeout(1400) + _log("Closed GST Aadhaar/E-KYC reminder popup using Later/Remind me later.") + return + except Exception: + pass try: - loc = page.locator(selector).first - if loc.is_visible(timeout=2500): - loc.click(timeout=10000) - page.wait_for_timeout(1200) - _log("Closed GST Aadhaar/E-KYC reminder popup.") + body = (page.locator("body").inner_text(timeout=1000) or "").lower() + if "aadhaar" not in body and "e-kyc" not in body and "remind me later" not in body: return except Exception: pass + page.wait_for_timeout(700) def is_logged_in_services_page(page) -> bool: @@ -593,6 +608,242 @@ def _store_gst_download_blob(raw_dir: Path, period: str, return_type: str, index return stored + +def _period_ui_parts(period: str) -> tuple[str, str, str]: + value = str(period or "").strip() + if not re.fullmatch(r"\d{6}", value): + raise ValueError(f"Invalid GST return period: {period!r}") + month = int(value[:2]) + year = int(value[2:]) + fy_start = year if month >= 4 else year - 1 + fy = f"{fy_start}-{str(fy_start + 1)[-2:]}" + month_name = datetime(year, month, 1).strftime("%B") + quarter_no = 1 if month in (4, 5, 6) else 2 if month in (7, 8, 9) else 3 if month in (10, 11, 12) else 4 + return fy, f"Quarter {quarter_no}", month_name + + +def _set_dashboard_select_by_text(page, preferred_text: str, *, contains: bool = False) -> bool: + """Select a visible Return Dashboard dropdown option by its displayed text.""" + target = preferred_text.strip().lower() + selects = page.locator("select:visible") + try: + count = selects.count() + except Exception: + count = 0 + for i in range(count): + select = selects.nth(i) + try: + options = select.locator("option").all_text_contents() + except Exception: + continue + for label in options: + norm = str(label or "").strip().lower() + match = target in norm if contains else norm == target + if not match: + continue + try: + select.select_option(label=label) + page.wait_for_timeout(400) + return True + except Exception: + try: + page.evaluate( + """({el,text})=>{const opt=[...el.options].find(o=>o.textContent.trim().toLowerCase()===text.toLowerCase()||o.textContent.trim().toLowerCase().includes(text.toLowerCase()));if(!opt)return false;el.value=opt.value;el.dispatchEvent(new Event('input',{bubbles:true}));el.dispatchEvent(new Event('change',{bubbles:true}));return true;}""", + {"el": select.element_handle(), "text": preferred_text}, + ) + page.wait_for_timeout(400) + return True + except Exception: + pass + return False + + +def configure_return_dashboard_period(page, period: str, progress) -> None: + return_to_return_dashboard(page, progress) + verify_return_dashboard_session(page) + fy, quarter, month_name = _period_ui_parts(period) + progress(stage=f"Selecting GST period {period}", message=f"Selecting {month_name} {period[2:]} on the GST Return Dashboard.") + show_status_overlay(page, f"Selecting {month_name} {period[2:]} on GST Return Dashboard...", "#0f766e") + if not _set_dashboard_select_by_text(page, fy): + raise RuntimeError(f"GST Return Dashboard Financial Year {fy} option was not found.") + if not _set_dashboard_select_by_text(page, quarter, contains=True): + raise RuntimeError(f"GST Return Dashboard {quarter} option was not found.") + if not _set_dashboard_select_by_text(page, month_name, contains=True): + raise RuntimeError(f"GST Return Dashboard month {month_name} option was not found.") + clicked = False + for selector in ("button:has-text('SEARCH')", "input[type='submit'][value*='SEARCH' i]", "text=SEARCH"): + try: + loc = page.locator(selector).first + if loc.is_visible(timeout=2500): + loc.click(timeout=10000) + clicked = True + break + except Exception: + pass + if not clicked: + raise RuntimeError("GST Return Dashboard Search button was not found.") + page.wait_for_timeout(2600) + if is_access_denied_page(page): + raise RuntimeError("GST returned Access Denied after Return Dashboard period Search.") + + +def open_return_card_naturally(page, period: str, return_label: str, progress) -> None: + """Open GSTR-1/GSTR-3B from the searched Return Dashboard card like a user.""" + configure_return_dashboard_period(page, period, progress) + label_variants = [return_label, return_label.replace("-", ""), return_label.replace("GSTR", "GSTR ")] + card = None + for label in label_variants: + try: + hit = page.get_by_text(re.compile(rf"^\s*{re.escape(label)}\s*$", re.I)).first + if not hit.is_visible(timeout=1800): + continue + for xpath in ( + "xpath=ancestor::tr[1]", + "xpath=ancestor::div[contains(@class,'card')][1]", + "xpath=ancestor::div[.//button or .//a][1]", + ): + try: + candidate = hit.locator(xpath) + if candidate.count() and candidate.first.is_visible(timeout=1200): + card = candidate.first + break + except Exception: + pass + if card is not None: + break + except Exception: + pass + if card is None: + raise RuntimeError(f"{return_label} card was not found for period {period} after GST Search.") + + actions = re.compile(r"PREPARE\s+ONLINE|VIEW|VIEW\s*/\s*DOWNLOAD|DOWNLOAD", re.I) + controls = card.locator("button:visible, a:visible") + chosen = None + try: + count = controls.count() + except Exception: + count = 0 + for i in range(count): + control = controls.nth(i) + try: + txt = (control.inner_text(timeout=1000) or "").strip() + except Exception: + txt = "" + if actions.search(txt): + chosen = control + break + if chosen is None: + raise RuntimeError(f"{return_label} action button (Prepare Online/View/Download) was not found for period {period}.") + + before = page.url + progress(stage=f"Opening {return_label} {period}", message=f"Opening {return_label} {period} through the GST portal card.") + show_status_overlay(page, f"Opening {return_label} {period} through GST portal...", "#0f766e") + chosen.click(timeout=15000) + try: + page.wait_for_load_state("domcontentloaded", timeout=30000) + except Exception: + pass + page.wait_for_timeout(2200) + if is_access_denied_page(page): + raise RuntimeError(f"GST returned Access Denied while opening {return_label} {period} naturally.") + if page.url == before: + # Angular routes can keep the same document URL while rendering the module. + _log(f"{return_label} card action remained on the same URL; continuing with the initialized page context.") + + +def _poll_generated_download_urls(page, period: str, return_type: str, initial_payload, progress, *, wait_seconds: int = 1500, interval_seconds: int = 60) -> tuple[list[str], object]: + """Wait for GST offline generation instead of treating 'wait 20 min' as failure.""" + payload = initial_payload + urls = _extract_download_urls(payload) + file_num = "1" + try: + file_num = extract_file_num(json.dumps(payload, ensure_ascii=False)) or "1" + except Exception: + file_num = "1" + if urls: + return urls, payload + + deadline = time.time() + max(60, int(wait_seconds)) + download_url = f"https://return.gst.gov.in/returns/auth/api/offline/download/url?rtn_prd={period}&rtn_typ={return_type}&file_num={file_num}" + last_message = _gst_response_message(payload) + while time.time() < deadline: + remaining = max(0, int(deadline - time.time())) + progress( + stage=f"Waiting for {return_type} {period}", + message=f"GST is preparing {return_type} {period}. Checking again in {interval_seconds} seconds; up to {remaining // 60 + 1} minute(s) remain.", + ) + show_status_overlay(page, f"{return_type} {period}: GST accepted generation. Waiting for the file; next check in {interval_seconds} seconds...", "#b45309") + page.wait_for_timeout(interval_seconds * 1000) + try: + response_text = page_fetch_text(page, download_url, referer=RETURN_DASHBOARD_URL) + candidate = validate_gst_json(response_text, f"{return_type} download status {period}", allow_empty=True) + candidate_urls = _extract_download_urls(candidate) + if candidate_urls: + return candidate_urls, candidate + message = _gst_response_message(candidate) + if message: + last_message = message + except Exception as exc: + _log(f"{return_type} download-status poll warning for {period}: {exc}") + # Re-read generation metadata occasionally because GST variants sometimes + # expose the final URL there instead of the /url endpoint. + try: + generate_url = GSTR2B_OFFLINE_URL.format(period=period) if return_type == "GSTR2B" else GSTR1_URL.format(period=period) + response_text = page_fetch_text(page, generate_url, referer=RETURN_DASHBOARD_URL) + candidate = validate_gst_json(response_text, f"{return_type} generation status {period}", allow_empty=True) + candidate_urls = _extract_download_urls(candidate) + if candidate_urls: + return candidate_urls, candidate + message = _gst_response_message(candidate) + if message: + last_message = message + except Exception as exc: + _log(f"{return_type} generation-status poll warning for {period}: {exc}") + + suffix = f" Last GST message: {last_message}" if last_message else "" + raise RuntimeError(f"{return_type} {period}: GST did not make the generated file available within 25 minutes.{suffix}") + + +def download_gstr1_natural_ui(page, raw_dir: Path, period: str, progress) -> dict: + open_return_card_naturally(page, period, "GSTR-1", progress) + response_text = page_fetch_text(page, GSTR1_URL.format(period=period), referer=page.url or RETURN_DASHBOARD_URL) + response_data = validate_gst_json(response_text, f"GSTR-1 generate {period}", allow_empty=True) + (raw_dir / f"{period}_GSTR1_GENERATE.json").write_text(json.dumps(response_data, ensure_ascii=False, indent=2), encoding="utf-8") + urls, final_meta = _poll_generated_download_urls(page, period, "GSTR1", response_data, progress) + files: list[Path] = [] + for index, url in enumerate(urls, 1): + body, ctype = fetch_binary(page, url, referer=page.url or RETURN_DASHBOARD_URL) + files.extend(_store_gst_download_blob(raw_dir, period, "GSTR1", index, body, ctype)) + json_files = [p for p in files if p.suffix.lower() == ".json"] + primary = json_files[0] if json_files else (files[0] if files else None) + if primary is None: + raise RuntimeError(f"GSTR-1 {period}: GST download completed but no file was saved.") + return {"return_type": "GSTR1", "path": str(primary.relative_to(raw_dir.parent.parent)), "bytes": sum(p.stat().st_size for p in files if p.exists()), "files": [str(p.relative_to(raw_dir.parent.parent)) for p in files], "source": "natural_return_card_then_offline_download"} + + +def download_gstr3b_natural_ui(page, raw_dir: Path, period: str, progress) -> dict: + open_return_card_naturally(page, period, "GSTR-3B", progress) + progress(stage=f"Reading GSTR-3B {period}", message=f"Reading GSTR-3B {period} after the GST portal initialized that return page.") + summary = page_fetch_text(page, GSTR3B_SUMMARY_URL.format(period=period), referer=page.url or RETURN_DASHBOARD_URL) + payable = page_fetch_text(page, GSTR3B_URL.format(period=period), referer=page.url or RETURN_DASHBOARD_URL) + validate_gst_json(summary, f"GSTR-3B summary {period}", allow_empty=True) + validate_gst_json(payable, f"GSTR-3B tax payable {period}", allow_empty=True) + (raw_dir / f"{period}_GSTR3B_SUMMARY.json").write_text(summary, encoding="utf-8") + (raw_dir / f"{period}_GSTR3B_TAXPAYBLE.json").write_text(payable, encoding="utf-8") + try: + combined = json.loads(summary or "{}") + except Exception: + combined = {"summary_raw": summary} + if not isinstance(combined, dict): + combined = {"summary": combined} + try: + combined["taxpayble"] = json.loads(payable or "{}") + except Exception: + combined["taxpayble"] = {"raw": payable} + path = raw_dir / f"{period}_GSTR3B.json" + path.write_text(json.dumps(combined, ensure_ascii=False, indent=2), encoding="utf-8") + return {"return_type": "GSTR3B", "path": str(path.relative_to(raw_dir.parent.parent)), "bytes": path.stat().st_size, "source": "natural_return_card_then_gstr3b_page_api"} + def download_gstr2b_octagst_style(page, raw_dir: Path, period: str, progress) -> dict: """Download GSTR-2B from Return Dashboard without entering gstr2b.gst.gov.in. @@ -617,10 +868,11 @@ def download_gstr2b_octagst_style(page, raw_dir: Path, period: str, progress) -> urls = _extract_download_urls(response_data) if not urls: message = _gst_response_message(response_data) - suffix = f" GST message: {message}" if message else "" - raise RuntimeError( - f"GSTR-2B {period}: GST did not return a downloadable file URL from the Return Dashboard API.{suffix}" + _log(f"GSTR-2B {period} generation pending. GST message: {message}") + urls, response_data = _poll_generated_download_urls( + page, period, "GSTR2B", response_data, progress, wait_seconds=1500, interval_seconds=60 ) + metadata_path.write_text(json.dumps(response_data, ensure_ascii=False, indent=2), encoding="utf-8") files: list[Path] = [] for index, url in enumerate(urls, 1): @@ -661,50 +913,18 @@ def download_period(page, work_root: Path, period: str, return_types: list[str], selected = {str(x or "").upper() for x in return_types} downloaded: list[dict] = [] - # GSTR-1, GSTR-3B and GSTR-2A use the return.gst.gov.in session. - # Always confirm/restore Return Dashboard before each of those calls. + # GSTR-1 and GSTR-3B are opened through the actual Return Dashboard card + # for the selected month before reading their return data. GSTR-2B retains + # the working Return Dashboard offline-generation flow with pending polling. if "GSTR1" in selected: - return_to_return_dashboard(page, progress) - progress(stage=f"Downloading GSTR-1 {period}", message=f"Generating and downloading GSTR-1 for {period}.") - generated = page_fetch_text(page, GSTR1_URL.format(period=period), referer=RETURN_DASHBOARD_URL) - validate_gst_json(generated, f"GSTR-1 generate {period}") - (raw_dir / f"{period}_GSTR1_GENERATE.json").write_text(generated, encoding="utf-8") - file_num = extract_file_num(generated) - content = extract_payload(page_fetch_text(page, GSTR1_DOWNLOAD_URL.format(period=period, file_num=file_num))) - validate_gst_json(content, f"GSTR-1 download {period}") - path = raw_dir / f"{period}_GSTR1.json" - path.write_text(content, encoding="utf-8") - downloaded.append({"return_type": "GSTR1", "path": str(path.relative_to(work_root)), "bytes": path.stat().st_size}) + downloaded.append(download_gstr1_natural_ui(page, raw_dir, period, progress)) if "GSTR2B" in selected: - # OctaGST-style path: remain on the authenticated Return Dashboard and use - # the return-domain offline-download API. Do not transfer the browser to - # gstr2b.gst.gov.in, which is the step GST rejected with Access Denied. return_to_return_dashboard(page, progress) downloaded.append(download_gstr2b_octagst_style(page, raw_dir, period, progress)) if "GSTR3B" in selected: - return_to_return_dashboard(page, progress) - progress(stage=f"Downloading GSTR-3B {period}", message=f"Downloading GSTR-3B summary and tax payable for {period}.") - summary = page_fetch_text(page, GSTR3B_SUMMARY_URL.format(period=period), referer=RETURN_DASHBOARD_URL) - payable = page_fetch_text(page, GSTR3B_URL.format(period=period), referer=RETURN_DASHBOARD_URL) - validate_gst_json(summary, f"GSTR-3B summary {period}", allow_empty=True) - validate_gst_json(payable, f"GSTR-3B tax payable {period}", allow_empty=True) - (raw_dir / f"{period}_GSTR3B_SUMMARY.json").write_text(summary, encoding="utf-8") - (raw_dir / f"{period}_GSTR3B_TAXPAYBLE.json").write_text(payable, encoding="utf-8") - try: - combined = json.loads(summary or "{}") - except Exception: - combined = {"summary_raw": summary} - if not isinstance(combined, dict): - combined = {"summary": combined} - try: - combined["taxpayble"] = json.loads(payable or "{}") - except Exception: - combined["taxpayble"] = {"raw": payable} - path = raw_dir / f"{period}_GSTR3B.json" - path.write_text(json.dumps(combined, ensure_ascii=False, indent=2), encoding="utf-8") - downloaded.append({"return_type": "GSTR3B", "path": str(path.relative_to(work_root)), "bytes": path.stat().st_size}) + downloaded.append(download_gstr3b_natural_ui(page, raw_dir, period, progress)) if "GSTR2A" in selected: return_to_return_dashboard(page, progress) @@ -724,7 +944,7 @@ def download_period(page, work_root: Path, period: str, return_types: list[str], write_json( work_root / period / "download_manifest.json", - {"period": period, "downloaded_at_utc": now(), "source": "arrr_gst_operator_agent_1.5.0", "downloaded": downloaded}, + {"period": period, "downloaded_at_utc": now(), "source": "arrr_gst_operator_agent_1.5.1", "downloaded": downloaded}, ) return downloaded diff --git a/app/modules/accounting/gst_operator_agent_runtime/install_gst_operator_agent.ps1 b/app/modules/accounting/gst_operator_agent_runtime/install_gst_operator_agent.ps1 index 1dfba4a..c18c388 100644 --- a/app/modules/accounting/gst_operator_agent_runtime/install_gst_operator_agent.ps1 +++ b/app/modules/accounting/gst_operator_agent_runtime/install_gst_operator_agent.ps1 @@ -9,7 +9,7 @@ $Startup = Join-Path $env:APPDATA "Microsoft\Windows\Start Menu\Programs\Startup $ThumbprintFile = Join-Path $InstallRoot "cert_thumbprint.txt" $ProtocolKey = "HKCU:\Software\Classes\arrrgst" -Write-Host "ARRR GST Operator Agent 1.5.0 - clean installation" -ForegroundColor Cyan +Write-Host "ARRR GST Operator Agent 1.5.1 - clean installation" -ForegroundColor Cyan Write-Host "Install root: $InstallRoot" # Clean legacy v1.0.x listener/startup/certificate/protocol state first. diff --git a/app/modules/accounting/gst_reconciliation_ui.py b/app/modules/accounting/gst_reconciliation_ui.py index 4ffc70a..eb6014d 100644 --- a/app/modules/accounting/gst_reconciliation_ui.py +++ b/app/modules/accounting/gst_reconciliation_ui.py @@ -147,7 +147,7 @@ router = APIRouter(prefix="/tools/accounting/gst-reconciliation", tags=["account _TOKEN_PURPOSE = "gst_lightweight_operator_v3" _TOKEN_MINUTES = 15 _UPLOAD_ROOT = Path(tempfile.gettempdir()) / "audit_firm_gst_operator_uploads" -_OPERATOR_AGENT_VERSION = "1.5.0" +_OPERATOR_AGENT_VERSION = "1.5.1" _OPERATOR_RUNTIME_ROOT = Path(__file__).resolve().parent / "gst_operator_agent_runtime" _OPERATOR_PACKAGE_FILES = ("gst_operator_agent.py", "requirements.txt", "README.txt", "install_gst_operator_agent.ps1", "uninstall_gst_operator_agent.ps1")