diff --git a/app/modules/accounting/gst_reconciliation_ui.py b/app/modules/accounting/gst_reconciliation_ui.py index e01444b..50e6c3b 100644 --- a/app/modules/accounting/gst_reconciliation_ui.py +++ b/app/modules/accounting/gst_reconciliation_ui.py @@ -2,16 +2,20 @@ from __future__ import annotations import calendar import re -from datetime import date +import tempfile +import uuid +from datetime import date, datetime, timedelta, timezone from pathlib import Path from urllib.parse import urlencode -from fastapi import APIRouter, Form, Request -from fastapi.responses import RedirectResponse +import jwt +from fastapi import APIRouter, File, Form, Request, UploadFile +from fastapi.responses import FileResponse, JSONResponse, RedirectResponse from sqlalchemy import select from app.core.db.common import CommonSessionLocal from app.core.security.csrf import get_or_create_csrf_token, validate_csrf +from app.core.settings import get_settings from app.core.templating import templates from app.modules.accounting.agent_bridge import request_agent_command from app.modules.accounting.ui import _find_visible_client, _require_partner, _visible_clients, _node_online @@ -19,10 +23,13 @@ from app.modules.core.rbac.deps import get_user_permissions, get_user_roles from app.modules.credential_vault.crypto import decrypt_value from app.modules.credential_vault.models import CredentialVaultEntry from app.modules.credential_vault.service import can_view_entry, log_access -from app.modules.documents.services import build_document_scope, client_folder_parts, get_active_storage_node_for_branch, sanitize_segment +from app.modules.documents.services import client_folder_parts, get_active_storage_node_for_branch, sanitize_segment from app.modules.registrations.models import ClientRegistration, RegistrationType router = APIRouter(prefix="/tools/accounting/gst-reconciliation", tags=["accounting-gst-reconciliation-ui"]) +_TOKEN_PURPOSE = "gst_operator_browser_v1" +_TOKEN_MINUTES = 30 +_UPLOAD_ROOT = Path(tempfile.gettempdir()) / "audit_firm_gst_operator_uploads" def _fy_bounds(fy: str) -> tuple[date, date]: @@ -33,6 +40,18 @@ def _fy_bounds(fy: str) -> tuple[date, date]: return date(y, 4, 1), date(y + 1, 3, 31) +def _periods_for_fy(fy: str) -> list[str]: + start, _ = _fy_bounds(fy) + periods=[] + y=start.year; m=4 + for _ in range(12): + periods.append(f"{m:02d}{y:04d}") + m += 1 + if m == 13: + m = 1; y += 1 + return periods + + def _fy_for_period(period: str) -> str: digits = re.sub(r"\D", "", period or "") if len(digits) != 6: @@ -75,24 +94,15 @@ def _credential_is_gst_portal(entry: CredentialVaultEntry) -> bool: title = str(entry.title or "").strip().lower() portal_url = str(entry.portal_url or "").strip().lower() reference = str(entry.reference_number or "").strip().lower() - if category == "gst_portal": return True - - # Older/client-level vault entries may have been saved under a generic category. - # Accept them only when the entry itself clearly identifies a GST portal login. haystack = " ".join((title, portal_url, reference)) gst_hint = ( - "gst portal" in haystack - or "gst login" in haystack - or "services.gst.gov.in" in portal_url - or "www.gst.gov.in" in portal_url + "gst portal" in haystack or "gst login" in haystack + or "services.gst.gov.in" in portal_url or "www.gst.gov.in" in portal_url ) if not gst_hint: return False - - # Do not silently substitute E-Way Bill, E-Invoice or API/provider secrets for - # a GST portal username/password unless the title explicitly says GST Portal/Login. if category in {"eway_bill", "einvoice", "api_key"}: return "gst portal" in title or "gst login" in title return True @@ -103,8 +113,6 @@ def _credential_matches_gstin(entry: CredentialVaultEntry, registration_id: int, return True if _norm_gstin(entry.reference_number) == _norm_gstin(gstin): return True - # Client-level GST Portal credentials created before registration-level vault - # linking remain valid candidates for the selected client's GST registration. return entry.registration_id is None @@ -117,25 +125,14 @@ def _vault_entries(db, user, request, tenant_id: int, client_id: int, registrati rows = db.execute(q.order_by(CredentialVaultEntry.title.asc())).scalars().all() branch_id = request.session.get("active_branch_id") or getattr(user, "branch_id", None) visible = [r for r in rows if can_view_entry(db, user, r, int(branch_id) if branch_id else None)] - if not registration_id: return [r for r in visible if _credential_is_gst_portal(r)] - - eligible = [ - r for r in visible - if _credential_is_gst_portal(r) and _credential_matches_gstin(r, int(registration_id), gstin) - ] - + eligible = [r for r in visible if _credential_is_gst_portal(r) and _credential_matches_gstin(r, int(registration_id), gstin)] def rank(entry: CredentialVaultEntry): exact_registration = int(entry.registration_id or 0) == int(registration_id) exact_gstin = _norm_gstin(entry.reference_number) == _norm_gstin(gstin) client_level = entry.registration_id is None - return ( - 0 if exact_registration else 1 if exact_gstin else 2 if client_level else 3, - str(entry.title or "").lower(), - int(entry.id or 0), - ) - + return (0 if exact_registration else 1 if exact_gstin else 2 if client_level else 3, str(entry.title or "").lower(), int(entry.id or 0)) return sorted(eligible, key=rank) @@ -143,9 +140,7 @@ def _storage_payload(client, fy: str, gstin: str) -> tuple[str, str]: fy_folder = sanitize_segment(f"FY{fy}", "FY") letter, client_folder = client_folder_parts(client, int(client.id)) root = Path(fy_folder) / "Clients" / letter / client_folder - accounting = root / "Accounting" - gst = root / "GST" / sanitize_segment(f"GSTIN_{gstin}", "GSTIN") - return accounting.as_posix(), gst.as_posix() + return (root / "Accounting").as_posix(), (root / "GST" / sanitize_segment(f"GSTIN_{gstin}", "GSTIN")).as_posix() def _redirect(client_id: int, **params): @@ -153,8 +148,30 @@ def _redirect(client_id: int, **params): return RedirectResponse("/tools/accounting/gst-reconciliation?" + urlencode(data), status_code=303) +def _encode_operator_token(payload: dict) -> str: + now = datetime.now(timezone.utc) + body = {**payload, "purpose": _TOKEN_PURPOSE, "iat": now, "exp": now + timedelta(minutes=_TOKEN_MINUTES)} + return jwt.encode(body, get_settings().SECRET_KEY, algorithm="HS256") + + +def _decode_operator_token(token: str) -> dict: + data = jwt.decode(token, get_settings().SECRET_KEY, algorithms=["HS256"]) + if data.get("purpose") != _TOKEN_PURPOSE: + raise ValueError("Invalid GST operator token.") + return data + + +def _selected_periods(download_mode: str, financial_year: str, period: str) -> list[str]: + if download_mode == "full_fy": + return _periods_for_fy(financial_year) + digits = re.sub(r"\D", "", period or "") + if len(digits) != 6: + raise ValueError("Enter a valid MMYYYY period for Single Month mode.") + return [digits] + + @router.get("") -def page(request: Request, client_id: int | None = None, registration_id: int | None = None, period: str = "", message: str = "", error: str = ""): +def page(request: Request, client_id: int | None = None, registration_id: int | None = None, period: str = "", financial_year: str = "", download_mode: str = "single", message: str = "", error: str = ""): db = CommonSessionLocal() try: user, response = _require_partner(request, db, "accounting.learning.view") @@ -162,40 +179,37 @@ def page(request: Request, client_id: int | None = None, registration_id: int | return response clients, scope = _visible_clients(db, request, user) selected = next((c for c in clients if client_id and int(c.id) == int(client_id)), None) - registrations=[]; selected_reg=None; credentials=[]; node=None; status={}; analysis={} + registrations=[]; selected_reg=None; credentials=[]; node=None + if not financial_year: + financial_year = str(request.session.get("active_financial_year") or request.session.get("active_fy") or "2025-26") + if download_mode not in {"single", "full_fy"}: + download_mode = "single" if selected: registrations=[(r,t) for r,t in _gst_regs(db,scope.tenant_id,selected.id) if _is_gstin(r,t)] selected_reg=next((r for r,t in registrations if registration_id and int(r.id)==int(registration_id)),None) if not selected_reg and registrations: selected_reg=registrations[0][0] selected_gstin = _norm_gstin(getattr(selected_reg, "registration_number", "")) if selected_reg else "" - credentials=_vault_entries( - db,user,request,scope.tenant_id,selected.id, - int(selected_reg.id) if selected_reg else None, selected_gstin, - ) + credentials=_vault_entries(db,user,request,scope.tenant_id,selected.id,int(selected_reg.id) if selected_reg else None, selected_gstin) node=get_active_storage_node_for_branch(db,scope.tenant_id,scope.branch_id) - if selected_reg and period and node and _node_online(node): - gstin=re.sub(r"\s+","",str(selected_reg.registration_number or "").upper()) - try: - status=(request_agent_command(node.node_code,"gst_return_download_status",{"client_id":selected.id,"gstin":gstin,"period":re.sub(r"\D","",period)},timeout_seconds=8).get("result") or {}).get("job") or {} - except Exception: - status={} - try: - job_result=status.get("result") or {} - # analysis is loaded only after an explicit Analyze action; status result is download manifest. - analysis={} - except Exception: - pass + operator_job = request.session.pop("gst_operator_job", None) return templates.TemplateResponse("modules/accounting/templates/accounting/gst_reconciliation.html",{ "request":request,"current_user":user,"current_user_roles":get_user_roles(db,user.id),"current_user_permissions":get_user_permissions(db,user.id),"csrf_token":get_or_create_csrf_token(request), - "clients":clients,"selected_client":selected,"registrations":registrations,"selected_registration":selected_reg,"credentials":credentials,"node":node,"node_online":_node_online(node) if node else False,"period":period,"status":status,"analysis":analysis,"message":message,"error":error,"title":"GST Return Reconciliation", + "clients":clients,"selected_client":selected,"registrations":registrations,"selected_registration":selected_reg,"credentials":credentials,"node":node,"node_online":_node_online(node) if node else False, + "period":period,"financial_year":financial_year,"download_mode":download_mode,"operator_job":operator_job,"message":message,"error":error,"title":"GST Return Reconciliation", }) finally: db.close() @router.post("/download/start") -def start_download(request: Request, client_id: int=Form(...), registration_id: int=Form(...), credential_id: int=Form(...), period: str=Form(...), include_2a: str=Form(""), csrf_token: str=Form(...)): +def start_download( + request: Request, + client_id: int=Form(...), registration_id: int=Form(...), credential_id: int=Form(...), + period: str=Form(""), financial_year: str=Form(...), download_mode: str=Form("single"), + gstr1: str=Form(""), gstr2b: str=Form(""), gstr3b: str=Form(""), gstr2a: str=Form(""), + csrf_token: str=Form(...), +): validate_csrf(request,csrf_token) db=CommonSessionLocal() try: @@ -205,36 +219,107 @@ def start_download(request: Request, client_id: int=Form(...), registration_id: if not client: return _redirect(client_id,error="Client is not available in your scope.") pair=next(((r,t) for r,t in _gst_regs(db,scope.tenant_id,client.id) if int(r.id)==registration_id and _is_gstin(r,t)),None) if not pair: return _redirect(client_id,error="Select a valid GSTIN registration.") - reg,_=pair; gstin=re.sub(r"\s+","",str(reg.registration_number or "").upper()) + reg,_=pair; gstin=_norm_gstin(reg.registration_number) cred=db.get(CredentialVaultEntry,credential_id) - credential_ok = bool( - cred - and int(cred.tenant_id or 0) == int(scope.tenant_id) - and int(cred.client_id or 0) == int(client.id) - and str(cred.status or "").lower() != "archived" - and can_view_entry(db,user,cred,scope.branch_id) - and _credential_is_gst_portal(cred) - and _credential_matches_gstin(cred,int(reg.id),gstin) - ) + credential_ok = bool(cred and int(cred.tenant_id or 0)==int(scope.tenant_id) and int(cred.client_id or 0)==int(client.id) and str(cred.status or "").lower()!="archived" and can_view_entry(db,user,cred,scope.branch_id) and _credential_is_gst_portal(cred) and _credential_matches_gstin(cred,int(reg.id),gstin)) if not credential_ok: - return _redirect(client_id,registration_id=registration_id,period=period,error="Selected GST Portal credential is not available for this client/GSTIN.") - username=decrypt_value(cred.tenant_id,cred.username_encrypted) or ""; password=decrypt_value(cred.tenant_id,cred.secret_encrypted) or "" - if not username or not password: return _redirect(client_id,registration_id=registration_id,period=period,error="GST username/password is missing in Credential Vault.") - fy=_fy_for_period(period); accounting_dir,gst_dir=_storage_payload(client,fy,gstin) + return _redirect(client_id,registration_id=registration_id,period=period,financial_year=financial_year,download_mode=download_mode,error="Selected GST Portal credential is not available for this client/GSTIN.") node=get_active_storage_node_for_branch(db,scope.tenant_id,scope.branch_id) - if not node or not _node_online(node): return _redirect(client_id,registration_id=registration_id,period=period,error="Local Storage Agent is offline.") - result=request_agent_command(node.node_code,"gst_return_download_start",{"client_id":client.id,"client_name":client.client_name,"gstin":gstin,"financial_year":fy,"period":re.sub(r"\D","",period),"gst_relative_dir":gst_dir,"accounting_relative_dir":accounting_dir,"username":username,"password":password,"include_2a":bool(include_2a),"login_timeout_seconds":900},timeout_seconds=15) - log_access(db,request,user,cred,"use_for_gst_download",reason=f"GST return download {period}",fields="username,secret",success=bool(result.get("ok"))) + if not node or not _node_online(node): + return _redirect(client_id,registration_id=registration_id,period=period,financial_year=financial_year,download_mode=download_mode,error="Local Storage Agent is offline.") + periods=_selected_periods(download_mode,financial_year,period) + return_types=[name for name,flag in (("GSTR1",gstr1),("GSTR2B",gstr2b),("GSTR3B",gstr3b),("GSTR2A",gstr2a)) if flag] + if not return_types: + return _redirect(client_id,registration_id=registration_id,period=period,financial_year=financial_year,download_mode=download_mode,error="Select at least one GST return to download.") + # Ensure FY is consistent in single-month mode. + if download_mode == "single": + financial_year=_fy_for_period(periods[0]) + accounting_dir,gst_dir=_storage_payload(client,financial_year,gstin) + jti=uuid.uuid4().hex + token=_encode_operator_token({ + "jti":jti,"user_id":int(user.id),"tenant_id":int(scope.tenant_id),"branch_id":int(scope.branch_id), + "node_code":str(node.node_code),"client_id":int(client.id),"client_name":str(client.client_name or ""), + "registration_id":int(reg.id),"credential_id":int(cred.id),"gstin":gstin,"financial_year":financial_year, + "periods":periods,"return_types":return_types,"gst_relative_dir":gst_dir,"accounting_relative_dir":accounting_dir, + }) + log_access(db,request,user,cred,"use_for_gst_download",reason=f"GST returns {financial_year}: {','.join(return_types)}",fields="username,secret",success=True) db.commit() - if not result.get("ok"): return _redirect(client_id,registration_id=registration_id,period=period,error=result.get("error") or "GST download could not be started.") - return _redirect(client_id,registration_id=registration_id,period=period,message="GST browser started on the Local Storage workstation. Complete captcha/OTP there; downloaded returns will be stored in the client GST directory.") + request.session["gst_operator_job"]={"token":token,"job_id":jti,"periods":periods,"return_types":return_types} + return _redirect(client_id,registration_id=registration_id,period=period,financial_year=financial_year,download_mode=download_mode,message="GST download prepared. The ERP will now ask the Local Agent on this computer to open the visible GST browser; CAPTCHA/OTP will appear here. Completed files will be transferred to the configured client local storage.") except Exception as exc: - db.rollback(); return _redirect(client_id,registration_id=registration_id,period=period,error=str(exc)) - finally: db.close() + db.rollback(); return _redirect(client_id,registration_id=registration_id,period=period,financial_year=financial_year,download_mode=download_mode,error=str(exc)) + finally: + db.close() + + +@router.post("/operator/redeem") +async def operator_redeem(request: Request): + body=await request.json(); token=str(body.get("token") or "") + try: + data=_decode_operator_token(token) + db=CommonSessionLocal() + try: + cred=db.get(CredentialVaultEntry,int(data["credential_id"])) + if not cred or int(cred.tenant_id or 0)!=int(data["tenant_id"]) or int(cred.client_id or 0)!=int(data["client_id"]) or str(cred.status or "").lower()=="archived": + raise ValueError("GST credential is no longer available.") + username=decrypt_value(cred.tenant_id,cred.username_encrypted) or "" + password=decrypt_value(cred.tenant_id,cred.secret_encrypted) or "" + if not username or not password: + raise ValueError("GST username/password is missing in Credential Vault.") + finally: + db.close() + return JSONResponse({"ok":True,"payload":{**{k:v for k,v in data.items() if k not in {"iat","exp","purpose"}},"username":username,"password":password,"upload_url":str(get_settings().ERP_PUBLIC_BASE_URL).rstrip("/")+"/tools/accounting/gst-reconciliation/operator/upload","login_timeout_seconds":900}}) + except Exception as exc: + return JSONResponse({"ok":False,"error":str(exc)},status_code=400) + + +@router.post("/operator/upload") +async def operator_upload(token: str=Form(...), package: UploadFile=File(...)): + try: + data=_decode_operator_token(token); jti=str(data.get("jti") or "") + if not jti: + raise ValueError("GST operator job id is missing.") + _UPLOAD_ROOT.mkdir(parents=True,exist_ok=True) + package_path=_UPLOAD_ROOT/f"{jti}.zip" + total=0 + with package_path.open("wb") as out: + while True: + chunk=await package.read(1024*1024) + if not chunk: break + total += len(chunk) + if total > 250*1024*1024: + raise ValueError("GST download package exceeds the 250 MB safety limit.") + out.write(chunk) + package_url=str(get_settings().ERP_PUBLIC_BASE_URL).rstrip("/")+f"/tools/accounting/gst-reconciliation/operator/package/{jti}?token={token}" + result=request_agent_command(str(data["node_code"]),"gst_return_package_store",{ + "client_id":int(data["client_id"]),"gstin":str(data["gstin"]),"financial_year":str(data["financial_year"]), + "gst_relative_dir":str(data["gst_relative_dir"]),"package_url":package_url,"periods":data.get("periods") or [],"return_types":data.get("return_types") or [], + },timeout_seconds=120) + if not result.get("ok"): + raise RuntimeError(result.get("error") or "Local Storage Agent could not store the GST return package.") + try: package_path.unlink(missing_ok=True) + except Exception: pass + return JSONResponse({"ok":True,"stored":result.get("result") or {}}) + except Exception as exc: + return JSONResponse({"ok":False,"error":str(exc)},status_code=400) + + +@router.get("/operator/package/{job_id}") +def operator_package(job_id: str, token: str): + try: + data=_decode_operator_token(token) + if str(data.get("jti") or "") != str(job_id): + raise ValueError("GST package token does not match the requested job.") + path=_UPLOAD_ROOT/f"{job_id}.zip" + if not path.is_file(): + return JSONResponse({"ok":False,"error":"GST package is no longer available."},status_code=404) + return FileResponse(path,media_type="application/zip",filename=f"gst_returns_{job_id}.zip") + except Exception as exc: + return JSONResponse({"ok":False,"error":str(exc)},status_code=400) @router.post("/analyze") -def analyze(request: Request, client_id: int=Form(...), registration_id: int=Form(...), period: str=Form(...), csrf_token: str=Form(...)): +def analyze(request: Request, client_id: int=Form(...), registration_id: int=Form(...), period: str=Form(""), financial_year: str=Form(...), analyze_mode: str=Form("single"), csrf_token: str=Form(...)): validate_csrf(request,csrf_token) db=CommonSessionLocal() try: @@ -244,15 +329,36 @@ def analyze(request: Request, client_id: int=Form(...), registration_id: int=For if not client: return _redirect(client_id,error="Client is not available in your scope.") pair=next(((r,t) for r,t in _gst_regs(db,scope.tenant_id,client.id) if int(r.id)==registration_id and _is_gstin(r,t)),None) if not pair: return _redirect(client_id,error="GSTIN registration was not found.") - reg,_=pair; gstin=re.sub(r"\s+","",str(reg.registration_number or "").upper()); fy=_fy_for_period(period) - accounting_dir,gst_dir=_storage_payload(client,fy,gstin); date_from,date_to=_period_bounds(period) + reg,_=pair; gstin=_norm_gstin(reg.registration_number) + periods=_periods_for_fy(financial_year) if analyze_mode=="full_fy" else [re.sub(r"\D","",period or "")] + if any(len(p)!=6 for p in periods): raise ValueError("Enter a valid MMYYYY period.") + accounting_dir,gst_dir=_storage_payload(client,financial_year,gstin) node=get_active_storage_node_for_branch(db,scope.tenant_id,scope.branch_id) - if not node or not _node_online(node): return _redirect(client_id,registration_id=registration_id,period=period,error="Local Storage Agent is offline.") - res=request_agent_command(node.node_code,"gst_reconciliation_analyze",{"client_id":client.id,"gstin":gstin,"financial_year":fy,"period":re.sub(r"\D","",period),"gst_relative_dir":gst_dir,"accounting_relative_dir":accounting_dir,"date_from":date_from,"date_to":date_to},timeout_seconds=25) - if not res.get("ok"): return _redirect(client_id,registration_id=registration_id,period=period,error=res.get("error") or "GST reconciliation failed.") - # Save compact analysis in session for immediate display; no GST raw data or credentials are stored on VPS. - request.session["gst_reconciliation_result"]=(res.get("result") or {}).get("analysis") or {} - return _redirect(client_id,registration_id=registration_id,period=period,message="GST Purchase, Sales and ITC reconciliation completed from local stored return data and Accounting Mirror.") + if not node or not _node_online(node): return _redirect(client_id,registration_id=registration_id,period=period,financial_year=financial_year,error="Local Storage Agent is offline.") + results=[] + for p in periods: + date_from,date_to=_period_bounds(p) + res=request_agent_command(node.node_code,"gst_reconciliation_analyze",{"client_id":client.id,"gstin":gstin,"financial_year":financial_year,"period":p,"gst_relative_dir":gst_dir,"accounting_relative_dir":accounting_dir,"date_from":date_from,"date_to":date_to},timeout_seconds=30) + if res.get("ok"): + results.append((res.get("result") or {}).get("analysis") or {}) + if not results: raise RuntimeError("No stored GST periods could be reconciled.") + if analyze_mode=="full_fy": + def sum_counts(section): + out={} + for r in results: + for k,v in (((r.get(section) or {}).get("counts") or {}).items()): out[k]=out.get(k,0)+int(v or 0) + return out + taxes={k:{"gstr2b":0.0,"gstr3b":0.0,"difference":0.0} for k in ("igst","cgst","sgst","cess")} + for r in results: + for k,row in (r.get("itc_reconciliation") or {}).items(): + if k in taxes: + for f in taxes[k]: taxes[k][f]=round(taxes[k][f]+float(row.get(f) or 0),2) + analysis={"period":financial_year,"sales_reconciliation":{"counts":sum_counts("sales_reconciliation")},"purchase_reconciliation":{"counts":sum_counts("purchase_reconciliation")},"itc_reconciliation":taxes} + else: + analysis=results[0] + request.session["gst_reconciliation_result"]=analysis + return _redirect(client_id,registration_id=registration_id,period=period,financial_year=financial_year,download_mode=("full_fy" if analyze_mode=="full_fy" else "single"),message="GST Purchase, Sales and ITC reconciliation completed from stored local return data and Accounting Mirror.") except Exception as exc: - return _redirect(client_id,registration_id=registration_id,period=period,error=str(exc)) - finally: db.close() + return _redirect(client_id,registration_id=registration_id,period=period,financial_year=financial_year,error=str(exc)) + finally: + db.close() diff --git a/app/modules/accounting/templates/accounting/gst_reconciliation.html b/app/modules/accounting/templates/accounting/gst_reconciliation.html index 4fb3e4d..ebf8198 100644 --- a/app/modules/accounting/templates/accounting/gst_reconciliation.html +++ b/app/modules/accounting/templates/accounting/gst_reconciliation.html @@ -1,49 +1,56 @@ {% extends "ui/templates/base/layout.html" %} {% block content %}
-

GST Return Reconciliation

Download GST portal data through Credential Vault, store it in client local storage, and reconcile against Accounting Mirror.

Back to Accounting
+
+

GST Return Reconciliation

Download GST returns on the workstation where you are using ERP, store them in the configured client local storage, and reconcile against Accounting Mirror.

+ Back to Accounting +
{% if message %}
{{ message }}
{% endif %} {% if error %}
{{ error }}
{% endif %} -
+ + - + +
+
+ {% if selected_client and selected_registration %}
- + +

1. Download from GST Portal

-

The Local Storage Agent opens GST portal on the workstation. Username/password are taken from Credential Vault; captcha/OTP remains interactive. Raw return data is saved under the client's FY/GST/GSTIN/period directory.

- - {% if not credentials %} -
- No usable GST Portal credential is linked to this client/GSTIN. - Add GST Portal credential. +

The visible GST browser opens on this computer through the ERP Local Agent at 127.0.0.1:8788. CAPTCHA/OTP remains visible here. After download, return data is transferred to the configured Local Storage Agent and stored under the client's FY/GST/GSTIN/period directory.

+ + {% if credentials|length==1 %}
The only eligible GST Portal credential has been selected automatically.
{% endif %} +
+
Returns to download
+
- {% elif credentials|length == 1 %} -
The only eligible GST Portal credential has been selected automatically.
- {% endif %} - - -
Storage Agent: {{ 'Online' if node_online else 'Offline' }}
+ +
Configured Storage Agent: {{ 'Online' if node_online else 'Offline' }}
+
The operator workstation also needs ERP Local Agent 1.26.15+ running so the browser can open here.
+
- +

2. Reconcile Stored Data

-

Sales: Accounting Mirror vs GSTR-1. Purchases: Accounting Mirror vs GSTR-2B. ITC: GSTR-2B vs GSTR-3B. Reconciliation JSON is retained in the same local client GST directory.

- +

Sales: Accounting Mirror vs GSTR-1. Purchases: Accounting Mirror vs GSTR-2B. ITC: GSTR-2B vs GSTR-3B. Analysis reads only stored local return data after download.

+ +
- {% if status %}

Download Status

Status: {{ status.status or '-' }}
Stage: {{ status.stage or '-' }}
{{ status.message or '' }}
{% endif %} + + + {% set a=request.session.get('gst_reconciliation_result') or {} %} {% if a %} {% set s=a.get('sales_reconciliation',{}).get('counts',{}) %}{% set p=a.get('purchase_reconciliation',{}).get('counts',{}) %} @@ -54,4 +61,34 @@ {% endif %} {% endif %}
+ +{% if operator_job %} + +{% endif %} {% endblock %} diff --git a/app/modules/documents/agent_package.py b/app/modules/documents/agent_package.py index be5a24f..4fdb1a3 100644 --- a/app/modules/documents/agent_package.py +++ b/app/modules/documents/agent_package.py @@ -4,7 +4,7 @@ import io from pathlib import Path import zipfile -ERP_LOCAL_AGENT_VERSION = "1.26.14" +ERP_LOCAL_AGENT_VERSION = "1.26.15" ERP_LOCAL_AGENT_NAME = "ERP Local Agent" RUNTIME_ROOT = Path(__file__).resolve().parent / "local_agent_runtime" _DETERMINISTIC_ZIP_TIMESTAMP = (2026, 1, 1, 0, 0, 0) @@ -65,7 +65,7 @@ def _build_zip(*, env_text: str | None, include_env: bool, include_admin_readme: text = ( f"ERP Local Agent {ERP_LOCAL_AGENT_VERSION}\n" "Existing storage, WebSocket tunnel, dashboard, Tally mapping and client .act functionality are preserved.\n" - "Existing master/transaction sync and accounting tools are preserved. Existing controlled write-back now persists approved instructions in the client .act and transfers them through the shared .NET batch writer with Tally read-back verification.\nAccounting Mirror uses the proven v3.0.1 ODBC runtime for ledgers, vouchers, stock movement, HSN and GST rates.\nTally TDL is exposed through a stable local path and localhost URL so later agent updates can repair/replace the TDL without changing the Tally link.\nFull Accounting Export runs asynchronously from the ERP, publishes live progress in both ERP and the Local Agent dashboard, and refreshes the client SQLite accounting datasets after the ODBC mirror is built.\nCash Payment Compliance reads only the local SQLite Accounting Mirror after mirroring; it no longer starts a Tally extraction job or requires TallyPrime during review.\nFull Accounting Export now enforces the ERP-selected financial-year transaction period. If Tally ODBC is restricted to a different active Tally period, the existing read-only exact-date Tally exporter corrects only the mirror transaction tables while preserving ODBC masters.\n" + "Existing master/transaction sync and accounting tools are preserved. Existing controlled write-back now persists approved instructions in the client .act and transfers them through the shared .NET batch writer with Tally read-back verification.\nAccounting Mirror uses the proven v3.0.1 ODBC runtime for ledgers, vouchers, stock movement, HSN and GST rates.\nTally TDL is exposed through a stable local path and localhost URL so later agent updates can repair/replace the TDL without changing the Tally link.\nFull Accounting Export runs asynchronously from the ERP, publishes live progress in both ERP and the Local Agent dashboard, and refreshes the client SQLite accounting datasets after the ODBC mirror is built.\nCash Payment Compliance reads only the local SQLite Accounting Mirror after mirroring; it no longer starts a Tally extraction job or requires TallyPrime during review.\nFull Accounting Export now enforces the ERP-selected financial-year transaction period. If Tally ODBC is restricted to a different active Tally period, the existing read-only exact-date Tally exporter corrects only the mirror transaction tables while preserving ODBC masters.\nGST return download supports an interactive browser on the ERP operator workstation, selectable GSTR-1/GSTR-2B/GSTR-3B/GSTR-2A, full-financial-year download, and transfer to configured client local storage.\n" "Dashboard: http://127.0.0.1:8788\nDesktop dashboard is single-instance: repeated shortcut clicks focus the existing window without starting another Local Agent.\nUpdate dashboard shows check, download, install, restart and completion progress.\n" ) _write_zip_bytes(dst, "README_ERP_LOCAL_AGENT.txt", text.encode("utf-8")) diff --git a/app/modules/documents/local_agent_runtime/erp_local_agent/__init__.py b/app/modules/documents/local_agent_runtime/erp_local_agent/__init__.py index ef5ee3e..404e11c 100644 --- a/app/modules/documents/local_agent_runtime/erp_local_agent/__init__.py +++ b/app/modules/documents/local_agent_runtime/erp_local_agent/__init__.py @@ -1,2 +1,2 @@ -__version__ = "1.26.14" +__version__ = "1.26.15" AGENT_NAME = "ERP Local Agent" diff --git a/app/modules/documents/local_agent_runtime/erp_local_agent/commands.py b/app/modules/documents/local_agent_runtime/erp_local_agent/commands.py index e3d9808..ffd95f0 100644 --- a/app/modules/documents/local_agent_runtime/erp_local_agent/commands.py +++ b/app/modules/documents/local_agent_runtime/erp_local_agent/commands.py @@ -6,6 +6,10 @@ import time import threading import json import uuid +import shutil +import zipfile +import tempfile +import requests from pathlib import Path from typing import Any @@ -189,6 +193,8 @@ class AgentCommandProcessor: result = self._gst_return_download_status(payload) elif action == "gst_reconciliation_analyze": result = self._gst_reconciliation_analyze(payload) + elif action == "gst_return_package_store": + result = self._gst_return_package_store(payload) elif action == "accounting_analysis_save": result = self._accounting_analysis_save(payload) elif action == "accounting_analysis_history": @@ -600,6 +606,52 @@ class AgentCommandProcessor: result = gst_download_status(payload, Path(__file__).resolve().parents[1] / "data") return {"job": result, "agent": self._agent_info()} + + def _gst_return_package_store(self, payload: dict[str, Any]) -> dict[str, Any]: + client_id = int(payload.get("client_id") or 0) + gstin = str(payload.get("gstin") or "").strip().upper() + gst_relative_dir = str(payload.get("gst_relative_dir") or "").strip() + package_url = str(payload.get("package_url") or "").strip() + if client_id <= 0 or len(gstin) != 15 or not gst_relative_dir or not package_url: + raise ValueError("Client, GSTIN, GST storage path and package URL are required.") + root = Path(self.config.storage_root).resolve() + target = (root / Path(gst_relative_dir)).resolve() + if root != target and root not in target.parents: + raise ValueError("GST storage path is outside the configured storage root.") + target.mkdir(parents=True, exist_ok=True) + temp_dir = Path(tempfile.mkdtemp(prefix="gst_store_")) + package_path = temp_dir / "gst_returns.zip" + try: + with requests.get(package_url, stream=True, timeout=180) as response: + response.raise_for_status() + total = 0 + with package_path.open("wb") as handle: + for chunk in response.iter_content(chunk_size=1024 * 1024): + if not chunk: + continue + total += len(chunk) + if total > 250 * 1024 * 1024: + raise ValueError("GST return package exceeds the 250 MB safety limit.") + handle.write(chunk) + stored=[] + with zipfile.ZipFile(package_path, "r") as archive: + for info in archive.infolist(): + if info.is_dir(): + continue + rel = Path(info.filename.replace("\\", "/")) + if rel.is_absolute() or ".." in rel.parts: + raise ValueError(f"Unsafe GST package path: {info.filename}") + destination = (target / rel).resolve() + if target != destination and target not in destination.parents: + raise ValueError(f"Unsafe GST package destination: {info.filename}") + destination.parent.mkdir(parents=True, exist_ok=True) + with archive.open(info, "r") as src, destination.open("wb") as dst: + shutil.copyfileobj(src, dst) + stored.append(str(destination)) + return {"stored_files": len(stored), "gst_storage_path": str(target), "periods": payload.get("periods") or [], "return_types": payload.get("return_types") or [], "agent": self._agent_info()} + finally: + shutil.rmtree(temp_dir, ignore_errors=True) + def _gst_reconciliation_analyze(self, payload: dict[str, Any]) -> dict[str, Any]: client_id = int(payload.get("client_id") or 0) period = re.sub(r"\D", "", str(payload.get("period") or "")) diff --git a/app/modules/documents/local_agent_runtime/erp_local_agent/dashboard.py b/app/modules/documents/local_agent_runtime/erp_local_agent/dashboard.py index 432e71d..03ac1c7 100644 --- a/app/modules/documents/local_agent_runtime/erp_local_agent/dashboard.py +++ b/app/modules/documents/local_agent_runtime/erp_local_agent/dashboard.py @@ -9,9 +9,11 @@ from http import HTTPStatus from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path from typing import Any +from urllib.parse import parse_qs, urlsplit from . import AGENT_NAME, __version__ from .tally import TallyLiveConnector +from .gst_portal_runtime import start_operator_download, operator_download_status class AgentDashboard: @@ -84,6 +86,16 @@ class AgentDashboard: self.send_response(status) self.send_header("Content-Type", "application/json; charset=utf-8") self.send_header("Cache-Control", "no-store") + origin = self.headers.get("Origin") or "" + erp_origin = "" + try: + parts = urlsplit(str(dashboard.config.erp_base_url or "")) + erp_origin = f"{parts.scheme}://{parts.netloc}" if parts.scheme and parts.netloc else "" + except Exception: + erp_origin = "" + if origin and erp_origin and origin.rstrip("/") == erp_origin.rstrip("/"): + self.send_header("Access-Control-Allow-Origin", origin) + self.send_header("Vary", "Origin") self.send_header("Content-Length", str(len(data))) self.end_headers() self.wfile.write(data) @@ -97,6 +109,22 @@ class AgentDashboard: self.end_headers() self.wfile.write(data) + def do_OPTIONS(self): + origin = self.headers.get("Origin") or "" + erp_origin = "" + try: + parts = urlsplit(str(dashboard.config.erp_base_url or "")) + erp_origin = f"{parts.scheme}://{parts.netloc}" if parts.scheme and parts.netloc else "" + except Exception: + erp_origin = "" + self.send_response(204) + if origin and erp_origin and origin.rstrip("/") == erp_origin.rstrip("/"): + self.send_header("Access-Control-Allow-Origin", origin) + self.send_header("Vary", "Origin") + self.send_header("Access-Control-Allow-Methods", "GET, POST, OPTIONS") + self.send_header("Access-Control-Allow-Headers", "Content-Type") + self.end_headers() + def do_GET(self): if self.path == "/" or self.path.startswith("/?"): return self._html(dashboard._page()) @@ -106,6 +134,11 @@ class AgentDashboard: return self._json(dashboard.history()) if self.path.startswith("/api/mirror-progress"): return self._json(dashboard.mirror_progress()) + if self.path.startswith("/api/gst/operator/status"): + query=parse_qs(urlsplit(self.path).query) + job_id=str((query.get("job_id") or [""])[0]) + data_root=Path(__file__).resolve().parents[1] / "data" + return self._json({"ok":True,"job":operator_download_status(job_id,data_root)}) if self.path.split("?", 1)[0] == "/tally/ERP_Accounting_Mirror.tdl": info = dashboard._ensure_managed_tdl() payload = Path(info["path"]).read_bytes() @@ -121,6 +154,14 @@ class AgentDashboard: def do_POST(self): try: + if self.path == "/api/gst/operator/start": + length=int(self.headers.get("Content-Length") or 0) + raw=self.rfile.read(length) if length else b"{}" + body=json.loads(raw.decode("utf-8") or "{}") + token=str(body.get("token") or "") + data_root=Path(__file__).resolve().parents[1] / "data" + result=start_operator_download(token,str(dashboard.config.erp_base_url or ""),data_root) + return self._json({"ok":True,**result}) if self.path == "/api/update/check": return self._json({"ok": True, "update": dashboard.updater.check_for_update(force=True)}) if self.path == "/api/update/download": diff --git a/app/modules/documents/local_agent_runtime/erp_local_agent/gst_portal_runtime.py b/app/modules/documents/local_agent_runtime/erp_local_agent/gst_portal_runtime.py index ccc1bc9..a1614b1 100644 --- a/app/modules/documents/local_agent_runtime/erp_local_agent/gst_portal_runtime.py +++ b/app/modules/documents/local_agent_runtime/erp_local_agent/gst_portal_runtime.py @@ -5,6 +5,9 @@ import re import sqlite3 import threading import time +import shutil +import zipfile +import requests from datetime import datetime, timezone from pathlib import Path from typing import Any @@ -343,3 +346,146 @@ def start_download(payload: dict[str, Any], storage_root: Path, progress_root: P def download_status(payload: dict[str, Any], progress_root: Path) -> dict[str, Any]: key=_safe(f"{payload.get('client_id')}_{payload.get('gstin')}_{payload.get('period')}") return _read_json(progress_root/f"gst_download_{key}.json") + +# --------------------------------------------------------------------------- +# Interactive GST browser mode (operator workstation) +# --------------------------------------------------------------------------- +_OPERATOR_THREADS: dict[str, threading.Thread] = {} + + +def _operator_job_path(progress_root: Path, job_id: str) -> Path: + return progress_root / f"gst_operator_{_safe(job_id)}.json" + + +def _fetch_text(page, url: str) -> str: + return page.evaluate("async (u)=>{const r=await fetch(u,{credentials:'include'}); return await r.text();}", url) or "" + + +def _download_selected_period(page, work_root: Path, period: str, return_types: list[str], progress) -> dict[str, Any]: + base_dir = work_root / period + raw_dir = base_dir / "raw" + raw_dir.mkdir(parents=True, exist_ok=True) + downloaded=[] + selected={str(x or '').upper() for x in return_types} + + if "GSTR1" in selected: + progress(stage=f"Downloading GSTR-1 {period}", message=f"Reading GSTR-1 for {period}.") + generate_text=_fetch_text(page,GSTR1_URL.format(period=period)) + (raw_dir/f"{period}_GSTR1_GENERATE.json").write_text(generate_text,encoding="utf-8",errors="ignore") + file_num=_extract_file_num(generate_text) + g1_text=_extract_payload(_fetch_text(page,GSTR1_DOWNLOAD_URL.format(period=period,file_num=file_num))) + g1_path=raw_dir/f"{period}_GSTR1.json"; g1_path.write_text(g1_text,encoding="utf-8",errors="ignore") + downloaded.append({"return_type":"GSTR1","path":str(g1_path.relative_to(work_root)),"bytes":g1_path.stat().st_size}) + + if "GSTR2B" in selected: + progress(stage=f"Downloading GSTR-2B {period}", message=f"Reading GSTR-2B for {period}.") + g2_text=_fetch_text(page,GSTR2B_URL.format(period=period)) + g2_path=raw_dir/f"{period}_GSTR2B.json"; g2_path.write_text(g2_text,encoding="utf-8",errors="ignore") + downloaded.append({"return_type":"GSTR2B","path":str(g2_path.relative_to(work_root)),"bytes":g2_path.stat().st_size}) + + if "GSTR2A" in selected: + progress(stage=f"Requesting GSTR-2A {period}", message=f"Requesting GSTR-2A for {period}.") + g2a_text=_fetch_text(page,GSTR2A_URL.format(period=period)) + g2a_path=raw_dir/f"{period}_GSTR2A.json"; g2a_path.write_text(g2a_text,encoding="utf-8",errors="ignore") + downloaded.append({"return_type":"GSTR2A_GENERATE","path":str(g2a_path.relative_to(work_root)),"bytes":g2a_path.stat().st_size}) + + if "GSTR3B" in selected: + progress(stage=f"Downloading GSTR-3B {period}", message=f"Reading GSTR-3B for {period}.") + g3_summary=_fetch_text(page,GSTR3B_SUMMARY_URL.format(period=period)) + g3_payable=_fetch_text(page,GSTR3B_URL.format(period=period)) + try: combined=json.loads(g3_summary or "{}") + except Exception: combined={"summary_raw":g3_summary or ""} + if not isinstance(combined,dict): combined={"summary":combined} + try: combined["taxpayble"]=json.loads(g3_payable or "{}") + except Exception: combined["taxpayble"]={"raw":g3_payable or ""} + (raw_dir/f"{period}_GSTR3B_SUMMARY.json").write_text(g3_summary,encoding="utf-8",errors="ignore") + (raw_dir/f"{period}_GSTR3B_TAXPAYBLE.json").write_text(g3_payable,encoding="utf-8",errors="ignore") + g3_path=raw_dir/f"{period}_GSTR3B.json"; g3_path.write_text(json.dumps(combined,ensure_ascii=False,indent=2),encoding="utf-8") + downloaded.append({"return_type":"GSTR3B","path":str(g3_path.relative_to(work_root)),"bytes":g3_path.stat().st_size}) + + normalized=normalize_downloads(base_dir,period) + _write_json(base_dir/"download_manifest.json",{"period":period,"downloaded_at_utc":_now(),"downloaded":downloaded,"normalized":normalized}) + return {"period":period,"downloaded":downloaded,"normalized":normalized} + + +def _operator_browser_worker(payload: dict[str, Any], progress_path: Path, progress_root: Path) -> None: + job_id=str(payload.get("jti") or payload.get("job_id") or "job") + work_root=progress_root / "gst_operator_work" / _safe(job_id) + try: + if work_root.exists(): shutil.rmtree(work_root,ignore_errors=True) + work_root.mkdir(parents=True,exist_ok=True) + def progress(**kw): + cur=_read_json(progress_path); cur.update(kw); cur["updated_at_utc"]=_now(); _write_json(progress_path,cur) + progress(status="running",stage="Opening GST Login",message="Opening GST Portal on this computer. Complete CAPTCHA/OTP in the visible browser.") + from playwright.sync_api import sync_playwright + with sync_playwright() as pw: + browser=None; errors=[] + profile=progress_root/"gst_operator_browser_profile"; profile.mkdir(parents=True,exist_ok=True) + for channel in ("chrome","msedge"): + try: + browser=pw.chromium.launch_persistent_context(user_data_dir=str(profile/channel),channel=channel,headless=False,no_viewport=True,accept_downloads=True,args=["--start-maximized","--no-first-run","--disable-blink-features=AutomationControlled"]) + break + except Exception as exc: errors.append(f"{channel}: {exc}") + if browser is None: raise RuntimeError("Could not open Chrome/Edge on this workstation. "+" | ".join(errors)) + page=browser.pages[0] if browser.pages else browser.new_page() + page.goto(GST_LOGIN_URL,wait_until="domcontentloaded",timeout=90000) + page.wait_for_selector("#username",timeout=30000) + page.fill("#username",str(payload.get("username") or "")); page.fill("#user_pass",str(payload.get("password") or "")) + progress(stage="Waiting for GST Login",message="Username/password filled. Complete CAPTCHA/OTP in this browser; download will continue automatically after login.") + deadline=time.time()+int(payload.get("login_timeout_seconds") or 900) + while time.time() dict[str, Any]: + if not job_token: raise ValueError("GST operator job token is required.") + redeem_url=str(erp_base_url or "").rstrip("/")+"/tools/accounting/gst-reconciliation/operator/redeem" + response=requests.post(redeem_url,json={"token":job_token},timeout=30) + try: body=response.json() + except Exception: body={"ok":False,"error":response.text[:1000]} + if not response.ok or not body.get("ok"): raise RuntimeError(body.get("error") or "ERP could not authorize the GST operator job.") + payload=dict(body.get("payload") or {}); payload["job_token"]=job_token + job_id=str(payload.get("jti") or "") + if not job_id: raise RuntimeError("ERP did not return a GST operator job id.") + progress_path=_operator_job_path(progress_root,job_id) + current=_read_json(progress_path) + if current.get("status") in {"queued","running"}: return {"started":False,"already_running":True,"job":current} + initial={"status":"queued","stage":"Queued","message":"Interactive GST browser job queued on this computer.","job_id":job_id,"periods":payload.get("periods") or [],"return_types":payload.get("return_types") or [],"started_at_utc":_now(),"updated_at_utc":_now()} + _write_json(progress_path,initial) + thread=threading.Thread(target=_operator_browser_worker,args=(payload,progress_path,progress_root),daemon=True,name=f"gst-operator-{job_id}") + _OPERATOR_THREADS[job_id]=thread; thread.start() + return {"started":True,"job":initial} + + +def operator_download_status(job_id: str, progress_root: Path) -> dict[str, Any]: + return _read_json(_operator_job_path(progress_root,job_id))