From 477ea79f766ff4779ff6dc3a7ab88920b3027799 Mon Sep 17 00:00:00 2001 From: A R R R Associates Date: Thu, 10 Sep 2026 21:46:10 +0530 Subject: [PATCH] Add GST return download local storage and reconciliation workflow --- .../accounting/gst_reconciliation_ui.py | 186 ++++++++++ .../accounting/gst_reconciliation.html | 42 +++ .../templates/accounting/tally.html | 1 + app/modules/documents/agent_package.py | 2 +- .../erp_local_agent/__init__.py | 3 +- .../erp_local_agent/commands.py | 58 +++ .../erp_local_agent/gst_portal_runtime.py | 345 ++++++++++++++++++ .../local_agent_runtime/requirements.txt | 1 + app/ui/app.py | 7 +- 9 files changed, 638 insertions(+), 7 deletions(-) create mode 100644 app/modules/accounting/gst_reconciliation_ui.py create mode 100644 app/modules/accounting/templates/accounting/gst_reconciliation.html create mode 100644 app/modules/documents/local_agent_runtime/erp_local_agent/gst_portal_runtime.py diff --git a/app/modules/accounting/gst_reconciliation_ui.py b/app/modules/accounting/gst_reconciliation_ui.py new file mode 100644 index 0000000..87d1cca --- /dev/null +++ b/app/modules/accounting/gst_reconciliation_ui.py @@ -0,0 +1,186 @@ +from __future__ import annotations + +import calendar +import re +from datetime import date +from pathlib import Path +from urllib.parse import urlencode + +from fastapi import APIRouter, Form, Request +from fastapi.responses import RedirectResponse +from sqlalchemy import select + +from app.core.db.common import CommonSessionLocal +from app.core.security.csrf import get_or_create_csrf_token, validate_csrf +from app.core.templating import templates +from app.modules.accounting.agent_bridge import request_agent_command +from app.modules.accounting.ui import _find_visible_client, _require_partner, _visible_clients, _node_online +from app.modules.core.rbac.deps import get_user_permissions, get_user_roles +from app.modules.credential_vault.crypto import decrypt_value +from app.modules.credential_vault.models import CredentialVaultEntry +from app.modules.credential_vault.service import can_view_entry, log_access +from app.modules.documents.services import build_document_scope, client_folder_parts, get_active_storage_node_for_branch, sanitize_segment +from app.modules.registrations.models import ClientRegistration, RegistrationType + +router = APIRouter(prefix="/tools/accounting/gst-reconciliation", tags=["accounting-gst-reconciliation-ui"]) + + +def _fy_bounds(fy: str) -> tuple[date, date]: + m = re.fullmatch(r"(\d{4})-(\d{2})", str(fy or "").strip()) + if not m: + raise ValueError("Invalid financial year.") + y = int(m.group(1)) + return date(y, 4, 1), date(y + 1, 3, 31) + + +def _fy_for_period(period: str) -> str: + digits = re.sub(r"\D", "", period or "") + if len(digits) != 6: + raise ValueError("Return period must be MMYYYY.") + month, year = int(digits[:2]), int(digits[2:]) + if month < 1 or month > 12: + raise ValueError("Invalid GST return month.") + sy = year if month >= 4 else year - 1 + return f"{sy}-{str(sy+1)[-2:]}" + + +def _period_bounds(period: str) -> tuple[str, str]: + digits = re.sub(r"\D", "", period or "") + month, year = int(digits[:2]), int(digits[2:]) + last = calendar.monthrange(year, month)[1] + return date(year, month, 1).isoformat(), date(year, month, last).isoformat() + + +def _gst_regs(db, tenant_id: int, client_id: int): + return db.execute( + select(ClientRegistration, RegistrationType) + .join(RegistrationType, RegistrationType.id == ClientRegistration.registration_type_id) + .where(ClientRegistration.tenant_id == tenant_id, ClientRegistration.client_id == client_id) + .order_by(ClientRegistration.id.asc()) + ).all() + + +def _is_gstin(reg, typ) -> bool: + code = str(getattr(typ, "code", "") or "").upper().strip() + num = re.sub(r"\s+", "", str(getattr(reg, "registration_number", "") or "").upper()) + return code == "GSTIN" and len(num) == 15 + + +def _vault_entries(db, user, request, tenant_id: int, client_id: int, registration_id: int | None = None): + q = select(CredentialVaultEntry).where( + CredentialVaultEntry.tenant_id == tenant_id, + CredentialVaultEntry.client_id == client_id, + CredentialVaultEntry.status != "archived", + ) + if registration_id: + q = q.where(CredentialVaultEntry.registration_id == registration_id) + rows = db.execute(q.order_by(CredentialVaultEntry.title.asc())).scalars().all() + visible = [r for r in rows if can_view_entry(db, user, r, getattr(user, "branch_id", None))] + gst = [r for r in visible if "gst" in (str(r.title or "") + " " + str(r.category or "") + " " + str(r.portal_url or "")).lower()] + return gst or visible + + +def _storage_payload(client, fy: str, gstin: str) -> tuple[str, str]: + fy_folder = sanitize_segment(f"FY{fy}", "FY") + letter, client_folder = client_folder_parts(client, int(client.id)) + root = Path(fy_folder) / "Clients" / letter / client_folder + accounting = root / "Accounting" + gst = root / "GST" / sanitize_segment(f"GSTIN_{gstin}", "GSTIN") + return accounting.as_posix(), gst.as_posix() + + +def _redirect(client_id: int, **params): + data = {"client_id": client_id, **{k: v for k, v in params.items() if v not in (None, "")}} + return RedirectResponse("/tools/accounting/gst-reconciliation?" + urlencode(data), status_code=303) + + +@router.get("") +def page(request: Request, client_id: int | None = None, registration_id: int | None = None, period: str = "", message: str = "", error: str = ""): + db = CommonSessionLocal() + try: + user, response = _require_partner(request, db, "accounting.learning.view") + if response: + return response + clients, scope = _visible_clients(db, request, user) + selected = next((c for c in clients if client_id and int(c.id) == int(client_id)), None) + registrations=[]; selected_reg=None; credentials=[]; node=None; status={}; analysis={} + if selected: + registrations=[(r,t) for r,t in _gst_regs(db,scope.tenant_id,selected.id) if _is_gstin(r,t)] + selected_reg=next((r for r,t in registrations if registration_id and int(r.id)==int(registration_id)),None) + if not selected_reg and registrations: + selected_reg=registrations[0][0] + credentials=_vault_entries(db,user,request,scope.tenant_id,selected.id,int(selected_reg.id) if selected_reg else None) + node=get_active_storage_node_for_branch(db,scope.tenant_id,scope.branch_id) + if selected_reg and period and node and _node_online(node): + gstin=re.sub(r"\s+","",str(selected_reg.registration_number or "").upper()) + try: + status=(request_agent_command(node.node_code,"gst_return_download_status",{"client_id":selected.id,"gstin":gstin,"period":re.sub(r"\D","",period)},timeout_seconds=8).get("result") or {}).get("job") or {} + except Exception: + status={} + try: + job_result=status.get("result") or {} + # analysis is loaded only after an explicit Analyze action; status result is download manifest. + analysis={} + except Exception: + pass + return templates.TemplateResponse("modules/accounting/templates/accounting/gst_reconciliation.html",{ + "request":request,"current_user":user,"current_user_roles":get_user_roles(db,user.id),"current_user_permissions":get_user_permissions(db,user.id),"csrf_token":get_or_create_csrf_token(request), + "clients":clients,"selected_client":selected,"registrations":registrations,"selected_registration":selected_reg,"credentials":credentials,"node":node,"node_online":_node_online(node) if node else False,"period":period,"status":status,"analysis":analysis,"message":message,"error":error,"title":"GST Return Reconciliation", + }) + finally: + db.close() + + +@router.post("/download/start") +def start_download(request: Request, client_id: int=Form(...), registration_id: int=Form(...), credential_id: int=Form(...), period: str=Form(...), include_2a: str=Form(""), csrf_token: str=Form(...)): + validate_csrf(request,csrf_token) + db=CommonSessionLocal() + try: + user,response=_require_partner(request,db,"accounting.learning.manage") + if response: return response + client,_,scope=_find_visible_client(db,request,user,client_id) + if not client: return _redirect(client_id,error="Client is not available in your scope.") + pair=next(((r,t) for r,t in _gst_regs(db,scope.tenant_id,client.id) if int(r.id)==registration_id and _is_gstin(r,t)),None) + if not pair: return _redirect(client_id,error="Select a valid GSTIN registration.") + reg,_=pair; gstin=re.sub(r"\s+","",str(reg.registration_number or "").upper()) + cred=db.get(CredentialVaultEntry,credential_id) + if not cred or int(cred.client_id or 0)!=int(client.id) or (cred.registration_id and int(cred.registration_id)!=int(reg.id)) or not can_view_entry(db,user,cred,scope.branch_id): + return _redirect(client_id,registration_id=registration_id,period=period,error="Selected GST credential is not available for this client/registration.") + username=decrypt_value(cred.tenant_id,cred.username_encrypted) or ""; password=decrypt_value(cred.tenant_id,cred.secret_encrypted) or "" + if not username or not password: return _redirect(client_id,registration_id=registration_id,period=period,error="GST username/password is missing in Credential Vault.") + fy=_fy_for_period(period); accounting_dir,gst_dir=_storage_payload(client,fy,gstin) + node=get_active_storage_node_for_branch(db,scope.tenant_id,scope.branch_id) + if not node or not _node_online(node): return _redirect(client_id,registration_id=registration_id,period=period,error="Local Storage Agent is offline.") + result=request_agent_command(node.node_code,"gst_return_download_start",{"client_id":client.id,"client_name":client.client_name,"gstin":gstin,"financial_year":fy,"period":re.sub(r"\D","",period),"gst_relative_dir":gst_dir,"accounting_relative_dir":accounting_dir,"username":username,"password":password,"include_2a":bool(include_2a),"login_timeout_seconds":900},timeout_seconds=15) + log_access(db,request,user,cred,"use_for_gst_download",reason=f"GST return download {period}",fields="username,secret",success=bool(result.get("ok"))) + db.commit() + if not result.get("ok"): return _redirect(client_id,registration_id=registration_id,period=period,error=result.get("error") or "GST download could not be started.") + return _redirect(client_id,registration_id=registration_id,period=period,message="GST browser started on the Local Storage workstation. Complete captcha/OTP there; downloaded returns will be stored in the client GST directory.") + except Exception as exc: + db.rollback(); return _redirect(client_id,registration_id=registration_id,period=period,error=str(exc)) + finally: db.close() + + +@router.post("/analyze") +def analyze(request: Request, client_id: int=Form(...), registration_id: int=Form(...), period: str=Form(...), csrf_token: str=Form(...)): + validate_csrf(request,csrf_token) + db=CommonSessionLocal() + try: + user,response=_require_partner(request,db,"accounting.learning.manage") + if response: return response + client,_,scope=_find_visible_client(db,request,user,client_id) + if not client: return _redirect(client_id,error="Client is not available in your scope.") + pair=next(((r,t) for r,t in _gst_regs(db,scope.tenant_id,client.id) if int(r.id)==registration_id and _is_gstin(r,t)),None) + if not pair: return _redirect(client_id,error="GSTIN registration was not found.") + reg,_=pair; gstin=re.sub(r"\s+","",str(reg.registration_number or "").upper()); fy=_fy_for_period(period) + accounting_dir,gst_dir=_storage_payload(client,fy,gstin); date_from,date_to=_period_bounds(period) + node=get_active_storage_node_for_branch(db,scope.tenant_id,scope.branch_id) + if not node or not _node_online(node): return _redirect(client_id,registration_id=registration_id,period=period,error="Local Storage Agent is offline.") + res=request_agent_command(node.node_code,"gst_reconciliation_analyze",{"client_id":client.id,"gstin":gstin,"financial_year":fy,"period":re.sub(r"\D","",period),"gst_relative_dir":gst_dir,"accounting_relative_dir":accounting_dir,"date_from":date_from,"date_to":date_to},timeout_seconds=25) + if not res.get("ok"): return _redirect(client_id,registration_id=registration_id,period=period,error=res.get("error") or "GST reconciliation failed.") + # Save compact analysis in session for immediate display; no GST raw data or credentials are stored on VPS. + request.session["gst_reconciliation_result"]=(res.get("result") or {}).get("analysis") or {} + return _redirect(client_id,registration_id=registration_id,period=period,message="GST Purchase, Sales and ITC reconciliation completed from local stored return data and Accounting Mirror.") + except Exception as exc: + return _redirect(client_id,registration_id=registration_id,period=period,error=str(exc)) + finally: db.close() diff --git a/app/modules/accounting/templates/accounting/gst_reconciliation.html b/app/modules/accounting/templates/accounting/gst_reconciliation.html new file mode 100644 index 0000000..faa9f6d --- /dev/null +++ b/app/modules/accounting/templates/accounting/gst_reconciliation.html @@ -0,0 +1,42 @@ +{% extends "ui/templates/base/layout.html" %} +{% block content %} +
+

GST Return Reconciliation

Download GST portal data through Credential Vault, store it in client local storage, and reconcile against Accounting Mirror.

Back to Accounting
+ {% if message %}
{{ message }}
{% endif %} + {% if error %}
{{ error }}
{% endif %} +
+ + + +
+
+ {% if selected_client and selected_registration %} +
+
+ +

1. Download from GST Portal

+

The Local Storage Agent opens GST portal on the workstation. Username/password are taken from Credential Vault; captcha/OTP remains interactive. Raw return data is saved under the client's FY/GST/GSTIN/period directory.

+ + + +
Storage Agent: {{ 'Online' if node_online else 'Offline' }}
+
+
+ +

2. Reconcile Stored Data

+

Sales: Accounting Mirror vs GSTR-1. Purchases: Accounting Mirror vs GSTR-2B. ITC: GSTR-2B vs GSTR-3B. Reconciliation JSON is retained in the same local client GST directory.

+ +
+
+ {% if status %}

Download Status

Status: {{ status.status or '-' }}
Stage: {{ status.stage or '-' }}
{{ status.message or '' }}
{% endif %} + {% set a=request.session.get('gst_reconciliation_result') or {} %} + {% if a %} + {% set s=a.get('sales_reconciliation',{}).get('counts',{}) %}{% set p=a.get('purchase_reconciliation',{}).get('counts',{}) %} +

Reconciliation Summary — {{ a.get('period','') }}

+

Sales vs GSTR-1

Books {{ s.get('books',0) }} · Portal {{ s.get('portal',0) }} · Matched {{ s.get('matched',0) }} · Missing in portal {{ s.get('missing_in_portal',0) }} · Missing in books {{ s.get('missing_in_books',0) }} · Value mismatch {{ s.get('value_mismatch',0) }}

Purchases vs GSTR-2B

Books {{ p.get('books',0) }} · 2B {{ p.get('portal',0) }} · Matched {{ p.get('matched',0) }} · Missing in 2B {{ p.get('missing_in_portal',0) }} · Missing in books {{ p.get('missing_in_books',0) }} · Value mismatch {{ p.get('value_mismatch',0) }}

+

ITC: GSTR-2B vs GSTR-3B

{% for tax,row in a.get('itc_reconciliation',{}).items() %}{% endfor %}
TaxGSTR-2BGSTR-3BDifference
{{ tax }}{{ '%.2f'|format(row.get('gstr2b',0)) }}{{ '%.2f'|format(row.get('gstr3b',0)) }}{{ '%.2f'|format(row.get('difference',0)) }}
+
+ {% endif %} + {% endif %} +
+{% endblock %} diff --git a/app/modules/accounting/templates/accounting/tally.html b/app/modules/accounting/templates/accounting/tally.html index a00e034..8a73391 100644 --- a/app/modules/accounting/templates/accounting/tally.html +++ b/app/modules/accounting/templates/accounting/tally.html @@ -33,6 +33,7 @@ Historical Learning Ledger Learning GSTR-2B Intelligence + GST Return Reconciliation E-Invoice / E-Way Bill Purchase Review Purchase → Tally diff --git a/app/modules/documents/agent_package.py b/app/modules/documents/agent_package.py index 0f6bd39..be5a24f 100644 --- a/app/modules/documents/agent_package.py +++ b/app/modules/documents/agent_package.py @@ -4,7 +4,7 @@ import io from pathlib import Path import zipfile -ERP_LOCAL_AGENT_VERSION = "1.26.13" +ERP_LOCAL_AGENT_VERSION = "1.26.14" ERP_LOCAL_AGENT_NAME = "ERP Local Agent" RUNTIME_ROOT = Path(__file__).resolve().parent / "local_agent_runtime" _DETERMINISTIC_ZIP_TIMESTAMP = (2026, 1, 1, 0, 0, 0) diff --git a/app/modules/documents/local_agent_runtime/erp_local_agent/__init__.py b/app/modules/documents/local_agent_runtime/erp_local_agent/__init__.py index b961242..ef5ee3e 100644 --- a/app/modules/documents/local_agent_runtime/erp_local_agent/__init__.py +++ b/app/modules/documents/local_agent_runtime/erp_local_agent/__init__.py @@ -1 +1,2 @@ -__version__ = "1.26.13" +__version__ = "1.26.14" +AGENT_NAME = "ERP Local Agent" diff --git a/app/modules/documents/local_agent_runtime/erp_local_agent/commands.py b/app/modules/documents/local_agent_runtime/erp_local_agent/commands.py index 86b9357..e3d9808 100644 --- a/app/modules/documents/local_agent_runtime/erp_local_agent/commands.py +++ b/app/modules/documents/local_agent_runtime/erp_local_agent/commands.py @@ -14,6 +14,7 @@ from .accounting_store import LocalAccountingStore from .mirror_tally import MirrorFirstTallyConnector from .native_voucher_engine import NativeVoucherEngine from .tally_batch_queue import TallyBatchQueue +from .gst_portal_runtime import start_download as gst_start_download, download_status as gst_download_status, analyze_gst_storage _CASH_TALLY_EXTRACTION_LOCK = threading.Lock() @@ -182,6 +183,12 @@ class AgentCommandProcessor: result = self._accounting_full_export_status(payload) elif action == "accounting_mirror_query": result = self._accounting_mirror_query(payload) + elif action == "gst_return_download_start": + result = self._gst_return_download_start(payload) + elif action == "gst_return_download_status": + result = self._gst_return_download_status(payload) + elif action == "gst_reconciliation_analyze": + result = self._gst_reconciliation_analyze(payload) elif action == "accounting_analysis_save": result = self._accounting_analysis_save(payload) elif action == "accounting_analysis_history": @@ -568,6 +575,57 @@ class AgentCommandProcessor: finally: db.close() + def _gst_return_download_start(self, payload: dict[str, Any]) -> dict[str, Any]: + client_id = int(payload.get("client_id") or 0) + gstin = str(payload.get("gstin") or "").strip().upper() + period = re.sub(r"\D", "", str(payload.get("period") or "")) + relative_dir = str(payload.get("gst_relative_dir") or "").strip() + username = str(payload.get("username") or "").strip() + password = str(payload.get("password") or "") + if client_id <= 0: + raise ValueError("client_id is required.") + if not re.fullmatch(r"\d{6}", period): + raise ValueError("GST return period must be MMYYYY.") + if len(gstin) != 15: + raise ValueError("A valid GSTIN is required.") + if not relative_dir: + raise ValueError("GST local storage directory is required.") + if not username or not password: + raise ValueError("GST portal username/password are missing from the selected Credential Vault entry.") + result = gst_start_download(payload, Path(self.config.storage_root), Path(__file__).resolve().parents[1] / "data") + # Never echo credentials back to ERP or write them into result/progress files. + return {**result, "agent": self._agent_info()} + + def _gst_return_download_status(self, payload: dict[str, Any]) -> dict[str, Any]: + result = gst_download_status(payload, Path(__file__).resolve().parents[1] / "data") + return {"job": result, "agent": self._agent_info()} + + def _gst_reconciliation_analyze(self, payload: dict[str, Any]) -> dict[str, Any]: + client_id = int(payload.get("client_id") or 0) + period = re.sub(r"\D", "", str(payload.get("period") or "")) + gst_relative_dir = str(payload.get("gst_relative_dir") or "").strip() + accounting_relative_dir = str(payload.get("accounting_relative_dir") or "").strip() + date_from = str(payload.get("date_from") or "").strip() + date_to = str(payload.get("date_to") or "").strip() + if client_id <= 0 or not gst_relative_dir or not accounting_relative_dir: + raise ValueError("Client GST and Accounting storage paths are required.") + root = Path(self.config.storage_root) + base_dir = root / Path(gst_relative_dir) / period + mirror_db = root / Path(accounting_relative_dir) / f"client_{client_id:08d}_mirror.act" + if not base_dir.exists(): + raise ValueError("GST data has not yet been downloaded for this client/period.") + if not mirror_db.exists(): + raise ValueError("Accounting Mirror is not available for the selected financial year. Run Full Accounting Export first.") + result = analyze_gst_storage(base_dir, mirror_db, period, date_from, date_to) + compact = { + "period": result.get("period"), + "generated_at_utc": result.get("generated_at_utc"), + "sales_reconciliation": {"counts": (result.get("sales_reconciliation") or {}).get("counts") or {}}, + "purchase_reconciliation": {"counts": (result.get("purchase_reconciliation") or {}).get("counts") or {}}, + "itc_reconciliation": result.get("itc_reconciliation") or {}, + } + return {"analysis": compact, "report_path": str(base_dir / "reports" / f"{period}_GST_reconciliation.json"), "agent": self._agent_info()} + def _accounting_analysis_save(self, payload: dict[str, Any]) -> dict[str, Any]: client_id = int(payload.get("client_id") or 0) if client_id <= 0: diff --git a/app/modules/documents/local_agent_runtime/erp_local_agent/gst_portal_runtime.py b/app/modules/documents/local_agent_runtime/erp_local_agent/gst_portal_runtime.py new file mode 100644 index 0000000..ccc1bc9 --- /dev/null +++ b/app/modules/documents/local_agent_runtime/erp_local_agent/gst_portal_runtime.py @@ -0,0 +1,345 @@ +from __future__ import annotations + +import json +import re +import sqlite3 +import threading +import time +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +GST_LOGIN_URL = "https://services.gst.gov.in/services/login" +GSTR1_URL = "https://return.gst.gov.in/returns/auth/api/offline/download/generate?flag=0&rtn_prd={period}&rtn_typ=GSTR1" +GSTR2A_URL = "https://return.gst.gov.in/returns/auth/api/offline/download/generate?flag=0&rtn_prd={period}&rtn_typ=GSTR2A" +GSTR1_DOWNLOAD_URL = "https://return.gst.gov.in/returns/auth/api/offline/download/url?rtn_prd={period}&rtn_typ=GSTR1&file_num={file_num}" +GSTR2B_URL = "https://gstr2b.gst.gov.in/gstr2b/auth/api/gstr2b/getjson?rtnprd={period}" +GSTR3B_SUMMARY_URL = "https://return.gst.gov.in/returns/auth/api/gstr3b/summary?rtn_prd={period}" +GSTR3B_URL = "https://return.gst.gov.in/returns/auth/api/gstr3b/taxpayble?rtn_prd={period}" + +_LOCK = threading.Lock() +_THREADS: dict[str, threading.Thread] = {} + + +def _now() -> str: + return datetime.now(timezone.utc).isoformat() + + +def _write_json(path: Path, payload: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_suffix(path.suffix + ".tmp") + tmp.write_text(json.dumps(payload, ensure_ascii=False, indent=2, default=str), encoding="utf-8") + tmp.replace(path) + + +def _read_json(path: Path) -> dict[str, Any]: + try: + data = json.loads(path.read_text(encoding="utf-8")) + return data if isinstance(data, dict) else {} + except Exception: + return {} + + +def _safe(value: str, fallback: str = "item") -> str: + value = re.sub(r"[^A-Za-z0-9._ -]+", "_", str(value or "").strip()).strip(" ._") + return value or fallback + + +def _parse_date(value: str) -> str: + text = str(value or "").strip() + for fmt in ("%d-%m-%Y", "%d/%m/%Y", "%Y-%m-%d", "%d-%b-%Y", "%d/%m/%y"): + try: + return datetime.strptime(text, fmt).date().isoformat() + except Exception: + pass + return text + + +def _f(value: Any) -> float: + try: + return float(value or 0) + except Exception: + return 0.0 + + +def _doc_key(value: str) -> str: + return "".join(ch for ch in str(value or "").upper() if ch.isalnum()) + + +def _walk(obj: Any): + if isinstance(obj, dict): + yield obj + for v in obj.values(): + yield from _walk(v) + elif isinstance(obj, list): + for v in obj: + yield from _walk(v) + + +def _tax_from_invoice(inv: dict[str, Any]) -> dict[str, float]: + out = {"taxable": 0.0, "igst": 0.0, "cgst": 0.0, "sgst": 0.0, "cess": 0.0} + for item in inv.get("itms") or inv.get("items") or []: + d = item.get("itm_det") or item.get("item_det") or item + out["taxable"] += _f(d.get("txval")) + out["igst"] += _f(d.get("iamt")) + out["cgst"] += _f(d.get("camt")) + out["sgst"] += _f(d.get("samt")) + out["cess"] += _f(d.get("csamt")) + if not (out["taxable"] or out["igst"] or out["cgst"] or out["sgst"] or out["cess"]): + out["taxable"] = _f(inv.get("txval") or inv.get("taxable_value")) + out["igst"] = _f(inv.get("iamt") or inv.get("igst")) + out["cgst"] = _f(inv.get("camt") or inv.get("cgst")) + out["sgst"] = _f(inv.get("samt") or inv.get("sgst")) + out["cess"] = _f(inv.get("csamt") or inv.get("cess")) + return out + + +def extract_invoice_rows(data: Any) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + seen: set[tuple[str, str, str]] = set() + for obj in _walk(data): + invno = str(obj.get("inum") or obj.get("inv_num") or obj.get("doc_no") or obj.get("invoiceNumber") or "").strip() + if not invno: + continue + gstin = str(obj.get("ctin") or obj.get("stin") or obj.get("supplier_gstin") or obj.get("recipientGstin") or "").strip().upper() + date = _parse_date(str(obj.get("idt") or obj.get("inv_date") or obj.get("doc_date") or obj.get("invoiceDate") or "")) + key = (gstin, _doc_key(invno), date) + if key in seen: + continue + seen.add(key) + tax = _tax_from_invoice(obj) + rows.append({ + "gstin": gstin, + "invoice_no": invno, + "invoice_key": _doc_key(invno), + "invoice_date": date, + "invoice_value": _f(obj.get("val") or obj.get("invoice_value") or obj.get("invoiceValue")), + **tax, + }) + return rows + + +def extract_gstr3b_itc(data: Any) -> dict[str, float]: + totals = {"igst": 0.0, "cgst": 0.0, "sgst": 0.0, "cess": 0.0} + if not isinstance(data, dict): + return totals + itc = data.get("itc_elg") or {} + for item in itc.get("itc_avl") or []: + totals["igst"] += _f(item.get("iamt")) + totals["cgst"] += _f(item.get("camt")) + totals["sgst"] += _f(item.get("samt")) + totals["cess"] += _f(item.get("csamt")) + return totals + + +def normalize_downloads(base_dir: Path, period: str) -> dict[str, Any]: + normalized = base_dir / "normalized" + normalized.mkdir(parents=True, exist_ok=True) + result: dict[str, Any] = {} + for rtype in ("GSTR1", "GSTR2A", "GSTR2B", "GSTR3B"): + raw_path = base_dir / "raw" / f"{period}_{rtype}.json" + if not raw_path.exists(): + continue + try: + data = json.loads(raw_path.read_text(encoding="utf-8", errors="ignore") or "{}") + except Exception: + data = {} + if rtype in {"GSTR1", "GSTR2A", "GSTR2B"}: + rows = extract_invoice_rows(data) + _write_json(normalized / f"{period}_{rtype}_invoices.json", {"period": period, "rows": rows}) + result[rtype] = {"invoice_rows": len(rows)} + else: + itc = extract_gstr3b_itc(data) + _write_json(normalized / f"{period}_{rtype}_itc.json", {"period": period, "itc": itc}) + result[rtype] = {"itc": itc} + return result + + +def _book_rows(mirror_db: Path, date_from: str, date_to: str, voucher_type: str) -> list[dict[str, Any]]: + if not mirror_db.exists(): + return [] + con = sqlite3.connect(str(mirror_db)) + con.row_factory = sqlite3.Row + try: + sql = """ + SELECT voucher_date, voucher_type, voucher_number, party_ledger, reference, ABS(COALESCE(voucher_amount,0)) amount + FROM voucher + WHERE voucher_date >= ? AND voucher_date <= ? AND UPPER(COALESCE(voucher_type,'')) LIKE ? + ORDER BY voucher_date, voucher_number + """ + rows = con.execute(sql, (date_from, date_to, f"%{voucher_type.upper()}%" )).fetchall() + return [{"date": r["voucher_date"], "voucher_type": r["voucher_type"], "invoice_no": r["voucher_number"] or r["reference"] or "", "invoice_key": _doc_key(r["voucher_number"] or r["reference"] or ""), "party": r["party_ledger"] or "", "amount": _f(r["amount"])} for r in rows] + finally: + con.close() + + +def _match_books_to_portal(books: list[dict[str, Any]], portal: list[dict[str, Any]]) -> dict[str, Any]: + pmap: dict[str, list[dict[str, Any]]] = {} + for row in portal: + if row.get("invoice_key"): + pmap.setdefault(row["invoice_key"], []).append(row) + matched=[]; missing_portal=[]; used=set() + for b in books: + candidates=pmap.get(b.get("invoice_key") or "", []) + best=None + for p in candidates: + pid=id(p) + if pid in used: continue + best=p; break + if best is None: + missing_portal.append(b); continue + used.add(id(best)) + pv=_f(best.get("invoice_value")); bv=_f(b.get("amount")); diff=round(bv-pv,2) + matched.append({"books":b,"portal":best,"difference":diff,"status":"Matched" if abs(diff)<=1 else "Value mismatch"}) + missing_books=[p for p in portal if id(p) not in used] + return {"matched":matched,"missing_in_portal":missing_portal,"missing_in_books":missing_books,"counts":{"books":len(books),"portal":len(portal),"matched":len(matched),"missing_in_portal":len(missing_portal),"missing_in_books":len(missing_books),"value_mismatch":sum(1 for r in matched if r["status"]!="Matched")}} + + +def analyze_gst_storage(base_dir: Path, mirror_db: Path, period: str, date_from: str, date_to: str) -> dict[str, Any]: + norm=base_dir/"normalized" + def rows(name): + p=norm/f"{period}_{name}_invoices.json" + d=_read_json(p) if p.exists() else {} + return d.get("rows") or [] + g1=rows("GSTR1"); g2b=rows("GSTR2B") + sales=_book_rows(mirror_db,date_from,date_to,"SALES") + purchases=_book_rows(mirror_db,date_from,date_to,"PURCHASE") + sales_rec=_match_books_to_portal(sales,g1) + purchase_rec=_match_books_to_portal(purchases,g2b) + g2b_itc={k:round(sum(_f(r.get(k)) for r in g2b),2) for k in ("igst","cgst","sgst","cess")} + g3p=norm/f"{period}_GSTR3B_itc.json"; g3=_read_json(g3p).get("itc") if g3p.exists() else {} + g3={k:round(_f((g3 or {}).get(k)),2) for k in ("igst","cgst","sgst","cess")} + itc={k:{"gstr2b":g2b_itc[k],"gstr3b":g3[k],"difference":round(g3[k]-g2b_itc[k],2)} for k in g2b_itc} + result={"period":period,"generated_at_utc":_now(),"sales_reconciliation":sales_rec,"purchase_reconciliation":purchase_rec,"itc_reconciliation":itc} + reports=base_dir/"reports"; reports.mkdir(parents=True,exist_ok=True); _write_json(reports/f"{period}_GST_reconciliation.json",result) + return result + + +def _extract_file_num(text: str) -> str: + try: + data=json.loads(text or "{}") + except Exception: + return "1" + vals=[] + for obj in _walk(data): + if isinstance(obj,dict): + for key in ("file_num","fileNum","filenum","file_number","fileNumber"): + if obj.get(key) not in (None,""): + vals.append(str(obj.get(key))) + return vals[0] if vals else "1" + + +def _extract_payload(text: str) -> str: + try: + data=json.loads(text or "{}") + except Exception: + return text or "" + if isinstance(data,dict): + for key in ("data","json","payload","content","fileContent","response"): + value=data.get(key) + if isinstance(value,(dict,list)): + return json.dumps(value,ensure_ascii=False) + if isinstance(value,str) and value.lstrip().startswith(("{","[")): + return value + return text or "" + + +def _browser_download(payload: dict[str, Any], progress_path: Path, storage_root: Path) -> None: + base_dir = storage_root / Path(str(payload["gst_relative_dir"])) / str(payload["period"]) + raw_dir = base_dir / "raw"; raw_dir.mkdir(parents=True, exist_ok=True) + def progress(**kw): + cur=_read_json(progress_path); cur.update(kw); cur["updated_at_utc"]=_now(); _write_json(progress_path,cur) + try: + progress(status="running",stage="Opening GST Login",message="Opening GST portal. Complete captcha/OTP in the browser.") + from playwright.sync_api import sync_playwright + with sync_playwright() as pw: + browser=None; errors=[] + profile=storage_root/".erp_gst_browser_profile"; profile.mkdir(parents=True,exist_ok=True) + for channel in ("chrome","msedge"): + try: + browser=pw.chromium.launch_persistent_context(user_data_dir=str(profile/channel),channel=channel,headless=False,no_viewport=True,accept_downloads=True,args=["--start-maximized","--no-first-run","--disable-blink-features=AutomationControlled"]) + break + except Exception as exc: errors.append(f"{channel}: {exc}") + if browser is None: raise RuntimeError("Could not open Chrome/Edge through Playwright. "+" | ".join(errors)) + page=browser.pages[0] if browser.pages else browser.new_page() + page.goto(GST_LOGIN_URL,wait_until="domcontentloaded",timeout=90000) + page.wait_for_selector("#username",timeout=30000) + page.fill("#username",str(payload.get("username") or "")); page.fill("#user_pass",str(payload.get("password") or "")) + progress(stage="Waiting for GST Login",message="Username/password filled. Enter captcha and OTP in the GST browser. The agent will continue after login.") + deadline=time.time()+int(payload.get("login_timeout_seconds") or 900) + while time.time() bool: + try: + return bool(page.evaluate("async()=>{try{const r=await fetch('https://return.gst.gov.in/returns/auth/api/returns/profile',{credentials:'include'}); return r.status!==401 && r.status!==403;}catch(e){return false;}}")) + except Exception: + return False + + +def start_download(payload: dict[str, Any], storage_root: Path, progress_root: Path) -> dict[str, Any]: + key=_safe(f"{payload.get('client_id')}_{payload.get('gstin')}_{payload.get('period')}") + progress_path=progress_root/f"gst_download_{key}.json" + current=_read_json(progress_path) + if current.get("status") in {"queued","running"}: + return {"started":False,"already_running":True,"job":current} + initial={"status":"queued","stage":"Queued","message":"GST return download queued.","client_id":payload.get("client_id"),"gstin":payload.get("gstin"),"period":payload.get("period"),"started_at_utc":_now(),"updated_at_utc":_now()} + _write_json(progress_path,initial) + thread=threading.Thread(target=_browser_download,args=(dict(payload),progress_path,storage_root),daemon=True,name=f"gst-download-{key}") + with _LOCK: _THREADS[key]=thread + thread.start() + return {"started":True,"job":initial} + + +def download_status(payload: dict[str, Any], progress_root: Path) -> dict[str, Any]: + key=_safe(f"{payload.get('client_id')}_{payload.get('gstin')}_{payload.get('period')}") + return _read_json(progress_root/f"gst_download_{key}.json") diff --git a/app/modules/documents/local_agent_runtime/requirements.txt b/app/modules/documents/local_agent_runtime/requirements.txt index e1d6097..bf63221 100644 --- a/app/modules/documents/local_agent_runtime/requirements.txt +++ b/app/modules/documents/local_agent_runtime/requirements.txt @@ -2,3 +2,4 @@ requests==2.32.3 python-dotenv==1.0.1 websockets==12.0 pywebview==5.3.2 +playwright==1.55.0 diff --git a/app/ui/app.py b/app/ui/app.py index 6b86bf9..8bfdc94 100644 --- a/app/ui/app.py +++ b/app/ui/app.py @@ -61,6 +61,7 @@ from app.modules.accounting.automatic_voucher_ui import router as accounting_aut from app.modules.accounting.master_integrity_ui import router as accounting_master_integrity_ui_router from app.modules.accounting.cash_payment_ui import router as accounting_cash_payment_ui_router from app.modules.accounting.creditors_aging_ui import router as accounting_creditors_aging_ui_router +from app.modules.accounting.gst_reconciliation_ui import router as accounting_gst_reconciliation_ui_router from app.modules.accounting.tds_ui import router as accounting_tds_ui_router from app.modules.registrations.ui import router as registrations_ui_router from app.modules.credential_vault.ui import router as credential_vault_ui_router @@ -107,6 +108,7 @@ def mount_ui(app: FastAPI) -> None: app.include_router(accounting_master_integrity_ui_router) app.include_router(accounting_cash_payment_ui_router) app.include_router(accounting_creditors_aging_ui_router) + app.include_router(accounting_gst_reconciliation_ui_router) app.include_router(accounting_tds_ui_router) app.include_router(work_tracker_ui_router) app.include_router(billing_ui_router) @@ -138,8 +140,3 @@ def mount_ui(app: FastAPI) -> None: app.include_router(task_category_import_router) app.include_router(client_portal_router) app.include_router(consultant_portal_router) - - - - -