diff --git a/app/modules/accounting/gst_reconciliation_ui.py b/app/modules/accounting/gst_reconciliation_ui.py index 50e6c3b..99647a4 100644 --- a/app/modules/accounting/gst_reconciliation_ui.py +++ b/app/modules/accounting/gst_reconciliation_ui.py @@ -4,6 +4,9 @@ import calendar import re import tempfile import uuid +import io +import json +import zipfile from datetime import date, datetime, timedelta, timezone from pathlib import Path from urllib.parse import urlencode @@ -24,6 +27,7 @@ from app.modules.credential_vault.crypto import decrypt_value from app.modules.credential_vault.models import CredentialVaultEntry from app.modules.credential_vault.service import can_view_entry, log_access from app.modules.documents.services import client_folder_parts, get_active_storage_node_for_branch, sanitize_segment +from app.modules.documents.local_agent_runtime.erp_local_agent.gst_portal_runtime import extract_invoice_rows, extract_gstr3b_itc from app.modules.registrations.models import ClientRegistration, RegistrationType router = APIRouter(prefix="/tools/accounting/gst-reconciliation", tags=["accounting-gst-reconciliation-ui"]) @@ -170,6 +174,103 @@ def _selected_periods(download_mode: str, financial_year: str, period: str) -> l return [digits] + +GST_LOGIN_URL = "https://services.gst.gov.in/services/login" +_MAX_IMPORT_BYTES = 250 * 1024 * 1024 + + +def _safe_upload_name(name: str) -> str: + cleaned = re.sub(r"[^A-Za-z0-9._ -]+", "_", str(name or "").strip()).strip(" ._") + return cleaned or "return.json" + + +def _period_from_name_or_payload(name: str, payload, fallback: str = "") -> str: + def valid(v: str) -> str: + d = re.sub(r"\D", "", str(v or "")) + if len(d) == 6: + mm, yy = int(d[:2]), int(d[2:]) + if 1 <= mm <= 12 and 2000 <= yy <= 2100: + return d + yy2, mm2 = int(d[:4]), int(d[4:]) + if 2000 <= yy2 <= 2100 and 1 <= mm2 <= 12: + return f"{mm2:02d}{yy2:04d}" + return "" + + if isinstance(payload, dict): + stack = [payload] + seen = 0 + while stack and seen < 5000: + obj = stack.pop(); seen += 1 + if not isinstance(obj, dict): + continue + for key in ("fp", "rtn_prd", "return_period", "period", "ret_period"): + if key in obj: + found = valid(obj.get(key)) + if found: + return found + for value in obj.values(): + if isinstance(value, dict): + stack.append(value) + elif isinstance(value, list): + stack.extend(x for x in value[:200] if isinstance(x, dict)) + + for pattern in (r"(? list[tuple[str, object, bytes]]: + lower = str(upload_name or "").lower() + if lower.endswith(".json"): + try: + payload = json.loads(content.decode("utf-8-sig", errors="strict")) + except Exception as exc: + raise ValueError(f"{upload_name}: invalid JSON file ({exc}).") from exc + return [(upload_name, payload, content)] + if lower.endswith(".zip"): + out = [] + try: + with zipfile.ZipFile(io.BytesIO(content), "r") as archive: + for info in archive.infolist(): + if info.is_dir() or not info.filename.lower().endswith(".json"): + continue + raw = archive.read(info) + try: + payload = json.loads(raw.decode("utf-8-sig", errors="strict")) + except Exception: + continue + out.append((Path(info.filename).name, payload, raw)) + except zipfile.BadZipFile as exc: + raise ValueError(f"{upload_name}: invalid ZIP file.") from exc + if not out: + raise ValueError(f"{upload_name}: ZIP does not contain readable JSON return data.") + return out + raise ValueError(f"{upload_name}: only GST JSON or ZIP files are supported.") + + +def _merge_invoice_rows(payloads: list[object]) -> list[dict]: + rows=[]; seen=set() + for payload in payloads: + for row in extract_invoice_rows(payload): + key=(str(row.get("gstin") or ""), str(row.get("invoice_key") or ""), str(row.get("invoice_date") or "")) + if key in seen: + continue + seen.add(key); rows.append(row) + return rows + + +def _merge_gstr3b_itc(payloads: list[object]) -> dict[str, float]: + totals={"igst":0.0,"cgst":0.0,"sgst":0.0,"cess":0.0} + for payload in payloads: + row=extract_gstr3b_itc(payload) + for k in totals: + totals[k]=round(totals[k]+float(row.get(k) or 0),2) + return totals + @router.get("") def page(request: Request, client_id: int | None = None, registration_id: int | None = None, period: str = "", financial_year: str = "", download_mode: str = "single", message: str = "", error: str = ""): db = CommonSessionLocal() @@ -196,7 +297,7 @@ def page(request: Request, client_id: int | None = None, registration_id: int | return templates.TemplateResponse("modules/accounting/templates/accounting/gst_reconciliation.html",{ "request":request,"current_user":user,"current_user_roles":get_user_roles(db,user.id),"current_user_permissions":get_user_permissions(db,user.id),"csrf_token":get_or_create_csrf_token(request), "clients":clients,"selected_client":selected,"registrations":registrations,"selected_registration":selected_reg,"credentials":credentials,"node":node,"node_online":_node_online(node) if node else False, - "period":period,"financial_year":financial_year,"download_mode":download_mode,"operator_job":operator_job,"message":message,"error":error,"title":"GST Return Reconciliation", + "period":period,"financial_year":financial_year,"download_mode":download_mode,"operator_job":operator_job,"gst_login_url":GST_LOGIN_URL,"message":message,"error":error,"title":"GST Return Reconciliation", }) finally: db.close() @@ -318,6 +419,136 @@ def operator_package(job_id: str, token: str): return JSONResponse({"ok":False,"error":str(exc)},status_code=400) + +@router.post("/import") +async def import_downloaded_returns( + request: Request, + client_id: int = Form(...), registration_id: int = Form(...), period: str = Form(""), + financial_year: str = Form(...), download_mode: str = Form("single"), + gstr1_files: list[UploadFile] = File(default=[]), + gstr2b_files: list[UploadFile] = File(default=[]), + gstr3b_files: list[UploadFile] = File(default=[]), + gstr2a_files: list[UploadFile] = File(default=[]), + csrf_token: str = Form(...), +): + validate_csrf(request, csrf_token) + db = CommonSessionLocal() + temp_root = None + package_path = None + try: + user, response = _require_partner(request, db, "accounting.learning.manage") + if response: + return response + client, _, scope = _find_visible_client(db, request, user, client_id) + if not client: + return _redirect(client_id, error="Client is not available in your scope.") + pair = next(((r,t) for r,t in _gst_regs(db, scope.tenant_id, client.id) if int(r.id)==registration_id and _is_gstin(r,t)), None) + if not pair: + return _redirect(client_id, error="Select a valid GSTIN registration.") + reg, _ = pair + gstin = _norm_gstin(reg.registration_number) + node = get_active_storage_node_for_branch(db, scope.tenant_id, scope.branch_id) + if not node or not _node_online(node): + return _redirect(client_id, registration_id=registration_id, period=period, financial_year=financial_year, download_mode=download_mode, error="Configured Local Storage Agent is offline.") + + allowed_periods = set(_selected_periods(download_mode, financial_year, period)) + if download_mode == "single": + financial_year = _fy_for_period(next(iter(allowed_periods))) + accounting_dir, gst_dir = _storage_payload(client, financial_year, gstin) + groups = { + "GSTR1": gstr1_files or [], "GSTR2B": gstr2b_files or [], + "GSTR3B": gstr3b_files or [], "GSTR2A": gstr2a_files or [], + } + if not any(groups.values()): + raise ValueError("Select at least one GST JSON/ZIP file to import.") + + temp_root = Path(tempfile.mkdtemp(prefix="gst_manual_import_")) + by_period: dict[str, dict[str, dict]] = {} + total_bytes = 0 + imported_files = 0 + for rtype, uploads in groups.items(): + for upload in uploads: + if not upload or not upload.filename: + continue + content = await upload.read() + total_bytes += len(content) + if total_bytes > _MAX_IMPORT_BYTES: + raise ValueError("GST import exceeds the 250 MB safety limit.") + for member_name, payload, raw in _json_members(upload.filename, content): + p = _period_from_name_or_payload(member_name, payload, period if download_mode == "single" else "") + if not p: + raise ValueError(f"Could not determine return period for {member_name}. Use GST JSON containing fp/return period, or Single Month mode.") + if p not in allowed_periods: + raise ValueError(f"{member_name} belongs to {p}, outside the selected {'financial year' if download_mode=='full_fy' else 'month'}.") + slot = by_period.setdefault(p, {}).setdefault(rtype, {"payloads": [], "raw": []}) + slot["payloads"].append(payload) + slot["raw"].append((member_name, raw)) + imported_files += 1 + + if not by_period: + raise ValueError("No readable GST JSON return data was found in the selected files.") + + now = datetime.now(timezone.utc).isoformat() + return_types = set() + for p, period_data in sorted(by_period.items()): + base = temp_root / p + raw_dir = base / "raw" + normalized_dir = base / "normalized" + raw_dir.mkdir(parents=True, exist_ok=True) + normalized_dir.mkdir(parents=True, exist_ok=True) + manifest_downloaded=[]; normalized={} + for rtype, data in period_data.items(): + return_types.add(rtype) + originals = raw_dir / "original" + originals.mkdir(parents=True, exist_ok=True) + payloads = data["payloads"] + for idx, (member_name, raw) in enumerate(data["raw"], 1): + safe = _safe_upload_name(member_name) + dest = originals / f"{rtype}_{idx:03d}_{safe}" + dest.write_bytes(raw) + manifest_downloaded.append({"return_type": rtype, "source_name": member_name, "path": str(dest.relative_to(temp_root).as_posix()), "bytes": len(raw)}) + canonical = payloads[0] if len(payloads) == 1 else {"period": p, "source_files": len(payloads), "payloads": payloads} + (raw_dir / f"{p}_{rtype}.json").write_text(json.dumps(canonical, ensure_ascii=False, indent=2, default=str), encoding="utf-8") + if rtype in {"GSTR1", "GSTR2A", "GSTR2B"}: + rows = _merge_invoice_rows(payloads) + (normalized_dir / f"{p}_{rtype}_invoices.json").write_text(json.dumps({"period":p,"rows":rows}, ensure_ascii=False, indent=2, default=str), encoding="utf-8") + normalized[rtype] = {"invoice_rows": len(rows)} + elif rtype == "GSTR3B": + itc = _merge_gstr3b_itc(payloads) + (normalized_dir / f"{p}_{rtype}_itc.json").write_text(json.dumps({"period":p,"itc":itc}, ensure_ascii=False, indent=2), encoding="utf-8") + normalized[rtype] = {"itc": itc} + (base / "download_manifest.json").write_text(json.dumps({"gstin":gstin,"period":p,"financial_year":financial_year,"downloaded_at_utc":now,"source":"manual_gst_portal_browser_import","downloaded":manifest_downloaded,"normalized":normalized}, ensure_ascii=False, indent=2), encoding="utf-8") + + jti = uuid.uuid4().hex + token = _encode_operator_token({"jti":jti,"user_id":int(user.id),"tenant_id":int(scope.tenant_id),"branch_id":int(scope.branch_id),"node_code":str(node.node_code),"client_id":int(client.id),"gstin":gstin,"financial_year":financial_year}) + _UPLOAD_ROOT.mkdir(parents=True, exist_ok=True) + package_path = _UPLOAD_ROOT / f"{jti}.zip" + with zipfile.ZipFile(package_path, "w", zipfile.ZIP_DEFLATED, compresslevel=6) as archive: + for f in sorted(temp_root.rglob("*")): + if f.is_file(): + archive.write(f, f.relative_to(temp_root).as_posix()) + package_url = str(get_settings().ERP_PUBLIC_BASE_URL).rstrip("/") + f"/tools/accounting/gst-reconciliation/operator/package/{jti}?token={token}" + result = request_agent_command(str(node.node_code), "gst_return_package_store", { + "client_id": int(client.id), "gstin": gstin, "financial_year": financial_year, + "gst_relative_dir": gst_dir, "package_url": package_url, + "periods": sorted(by_period), "return_types": sorted(return_types), + }, timeout_seconds=180) + if not result.get("ok"): + raise RuntimeError(result.get("error") or "Local Storage Agent could not store the imported GST return package.") + return _redirect(client_id, registration_id=registration_id, period=period, financial_year=financial_year, download_mode=download_mode, message=f"Imported {imported_files} GST JSON file(s) for {len(by_period)} period(s) and stored them in the client GST directory. You can now run reconciliation.") + except Exception as exc: + db.rollback() + return _redirect(client_id, registration_id=registration_id, period=period, financial_year=financial_year, download_mode=download_mode, error=str(exc)) + finally: + db.close() + if package_path: + try: package_path.unlink(missing_ok=True) + except Exception: pass + if temp_root: + import shutil + shutil.rmtree(temp_root, ignore_errors=True) + + @router.post("/analyze") def analyze(request: Request, client_id: int=Form(...), registration_id: int=Form(...), period: str=Form(""), financial_year: str=Form(...), analyze_mode: str=Form("single"), csrf_token: str=Form(...)): validate_csrf(request,csrf_token) diff --git a/app/modules/accounting/templates/accounting/gst_reconciliation.html b/app/modules/accounting/templates/accounting/gst_reconciliation.html index ebf8198..d533b3c 100644 --- a/app/modules/accounting/templates/accounting/gst_reconciliation.html +++ b/app/modules/accounting/templates/accounting/gst_reconciliation.html @@ -2,7 +2,7 @@ {% block content %}
Download GST returns on the workstation where you are using ERP, store them in the configured client local storage, and reconcile against Accounting Mirror.
Open GST Portal directly in your browser, import downloaded return files into the configured client local storage, and reconcile against Accounting Mirror.
Sales: Accounting Mirror vs GSTR-1. Purchases: Accounting Mirror vs GSTR-2B. ITC: GSTR-2B vs GSTR-3B. Analysis reads only stored local return data after download.
- +Sales: Accounting Mirror vs GSTR-1. Purchases: Accounting Mirror vs GSTR-2B. ITC: GSTR-2B vs GSTR-3B. Analysis reads only return data already stored in the client's local-storage GST directory.
+