A R R R Associates
735 lines
37 KiB
Python
735 lines
37 KiB
Python
from __future__ import annotations
|
|
|
|
import calendar
|
|
import re
|
|
import tempfile
|
|
import uuid
|
|
import io
|
|
import json
|
|
import zipfile
|
|
from datetime import date, datetime, timedelta, timezone
|
|
from pathlib import Path
|
|
from urllib.parse import urlencode
|
|
|
|
import jwt
|
|
from fastapi import APIRouter, File, Form, Request, UploadFile
|
|
from fastapi.responses import FileResponse, JSONResponse, RedirectResponse, StreamingResponse
|
|
from sqlalchemy import select
|
|
|
|
from app.core.db.common import CommonSessionLocal
|
|
from app.core.security.csrf import get_or_create_csrf_token, validate_csrf
|
|
from app.core.settings import get_settings
|
|
from app.core.templating import templates
|
|
from app.modules.accounting.agent_bridge import request_agent_command
|
|
from app.modules.accounting.ui import _find_visible_client, _require_partner, _visible_clients, _node_online
|
|
from app.modules.core.rbac.deps import get_user_permissions, get_user_roles
|
|
from app.modules.credential_vault.crypto import decrypt_value
|
|
from app.modules.credential_vault.models import CredentialVaultEntry
|
|
from app.modules.credential_vault.service import can_view_entry, log_access
|
|
from app.modules.documents.services import client_folder_parts, get_active_storage_node_for_branch, sanitize_segment
|
|
from app.modules.registrations.models import ClientRegistration, RegistrationType
|
|
|
|
|
|
|
|
def _gst_parse_date(value: str) -> str:
|
|
text = str(value or "").strip()
|
|
for fmt in ("%d-%m-%Y", "%d/%m/%Y", "%Y-%m-%d", "%d-%b-%Y", "%d/%m/%y"):
|
|
try:
|
|
return datetime.strptime(text, fmt).date().isoformat()
|
|
except Exception:
|
|
pass
|
|
return text
|
|
|
|
|
|
def _gst_float(value) -> float:
|
|
try:
|
|
return float(value or 0)
|
|
except Exception:
|
|
return 0.0
|
|
|
|
|
|
def _gst_doc_key(value: str) -> str:
|
|
return "".join(ch for ch in str(value or "").upper() if ch.isalnum())
|
|
|
|
|
|
def _gst_walk(obj):
|
|
if isinstance(obj, dict):
|
|
yield obj
|
|
for value in obj.values():
|
|
yield from _gst_walk(value)
|
|
elif isinstance(obj, list):
|
|
for value in obj:
|
|
yield from _gst_walk(value)
|
|
|
|
|
|
def _gst_tax_from_invoice(inv: dict) -> dict[str, float]:
|
|
out = {"taxable": 0.0, "igst": 0.0, "cgst": 0.0, "sgst": 0.0, "cess": 0.0}
|
|
for item in inv.get("itms") or inv.get("items") or []:
|
|
detail = item.get("itm_det") or item.get("item_det") or item
|
|
out["taxable"] += _gst_float(detail.get("txval"))
|
|
out["igst"] += _gst_float(detail.get("iamt"))
|
|
out["cgst"] += _gst_float(detail.get("camt"))
|
|
out["sgst"] += _gst_float(detail.get("samt"))
|
|
out["cess"] += _gst_float(detail.get("csamt"))
|
|
if not any(out.values()):
|
|
out["taxable"] = _gst_float(inv.get("txval") or inv.get("taxable_value"))
|
|
out["igst"] = _gst_float(inv.get("iamt") or inv.get("igst"))
|
|
out["cgst"] = _gst_float(inv.get("camt") or inv.get("cgst"))
|
|
out["sgst"] = _gst_float(inv.get("samt") or inv.get("sgst"))
|
|
out["cess"] = _gst_float(inv.get("csamt") or inv.get("cess"))
|
|
return out
|
|
|
|
|
|
def extract_invoice_rows(data) -> list[dict]:
|
|
"""Normalize invoice-like rows from GST JSON without importing workstation runtime code."""
|
|
rows: list[dict] = []
|
|
seen: set[tuple[str, str, str]] = set()
|
|
for obj in _gst_walk(data):
|
|
invno = str(
|
|
obj.get("inum")
|
|
or obj.get("inv_num")
|
|
or obj.get("doc_no")
|
|
or obj.get("invoiceNumber")
|
|
or ""
|
|
).strip()
|
|
if not invno:
|
|
continue
|
|
gstin = str(
|
|
obj.get("ctin")
|
|
or obj.get("stin")
|
|
or obj.get("supplier_gstin")
|
|
or obj.get("recipientGstin")
|
|
or ""
|
|
).strip().upper()
|
|
invoice_date = _gst_parse_date(
|
|
str(
|
|
obj.get("idt")
|
|
or obj.get("inv_date")
|
|
or obj.get("doc_date")
|
|
or obj.get("invoiceDate")
|
|
or ""
|
|
)
|
|
)
|
|
key = (gstin, _gst_doc_key(invno), invoice_date)
|
|
if key in seen:
|
|
continue
|
|
seen.add(key)
|
|
tax = _gst_tax_from_invoice(obj)
|
|
rows.append(
|
|
{
|
|
"gstin": gstin,
|
|
"invoice_no": invno,
|
|
"invoice_key": _gst_doc_key(invno),
|
|
"invoice_date": invoice_date,
|
|
"invoice_value": _gst_float(
|
|
obj.get("val") or obj.get("invoice_value") or obj.get("invoiceValue")
|
|
),
|
|
**tax,
|
|
}
|
|
)
|
|
return rows
|
|
|
|
|
|
def extract_gstr3b_itc(data) -> dict[str, float]:
|
|
"""Normalize GSTR-3B eligible-ITC totals from uploaded portal JSON."""
|
|
totals = {"igst": 0.0, "cgst": 0.0, "sgst": 0.0, "cess": 0.0}
|
|
if not isinstance(data, dict):
|
|
return totals
|
|
itc = data.get("itc_elg") or {}
|
|
for item in itc.get("itc_avl") or []:
|
|
totals["igst"] += _gst_float(item.get("iamt"))
|
|
totals["cgst"] += _gst_float(item.get("camt"))
|
|
totals["sgst"] += _gst_float(item.get("samt"))
|
|
totals["cess"] += _gst_float(item.get("csamt"))
|
|
return totals
|
|
|
|
router = APIRouter(prefix="/tools/accounting/gst-reconciliation", tags=["accounting-gst-reconciliation-ui"])
|
|
_TOKEN_PURPOSE = "gst_lightweight_operator_v2"
|
|
_TOKEN_MINUTES = 15
|
|
_UPLOAD_ROOT = Path(tempfile.gettempdir()) / "audit_firm_gst_operator_uploads"
|
|
_OPERATOR_AGENT_VERSION = "1.0.0"
|
|
_OPERATOR_AGENT_PORT = 8791
|
|
_OPERATOR_RUNTIME_ROOT = Path(__file__).resolve().parent / "gst_operator_agent_runtime"
|
|
|
|
|
|
def _fy_bounds(fy: str) -> tuple[date, date]:
|
|
m = re.fullmatch(r"(\d{4})-(\d{2})", str(fy or "").strip())
|
|
if not m:
|
|
raise ValueError("Invalid financial year.")
|
|
y = int(m.group(1))
|
|
return date(y, 4, 1), date(y + 1, 3, 31)
|
|
|
|
|
|
def _periods_for_fy(fy: str) -> list[str]:
|
|
start, _ = _fy_bounds(fy)
|
|
periods=[]
|
|
y=start.year; m=4
|
|
for _ in range(12):
|
|
periods.append(f"{m:02d}{y:04d}")
|
|
m += 1
|
|
if m == 13:
|
|
m = 1; y += 1
|
|
return periods
|
|
|
|
|
|
def _fy_for_period(period: str) -> str:
|
|
digits = re.sub(r"\D", "", period or "")
|
|
if len(digits) != 6:
|
|
raise ValueError("Return period must be MMYYYY.")
|
|
month, year = int(digits[:2]), int(digits[2:])
|
|
if month < 1 or month > 12:
|
|
raise ValueError("Invalid GST return month.")
|
|
sy = year if month >= 4 else year - 1
|
|
return f"{sy}-{str(sy+1)[-2:]}"
|
|
|
|
|
|
def _period_bounds(period: str) -> tuple[str, str]:
|
|
digits = re.sub(r"\D", "", period or "")
|
|
month, year = int(digits[:2]), int(digits[2:])
|
|
last = calendar.monthrange(year, month)[1]
|
|
return date(year, month, 1).isoformat(), date(year, month, last).isoformat()
|
|
|
|
|
|
def _gst_regs(db, tenant_id: int, client_id: int):
|
|
return db.execute(
|
|
select(ClientRegistration, RegistrationType)
|
|
.join(RegistrationType, RegistrationType.id == ClientRegistration.registration_type_id)
|
|
.where(ClientRegistration.tenant_id == tenant_id, ClientRegistration.client_id == client_id)
|
|
.order_by(ClientRegistration.id.asc())
|
|
).all()
|
|
|
|
|
|
def _is_gstin(reg, typ) -> bool:
|
|
code = str(getattr(typ, "code", "") or "").upper().strip()
|
|
num = re.sub(r"\s+", "", str(getattr(reg, "registration_number", "") or "").upper())
|
|
return code == "GSTIN" and len(num) == 15
|
|
|
|
|
|
def _norm_gstin(value: str | None) -> str:
|
|
return re.sub(r"[^0-9A-Z]", "", str(value or "").upper())
|
|
|
|
|
|
def _credential_is_gst_portal(entry: CredentialVaultEntry) -> bool:
|
|
category = str(entry.category or "").strip().lower()
|
|
title = str(entry.title or "").strip().lower()
|
|
portal_url = str(entry.portal_url or "").strip().lower()
|
|
reference = str(entry.reference_number or "").strip().lower()
|
|
if category == "gst_portal":
|
|
return True
|
|
haystack = " ".join((title, portal_url, reference))
|
|
gst_hint = (
|
|
"gst portal" in haystack or "gst login" in haystack
|
|
or "services.gst.gov.in" in portal_url or "www.gst.gov.in" in portal_url
|
|
)
|
|
if not gst_hint:
|
|
return False
|
|
if category in {"eway_bill", "einvoice", "api_key"}:
|
|
return "gst portal" in title or "gst login" in title
|
|
return True
|
|
|
|
|
|
def _credential_matches_gstin(entry: CredentialVaultEntry, registration_id: int, gstin: str) -> bool:
|
|
if int(entry.registration_id or 0) == int(registration_id):
|
|
return True
|
|
if _norm_gstin(entry.reference_number) == _norm_gstin(gstin):
|
|
return True
|
|
return entry.registration_id is None
|
|
|
|
|
|
def _vault_entries(db, user, request, tenant_id: int, client_id: int, registration_id: int | None = None, gstin: str = ""):
|
|
q = select(CredentialVaultEntry).where(
|
|
CredentialVaultEntry.tenant_id == tenant_id,
|
|
CredentialVaultEntry.client_id == client_id,
|
|
CredentialVaultEntry.status != "archived",
|
|
)
|
|
rows = db.execute(q.order_by(CredentialVaultEntry.title.asc())).scalars().all()
|
|
branch_id = request.session.get("active_branch_id") or getattr(user, "branch_id", None)
|
|
visible = [r for r in rows if can_view_entry(db, user, r, int(branch_id) if branch_id else None)]
|
|
if not registration_id:
|
|
return [r for r in visible if _credential_is_gst_portal(r)]
|
|
eligible = [r for r in visible if _credential_is_gst_portal(r) and _credential_matches_gstin(r, int(registration_id), gstin)]
|
|
def rank(entry: CredentialVaultEntry):
|
|
exact_registration = int(entry.registration_id or 0) == int(registration_id)
|
|
exact_gstin = _norm_gstin(entry.reference_number) == _norm_gstin(gstin)
|
|
client_level = entry.registration_id is None
|
|
return (0 if exact_registration else 1 if exact_gstin else 2 if client_level else 3, str(entry.title or "").lower(), int(entry.id or 0))
|
|
return sorted(eligible, key=rank)
|
|
|
|
|
|
def _storage_payload(client, fy: str, gstin: str) -> tuple[str, str]:
|
|
fy_folder = sanitize_segment(f"FY{fy}", "FY")
|
|
letter, client_folder = client_folder_parts(client, int(client.id))
|
|
root = Path(fy_folder) / "Clients" / letter / client_folder
|
|
return (root / "Accounting").as_posix(), (root / "GST" / sanitize_segment(f"GSTIN_{gstin}", "GSTIN")).as_posix()
|
|
|
|
|
|
def _redirect(client_id: int, **params):
|
|
data = {"client_id": client_id, **{k: v for k, v in params.items() if v not in (None, "")}}
|
|
return RedirectResponse("/tools/accounting/gst-reconciliation?" + urlencode(data), status_code=303)
|
|
|
|
|
|
def _encode_operator_token(payload: dict) -> str:
|
|
now = datetime.now(timezone.utc)
|
|
body = {**payload, "purpose": _TOKEN_PURPOSE, "iat": now, "exp": now + timedelta(minutes=_TOKEN_MINUTES)}
|
|
return jwt.encode(body, get_settings().SECRET_KEY, algorithm="HS256")
|
|
|
|
|
|
def _decode_operator_token(token: str) -> dict:
|
|
data = jwt.decode(token, get_settings().SECRET_KEY, algorithms=["HS256"])
|
|
if data.get("purpose") != _TOKEN_PURPOSE:
|
|
raise ValueError("Invalid GST operator token.")
|
|
return data
|
|
|
|
|
|
def _selected_periods(download_mode: str, financial_year: str, period: str) -> list[str]:
|
|
if download_mode == "full_fy":
|
|
return _periods_for_fy(financial_year)
|
|
digits = re.sub(r"\D", "", period or "")
|
|
if len(digits) != 6:
|
|
raise ValueError("Enter a valid MMYYYY period for Single Month mode.")
|
|
return [digits]
|
|
|
|
|
|
|
|
GST_LOGIN_URL = "https://services.gst.gov.in/services/login"
|
|
_MAX_IMPORT_BYTES = 250 * 1024 * 1024
|
|
|
|
|
|
def _safe_upload_name(name: str) -> str:
|
|
cleaned = re.sub(r"[^A-Za-z0-9._ -]+", "_", str(name or "").strip()).strip(" ._")
|
|
return cleaned or "return.json"
|
|
|
|
|
|
def _period_from_name_or_payload(name: str, payload, fallback: str = "") -> str:
|
|
def valid(v: str) -> str:
|
|
d = re.sub(r"\D", "", str(v or ""))
|
|
if len(d) == 6:
|
|
mm, yy = int(d[:2]), int(d[2:])
|
|
if 1 <= mm <= 12 and 2000 <= yy <= 2100:
|
|
return d
|
|
yy2, mm2 = int(d[:4]), int(d[4:])
|
|
if 2000 <= yy2 <= 2100 and 1 <= mm2 <= 12:
|
|
return f"{mm2:02d}{yy2:04d}"
|
|
return ""
|
|
|
|
if isinstance(payload, dict):
|
|
stack = [payload]
|
|
seen = 0
|
|
while stack and seen < 5000:
|
|
obj = stack.pop(); seen += 1
|
|
if not isinstance(obj, dict):
|
|
continue
|
|
for key in ("fp", "rtn_prd", "return_period", "period", "ret_period"):
|
|
if key in obj:
|
|
found = valid(obj.get(key))
|
|
if found:
|
|
return found
|
|
for value in obj.values():
|
|
if isinstance(value, dict):
|
|
stack.append(value)
|
|
elif isinstance(value, list):
|
|
stack.extend(x for x in value[:200] if isinstance(x, dict))
|
|
|
|
for pattern in (r"(?<!\d)(0[1-9]|1[0-2])(20\d{2})(?!\d)", r"(?<!\d)(20\d{2})(0[1-9]|1[0-2])(?!\d)"):
|
|
m = re.search(pattern, str(name or ""))
|
|
if m:
|
|
if len(m.group(1)) == 2:
|
|
return f"{m.group(1)}{m.group(2)}"
|
|
return f"{m.group(2)}{m.group(1)}"
|
|
return valid(fallback)
|
|
|
|
|
|
def _json_members(upload_name: str, content: bytes) -> list[tuple[str, object, bytes]]:
|
|
lower = str(upload_name or "").lower()
|
|
if lower.endswith(".json"):
|
|
try:
|
|
payload = json.loads(content.decode("utf-8-sig", errors="strict"))
|
|
except Exception as exc:
|
|
raise ValueError(f"{upload_name}: invalid JSON file ({exc}).") from exc
|
|
return [(upload_name, payload, content)]
|
|
if lower.endswith(".zip"):
|
|
out = []
|
|
try:
|
|
with zipfile.ZipFile(io.BytesIO(content), "r") as archive:
|
|
for info in archive.infolist():
|
|
if info.is_dir() or not info.filename.lower().endswith(".json"):
|
|
continue
|
|
raw = archive.read(info)
|
|
try:
|
|
payload = json.loads(raw.decode("utf-8-sig", errors="strict"))
|
|
except Exception:
|
|
continue
|
|
out.append((Path(info.filename).name, payload, raw))
|
|
except zipfile.BadZipFile as exc:
|
|
raise ValueError(f"{upload_name}: invalid ZIP file.") from exc
|
|
if not out:
|
|
raise ValueError(f"{upload_name}: ZIP does not contain readable JSON return data.")
|
|
return out
|
|
raise ValueError(f"{upload_name}: only GST JSON or ZIP files are supported.")
|
|
|
|
|
|
def _merge_invoice_rows(payloads: list[object]) -> list[dict]:
|
|
rows=[]; seen=set()
|
|
for payload in payloads:
|
|
for row in extract_invoice_rows(payload):
|
|
key=(str(row.get("gstin") or ""), str(row.get("invoice_key") or ""), str(row.get("invoice_date") or ""))
|
|
if key in seen:
|
|
continue
|
|
seen.add(key); rows.append(row)
|
|
return rows
|
|
|
|
|
|
def _merge_gstr3b_itc(payloads: list[object]) -> dict[str, float]:
|
|
totals={"igst":0.0,"cgst":0.0,"sgst":0.0,"cess":0.0}
|
|
for payload in payloads:
|
|
row=extract_gstr3b_itc(payload)
|
|
for k in totals:
|
|
totals[k]=round(totals[k]+float(row.get(k) or 0),2)
|
|
return totals
|
|
|
|
@router.get("")
|
|
def page(request: Request, client_id: int | None = None, registration_id: int | None = None, period: str = "", financial_year: str = "", download_mode: str = "single", message: str = "", error: str = ""):
|
|
db = CommonSessionLocal()
|
|
try:
|
|
user, response = _require_partner(request, db, "accounting.learning.view")
|
|
if response:
|
|
return response
|
|
clients, scope = _visible_clients(db, request, user)
|
|
selected = next((c for c in clients if client_id and int(c.id) == int(client_id)), None)
|
|
registrations=[]; selected_reg=None; credentials=[]; node=None
|
|
if not financial_year:
|
|
financial_year = str(request.session.get("active_financial_year") or request.session.get("active_fy") or "2025-26")
|
|
if download_mode not in {"single", "full_fy"}:
|
|
download_mode = "single"
|
|
if selected:
|
|
registrations=[(r,t) for r,t in _gst_regs(db,scope.tenant_id,selected.id) if _is_gstin(r,t)]
|
|
selected_reg=next((r for r,t in registrations if registration_id and int(r.id)==int(registration_id)),None)
|
|
if not selected_reg and registrations:
|
|
selected_reg=registrations[0][0]
|
|
selected_gstin = _norm_gstin(getattr(selected_reg, "registration_number", "")) if selected_reg else ""
|
|
credentials=_vault_entries(db,user,request,scope.tenant_id,selected.id,int(selected_reg.id) if selected_reg else None, selected_gstin)
|
|
node=get_active_storage_node_for_branch(db,scope.tenant_id,scope.branch_id)
|
|
operator_job = request.session.pop("gst_operator_job", None)
|
|
return templates.TemplateResponse("modules/accounting/templates/accounting/gst_reconciliation.html",{
|
|
"request":request,"current_user":user,"current_user_roles":get_user_roles(db,user.id),"current_user_permissions":get_user_permissions(db,user.id),"csrf_token":get_or_create_csrf_token(request),
|
|
"clients":clients,"selected_client":selected,"registrations":registrations,"selected_registration":selected_reg,"credentials":credentials,"node":node,"node_online":_node_online(node) if node else False,
|
|
"period":period,"financial_year":financial_year,"download_mode":download_mode,"operator_job":operator_job,"gst_login_url":GST_LOGIN_URL,
|
|
"operator_agent_version":_OPERATOR_AGENT_VERSION,"operator_agent_port":_OPERATOR_AGENT_PORT,
|
|
"message":message,"error":error,"title":"GST Return Reconciliation",
|
|
})
|
|
finally:
|
|
db.close()
|
|
|
|
|
|
@router.get("/operator-agent/download")
|
|
def download_operator_agent(request: Request):
|
|
db = CommonSessionLocal()
|
|
try:
|
|
user, response = _require_partner(request, db, "accounting.learning.view")
|
|
if response:
|
|
return response
|
|
if not _OPERATOR_RUNTIME_ROOT.is_dir():
|
|
return JSONResponse({"ok": False, "error": "GST Operator Agent runtime is missing from this ERP build."}, status_code=404)
|
|
memory = io.BytesIO()
|
|
with zipfile.ZipFile(memory, "w", zipfile.ZIP_DEFLATED, compresslevel=6) as archive:
|
|
for path in sorted(_OPERATOR_RUNTIME_ROOT.rglob("*")):
|
|
if path.is_file() and "__pycache__" not in path.parts:
|
|
archive.write(path, Path("ARRR_GST_Operator_Agent") / path.relative_to(_OPERATOR_RUNTIME_ROOT))
|
|
memory.seek(0)
|
|
headers = {"Content-Disposition": f'attachment; filename="ARRR_GST_Operator_Agent_{_OPERATOR_AGENT_VERSION}.zip"'}
|
|
return StreamingResponse(memory, media_type="application/zip", headers=headers)
|
|
finally:
|
|
db.close()
|
|
|
|
|
|
@router.post("/download/start")
|
|
def start_download(
|
|
request: Request,
|
|
client_id: int=Form(...), registration_id: int=Form(...), credential_id: int=Form(...),
|
|
period: str=Form(""), financial_year: str=Form(...), download_mode: str=Form("single"),
|
|
gstr1: str=Form(""), gstr2b: str=Form(""), gstr3b: str=Form(""), gstr2a: str=Form(""),
|
|
csrf_token: str=Form(...),
|
|
):
|
|
validate_csrf(request,csrf_token)
|
|
db=CommonSessionLocal()
|
|
try:
|
|
user,response=_require_partner(request,db,"accounting.learning.manage")
|
|
if response: return response
|
|
client,_,scope=_find_visible_client(db,request,user,client_id)
|
|
if not client: return _redirect(client_id,error="Client is not available in your scope.")
|
|
pair=next(((r,t) for r,t in _gst_regs(db,scope.tenant_id,client.id) if int(r.id)==registration_id and _is_gstin(r,t)),None)
|
|
if not pair: return _redirect(client_id,error="Select a valid GSTIN registration.")
|
|
reg,_=pair; gstin=_norm_gstin(reg.registration_number)
|
|
cred=db.get(CredentialVaultEntry,credential_id)
|
|
credential_ok = bool(cred and int(cred.tenant_id or 0)==int(scope.tenant_id) and int(cred.client_id or 0)==int(client.id) and str(cred.status or "").lower()!="archived" and can_view_entry(db,user,cred,scope.branch_id) and _credential_is_gst_portal(cred) and _credential_matches_gstin(cred,int(reg.id),gstin))
|
|
if not credential_ok:
|
|
return _redirect(client_id,registration_id=registration_id,period=period,financial_year=financial_year,download_mode=download_mode,error="Selected GST Portal credential is not available for this client/GSTIN.")
|
|
node=get_active_storage_node_for_branch(db,scope.tenant_id,scope.branch_id)
|
|
if not node or not _node_online(node):
|
|
return _redirect(client_id,registration_id=registration_id,period=period,financial_year=financial_year,download_mode=download_mode,error="Local Storage Agent is offline.")
|
|
periods=_selected_periods(download_mode,financial_year,period)
|
|
return_types=[name for name,flag in (("GSTR1",gstr1),("GSTR2B",gstr2b),("GSTR3B",gstr3b),("GSTR2A",gstr2a)) if flag]
|
|
if not return_types:
|
|
return _redirect(client_id,registration_id=registration_id,period=period,financial_year=financial_year,download_mode=download_mode,error="Select at least one GST return to download.")
|
|
# Ensure FY is consistent in single-month mode.
|
|
if download_mode == "single":
|
|
financial_year=_fy_for_period(periods[0])
|
|
accounting_dir,gst_dir=_storage_payload(client,financial_year,gstin)
|
|
jti=uuid.uuid4().hex
|
|
token=_encode_operator_token({
|
|
"jti":jti,"user_id":int(user.id),"tenant_id":int(scope.tenant_id),"branch_id":int(scope.branch_id),
|
|
"node_code":str(node.node_code),"client_id":int(client.id),"client_name":str(client.client_name or ""),
|
|
"registration_id":int(reg.id),"credential_id":int(cred.id),"gstin":gstin,"financial_year":financial_year,
|
|
"periods":periods,"return_types":return_types,"gst_relative_dir":gst_dir,"accounting_relative_dir":accounting_dir,
|
|
})
|
|
log_access(db,request,user,cred,"use_for_gst_download",reason=f"GST returns {financial_year}: {','.join(return_types)}",fields="username,secret",success=True)
|
|
db.commit()
|
|
request.session["gst_operator_job"]={"token":token,"job_id":jti,"periods":periods,"return_types":return_types}
|
|
return _redirect(client_id,registration_id=registration_id,period=period,financial_year=financial_year,download_mode=download_mode,message="GST download prepared. The lightweight GST Operator Agent on this computer will open the visible GST browser and autofill the selected Credential Vault login. Complete CAPTCHA/OTP there; downloaded return data will then be transferred to the configured client storage.")
|
|
except Exception as exc:
|
|
db.rollback(); return _redirect(client_id,registration_id=registration_id,period=period,financial_year=financial_year,download_mode=download_mode,error=str(exc))
|
|
finally:
|
|
db.close()
|
|
|
|
|
|
@router.post("/operator/redeem")
|
|
async def operator_redeem(request: Request):
|
|
body=await request.json(); token=str(body.get("token") or "")
|
|
try:
|
|
data=_decode_operator_token(token)
|
|
db=CommonSessionLocal()
|
|
try:
|
|
cred=db.get(CredentialVaultEntry,int(data["credential_id"]))
|
|
if not cred or int(cred.tenant_id or 0)!=int(data["tenant_id"]) or int(cred.client_id or 0)!=int(data["client_id"]) or str(cred.status or "").lower()=="archived":
|
|
raise ValueError("GST credential is no longer available.")
|
|
username=decrypt_value(cred.tenant_id,cred.username_encrypted) or ""
|
|
password=decrypt_value(cred.tenant_id,cred.secret_encrypted) or ""
|
|
if not username or not password:
|
|
raise ValueError("GST username/password is missing in Credential Vault.")
|
|
finally:
|
|
db.close()
|
|
return JSONResponse({"ok":True,"payload":{**{k:v for k,v in data.items() if k not in {"iat","exp","purpose"}},"username":username,"password":password,"upload_url":str(get_settings().ERP_PUBLIC_BASE_URL).rstrip("/")+"/tools/accounting/gst-reconciliation/operator/upload","login_timeout_seconds":900,"operator_agent_min_version":_OPERATOR_AGENT_VERSION}})
|
|
except Exception as exc:
|
|
return JSONResponse({"ok":False,"error":str(exc)},status_code=400)
|
|
|
|
|
|
@router.post("/operator/upload")
|
|
async def operator_upload(token: str=Form(...), package: UploadFile=File(...)):
|
|
try:
|
|
data=_decode_operator_token(token); jti=str(data.get("jti") or "")
|
|
if not jti:
|
|
raise ValueError("GST operator job id is missing.")
|
|
_UPLOAD_ROOT.mkdir(parents=True,exist_ok=True)
|
|
package_path=_UPLOAD_ROOT/f"{jti}.zip"
|
|
total=0
|
|
with package_path.open("wb") as out:
|
|
while True:
|
|
chunk=await package.read(1024*1024)
|
|
if not chunk: break
|
|
total += len(chunk)
|
|
if total > 250*1024*1024:
|
|
raise ValueError("GST download package exceeds the 250 MB safety limit.")
|
|
out.write(chunk)
|
|
package_url=str(get_settings().ERP_PUBLIC_BASE_URL).rstrip("/")+f"/tools/accounting/gst-reconciliation/operator/package/{jti}?token={token}"
|
|
result=request_agent_command(str(data["node_code"]),"gst_return_package_store",{
|
|
"client_id":int(data["client_id"]),"gstin":str(data["gstin"]),"financial_year":str(data["financial_year"]),
|
|
"gst_relative_dir":str(data["gst_relative_dir"]),"package_url":package_url,"periods":data.get("periods") or [],"return_types":data.get("return_types") or [],
|
|
},timeout_seconds=120)
|
|
if not result.get("ok"):
|
|
raise RuntimeError(result.get("error") or "Local Storage Agent could not store the GST return package.")
|
|
try: package_path.unlink(missing_ok=True)
|
|
except Exception: pass
|
|
return JSONResponse({"ok":True,"stored":result.get("result") or {}})
|
|
except Exception as exc:
|
|
return JSONResponse({"ok":False,"error":str(exc)},status_code=400)
|
|
|
|
|
|
@router.get("/operator/package/{job_id}")
|
|
def operator_package(job_id: str, token: str):
|
|
try:
|
|
data=_decode_operator_token(token)
|
|
if str(data.get("jti") or "") != str(job_id):
|
|
raise ValueError("GST package token does not match the requested job.")
|
|
path=_UPLOAD_ROOT/f"{job_id}.zip"
|
|
if not path.is_file():
|
|
return JSONResponse({"ok":False,"error":"GST package is no longer available."},status_code=404)
|
|
return FileResponse(path,media_type="application/zip",filename=f"gst_returns_{job_id}.zip")
|
|
except Exception as exc:
|
|
return JSONResponse({"ok":False,"error":str(exc)},status_code=400)
|
|
|
|
|
|
|
|
@router.post("/import")
|
|
async def import_downloaded_returns(
|
|
request: Request,
|
|
client_id: int = Form(...), registration_id: int = Form(...), period: str = Form(""),
|
|
financial_year: str = Form(...), download_mode: str = Form("single"),
|
|
gstr1_files: list[UploadFile] = File(default=[]),
|
|
gstr2b_files: list[UploadFile] = File(default=[]),
|
|
gstr3b_files: list[UploadFile] = File(default=[]),
|
|
gstr2a_files: list[UploadFile] = File(default=[]),
|
|
csrf_token: str = Form(...),
|
|
):
|
|
validate_csrf(request, csrf_token)
|
|
db = CommonSessionLocal()
|
|
temp_root = None
|
|
package_path = None
|
|
try:
|
|
user, response = _require_partner(request, db, "accounting.learning.manage")
|
|
if response:
|
|
return response
|
|
client, _, scope = _find_visible_client(db, request, user, client_id)
|
|
if not client:
|
|
return _redirect(client_id, error="Client is not available in your scope.")
|
|
pair = next(((r,t) for r,t in _gst_regs(db, scope.tenant_id, client.id) if int(r.id)==registration_id and _is_gstin(r,t)), None)
|
|
if not pair:
|
|
return _redirect(client_id, error="Select a valid GSTIN registration.")
|
|
reg, _ = pair
|
|
gstin = _norm_gstin(reg.registration_number)
|
|
node = get_active_storage_node_for_branch(db, scope.tenant_id, scope.branch_id)
|
|
if not node or not _node_online(node):
|
|
return _redirect(client_id, registration_id=registration_id, period=period, financial_year=financial_year, download_mode=download_mode, error="Configured Local Storage Agent is offline.")
|
|
|
|
allowed_periods = set(_selected_periods(download_mode, financial_year, period))
|
|
if download_mode == "single":
|
|
financial_year = _fy_for_period(next(iter(allowed_periods)))
|
|
accounting_dir, gst_dir = _storage_payload(client, financial_year, gstin)
|
|
groups = {
|
|
"GSTR1": gstr1_files or [], "GSTR2B": gstr2b_files or [],
|
|
"GSTR3B": gstr3b_files or [], "GSTR2A": gstr2a_files or [],
|
|
}
|
|
if not any(groups.values()):
|
|
raise ValueError("Select at least one GST JSON/ZIP file to import.")
|
|
|
|
temp_root = Path(tempfile.mkdtemp(prefix="gst_manual_import_"))
|
|
by_period: dict[str, dict[str, dict]] = {}
|
|
total_bytes = 0
|
|
imported_files = 0
|
|
for rtype, uploads in groups.items():
|
|
for upload in uploads:
|
|
if not upload or not upload.filename:
|
|
continue
|
|
content = await upload.read()
|
|
total_bytes += len(content)
|
|
if total_bytes > _MAX_IMPORT_BYTES:
|
|
raise ValueError("GST import exceeds the 250 MB safety limit.")
|
|
for member_name, payload, raw in _json_members(upload.filename, content):
|
|
p = _period_from_name_or_payload(member_name, payload, period if download_mode == "single" else "")
|
|
if not p:
|
|
raise ValueError(f"Could not determine return period for {member_name}. Use GST JSON containing fp/return period, or Single Month mode.")
|
|
if p not in allowed_periods:
|
|
raise ValueError(f"{member_name} belongs to {p}, outside the selected {'financial year' if download_mode=='full_fy' else 'month'}.")
|
|
slot = by_period.setdefault(p, {}).setdefault(rtype, {"payloads": [], "raw": []})
|
|
slot["payloads"].append(payload)
|
|
slot["raw"].append((member_name, raw))
|
|
imported_files += 1
|
|
|
|
if not by_period:
|
|
raise ValueError("No readable GST JSON return data was found in the selected files.")
|
|
|
|
now = datetime.now(timezone.utc).isoformat()
|
|
return_types = set()
|
|
for p, period_data in sorted(by_period.items()):
|
|
base = temp_root / p
|
|
raw_dir = base / "raw"
|
|
normalized_dir = base / "normalized"
|
|
raw_dir.mkdir(parents=True, exist_ok=True)
|
|
normalized_dir.mkdir(parents=True, exist_ok=True)
|
|
manifest_downloaded=[]; normalized={}
|
|
for rtype, data in period_data.items():
|
|
return_types.add(rtype)
|
|
originals = raw_dir / "original"
|
|
originals.mkdir(parents=True, exist_ok=True)
|
|
payloads = data["payloads"]
|
|
for idx, (member_name, raw) in enumerate(data["raw"], 1):
|
|
safe = _safe_upload_name(member_name)
|
|
dest = originals / f"{rtype}_{idx:03d}_{safe}"
|
|
dest.write_bytes(raw)
|
|
manifest_downloaded.append({"return_type": rtype, "source_name": member_name, "path": str(dest.relative_to(temp_root).as_posix()), "bytes": len(raw)})
|
|
canonical = payloads[0] if len(payloads) == 1 else {"period": p, "source_files": len(payloads), "payloads": payloads}
|
|
(raw_dir / f"{p}_{rtype}.json").write_text(json.dumps(canonical, ensure_ascii=False, indent=2, default=str), encoding="utf-8")
|
|
if rtype in {"GSTR1", "GSTR2A", "GSTR2B"}:
|
|
rows = _merge_invoice_rows(payloads)
|
|
(normalized_dir / f"{p}_{rtype}_invoices.json").write_text(json.dumps({"period":p,"rows":rows}, ensure_ascii=False, indent=2, default=str), encoding="utf-8")
|
|
normalized[rtype] = {"invoice_rows": len(rows)}
|
|
elif rtype == "GSTR3B":
|
|
itc = _merge_gstr3b_itc(payloads)
|
|
(normalized_dir / f"{p}_{rtype}_itc.json").write_text(json.dumps({"period":p,"itc":itc}, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
normalized[rtype] = {"itc": itc}
|
|
(base / "download_manifest.json").write_text(json.dumps({"gstin":gstin,"period":p,"financial_year":financial_year,"downloaded_at_utc":now,"source":"manual_gst_portal_browser_import","downloaded":manifest_downloaded,"normalized":normalized}, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
|
|
jti = uuid.uuid4().hex
|
|
token = _encode_operator_token({"jti":jti,"user_id":int(user.id),"tenant_id":int(scope.tenant_id),"branch_id":int(scope.branch_id),"node_code":str(node.node_code),"client_id":int(client.id),"gstin":gstin,"financial_year":financial_year})
|
|
_UPLOAD_ROOT.mkdir(parents=True, exist_ok=True)
|
|
package_path = _UPLOAD_ROOT / f"{jti}.zip"
|
|
with zipfile.ZipFile(package_path, "w", zipfile.ZIP_DEFLATED, compresslevel=6) as archive:
|
|
for f in sorted(temp_root.rglob("*")):
|
|
if f.is_file():
|
|
archive.write(f, f.relative_to(temp_root).as_posix())
|
|
package_url = str(get_settings().ERP_PUBLIC_BASE_URL).rstrip("/") + f"/tools/accounting/gst-reconciliation/operator/package/{jti}?token={token}"
|
|
result = request_agent_command(str(node.node_code), "gst_return_package_store", {
|
|
"client_id": int(client.id), "gstin": gstin, "financial_year": financial_year,
|
|
"gst_relative_dir": gst_dir, "package_url": package_url,
|
|
"periods": sorted(by_period), "return_types": sorted(return_types),
|
|
}, timeout_seconds=180)
|
|
if not result.get("ok"):
|
|
raise RuntimeError(result.get("error") or "Local Storage Agent could not store the imported GST return package.")
|
|
return _redirect(client_id, registration_id=registration_id, period=period, financial_year=financial_year, download_mode=download_mode, message=f"Imported {imported_files} GST JSON file(s) for {len(by_period)} period(s) and stored them in the client GST directory. You can now run reconciliation.")
|
|
except Exception as exc:
|
|
db.rollback()
|
|
return _redirect(client_id, registration_id=registration_id, period=period, financial_year=financial_year, download_mode=download_mode, error=str(exc))
|
|
finally:
|
|
db.close()
|
|
if package_path:
|
|
try: package_path.unlink(missing_ok=True)
|
|
except Exception: pass
|
|
if temp_root:
|
|
import shutil
|
|
shutil.rmtree(temp_root, ignore_errors=True)
|
|
|
|
|
|
@router.post("/analyze")
|
|
def analyze(request: Request, client_id: int=Form(...), registration_id: int=Form(...), period: str=Form(""), financial_year: str=Form(...), analyze_mode: str=Form("single"), csrf_token: str=Form(...)):
|
|
validate_csrf(request,csrf_token)
|
|
db=CommonSessionLocal()
|
|
try:
|
|
user,response=_require_partner(request,db,"accounting.learning.manage")
|
|
if response: return response
|
|
client,_,scope=_find_visible_client(db,request,user,client_id)
|
|
if not client: return _redirect(client_id,error="Client is not available in your scope.")
|
|
pair=next(((r,t) for r,t in _gst_regs(db,scope.tenant_id,client.id) if int(r.id)==registration_id and _is_gstin(r,t)),None)
|
|
if not pair: return _redirect(client_id,error="GSTIN registration was not found.")
|
|
reg,_=pair; gstin=_norm_gstin(reg.registration_number)
|
|
periods=_periods_for_fy(financial_year) if analyze_mode=="full_fy" else [re.sub(r"\D","",period or "")]
|
|
if any(len(p)!=6 for p in periods): raise ValueError("Enter a valid MMYYYY period.")
|
|
accounting_dir,gst_dir=_storage_payload(client,financial_year,gstin)
|
|
node=get_active_storage_node_for_branch(db,scope.tenant_id,scope.branch_id)
|
|
if not node or not _node_online(node): return _redirect(client_id,registration_id=registration_id,period=period,financial_year=financial_year,error="Local Storage Agent is offline.")
|
|
results=[]
|
|
for p in periods:
|
|
date_from,date_to=_period_bounds(p)
|
|
res=request_agent_command(node.node_code,"gst_reconciliation_analyze",{"client_id":client.id,"gstin":gstin,"financial_year":financial_year,"period":p,"gst_relative_dir":gst_dir,"accounting_relative_dir":accounting_dir,"date_from":date_from,"date_to":date_to},timeout_seconds=30)
|
|
if res.get("ok"):
|
|
results.append((res.get("result") or {}).get("analysis") or {})
|
|
if not results: raise RuntimeError("No stored GST periods could be reconciled.")
|
|
if analyze_mode=="full_fy":
|
|
def sum_counts(section):
|
|
out={}
|
|
for r in results:
|
|
for k,v in (((r.get(section) or {}).get("counts") or {}).items()): out[k]=out.get(k,0)+int(v or 0)
|
|
return out
|
|
taxes={k:{"gstr2b":0.0,"gstr3b":0.0,"difference":0.0} for k in ("igst","cgst","sgst","cess")}
|
|
for r in results:
|
|
for k,row in (r.get("itc_reconciliation") or {}).items():
|
|
if k in taxes:
|
|
for f in taxes[k]: taxes[k][f]=round(taxes[k][f]+float(row.get(f) or 0),2)
|
|
analysis={"period":financial_year,"sales_reconciliation":{"counts":sum_counts("sales_reconciliation")},"purchase_reconciliation":{"counts":sum_counts("purchase_reconciliation")},"itc_reconciliation":taxes}
|
|
else:
|
|
analysis=results[0]
|
|
request.session["gst_reconciliation_result"]=analysis
|
|
return _redirect(client_id,registration_id=registration_id,period=period,financial_year=financial_year,download_mode=("full_fy" if analyze_mode=="full_fy" else "single"),message="GST Purchase, Sales and ITC reconciliation completed from stored local return data and Accounting Mirror.")
|
|
except Exception as exc:
|
|
return _redirect(client_id,registration_id=registration_id,period=period,financial_year=financial_year,error=str(exc))
|
|
finally:
|
|
db.close()
|