A R R R Associates
752 lines
39 KiB
Python
752 lines
39 KiB
Python
from __future__ import annotations
|
|
|
|
import calendar
|
|
import re
|
|
import tempfile
|
|
import uuid
|
|
import io
|
|
import json
|
|
import zipfile
|
|
from datetime import date, datetime, timedelta, timezone
|
|
from pathlib import Path
|
|
from urllib.parse import urlencode
|
|
|
|
import jwt
|
|
from fastapi import APIRouter, File, Form, Request, UploadFile
|
|
from fastapi.responses import FileResponse, JSONResponse, RedirectResponse, StreamingResponse
|
|
from sqlalchemy import select
|
|
|
|
from app.core.db.common import CommonSessionLocal
|
|
from app.core.security.csrf import get_or_create_csrf_token, validate_csrf
|
|
from app.core.settings import get_settings
|
|
from app.core.templating import templates
|
|
from app.modules.accounting.agent_bridge import request_agent_command
|
|
from app.modules.accounting.ui import _accounting_storage_payload, _find_visible_client, _require_partner, _visible_clients, _node_online
|
|
from app.modules.core.rbac.deps import get_user_permissions, get_user_roles
|
|
from app.modules.credential_vault.crypto import decrypt_value
|
|
from app.modules.credential_vault.models import CredentialVaultEntry
|
|
from app.modules.credential_vault.service import can_view_entry, log_access
|
|
from app.modules.documents.services import client_folder_parts, get_active_storage_node_for_branch, sanitize_segment
|
|
from app.modules.registrations.models import ClientRegistration, RegistrationType
|
|
|
|
|
|
|
|
def _gst_parse_date(value: str) -> str:
|
|
text = str(value or "").strip()
|
|
for fmt in ("%d-%m-%Y", "%d/%m/%Y", "%Y-%m-%d", "%d-%b-%Y", "%d/%m/%y"):
|
|
try:
|
|
return datetime.strptime(text, fmt).date().isoformat()
|
|
except Exception:
|
|
pass
|
|
return text
|
|
|
|
|
|
def _gst_float(value) -> float:
|
|
try:
|
|
return float(value or 0)
|
|
except Exception:
|
|
return 0.0
|
|
|
|
|
|
def _gst_doc_key(value: str) -> str:
|
|
return "".join(ch for ch in str(value or "").upper() if ch.isalnum())
|
|
|
|
|
|
def _gst_walk(obj):
|
|
if isinstance(obj, dict):
|
|
yield obj
|
|
for value in obj.values():
|
|
yield from _gst_walk(value)
|
|
elif isinstance(obj, list):
|
|
for value in obj:
|
|
yield from _gst_walk(value)
|
|
|
|
|
|
def _gst_tax_from_invoice(inv: dict) -> dict[str, float]:
|
|
out = {"taxable": 0.0, "igst": 0.0, "cgst": 0.0, "sgst": 0.0, "cess": 0.0}
|
|
for item in inv.get("itms") or inv.get("items") or []:
|
|
detail = item.get("itm_det") or item.get("item_det") or item
|
|
out["taxable"] += _gst_float(detail.get("txval"))
|
|
out["igst"] += _gst_float(detail.get("iamt"))
|
|
out["cgst"] += _gst_float(detail.get("camt"))
|
|
out["sgst"] += _gst_float(detail.get("samt"))
|
|
out["cess"] += _gst_float(detail.get("csamt"))
|
|
if not any(out.values()):
|
|
out["taxable"] = _gst_float(inv.get("txval") or inv.get("taxable_value"))
|
|
out["igst"] = _gst_float(inv.get("iamt") or inv.get("igst"))
|
|
out["cgst"] = _gst_float(inv.get("camt") or inv.get("cgst"))
|
|
out["sgst"] = _gst_float(inv.get("samt") or inv.get("sgst"))
|
|
out["cess"] = _gst_float(inv.get("csamt") or inv.get("cess"))
|
|
return out
|
|
|
|
|
|
def extract_invoice_rows(data) -> list[dict]:
|
|
"""Normalize invoice-like rows from GST JSON without importing workstation runtime code."""
|
|
rows: list[dict] = []
|
|
seen: set[tuple[str, str, str]] = set()
|
|
for obj in _gst_walk(data):
|
|
invno = str(
|
|
obj.get("inum")
|
|
or obj.get("inv_num")
|
|
or obj.get("doc_no")
|
|
or obj.get("invoiceNumber")
|
|
or ""
|
|
).strip()
|
|
if not invno:
|
|
continue
|
|
gstin = str(
|
|
obj.get("ctin")
|
|
or obj.get("stin")
|
|
or obj.get("supplier_gstin")
|
|
or obj.get("recipientGstin")
|
|
or ""
|
|
).strip().upper()
|
|
invoice_date = _gst_parse_date(
|
|
str(
|
|
obj.get("idt")
|
|
or obj.get("inv_date")
|
|
or obj.get("doc_date")
|
|
or obj.get("invoiceDate")
|
|
or ""
|
|
)
|
|
)
|
|
key = (gstin, _gst_doc_key(invno), invoice_date)
|
|
if key in seen:
|
|
continue
|
|
seen.add(key)
|
|
tax = _gst_tax_from_invoice(obj)
|
|
rows.append(
|
|
{
|
|
"gstin": gstin,
|
|
"invoice_no": invno,
|
|
"invoice_key": _gst_doc_key(invno),
|
|
"invoice_date": invoice_date,
|
|
"invoice_value": _gst_float(
|
|
obj.get("val") or obj.get("invoice_value") or obj.get("invoiceValue")
|
|
),
|
|
**tax,
|
|
}
|
|
)
|
|
return rows
|
|
|
|
|
|
def extract_gstr3b_itc(data) -> dict[str, float]:
|
|
"""Normalize GSTR-3B eligible-ITC totals from uploaded portal JSON."""
|
|
totals = {"igst": 0.0, "cgst": 0.0, "sgst": 0.0, "cess": 0.0}
|
|
if not isinstance(data, dict):
|
|
return totals
|
|
itc = data.get("itc_elg") or {}
|
|
for item in itc.get("itc_avl") or []:
|
|
totals["igst"] += _gst_float(item.get("iamt"))
|
|
totals["cgst"] += _gst_float(item.get("camt"))
|
|
totals["sgst"] += _gst_float(item.get("samt"))
|
|
totals["cess"] += _gst_float(item.get("csamt"))
|
|
return totals
|
|
|
|
router = APIRouter(prefix="/tools/accounting/gst-reconciliation", tags=["accounting-gst-reconciliation-ui"])
|
|
_TOKEN_PURPOSE = "gst_lightweight_operator_v3"
|
|
_TOKEN_MINUTES = 15
|
|
_UPLOAD_ROOT = Path(tempfile.gettempdir()) / "audit_firm_gst_operator_uploads"
|
|
_OPERATOR_AGENT_VERSION = "1.5.1"
|
|
_OPERATOR_RUNTIME_ROOT = Path(__file__).resolve().parent / "gst_operator_agent_runtime"
|
|
_OPERATOR_PACKAGE_FILES = ("gst_operator_agent.py", "requirements.txt", "README.txt", "install_gst_operator_agent.ps1", "uninstall_gst_operator_agent.ps1")
|
|
|
|
|
|
def _fy_bounds(fy: str) -> tuple[date, date]:
|
|
m = re.fullmatch(r"(\d{4})-(\d{2})", str(fy or "").strip())
|
|
if not m:
|
|
raise ValueError("Invalid financial year.")
|
|
y = int(m.group(1))
|
|
return date(y, 4, 1), date(y + 1, 3, 31)
|
|
|
|
|
|
def _periods_for_fy(fy: str) -> list[str]:
|
|
start, _ = _fy_bounds(fy)
|
|
periods=[]
|
|
y=start.year; m=4
|
|
for _ in range(12):
|
|
periods.append(f"{m:02d}{y:04d}")
|
|
m += 1
|
|
if m == 13:
|
|
m = 1; y += 1
|
|
return periods
|
|
|
|
|
|
def _fy_for_period(period: str) -> str:
|
|
digits = re.sub(r"\D", "", period or "")
|
|
if len(digits) != 6:
|
|
raise ValueError("Return period must be MMYYYY.")
|
|
month, year = int(digits[:2]), int(digits[2:])
|
|
if month < 1 or month > 12:
|
|
raise ValueError("Invalid GST return month.")
|
|
sy = year if month >= 4 else year - 1
|
|
return f"{sy}-{str(sy+1)[-2:]}"
|
|
|
|
|
|
def _period_bounds(period: str) -> tuple[str, str]:
|
|
digits = re.sub(r"\D", "", period or "")
|
|
month, year = int(digits[:2]), int(digits[2:])
|
|
last = calendar.monthrange(year, month)[1]
|
|
return date(year, month, 1).isoformat(), date(year, month, last).isoformat()
|
|
|
|
|
|
def _gst_regs(db, tenant_id: int, client_id: int):
|
|
return db.execute(
|
|
select(ClientRegistration, RegistrationType)
|
|
.join(RegistrationType, RegistrationType.id == ClientRegistration.registration_type_id)
|
|
.where(ClientRegistration.tenant_id == tenant_id, ClientRegistration.client_id == client_id)
|
|
.order_by(ClientRegistration.id.asc())
|
|
).all()
|
|
|
|
|
|
def _is_gstin(reg, typ) -> bool:
|
|
code = str(getattr(typ, "code", "") or "").upper().strip()
|
|
num = re.sub(r"\s+", "", str(getattr(reg, "registration_number", "") or "").upper())
|
|
return code == "GSTIN" and len(num) == 15
|
|
|
|
|
|
def _norm_gstin(value: str | None) -> str:
|
|
return re.sub(r"[^0-9A-Z]", "", str(value or "").upper())
|
|
|
|
|
|
def _credential_is_gst_portal(entry: CredentialVaultEntry) -> bool:
|
|
category = str(entry.category or "").strip().lower()
|
|
title = str(entry.title or "").strip().lower()
|
|
portal_url = str(entry.portal_url or "").strip().lower()
|
|
reference = str(entry.reference_number or "").strip().lower()
|
|
if category == "gst_portal":
|
|
return True
|
|
haystack = " ".join((title, portal_url, reference))
|
|
gst_hint = (
|
|
"gst portal" in haystack or "gst login" in haystack
|
|
or "services.gst.gov.in" in portal_url or "www.gst.gov.in" in portal_url
|
|
)
|
|
if not gst_hint:
|
|
return False
|
|
if category in {"eway_bill", "einvoice", "api_key"}:
|
|
return "gst portal" in title or "gst login" in title
|
|
return True
|
|
|
|
|
|
def _credential_matches_gstin(entry: CredentialVaultEntry, registration_id: int, gstin: str) -> bool:
|
|
if int(entry.registration_id or 0) == int(registration_id):
|
|
return True
|
|
if _norm_gstin(entry.reference_number) == _norm_gstin(gstin):
|
|
return True
|
|
return entry.registration_id is None
|
|
|
|
|
|
def _vault_entries(db, user, request, tenant_id: int, client_id: int, registration_id: int | None = None, gstin: str = ""):
|
|
q = select(CredentialVaultEntry).where(
|
|
CredentialVaultEntry.tenant_id == tenant_id,
|
|
CredentialVaultEntry.client_id == client_id,
|
|
CredentialVaultEntry.status != "archived",
|
|
)
|
|
rows = db.execute(q.order_by(CredentialVaultEntry.title.asc())).scalars().all()
|
|
branch_id = request.session.get("active_branch_id") or getattr(user, "branch_id", None)
|
|
visible = [r for r in rows if can_view_entry(db, user, r, int(branch_id) if branch_id else None)]
|
|
if not registration_id:
|
|
return [r for r in visible if _credential_is_gst_portal(r)]
|
|
eligible = [r for r in visible if _credential_is_gst_portal(r) and _credential_matches_gstin(r, int(registration_id), gstin)]
|
|
def rank(entry: CredentialVaultEntry):
|
|
exact_registration = int(entry.registration_id or 0) == int(registration_id)
|
|
exact_gstin = _norm_gstin(entry.reference_number) == _norm_gstin(gstin)
|
|
client_level = entry.registration_id is None
|
|
return (0 if exact_registration else 1 if exact_gstin else 2 if client_level else 3, str(entry.title or "").lower(), int(entry.id or 0))
|
|
return sorted(eligible, key=rank)
|
|
|
|
|
|
def _storage_payload(client, fy: str, gstin: str) -> tuple[str, str]:
|
|
fy_folder = sanitize_segment(f"FY{fy}", "FY")
|
|
letter, client_folder = client_folder_parts(client, int(client.id))
|
|
root = Path(fy_folder) / "Clients" / letter / client_folder
|
|
accounting = _accounting_storage_payload(client, fy)
|
|
accounting_dir = str(accounting.get("accounting_relative_dir") or (root / "Accounting").as_posix()).strip()
|
|
return accounting_dir, (root / "GST" / sanitize_segment(f"GSTIN_{gstin}", "GSTIN")).as_posix()
|
|
|
|
|
|
def _redirect(client_id: int, **params):
|
|
data = {"client_id": client_id, **{k: v for k, v in params.items() if v not in (None, "")}}
|
|
return RedirectResponse("/tools/accounting/gst-reconciliation?" + urlencode(data), status_code=303)
|
|
|
|
|
|
def _encode_operator_token(payload: dict) -> str:
|
|
now = datetime.now(timezone.utc)
|
|
body = {**payload, "purpose": _TOKEN_PURPOSE, "iat": now, "exp": now + timedelta(minutes=_TOKEN_MINUTES)}
|
|
return jwt.encode(body, get_settings().SECRET_KEY, algorithm="HS256")
|
|
|
|
|
|
def _decode_operator_token(token: str) -> dict:
|
|
data = jwt.decode(token, get_settings().SECRET_KEY, algorithms=["HS256"])
|
|
if data.get("purpose") != _TOKEN_PURPOSE:
|
|
raise ValueError("Invalid GST operator token.")
|
|
return data
|
|
|
|
|
|
def _selected_periods(download_mode: str, financial_year: str, period: str) -> list[str]:
|
|
if download_mode == "full_fy":
|
|
return _periods_for_fy(financial_year)
|
|
digits = re.sub(r"\D", "", period or "")
|
|
if len(digits) != 6:
|
|
raise ValueError("Enter a valid MMYYYY period for Single Month mode.")
|
|
return [digits]
|
|
|
|
|
|
|
|
GST_LOGIN_URL = "https://services.gst.gov.in/services/login"
|
|
_MAX_IMPORT_BYTES = 250 * 1024 * 1024
|
|
|
|
|
|
def _safe_upload_name(name: str) -> str:
|
|
cleaned = re.sub(r"[^A-Za-z0-9._ -]+", "_", str(name or "").strip()).strip(" ._")
|
|
return cleaned or "return.json"
|
|
|
|
|
|
def _period_from_name_or_payload(name: str, payload, fallback: str = "") -> str:
|
|
def valid(v: str) -> str:
|
|
d = re.sub(r"\D", "", str(v or ""))
|
|
if len(d) == 6:
|
|
mm, yy = int(d[:2]), int(d[2:])
|
|
if 1 <= mm <= 12 and 2000 <= yy <= 2100:
|
|
return d
|
|
yy2, mm2 = int(d[:4]), int(d[4:])
|
|
if 2000 <= yy2 <= 2100 and 1 <= mm2 <= 12:
|
|
return f"{mm2:02d}{yy2:04d}"
|
|
return ""
|
|
|
|
if isinstance(payload, dict):
|
|
stack = [payload]
|
|
seen = 0
|
|
while stack and seen < 5000:
|
|
obj = stack.pop(); seen += 1
|
|
if not isinstance(obj, dict):
|
|
continue
|
|
for key in ("fp", "rtn_prd", "return_period", "period", "ret_period"):
|
|
if key in obj:
|
|
found = valid(obj.get(key))
|
|
if found:
|
|
return found
|
|
for value in obj.values():
|
|
if isinstance(value, dict):
|
|
stack.append(value)
|
|
elif isinstance(value, list):
|
|
stack.extend(x for x in value[:200] if isinstance(x, dict))
|
|
|
|
for pattern in (r"(?<!\d)(0[1-9]|1[0-2])(20\d{2})(?!\d)", r"(?<!\d)(20\d{2})(0[1-9]|1[0-2])(?!\d)"):
|
|
m = re.search(pattern, str(name or ""))
|
|
if m:
|
|
if len(m.group(1)) == 2:
|
|
return f"{m.group(1)}{m.group(2)}"
|
|
return f"{m.group(2)}{m.group(1)}"
|
|
return valid(fallback)
|
|
|
|
|
|
def _json_members(upload_name: str, content: bytes) -> list[tuple[str, object, bytes]]:
|
|
lower = str(upload_name or "").lower()
|
|
if lower.endswith(".json"):
|
|
try:
|
|
payload = json.loads(content.decode("utf-8-sig", errors="strict"))
|
|
except Exception as exc:
|
|
raise ValueError(f"{upload_name}: invalid JSON file ({exc}).") from exc
|
|
return [(upload_name, payload, content)]
|
|
if lower.endswith(".zip"):
|
|
out = []
|
|
try:
|
|
with zipfile.ZipFile(io.BytesIO(content), "r") as archive:
|
|
for info in archive.infolist():
|
|
if info.is_dir() or not info.filename.lower().endswith(".json"):
|
|
continue
|
|
raw = archive.read(info)
|
|
try:
|
|
payload = json.loads(raw.decode("utf-8-sig", errors="strict"))
|
|
except Exception:
|
|
continue
|
|
out.append((Path(info.filename).name, payload, raw))
|
|
except zipfile.BadZipFile as exc:
|
|
raise ValueError(f"{upload_name}: invalid ZIP file.") from exc
|
|
if not out:
|
|
raise ValueError(f"{upload_name}: ZIP does not contain readable JSON return data.")
|
|
return out
|
|
raise ValueError(f"{upload_name}: only GST JSON or ZIP files are supported.")
|
|
|
|
|
|
def _merge_invoice_rows(payloads: list[object]) -> list[dict]:
|
|
rows=[]; seen=set()
|
|
for payload in payloads:
|
|
for row in extract_invoice_rows(payload):
|
|
key=(str(row.get("gstin") or ""), str(row.get("invoice_key") or ""), str(row.get("invoice_date") or ""))
|
|
if key in seen:
|
|
continue
|
|
seen.add(key); rows.append(row)
|
|
return rows
|
|
|
|
|
|
def _merge_gstr3b_itc(payloads: list[object]) -> dict[str, float]:
|
|
totals={"igst":0.0,"cgst":0.0,"sgst":0.0,"cess":0.0}
|
|
for payload in payloads:
|
|
row=extract_gstr3b_itc(payload)
|
|
for k in totals:
|
|
totals[k]=round(totals[k]+float(row.get(k) or 0),2)
|
|
return totals
|
|
|
|
@router.get("")
|
|
def page(request: Request, client_id: str = "", registration_id: str = "", period: str = "", financial_year: str = "", download_mode: str = "single", message: str = "", error: str = ""):
|
|
db = CommonSessionLocal()
|
|
try:
|
|
user, response = _require_partner(request, db, "accounting.learning.view")
|
|
if response:
|
|
return response
|
|
clients, scope = _visible_clients(db, request, user)
|
|
client_id_text = str(client_id or "").strip()
|
|
registration_id_text = str(registration_id or "").strip()
|
|
client_id_value = int(client_id_text) if client_id_text.isdigit() else None
|
|
registration_id_value = int(registration_id_text) if registration_id_text.isdigit() else None
|
|
selected = next((c for c in clients if client_id_value and int(c.id) == client_id_value), None)
|
|
registrations=[]; selected_reg=None; credentials=[]; node=None
|
|
if not financial_year:
|
|
financial_year = str(request.session.get("active_financial_year") or request.session.get("active_fy") or "2025-26")
|
|
if download_mode not in {"single", "full_fy"}:
|
|
download_mode = "single"
|
|
if selected:
|
|
registrations=[(r,t) for r,t in _gst_regs(db,scope.tenant_id,selected.id) if _is_gstin(r,t)]
|
|
selected_reg=next((r for r,t in registrations if registration_id_value and int(r.id)==registration_id_value),None)
|
|
if not selected_reg and registrations:
|
|
selected_reg=registrations[0][0]
|
|
selected_gstin = _norm_gstin(getattr(selected_reg, "registration_number", "")) if selected_reg else ""
|
|
credentials=_vault_entries(db,user,request,scope.tenant_id,selected.id,int(selected_reg.id) if selected_reg else None, selected_gstin)
|
|
node=get_active_storage_node_for_branch(db,scope.tenant_id,scope.branch_id)
|
|
operator_job = request.session.pop("gst_operator_job", None)
|
|
return templates.TemplateResponse("modules/accounting/templates/accounting/gst_reconciliation.html",{
|
|
"request":request,"current_user":user,"current_user_roles":get_user_roles(db,user.id),"current_user_permissions":get_user_permissions(db,user.id),"csrf_token":get_or_create_csrf_token(request),
|
|
"clients":clients,"selected_client":selected,"registrations":registrations,"selected_registration":selected_reg,"credentials":credentials,"node":node,"node_online":_node_online(node) if node else False,
|
|
"period":period,"financial_year":financial_year,"download_mode":download_mode,"operator_job":operator_job,"gst_login_url":GST_LOGIN_URL,
|
|
"operator_agent_version":_OPERATOR_AGENT_VERSION,
|
|
"message":message,"error":error,"title":"GST Return Reconciliation",
|
|
})
|
|
finally:
|
|
db.close()
|
|
|
|
|
|
@router.get("/operator-agent/download")
|
|
def download_operator_agent(request: Request):
|
|
db = CommonSessionLocal()
|
|
try:
|
|
user, response = _require_partner(request, db, "accounting.learning.view")
|
|
if response:
|
|
return response
|
|
if not _OPERATOR_RUNTIME_ROOT.is_dir():
|
|
return JSONResponse({"ok": False, "error": "GST Operator Agent runtime is missing from this ERP build."}, status_code=404)
|
|
memory = io.BytesIO()
|
|
with zipfile.ZipFile(memory, "w", zipfile.ZIP_DEFLATED, compresslevel=6) as archive:
|
|
for name in _OPERATOR_PACKAGE_FILES:
|
|
path = _OPERATOR_RUNTIME_ROOT / name
|
|
if not path.is_file():
|
|
return JSONResponse({"ok": False, "error": f"GST Operator Agent package file is missing: {name}"}, status_code=500)
|
|
archive.write(path, Path("ARRR_GST_Operator_Agent") / name)
|
|
memory.seek(0)
|
|
headers = {"Content-Disposition": f'attachment; filename="ARRR_GST_Operator_Agent_{_OPERATOR_AGENT_VERSION}.zip"'}
|
|
return StreamingResponse(memory, media_type="application/zip", headers=headers)
|
|
finally:
|
|
db.close()
|
|
|
|
|
|
def _operator_public_base_url(request: Request) -> str:
|
|
proto = str(request.headers.get("x-forwarded-proto") or request.url.scheme or "https").split(",")[0].strip()
|
|
host = str(request.headers.get("x-forwarded-host") or request.headers.get("host") or request.url.netloc).split(",")[0].strip()
|
|
if not host:
|
|
return "https://office.arrrassociates.com"
|
|
return f"{proto}://{host}".rstrip("/")
|
|
|
|
|
|
@router.post("/download/start")
|
|
def start_download(
|
|
request: Request,
|
|
client_id: int=Form(...), registration_id: int=Form(...), credential_id: int=Form(...),
|
|
period: str=Form(""), financial_year: str=Form(...), download_mode: str=Form("single"),
|
|
gstr1: str=Form(""), gstr2b: str=Form(""), gstr3b: str=Form(""), gstr2a: str=Form(""),
|
|
csrf_token: str=Form(...),
|
|
):
|
|
validate_csrf(request,csrf_token)
|
|
db=CommonSessionLocal()
|
|
try:
|
|
user,response=_require_partner(request,db,"accounting.learning.manage")
|
|
if response: return response
|
|
client,_,scope=_find_visible_client(db,request,user,client_id)
|
|
if not client: return _redirect(client_id,error="Client is not available in your scope.")
|
|
pair=next(((r,t) for r,t in _gst_regs(db,scope.tenant_id,client.id) if int(r.id)==registration_id and _is_gstin(r,t)),None)
|
|
if not pair: return _redirect(client_id,error="Select a valid GSTIN registration.")
|
|
reg,_=pair; gstin=_norm_gstin(reg.registration_number)
|
|
cred=db.get(CredentialVaultEntry,credential_id)
|
|
credential_ok = bool(cred and int(cred.tenant_id or 0)==int(scope.tenant_id) and int(cred.client_id or 0)==int(client.id) and str(cred.status or "").lower()!="archived" and can_view_entry(db,user,cred,scope.branch_id) and _credential_is_gst_portal(cred) and _credential_matches_gstin(cred,int(reg.id),gstin))
|
|
if not credential_ok:
|
|
return _redirect(client_id,registration_id=registration_id,period=period,financial_year=financial_year,download_mode=download_mode,error="Selected GST Portal credential is not available for this client/GSTIN.")
|
|
node=get_active_storage_node_for_branch(db,scope.tenant_id,scope.branch_id)
|
|
if not node or not _node_online(node):
|
|
return _redirect(client_id,registration_id=registration_id,period=period,financial_year=financial_year,download_mode=download_mode,error="Local Storage Agent is offline.")
|
|
periods=_selected_periods(download_mode,financial_year,period)
|
|
return_types=[name for name,flag in (("GSTR1",gstr1),("GSTR2B",gstr2b),("GSTR3B",gstr3b),("GSTR2A",gstr2a)) if flag]
|
|
if not return_types:
|
|
return _redirect(client_id,registration_id=registration_id,period=period,financial_year=financial_year,download_mode=download_mode,error="Select at least one GST return to download.")
|
|
# Ensure FY is consistent in single-month mode.
|
|
if download_mode == "single":
|
|
financial_year=_fy_for_period(periods[0])
|
|
accounting_dir,gst_dir=_storage_payload(client,financial_year,gstin)
|
|
jti=uuid.uuid4().hex
|
|
token=_encode_operator_token({
|
|
"jti":jti,"user_id":int(user.id),"tenant_id":int(scope.tenant_id),"branch_id":int(scope.branch_id),
|
|
"node_code":str(node.node_code),"client_id":int(client.id),"client_name":str(client.client_name or ""),
|
|
"registration_id":int(reg.id),"credential_id":int(cred.id),"gstin":gstin,"financial_year":financial_year,
|
|
"periods":periods,"return_types":return_types,"gst_relative_dir":gst_dir,"accounting_relative_dir":accounting_dir,
|
|
"erp_base_url":_operator_public_base_url(request),
|
|
})
|
|
log_access(db,request,user,cred,"use_for_gst_download",reason=f"GST returns {financial_year}: {','.join(return_types)}",fields="username,secret",success=True)
|
|
db.commit()
|
|
request.session["gst_operator_job"]={"job_id":jti,"periods":periods,"return_types":return_types,"launch_url":"arrrgst://start?"+urlencode({"token":token})}
|
|
return _redirect(client_id,registration_id=registration_id,period=period,financial_year=financial_year,download_mode=download_mode,message="GST download prepared. Windows will launch the lightweight GST Operator Agent through the ARRR GST protocol. The agent will open the visible GST browser and autofill the selected Credential Vault login. Complete CAPTCHA/OTP there; the agent will enter Return Dashboard through the normal GST portal sequence before downloading return data and transferring it to configured client storage.")
|
|
except Exception as exc:
|
|
db.rollback(); return _redirect(client_id,registration_id=registration_id,period=period,financial_year=financial_year,download_mode=download_mode,error=str(exc))
|
|
finally:
|
|
db.close()
|
|
|
|
|
|
@router.post("/operator/redeem")
|
|
async def operator_redeem(request: Request):
|
|
body=await request.json(); token=str(body.get("token") or "")
|
|
try:
|
|
data=_decode_operator_token(token)
|
|
db=CommonSessionLocal()
|
|
try:
|
|
cred=db.get(CredentialVaultEntry,int(data["credential_id"]))
|
|
if not cred or int(cred.tenant_id or 0)!=int(data["tenant_id"]) or int(cred.client_id or 0)!=int(data["client_id"]) or str(cred.status or "").lower()=="archived":
|
|
raise ValueError("GST credential is no longer available.")
|
|
username=decrypt_value(cred.tenant_id,cred.username_encrypted) or ""
|
|
password=decrypt_value(cred.tenant_id,cred.secret_encrypted) or ""
|
|
if not username or not password:
|
|
raise ValueError("GST username/password is missing in Credential Vault.")
|
|
finally:
|
|
db.close()
|
|
return JSONResponse({"ok":True,"payload":{**{k:v for k,v in data.items() if k not in {"iat","exp","purpose"}},"username":username,"password":password,"upload_url":str(data.get("erp_base_url") or "https://office.arrrassociates.com").rstrip("/")+"/tools/accounting/gst-reconciliation/operator/upload","login_timeout_seconds":900,"operator_agent_min_version":_OPERATOR_AGENT_VERSION}})
|
|
except Exception as exc:
|
|
return JSONResponse({"ok":False,"error":str(exc)},status_code=400)
|
|
|
|
|
|
@router.post("/operator/upload")
|
|
async def operator_upload(token: str=Form(...), package: UploadFile=File(...)):
|
|
try:
|
|
data=_decode_operator_token(token); jti=str(data.get("jti") or "")
|
|
if not jti:
|
|
raise ValueError("GST operator job id is missing.")
|
|
_UPLOAD_ROOT.mkdir(parents=True,exist_ok=True)
|
|
package_path=_UPLOAD_ROOT/f"{jti}.zip"
|
|
total=0
|
|
with package_path.open("wb") as out:
|
|
while True:
|
|
chunk=await package.read(1024*1024)
|
|
if not chunk: break
|
|
total += len(chunk)
|
|
if total > 250*1024*1024:
|
|
raise ValueError("GST download package exceeds the 250 MB safety limit.")
|
|
out.write(chunk)
|
|
package_url=str(get_settings().ERP_PUBLIC_BASE_URL).rstrip("/")+f"/tools/accounting/gst-reconciliation/operator/package/{jti}?token={token}"
|
|
result=request_agent_command(str(data["node_code"]),"gst_return_package_store",{
|
|
"client_id":int(data["client_id"]),"gstin":str(data["gstin"]),"financial_year":str(data["financial_year"]),
|
|
"gst_relative_dir":str(data["gst_relative_dir"]),"package_url":package_url,"periods":data.get("periods") or [],"return_types":data.get("return_types") or [],
|
|
},timeout_seconds=120)
|
|
if not result.get("ok"):
|
|
raise RuntimeError(result.get("error") or "Local Storage Agent could not store the GST return package.")
|
|
try: package_path.unlink(missing_ok=True)
|
|
except Exception: pass
|
|
return JSONResponse({"ok":True,"stored":result.get("result") or {}})
|
|
except Exception as exc:
|
|
return JSONResponse({"ok":False,"error":str(exc)},status_code=400)
|
|
|
|
|
|
@router.get("/operator/package/{job_id}")
|
|
def operator_package(job_id: str, token: str):
|
|
try:
|
|
data=_decode_operator_token(token)
|
|
if str(data.get("jti") or "") != str(job_id):
|
|
raise ValueError("GST package token does not match the requested job.")
|
|
path=_UPLOAD_ROOT/f"{job_id}.zip"
|
|
if not path.is_file():
|
|
return JSONResponse({"ok":False,"error":"GST package is no longer available."},status_code=404)
|
|
return FileResponse(path,media_type="application/zip",filename=f"gst_returns_{job_id}.zip")
|
|
except Exception as exc:
|
|
return JSONResponse({"ok":False,"error":str(exc)},status_code=400)
|
|
|
|
|
|
|
|
@router.post("/import")
|
|
async def import_downloaded_returns(
|
|
request: Request,
|
|
client_id: int = Form(...), registration_id: int = Form(...), period: str = Form(""),
|
|
financial_year: str = Form(...), download_mode: str = Form("single"),
|
|
gstr1_files: list[UploadFile] = File(default=[]),
|
|
gstr2b_files: list[UploadFile] = File(default=[]),
|
|
gstr3b_files: list[UploadFile] = File(default=[]),
|
|
gstr2a_files: list[UploadFile] = File(default=[]),
|
|
csrf_token: str = Form(...),
|
|
):
|
|
validate_csrf(request, csrf_token)
|
|
db = CommonSessionLocal()
|
|
temp_root = None
|
|
package_path = None
|
|
try:
|
|
user, response = _require_partner(request, db, "accounting.learning.manage")
|
|
if response:
|
|
return response
|
|
client, _, scope = _find_visible_client(db, request, user, client_id)
|
|
if not client:
|
|
return _redirect(client_id, error="Client is not available in your scope.")
|
|
pair = next(((r,t) for r,t in _gst_regs(db, scope.tenant_id, client.id) if int(r.id)==registration_id and _is_gstin(r,t)), None)
|
|
if not pair:
|
|
return _redirect(client_id, error="Select a valid GSTIN registration.")
|
|
reg, _ = pair
|
|
gstin = _norm_gstin(reg.registration_number)
|
|
node = get_active_storage_node_for_branch(db, scope.tenant_id, scope.branch_id)
|
|
if not node or not _node_online(node):
|
|
return _redirect(client_id, registration_id=registration_id, period=period, financial_year=financial_year, download_mode=download_mode, error="Configured Local Storage Agent is offline.")
|
|
|
|
allowed_periods = set(_selected_periods(download_mode, financial_year, period))
|
|
if download_mode == "single":
|
|
financial_year = _fy_for_period(next(iter(allowed_periods)))
|
|
accounting_dir, gst_dir = _storage_payload(client, financial_year, gstin)
|
|
groups = {
|
|
"GSTR1": gstr1_files or [], "GSTR2B": gstr2b_files or [],
|
|
"GSTR3B": gstr3b_files or [], "GSTR2A": gstr2a_files or [],
|
|
}
|
|
if not any(groups.values()):
|
|
raise ValueError("Select at least one GST JSON/ZIP file to import.")
|
|
|
|
temp_root = Path(tempfile.mkdtemp(prefix="gst_manual_import_"))
|
|
by_period: dict[str, dict[str, dict]] = {}
|
|
total_bytes = 0
|
|
imported_files = 0
|
|
for rtype, uploads in groups.items():
|
|
for upload in uploads:
|
|
if not upload or not upload.filename:
|
|
continue
|
|
content = await upload.read()
|
|
total_bytes += len(content)
|
|
if total_bytes > _MAX_IMPORT_BYTES:
|
|
raise ValueError("GST import exceeds the 250 MB safety limit.")
|
|
for member_name, payload, raw in _json_members(upload.filename, content):
|
|
p = _period_from_name_or_payload(member_name, payload, period if download_mode == "single" else "")
|
|
if not p:
|
|
raise ValueError(f"Could not determine return period for {member_name}. Use GST JSON containing fp/return period, or Single Month mode.")
|
|
if p not in allowed_periods:
|
|
raise ValueError(f"{member_name} belongs to {p}, outside the selected {'financial year' if download_mode=='full_fy' else 'month'}.")
|
|
slot = by_period.setdefault(p, {}).setdefault(rtype, {"payloads": [], "raw": []})
|
|
slot["payloads"].append(payload)
|
|
slot["raw"].append((member_name, raw))
|
|
imported_files += 1
|
|
|
|
if not by_period:
|
|
raise ValueError("No readable GST JSON return data was found in the selected files.")
|
|
|
|
now = datetime.now(timezone.utc).isoformat()
|
|
return_types = set()
|
|
for p, period_data in sorted(by_period.items()):
|
|
base = temp_root / p
|
|
raw_dir = base / "raw"
|
|
normalized_dir = base / "normalized"
|
|
raw_dir.mkdir(parents=True, exist_ok=True)
|
|
normalized_dir.mkdir(parents=True, exist_ok=True)
|
|
manifest_downloaded=[]; normalized={}
|
|
for rtype, data in period_data.items():
|
|
return_types.add(rtype)
|
|
originals = raw_dir / "original"
|
|
originals.mkdir(parents=True, exist_ok=True)
|
|
payloads = data["payloads"]
|
|
for idx, (member_name, raw) in enumerate(data["raw"], 1):
|
|
safe = _safe_upload_name(member_name)
|
|
dest = originals / f"{rtype}_{idx:03d}_{safe}"
|
|
dest.write_bytes(raw)
|
|
manifest_downloaded.append({"return_type": rtype, "source_name": member_name, "path": str(dest.relative_to(temp_root).as_posix()), "bytes": len(raw)})
|
|
canonical = payloads[0] if len(payloads) == 1 else {"period": p, "source_files": len(payloads), "payloads": payloads}
|
|
(raw_dir / f"{p}_{rtype}.json").write_text(json.dumps(canonical, ensure_ascii=False, indent=2, default=str), encoding="utf-8")
|
|
if rtype in {"GSTR1", "GSTR2A", "GSTR2B"}:
|
|
rows = _merge_invoice_rows(payloads)
|
|
(normalized_dir / f"{p}_{rtype}_invoices.json").write_text(json.dumps({"period":p,"rows":rows}, ensure_ascii=False, indent=2, default=str), encoding="utf-8")
|
|
normalized[rtype] = {"invoice_rows": len(rows)}
|
|
elif rtype == "GSTR3B":
|
|
itc = _merge_gstr3b_itc(payloads)
|
|
(normalized_dir / f"{p}_{rtype}_itc.json").write_text(json.dumps({"period":p,"itc":itc}, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
normalized[rtype] = {"itc": itc}
|
|
(base / "download_manifest.json").write_text(json.dumps({"gstin":gstin,"period":p,"financial_year":financial_year,"downloaded_at_utc":now,"source":"manual_gst_portal_browser_import","downloaded":manifest_downloaded,"normalized":normalized}, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
|
|
jti = uuid.uuid4().hex
|
|
token = _encode_operator_token({"jti":jti,"user_id":int(user.id),"tenant_id":int(scope.tenant_id),"branch_id":int(scope.branch_id),"node_code":str(node.node_code),"client_id":int(client.id),"gstin":gstin,"financial_year":financial_year})
|
|
_UPLOAD_ROOT.mkdir(parents=True, exist_ok=True)
|
|
package_path = _UPLOAD_ROOT / f"{jti}.zip"
|
|
with zipfile.ZipFile(package_path, "w", zipfile.ZIP_DEFLATED, compresslevel=6) as archive:
|
|
for f in sorted(temp_root.rglob("*")):
|
|
if f.is_file():
|
|
archive.write(f, f.relative_to(temp_root).as_posix())
|
|
package_url = str(get_settings().ERP_PUBLIC_BASE_URL).rstrip("/") + f"/tools/accounting/gst-reconciliation/operator/package/{jti}?token={token}"
|
|
result = request_agent_command(str(node.node_code), "gst_return_package_store", {
|
|
"client_id": int(client.id), "gstin": gstin, "financial_year": financial_year,
|
|
"gst_relative_dir": gst_dir, "package_url": package_url,
|
|
"periods": sorted(by_period), "return_types": sorted(return_types),
|
|
}, timeout_seconds=180)
|
|
if not result.get("ok"):
|
|
raise RuntimeError(result.get("error") or "Local Storage Agent could not store the imported GST return package.")
|
|
return _redirect(client_id, registration_id=registration_id, period=period, financial_year=financial_year, download_mode=download_mode, message=f"Imported {imported_files} GST JSON file(s) for {len(by_period)} period(s) and stored them in the client GST directory. You can now run reconciliation.")
|
|
except Exception as exc:
|
|
db.rollback()
|
|
return _redirect(client_id, registration_id=registration_id, period=period, financial_year=financial_year, download_mode=download_mode, error=str(exc))
|
|
finally:
|
|
db.close()
|
|
if package_path:
|
|
try: package_path.unlink(missing_ok=True)
|
|
except Exception: pass
|
|
if temp_root:
|
|
import shutil
|
|
shutil.rmtree(temp_root, ignore_errors=True)
|
|
|
|
|
|
@router.post("/analyze")
|
|
def analyze(request: Request, client_id: int=Form(...), registration_id: int=Form(...), period: str=Form(""), financial_year: str=Form(...), analyze_mode: str=Form("single"), csrf_token: str=Form(...)):
|
|
validate_csrf(request,csrf_token)
|
|
db=CommonSessionLocal()
|
|
try:
|
|
user,response=_require_partner(request,db,"accounting.learning.manage")
|
|
if response: return response
|
|
client,_,scope=_find_visible_client(db,request,user,client_id)
|
|
if not client: return _redirect(client_id,error="Client is not available in your scope.")
|
|
pair=next(((r,t) for r,t in _gst_regs(db,scope.tenant_id,client.id) if int(r.id)==registration_id and _is_gstin(r,t)),None)
|
|
if not pair: return _redirect(client_id,error="GSTIN registration was not found.")
|
|
reg,_=pair; gstin=_norm_gstin(reg.registration_number)
|
|
periods=_periods_for_fy(financial_year) if analyze_mode=="full_fy" else [re.sub(r"\D","",period or "")]
|
|
if any(len(p)!=6 for p in periods): raise ValueError("Enter a valid MMYYYY period.")
|
|
accounting_dir,gst_dir=_storage_payload(client,financial_year,gstin)
|
|
node=get_active_storage_node_for_branch(db,scope.tenant_id,scope.branch_id)
|
|
if not node or not _node_online(node): return _redirect(client_id,registration_id=registration_id,period=period,financial_year=financial_year,error="Local Storage Agent is offline.")
|
|
results=[]
|
|
for p in periods:
|
|
date_from,date_to=_period_bounds(p)
|
|
res=request_agent_command(node.node_code,"gst_reconciliation_analyze",{"client_id":client.id,"gstin":gstin,"financial_year":financial_year,"period":p,"gst_relative_dir":gst_dir,"accounting_relative_dir":accounting_dir,"date_from":date_from,"date_to":date_to},timeout_seconds=30)
|
|
if res.get("ok"):
|
|
results.append((res.get("result") or {}).get("analysis") or {})
|
|
if not results: raise RuntimeError("No stored GST periods could be reconciled.")
|
|
if analyze_mode=="full_fy":
|
|
def sum_counts(section):
|
|
out={}
|
|
for r in results:
|
|
for k,v in (((r.get(section) or {}).get("counts") or {}).items()): out[k]=out.get(k,0)+int(v or 0)
|
|
return out
|
|
taxes={k:{"gstr2b":0.0,"gstr3b":0.0,"difference":0.0} for k in ("igst","cgst","sgst","cess")}
|
|
for r in results:
|
|
for k,row in (r.get("itc_reconciliation") or {}).items():
|
|
if k in taxes:
|
|
for f in taxes[k]: taxes[k][f]=round(taxes[k][f]+float(row.get(f) or 0),2)
|
|
analysis={"period":financial_year,"sales_reconciliation":{"counts":sum_counts("sales_reconciliation")},"purchase_reconciliation":{"counts":sum_counts("purchase_reconciliation")},"itc_reconciliation":taxes}
|
|
else:
|
|
analysis=results[0]
|
|
request.session["gst_reconciliation_result"]=analysis
|
|
return _redirect(client_id,registration_id=registration_id,period=period,financial_year=financial_year,download_mode=("full_fy" if analyze_mode=="full_fy" else "single"),message="GST Purchase, Sales and ITC reconciliation completed from stored local return data and Accounting Mirror.")
|
|
except Exception as exc:
|
|
return _redirect(client_id,registration_id=registration_id,period=period,financial_year=financial_year,error=str(exc))
|
|
finally:
|
|
db.close()
|