Switch GST reconciliation to direct browser and manual return import
This commit is contained in:
A R R R Associates
@@ -4,6 +4,9 @@ import calendar
|
||||
import re
|
||||
import tempfile
|
||||
import uuid
|
||||
import io
|
||||
import json
|
||||
import zipfile
|
||||
from datetime import date, datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlencode
|
||||
@@ -24,6 +27,7 @@ from app.modules.credential_vault.crypto import decrypt_value
|
||||
from app.modules.credential_vault.models import CredentialVaultEntry
|
||||
from app.modules.credential_vault.service import can_view_entry, log_access
|
||||
from app.modules.documents.services import client_folder_parts, get_active_storage_node_for_branch, sanitize_segment
|
||||
from app.modules.documents.local_agent_runtime.erp_local_agent.gst_portal_runtime import extract_invoice_rows, extract_gstr3b_itc
|
||||
from app.modules.registrations.models import ClientRegistration, RegistrationType
|
||||
|
||||
router = APIRouter(prefix="/tools/accounting/gst-reconciliation", tags=["accounting-gst-reconciliation-ui"])
|
||||
@@ -170,6 +174,103 @@ def _selected_periods(download_mode: str, financial_year: str, period: str) -> l
|
||||
return [digits]
|
||||
|
||||
|
||||
|
||||
GST_LOGIN_URL = "https://services.gst.gov.in/services/login"
|
||||
_MAX_IMPORT_BYTES = 250 * 1024 * 1024
|
||||
|
||||
|
||||
def _safe_upload_name(name: str) -> str:
|
||||
cleaned = re.sub(r"[^A-Za-z0-9._ -]+", "_", str(name or "").strip()).strip(" ._")
|
||||
return cleaned or "return.json"
|
||||
|
||||
|
||||
def _period_from_name_or_payload(name: str, payload, fallback: str = "") -> str:
|
||||
def valid(v: str) -> str:
|
||||
d = re.sub(r"\D", "", str(v or ""))
|
||||
if len(d) == 6:
|
||||
mm, yy = int(d[:2]), int(d[2:])
|
||||
if 1 <= mm <= 12 and 2000 <= yy <= 2100:
|
||||
return d
|
||||
yy2, mm2 = int(d[:4]), int(d[4:])
|
||||
if 2000 <= yy2 <= 2100 and 1 <= mm2 <= 12:
|
||||
return f"{mm2:02d}{yy2:04d}"
|
||||
return ""
|
||||
|
||||
if isinstance(payload, dict):
|
||||
stack = [payload]
|
||||
seen = 0
|
||||
while stack and seen < 5000:
|
||||
obj = stack.pop(); seen += 1
|
||||
if not isinstance(obj, dict):
|
||||
continue
|
||||
for key in ("fp", "rtn_prd", "return_period", "period", "ret_period"):
|
||||
if key in obj:
|
||||
found = valid(obj.get(key))
|
||||
if found:
|
||||
return found
|
||||
for value in obj.values():
|
||||
if isinstance(value, dict):
|
||||
stack.append(value)
|
||||
elif isinstance(value, list):
|
||||
stack.extend(x for x in value[:200] if isinstance(x, dict))
|
||||
|
||||
for pattern in (r"(?<!\d)(0[1-9]|1[0-2])(20\d{2})(?!\d)", r"(?<!\d)(20\d{2})(0[1-9]|1[0-2])(?!\d)"):
|
||||
m = re.search(pattern, str(name or ""))
|
||||
if m:
|
||||
if len(m.group(1)) == 2:
|
||||
return f"{m.group(1)}{m.group(2)}"
|
||||
return f"{m.group(2)}{m.group(1)}"
|
||||
return valid(fallback)
|
||||
|
||||
|
||||
def _json_members(upload_name: str, content: bytes) -> list[tuple[str, object, bytes]]:
|
||||
lower = str(upload_name or "").lower()
|
||||
if lower.endswith(".json"):
|
||||
try:
|
||||
payload = json.loads(content.decode("utf-8-sig", errors="strict"))
|
||||
except Exception as exc:
|
||||
raise ValueError(f"{upload_name}: invalid JSON file ({exc}).") from exc
|
||||
return [(upload_name, payload, content)]
|
||||
if lower.endswith(".zip"):
|
||||
out = []
|
||||
try:
|
||||
with zipfile.ZipFile(io.BytesIO(content), "r") as archive:
|
||||
for info in archive.infolist():
|
||||
if info.is_dir() or not info.filename.lower().endswith(".json"):
|
||||
continue
|
||||
raw = archive.read(info)
|
||||
try:
|
||||
payload = json.loads(raw.decode("utf-8-sig", errors="strict"))
|
||||
except Exception:
|
||||
continue
|
||||
out.append((Path(info.filename).name, payload, raw))
|
||||
except zipfile.BadZipFile as exc:
|
||||
raise ValueError(f"{upload_name}: invalid ZIP file.") from exc
|
||||
if not out:
|
||||
raise ValueError(f"{upload_name}: ZIP does not contain readable JSON return data.")
|
||||
return out
|
||||
raise ValueError(f"{upload_name}: only GST JSON or ZIP files are supported.")
|
||||
|
||||
|
||||
def _merge_invoice_rows(payloads: list[object]) -> list[dict]:
|
||||
rows=[]; seen=set()
|
||||
for payload in payloads:
|
||||
for row in extract_invoice_rows(payload):
|
||||
key=(str(row.get("gstin") or ""), str(row.get("invoice_key") or ""), str(row.get("invoice_date") or ""))
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key); rows.append(row)
|
||||
return rows
|
||||
|
||||
|
||||
def _merge_gstr3b_itc(payloads: list[object]) -> dict[str, float]:
|
||||
totals={"igst":0.0,"cgst":0.0,"sgst":0.0,"cess":0.0}
|
||||
for payload in payloads:
|
||||
row=extract_gstr3b_itc(payload)
|
||||
for k in totals:
|
||||
totals[k]=round(totals[k]+float(row.get(k) or 0),2)
|
||||
return totals
|
||||
|
||||
@router.get("")
|
||||
def page(request: Request, client_id: int | None = None, registration_id: int | None = None, period: str = "", financial_year: str = "", download_mode: str = "single", message: str = "", error: str = ""):
|
||||
db = CommonSessionLocal()
|
||||
@@ -196,7 +297,7 @@ def page(request: Request, client_id: int | None = None, registration_id: int |
|
||||
return templates.TemplateResponse("modules/accounting/templates/accounting/gst_reconciliation.html",{
|
||||
"request":request,"current_user":user,"current_user_roles":get_user_roles(db,user.id),"current_user_permissions":get_user_permissions(db,user.id),"csrf_token":get_or_create_csrf_token(request),
|
||||
"clients":clients,"selected_client":selected,"registrations":registrations,"selected_registration":selected_reg,"credentials":credentials,"node":node,"node_online":_node_online(node) if node else False,
|
||||
"period":period,"financial_year":financial_year,"download_mode":download_mode,"operator_job":operator_job,"message":message,"error":error,"title":"GST Return Reconciliation",
|
||||
"period":period,"financial_year":financial_year,"download_mode":download_mode,"operator_job":operator_job,"gst_login_url":GST_LOGIN_URL,"message":message,"error":error,"title":"GST Return Reconciliation",
|
||||
})
|
||||
finally:
|
||||
db.close()
|
||||
@@ -318,6 +419,136 @@ def operator_package(job_id: str, token: str):
|
||||
return JSONResponse({"ok":False,"error":str(exc)},status_code=400)
|
||||
|
||||
|
||||
|
||||
@router.post("/import")
|
||||
async def import_downloaded_returns(
|
||||
request: Request,
|
||||
client_id: int = Form(...), registration_id: int = Form(...), period: str = Form(""),
|
||||
financial_year: str = Form(...), download_mode: str = Form("single"),
|
||||
gstr1_files: list[UploadFile] = File(default=[]),
|
||||
gstr2b_files: list[UploadFile] = File(default=[]),
|
||||
gstr3b_files: list[UploadFile] = File(default=[]),
|
||||
gstr2a_files: list[UploadFile] = File(default=[]),
|
||||
csrf_token: str = Form(...),
|
||||
):
|
||||
validate_csrf(request, csrf_token)
|
||||
db = CommonSessionLocal()
|
||||
temp_root = None
|
||||
package_path = None
|
||||
try:
|
||||
user, response = _require_partner(request, db, "accounting.learning.manage")
|
||||
if response:
|
||||
return response
|
||||
client, _, scope = _find_visible_client(db, request, user, client_id)
|
||||
if not client:
|
||||
return _redirect(client_id, error="Client is not available in your scope.")
|
||||
pair = next(((r,t) for r,t in _gst_regs(db, scope.tenant_id, client.id) if int(r.id)==registration_id and _is_gstin(r,t)), None)
|
||||
if not pair:
|
||||
return _redirect(client_id, error="Select a valid GSTIN registration.")
|
||||
reg, _ = pair
|
||||
gstin = _norm_gstin(reg.registration_number)
|
||||
node = get_active_storage_node_for_branch(db, scope.tenant_id, scope.branch_id)
|
||||
if not node or not _node_online(node):
|
||||
return _redirect(client_id, registration_id=registration_id, period=period, financial_year=financial_year, download_mode=download_mode, error="Configured Local Storage Agent is offline.")
|
||||
|
||||
allowed_periods = set(_selected_periods(download_mode, financial_year, period))
|
||||
if download_mode == "single":
|
||||
financial_year = _fy_for_period(next(iter(allowed_periods)))
|
||||
accounting_dir, gst_dir = _storage_payload(client, financial_year, gstin)
|
||||
groups = {
|
||||
"GSTR1": gstr1_files or [], "GSTR2B": gstr2b_files or [],
|
||||
"GSTR3B": gstr3b_files or [], "GSTR2A": gstr2a_files or [],
|
||||
}
|
||||
if not any(groups.values()):
|
||||
raise ValueError("Select at least one GST JSON/ZIP file to import.")
|
||||
|
||||
temp_root = Path(tempfile.mkdtemp(prefix="gst_manual_import_"))
|
||||
by_period: dict[str, dict[str, dict]] = {}
|
||||
total_bytes = 0
|
||||
imported_files = 0
|
||||
for rtype, uploads in groups.items():
|
||||
for upload in uploads:
|
||||
if not upload or not upload.filename:
|
||||
continue
|
||||
content = await upload.read()
|
||||
total_bytes += len(content)
|
||||
if total_bytes > _MAX_IMPORT_BYTES:
|
||||
raise ValueError("GST import exceeds the 250 MB safety limit.")
|
||||
for member_name, payload, raw in _json_members(upload.filename, content):
|
||||
p = _period_from_name_or_payload(member_name, payload, period if download_mode == "single" else "")
|
||||
if not p:
|
||||
raise ValueError(f"Could not determine return period for {member_name}. Use GST JSON containing fp/return period, or Single Month mode.")
|
||||
if p not in allowed_periods:
|
||||
raise ValueError(f"{member_name} belongs to {p}, outside the selected {'financial year' if download_mode=='full_fy' else 'month'}.")
|
||||
slot = by_period.setdefault(p, {}).setdefault(rtype, {"payloads": [], "raw": []})
|
||||
slot["payloads"].append(payload)
|
||||
slot["raw"].append((member_name, raw))
|
||||
imported_files += 1
|
||||
|
||||
if not by_period:
|
||||
raise ValueError("No readable GST JSON return data was found in the selected files.")
|
||||
|
||||
now = datetime.now(timezone.utc).isoformat()
|
||||
return_types = set()
|
||||
for p, period_data in sorted(by_period.items()):
|
||||
base = temp_root / p
|
||||
raw_dir = base / "raw"
|
||||
normalized_dir = base / "normalized"
|
||||
raw_dir.mkdir(parents=True, exist_ok=True)
|
||||
normalized_dir.mkdir(parents=True, exist_ok=True)
|
||||
manifest_downloaded=[]; normalized={}
|
||||
for rtype, data in period_data.items():
|
||||
return_types.add(rtype)
|
||||
originals = raw_dir / "original"
|
||||
originals.mkdir(parents=True, exist_ok=True)
|
||||
payloads = data["payloads"]
|
||||
for idx, (member_name, raw) in enumerate(data["raw"], 1):
|
||||
safe = _safe_upload_name(member_name)
|
||||
dest = originals / f"{rtype}_{idx:03d}_{safe}"
|
||||
dest.write_bytes(raw)
|
||||
manifest_downloaded.append({"return_type": rtype, "source_name": member_name, "path": str(dest.relative_to(temp_root).as_posix()), "bytes": len(raw)})
|
||||
canonical = payloads[0] if len(payloads) == 1 else {"period": p, "source_files": len(payloads), "payloads": payloads}
|
||||
(raw_dir / f"{p}_{rtype}.json").write_text(json.dumps(canonical, ensure_ascii=False, indent=2, default=str), encoding="utf-8")
|
||||
if rtype in {"GSTR1", "GSTR2A", "GSTR2B"}:
|
||||
rows = _merge_invoice_rows(payloads)
|
||||
(normalized_dir / f"{p}_{rtype}_invoices.json").write_text(json.dumps({"period":p,"rows":rows}, ensure_ascii=False, indent=2, default=str), encoding="utf-8")
|
||||
normalized[rtype] = {"invoice_rows": len(rows)}
|
||||
elif rtype == "GSTR3B":
|
||||
itc = _merge_gstr3b_itc(payloads)
|
||||
(normalized_dir / f"{p}_{rtype}_itc.json").write_text(json.dumps({"period":p,"itc":itc}, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
normalized[rtype] = {"itc": itc}
|
||||
(base / "download_manifest.json").write_text(json.dumps({"gstin":gstin,"period":p,"financial_year":financial_year,"downloaded_at_utc":now,"source":"manual_gst_portal_browser_import","downloaded":manifest_downloaded,"normalized":normalized}, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
|
||||
jti = uuid.uuid4().hex
|
||||
token = _encode_operator_token({"jti":jti,"user_id":int(user.id),"tenant_id":int(scope.tenant_id),"branch_id":int(scope.branch_id),"node_code":str(node.node_code),"client_id":int(client.id),"gstin":gstin,"financial_year":financial_year})
|
||||
_UPLOAD_ROOT.mkdir(parents=True, exist_ok=True)
|
||||
package_path = _UPLOAD_ROOT / f"{jti}.zip"
|
||||
with zipfile.ZipFile(package_path, "w", zipfile.ZIP_DEFLATED, compresslevel=6) as archive:
|
||||
for f in sorted(temp_root.rglob("*")):
|
||||
if f.is_file():
|
||||
archive.write(f, f.relative_to(temp_root).as_posix())
|
||||
package_url = str(get_settings().ERP_PUBLIC_BASE_URL).rstrip("/") + f"/tools/accounting/gst-reconciliation/operator/package/{jti}?token={token}"
|
||||
result = request_agent_command(str(node.node_code), "gst_return_package_store", {
|
||||
"client_id": int(client.id), "gstin": gstin, "financial_year": financial_year,
|
||||
"gst_relative_dir": gst_dir, "package_url": package_url,
|
||||
"periods": sorted(by_period), "return_types": sorted(return_types),
|
||||
}, timeout_seconds=180)
|
||||
if not result.get("ok"):
|
||||
raise RuntimeError(result.get("error") or "Local Storage Agent could not store the imported GST return package.")
|
||||
return _redirect(client_id, registration_id=registration_id, period=period, financial_year=financial_year, download_mode=download_mode, message=f"Imported {imported_files} GST JSON file(s) for {len(by_period)} period(s) and stored them in the client GST directory. You can now run reconciliation.")
|
||||
except Exception as exc:
|
||||
db.rollback()
|
||||
return _redirect(client_id, registration_id=registration_id, period=period, financial_year=financial_year, download_mode=download_mode, error=str(exc))
|
||||
finally:
|
||||
db.close()
|
||||
if package_path:
|
||||
try: package_path.unlink(missing_ok=True)
|
||||
except Exception: pass
|
||||
if temp_root:
|
||||
import shutil
|
||||
shutil.rmtree(temp_root, ignore_errors=True)
|
||||
|
||||
|
||||
@router.post("/analyze")
|
||||
def analyze(request: Request, client_id: int=Form(...), registration_id: int=Form(...), period: str=Form(""), financial_year: str=Form(...), analyze_mode: str=Form("single"), csrf_token: str=Form(...)):
|
||||
validate_csrf(request,csrf_token)
|
||||
|
||||
Reference in New Issue
Block a user