Add Phase 11B client aware multi bank ownership and engagement archive
This commit is contained in:
@@ -615,6 +615,10 @@ def analyze_files(paths, customer_override="", account_override="", bank_hint="a
|
||||
frames = []
|
||||
for path in paths:
|
||||
meta, df = parse_pdf(path, bank_hint=bank_hint)
|
||||
# Preserve the bank-extracted owner before an optional manual correction.
|
||||
# Phase 11B ownership validation always prefers this original value.
|
||||
meta.extracted_customer_name = getattr(meta, "customer_name", "") or ""
|
||||
df["extracted_customer_name"] = meta.extracted_customer_name
|
||||
if customer_override:
|
||||
meta.customer_name = customer_override
|
||||
df["customer_name"] = customer_override
|
||||
|
||||
@@ -0,0 +1,438 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from difflib import SequenceMatcher
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
from sqlalchemy import or_, select
|
||||
|
||||
from app.modules.clients.models import Client, ClientBranch, ClientBusinessUnit
|
||||
from app.modules.consultants.models import ClientConsultantLink
|
||||
from app.modules.consultants.service import get_consultant_by_user
|
||||
from app.modules.documents.services import (
|
||||
build_document_scope,
|
||||
list_visible_engagements,
|
||||
save_uploaded_revision,
|
||||
user_can_upload_to_engagement,
|
||||
)
|
||||
from app.modules.registrations.models import ClientRegistration
|
||||
from app.modules.services.models import ClientServiceSubscription
|
||||
|
||||
|
||||
_LEGAL_NOISE = {
|
||||
"M", "S", "MS", "MR", "MRS", "MISS",
|
||||
"PRIVATE", "PVT", "LIMITED", "LTD", "LLP",
|
||||
"PROPRIETOR", "PROPRIETORSHIP", "PROP",
|
||||
"THE", "INDIA",
|
||||
}
|
||||
|
||||
|
||||
def _text(value) -> str:
|
||||
return str(value or "").strip()
|
||||
|
||||
|
||||
def _normalise_name(value) -> str:
|
||||
raw = re.sub(r"[^A-Z0-9]+", " ", _text(value).upper()).strip()
|
||||
tokens = [token for token in raw.split() if token and token not in _LEGAL_NOISE]
|
||||
return " ".join(tokens)
|
||||
|
||||
|
||||
def _name_match(extracted: str, alias: str) -> tuple[bool, int, str]:
|
||||
left = _normalise_name(extracted)
|
||||
right = _normalise_name(alias)
|
||||
if not left or not right:
|
||||
return False, 0, ""
|
||||
if left == right:
|
||||
return True, 100, "exact normalized account-holder match"
|
||||
|
||||
# Conservative containment is allowed only for reasonably descriptive names.
|
||||
if min(len(left), len(right)) >= 8 and (left in right or right in left):
|
||||
score = int(round(100 * min(len(left), len(right)) / max(len(left), len(right))))
|
||||
if score >= 82:
|
||||
return True, max(90, score), "legal/trade-name containment match"
|
||||
|
||||
ratio = int(round(SequenceMatcher(None, left, right).ratio() * 100))
|
||||
if ratio >= 92:
|
||||
return True, ratio, "high-confidence legal/trade-name similarity match"
|
||||
return False, ratio, ""
|
||||
|
||||
|
||||
def client_owner_aliases(db, *, tenant_id: int, client_id: int) -> list[str]:
|
||||
client = db.get(Client, int(client_id))
|
||||
if not client or client.tenant_id != int(tenant_id) or client.is_archived:
|
||||
raise ValueError("Selected client was not found.")
|
||||
|
||||
values = [client.client_name, client.trade_name]
|
||||
|
||||
for unit in db.execute(
|
||||
select(ClientBusinessUnit).where(
|
||||
ClientBusinessUnit.tenant_id == tenant_id,
|
||||
ClientBusinessUnit.client_id == client.id,
|
||||
ClientBusinessUnit.is_active.is_(True),
|
||||
)
|
||||
).scalars().all():
|
||||
values.extend([unit.business_name, unit.trade_name])
|
||||
|
||||
for branch in db.execute(
|
||||
select(ClientBranch).where(
|
||||
ClientBranch.tenant_id == tenant_id,
|
||||
ClientBranch.client_id == client.id,
|
||||
ClientBranch.is_active.is_(True),
|
||||
)
|
||||
).scalars().all():
|
||||
values.append(branch.branch_name)
|
||||
|
||||
for registration in db.execute(
|
||||
select(ClientRegistration).where(
|
||||
ClientRegistration.tenant_id == tenant_id,
|
||||
ClientRegistration.client_id == client.id,
|
||||
ClientRegistration.status.in_(["active", "valid", "registered"]),
|
||||
)
|
||||
).scalars().all():
|
||||
values.extend([registration.legal_name, registration.trade_name])
|
||||
|
||||
result = []
|
||||
seen = set()
|
||||
for value in values:
|
||||
value = _text(value)
|
||||
key = _normalise_name(value)
|
||||
if value and key and key not in seen:
|
||||
seen.add(key)
|
||||
result.append(value)
|
||||
return result
|
||||
|
||||
|
||||
def _consultant_context(db, user, tenant_id: int):
|
||||
consultant = get_consultant_by_user(
|
||||
db,
|
||||
tenant_id=tenant_id,
|
||||
user_id=int(user.id),
|
||||
)
|
||||
return consultant
|
||||
|
||||
|
||||
def analyzer_client_context(db, *, request, user, roles: list[str]) -> dict:
|
||||
"""Return role-scoped clients and archive-capable engagements.
|
||||
|
||||
Partner/Manager/Staff reuse the existing Document module visibility rules.
|
||||
Consultant reuses the existing ClientConsultantLink permissions.
|
||||
"""
|
||||
tenant_id = int(
|
||||
request.session.get("active_tenant_id")
|
||||
or getattr(user, "tenant_id", 0)
|
||||
or 0
|
||||
)
|
||||
role_set = set(roles or [])
|
||||
clients: dict[int, Client] = {}
|
||||
engagements: dict[int, ClientServiceSubscription] = {}
|
||||
|
||||
if "Consultant" in role_set:
|
||||
consultant = _consultant_context(db, user, tenant_id)
|
||||
if not consultant:
|
||||
return {"clients": [], "engagements": [], "consultant": None}
|
||||
|
||||
links = list(db.execute(
|
||||
select(ClientConsultantLink).where(
|
||||
ClientConsultantLink.tenant_id == tenant_id,
|
||||
ClientConsultantLink.consultant_id == consultant.id,
|
||||
ClientConsultantLink.is_active.is_(True),
|
||||
ClientConsultantLink.can_view_client.is_(True),
|
||||
)
|
||||
).scalars().all())
|
||||
|
||||
client_ids = {int(link.client_id) for link in links}
|
||||
upload_ids = {
|
||||
int(link.client_id)
|
||||
for link in links
|
||||
if bool(link.can_upload_documents)
|
||||
}
|
||||
|
||||
if client_ids:
|
||||
for client in db.execute(
|
||||
select(Client).where(
|
||||
Client.tenant_id == tenant_id,
|
||||
Client.id.in_(client_ids),
|
||||
Client.is_archived.is_(False),
|
||||
Client.is_active.is_(True),
|
||||
).order_by(Client.client_name.asc())
|
||||
).scalars().all():
|
||||
clients[int(client.id)] = client
|
||||
|
||||
if upload_ids:
|
||||
for engagement in db.execute(
|
||||
select(ClientServiceSubscription).where(
|
||||
ClientServiceSubscription.tenant_id == tenant_id,
|
||||
ClientServiceSubscription.client_id.in_(upload_ids),
|
||||
ClientServiceSubscription.is_active.is_(True),
|
||||
).order_by(
|
||||
ClientServiceSubscription.financial_year.desc(),
|
||||
ClientServiceSubscription.id.desc(),
|
||||
)
|
||||
).scalars().all():
|
||||
engagements[int(engagement.id)] = engagement
|
||||
|
||||
return {
|
||||
"clients": list(clients.values()),
|
||||
"engagements": list(engagements.values()),
|
||||
"consultant": consultant,
|
||||
}
|
||||
|
||||
# Firm Partner / Manager / Staff use the existing engagement-document scope.
|
||||
scope = build_document_scope(request, db, user)
|
||||
visible = list_visible_engagements(db, user, scope, limit=1000)
|
||||
for engagement in visible:
|
||||
client = getattr(engagement, "client", None) or db.get(Client, engagement.client_id)
|
||||
if client and client.is_active and not client.is_archived:
|
||||
clients[int(client.id)] = client
|
||||
if user_can_upload_to_engagement(db, user, engagement, scope):
|
||||
engagements[int(engagement.id)] = engagement
|
||||
|
||||
return {
|
||||
"clients": sorted(clients.values(), key=lambda c: (c.client_name or "").casefold()),
|
||||
"engagements": list(engagements.values()),
|
||||
"consultant": None,
|
||||
}
|
||||
|
||||
|
||||
def validate_selected_client_and_engagement(
|
||||
db,
|
||||
*,
|
||||
request,
|
||||
user,
|
||||
roles: list[str],
|
||||
client_id: int | None,
|
||||
engagement_id: int | None,
|
||||
):
|
||||
if not client_id:
|
||||
if engagement_id:
|
||||
raise ValueError("Select a client before selecting an engagement.")
|
||||
return None, None
|
||||
|
||||
context = analyzer_client_context(db, request=request, user=user, roles=roles)
|
||||
clients = {int(row.id): row for row in context["clients"]}
|
||||
engagements = {int(row.id): row for row in context["engagements"]}
|
||||
|
||||
client = clients.get(int(client_id))
|
||||
if not client:
|
||||
raise ValueError("Selected client is not available in your Bank Analyzer scope.")
|
||||
|
||||
if not engagement_id:
|
||||
raise ValueError(
|
||||
"Select an engagement document destination when a client is selected."
|
||||
)
|
||||
|
||||
engagement = engagements.get(int(engagement_id))
|
||||
if not engagement or int(engagement.client_id) != int(client.id):
|
||||
raise ValueError(
|
||||
"Selected engagement is not available for document upload for this client."
|
||||
)
|
||||
return client, engagement
|
||||
|
||||
|
||||
def validate_statement_ownership(
|
||||
db,
|
||||
*,
|
||||
tenant_id: int,
|
||||
client_id: int | None,
|
||||
metas: list,
|
||||
owner_override: str = "",
|
||||
ownership_confirmation: bool = False,
|
||||
) -> dict:
|
||||
"""Validate that all statements belong to one ERP client/legal/trade-name family.
|
||||
|
||||
With an ERP client selected the check is strict. Every statement must expose
|
||||
an owner name or use a manually confirmed owner override, and that name must
|
||||
match one of the ERP legal/trade aliases.
|
||||
"""
|
||||
extracted = []
|
||||
for position, meta in enumerate(metas, start=1):
|
||||
raw = _text(
|
||||
getattr(meta, "extracted_customer_name", None)
|
||||
or getattr(meta, "customer_name", None)
|
||||
)
|
||||
extracted.append({
|
||||
"statement_id": getattr(meta, "statement_id", f"STMT-{position:03d}"),
|
||||
"source_file": _text(getattr(meta, "source_file", "")),
|
||||
"owner_name": raw,
|
||||
})
|
||||
|
||||
if not client_id:
|
||||
# Preserve standalone behavior. We record the owner names for review but
|
||||
# do not change current storage or reject mixed standalone analyses.
|
||||
return {
|
||||
"status": "standalone",
|
||||
"client_id": None,
|
||||
"aliases": [],
|
||||
"statements": extracted,
|
||||
"message": "Standalone analysis; no ERP client ownership binding requested.",
|
||||
}
|
||||
|
||||
aliases = client_owner_aliases(
|
||||
db,
|
||||
tenant_id=int(tenant_id),
|
||||
client_id=int(client_id),
|
||||
)
|
||||
if not aliases:
|
||||
raise ValueError(
|
||||
"Selected client does not have a usable legal/trade name for bank ownership validation."
|
||||
)
|
||||
|
||||
override = _text(owner_override)
|
||||
if override:
|
||||
matched_override = any(_name_match(override, alias)[0] for alias in aliases)
|
||||
if not matched_override:
|
||||
raise ValueError(
|
||||
f"Account-holder override '{override}' does not match the selected client's "
|
||||
"legal/trade names."
|
||||
)
|
||||
|
||||
results = []
|
||||
failures = []
|
||||
|
||||
for statement in extracted:
|
||||
owner = statement["owner_name"] or override
|
||||
if not owner:
|
||||
failures.append(
|
||||
f"{statement['statement_id']}: account-holder name could not be extracted. "
|
||||
"Use Account holder override only after verifying the statement manually."
|
||||
)
|
||||
results.append({**statement, "matched": False, "score": 0, "matched_alias": ""})
|
||||
continue
|
||||
|
||||
best = (False, 0, "", "")
|
||||
for alias in aliases:
|
||||
matched, score, reason = _name_match(owner, alias)
|
||||
if score > best[1]:
|
||||
best = (matched, score, alias, reason)
|
||||
|
||||
matched, score, alias, reason = best
|
||||
results.append({
|
||||
**statement,
|
||||
"effective_owner_name": owner,
|
||||
"matched": bool(matched),
|
||||
"score": int(score),
|
||||
"matched_alias": alias,
|
||||
"reason": reason,
|
||||
})
|
||||
if not matched:
|
||||
failures.append(
|
||||
f"{statement['statement_id']}: '{owner}' does not match the selected client's "
|
||||
"legal/trade names."
|
||||
)
|
||||
|
||||
if failures:
|
||||
raise ValueError(
|
||||
"Bank account ownership validation failed. " + " ".join(failures)
|
||||
)
|
||||
|
||||
if len(metas) > 1 and not ownership_confirmation:
|
||||
raise ValueError(
|
||||
"Confirm that all uploaded bank statements belong to the selected client "
|
||||
"or its approved trade/business names."
|
||||
)
|
||||
|
||||
return {
|
||||
"status": "confirmed",
|
||||
"client_id": int(client_id),
|
||||
"aliases": aliases,
|
||||
"statements": results,
|
||||
"message": f"Ownership confirmed for {len(results)} statement(s).",
|
||||
}
|
||||
|
||||
|
||||
def _upload_proxy(path: Path, content_type: str):
|
||||
handle = path.open("rb")
|
||||
return SimpleNamespace(
|
||||
filename=path.name,
|
||||
content_type=content_type,
|
||||
file=handle,
|
||||
)
|
||||
|
||||
|
||||
def archive_analysis_to_engagement(
|
||||
db,
|
||||
*,
|
||||
job,
|
||||
engagement: ClientServiceSubscription | None,
|
||||
metas: list,
|
||||
source_paths: list[Path],
|
||||
output_path: Path,
|
||||
user,
|
||||
) -> dict:
|
||||
if not job.client_id or not job.engagement_id or engagement is None:
|
||||
return {"status": "not_requested", "documents": []}
|
||||
|
||||
archived = []
|
||||
meta_by_source = {
|
||||
Path(_text(getattr(meta, "source_file", ""))).name: meta
|
||||
for meta in metas
|
||||
}
|
||||
|
||||
for source in source_paths:
|
||||
upload = _upload_proxy(source, "application/pdf")
|
||||
try:
|
||||
meta = meta_by_source.get(source.name)
|
||||
bank = _text(getattr(meta, "bank_name", "")) if meta else ""
|
||||
account = _text(getattr(meta, "account_number", "")) if meta else ""
|
||||
title_bits = ["Bank Statement"]
|
||||
if bank:
|
||||
title_bits.append(bank)
|
||||
if account:
|
||||
title_bits.append(account[-6:])
|
||||
document = save_uploaded_revision(
|
||||
db,
|
||||
engagement=engagement,
|
||||
upload_file=upload,
|
||||
title=" - ".join(title_bits),
|
||||
document_type="BANK_STATEMENT",
|
||||
description=(
|
||||
f"Bank Statement Analyzer source. Job {job.id}. "
|
||||
f"Ownership status: {job.ownership_status}."
|
||||
),
|
||||
remarks="Automatically archived from Bank Statement Analyzer.",
|
||||
user=user,
|
||||
udin_required=False,
|
||||
)
|
||||
archived.append({
|
||||
"document_id": int(document.id),
|
||||
"filename": source.name,
|
||||
"type": "BANK_STATEMENT",
|
||||
})
|
||||
finally:
|
||||
upload.file.close()
|
||||
|
||||
workbook_upload = _upload_proxy(
|
||||
output_path,
|
||||
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||
)
|
||||
try:
|
||||
document = save_uploaded_revision(
|
||||
db,
|
||||
engagement=engagement,
|
||||
upload_file=workbook_upload,
|
||||
title="Bank Statement Analyzer - Analysis Workbook",
|
||||
document_type="WORKING_PAPER",
|
||||
description=(
|
||||
f"Generated Bank Statement Analyzer workbook for job {job.id}; "
|
||||
f"includes multi-bank contra review where applicable."
|
||||
),
|
||||
remarks="Automatically archived from Bank Statement Analyzer.",
|
||||
user=user,
|
||||
udin_required=False,
|
||||
)
|
||||
archived.append({
|
||||
"document_id": int(document.id),
|
||||
"filename": output_path.name,
|
||||
"type": "WORKING_PAPER",
|
||||
})
|
||||
finally:
|
||||
workbook_upload.file.close()
|
||||
|
||||
return {
|
||||
"status": "archived",
|
||||
"client_id": int(job.client_id),
|
||||
"engagement_id": int(job.engagement_id),
|
||||
"documents": archived,
|
||||
}
|
||||
@@ -17,6 +17,16 @@ class BankStatementAnalysisJob(CommonBase):
|
||||
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True)
|
||||
role_bucket: Mapped[str] = mapped_column(String(30), nullable=False)
|
||||
|
||||
# Phase 11B: optional client-aware analysis. Null preserves the existing
|
||||
# standalone Work-folder behavior exactly.
|
||||
client_id: Mapped[int | None] = mapped_column(ForeignKey("clients.id", ondelete="SET NULL"), nullable=True, index=True)
|
||||
engagement_id: Mapped[int | None] = mapped_column(ForeignKey("client_service_subscriptions.id", ondelete="SET NULL"), nullable=True, index=True)
|
||||
ownership_confirmation: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
||||
ownership_status: Mapped[str] = mapped_column(String(30), nullable=False, default="not_checked", index=True)
|
||||
ownership_validation_json: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
engagement_archive_status: Mapped[str] = mapped_column(String(30), nullable=False, default="not_requested", index=True)
|
||||
engagement_archive_json: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
|
||||
selected_bank: Mapped[str] = mapped_column(String(40), nullable=False, default="auto")
|
||||
financial_year: Mapped[str | None] = mapped_column(String(20), nullable=True)
|
||||
customer_override: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
|
||||
@@ -20,9 +20,16 @@ from app.core.db.common import CommonSessionLocal
|
||||
|
||||
from .analyzer import analyze_files, export_excel
|
||||
from .models import BankStatementAnalysisJob
|
||||
from .client_context import (
|
||||
archive_analysis_to_engagement,
|
||||
validate_statement_ownership,
|
||||
)
|
||||
from app.modules.core.iam.models import User
|
||||
from app.modules.services.models import ClientServiceSubscription
|
||||
|
||||
ALLOWED_ROLES = {"Partner", "Manager", "Branch Manager", "Staff", "Employee", "Consultant"}
|
||||
MAX_FILES = int(os.getenv("BANK_ANALYZER_MAX_FILES", "24"))
|
||||
# 0 means no fixed statement-count ceiling; per-file size and queue controls remain.
|
||||
MAX_FILES = int(os.getenv("BANK_ANALYZER_MAX_FILES", "0"))
|
||||
MAX_FILE_BYTES = int(os.getenv("BANK_ANALYZER_MAX_FILE_MB", "50")) * 1024 * 1024
|
||||
MAX_GLOBAL_PROCESSING = 3
|
||||
MAX_PENDING_PER_USER = 3
|
||||
@@ -117,7 +124,7 @@ async def save_uploads(files: list[UploadFile], input_dir: Path) -> list[Path]:
|
||||
usable = [item for item in files if item and (item.filename or "").strip()]
|
||||
if not usable:
|
||||
raise ValueError("Please select at least one PDF bank statement.")
|
||||
if len(usable) > MAX_FILES:
|
||||
if MAX_FILES > 0 and len(usable) > MAX_FILES:
|
||||
raise ValueError(f"A maximum of {MAX_FILES} PDF files can be analyzed in one job.")
|
||||
saved: list[Path] = []
|
||||
try:
|
||||
@@ -155,7 +162,7 @@ def pending_count_for_user(user_id: int) -> int:
|
||||
db.close()
|
||||
|
||||
|
||||
def enqueue_job(*, user, roles: Iterable[str], job_id: str, paths: list[Path], job_dir: Path, bank_selection: str, financial_year: str, customer_override: str, account_override: str, classification_enabled: bool) -> BankStatementAnalysisJob:
|
||||
def enqueue_job(*, user, roles: Iterable[str], job_id: str, paths: list[Path], job_dir: Path, bank_selection: str, financial_year: str, customer_override: str, account_override: str, classification_enabled: bool, client_id: int | None = None, engagement_id: int | None = None, ownership_confirmation: bool = False) -> BankStatementAnalysisJob:
|
||||
if pending_count_for_user(int(user.id)) >= MAX_PENDING_PER_USER:
|
||||
shutil.rmtree(job_dir, ignore_errors=True)
|
||||
raise ValueError("You already have three queued or processing analyses. Please wait for one to complete before submitting another.")
|
||||
@@ -169,6 +176,11 @@ def enqueue_job(*, user, roles: Iterable[str], job_id: str, paths: list[Path], j
|
||||
branch_id=getattr(user, "branch_id", None),
|
||||
user_id=int(user.id),
|
||||
role_bucket=bucket,
|
||||
client_id=int(client_id) if client_id else None,
|
||||
engagement_id=int(engagement_id) if engagement_id else None,
|
||||
ownership_confirmation=bool(ownership_confirmation),
|
||||
ownership_status="pending" if client_id else "not_checked",
|
||||
engagement_archive_status="pending" if engagement_id else "not_requested",
|
||||
selected_bank=bank_selection,
|
||||
financial_year=(financial_year or "").strip() or None,
|
||||
customer_override=(customer_override or "").strip() or None,
|
||||
@@ -245,6 +257,18 @@ def _process_job(job_id: str) -> None:
|
||||
bank_hint=job.selected_bank,
|
||||
classification_enabled=job.classification_enabled,
|
||||
)
|
||||
|
||||
ownership = validate_statement_ownership(
|
||||
db,
|
||||
tenant_id=int(job.tenant_id or 0),
|
||||
client_id=job.client_id,
|
||||
metas=metas,
|
||||
owner_override=job.customer_override or "",
|
||||
ownership_confirmation=bool(job.ownership_confirmation),
|
||||
)
|
||||
job.ownership_status = str(ownership.get("status") or "not_checked")
|
||||
job.ownership_validation_json = json.dumps(ownership, ensure_ascii=False)
|
||||
|
||||
job.progress_percent = 75
|
||||
db.commit()
|
||||
output = output_dir / _workbook_filename(metas)
|
||||
@@ -257,6 +281,28 @@ def _process_job(job_id: str) -> None:
|
||||
selected_bank=job.selected_bank,
|
||||
classification_enabled=job.classification_enabled,
|
||||
)
|
||||
|
||||
archive_result = {"status": "not_requested", "documents": []}
|
||||
if job.client_id and job.engagement_id:
|
||||
engagement = db.get(ClientServiceSubscription, int(job.engagement_id))
|
||||
uploader = db.get(User, int(job.user_id))
|
||||
if not engagement or int(engagement.client_id) != int(job.client_id):
|
||||
raise ValueError("Selected engagement is no longer valid for the selected client.")
|
||||
if not uploader:
|
||||
raise ValueError("Bank Analyzer uploader could not be resolved for document archive.")
|
||||
archive_result = archive_analysis_to_engagement(
|
||||
db,
|
||||
job=job,
|
||||
engagement=engagement,
|
||||
metas=metas,
|
||||
source_paths=paths,
|
||||
output_path=output,
|
||||
user=uploader,
|
||||
)
|
||||
job.engagement_archive_status = str(archive_result.get("status") or "failed")
|
||||
job.engagement_archive_json = json.dumps(archive_result, ensure_ascii=False)
|
||||
db.commit()
|
||||
|
||||
summary = {
|
||||
"job_id": job.id,
|
||||
"statement_count": len(metas),
|
||||
@@ -272,6 +318,10 @@ def _process_job(job_id: str) -> None:
|
||||
"account_number": next((meta.account_number for meta in metas if meta.account_number), ""),
|
||||
"financial_year": job.financial_year or "",
|
||||
"classification_enabled": job.classification_enabled,
|
||||
"client_id": job.client_id,
|
||||
"engagement_id": job.engagement_id,
|
||||
"ownership_status": job.ownership_status,
|
||||
"engagement_archive_status": job.engagement_archive_status,
|
||||
}
|
||||
# Keep the original uploaded statements until the job expiry time.
|
||||
# This applies equally to completed and failed jobs and allows the
|
||||
|
||||
@@ -13,13 +13,44 @@
|
||||
<form action="/tools/bank-statement-analyzer/analyze" method="post" enctype="multipart/form-data" class="rounded-2xl border border-slate-200 bg-white p-6 shadow-soft">
|
||||
<input type="hidden" name="csrf_token" value="{{ csrf_token }}">
|
||||
<div class="grid gap-5 md:grid-cols-2">
|
||||
{% if analyzer_clients %}
|
||||
<div class="md:col-span-2 rounded-xl border border-indigo-200 bg-indigo-50 p-4">
|
||||
<div class="grid gap-4 md:grid-cols-2">
|
||||
<label class="text-sm font-semibold text-slate-700">
|
||||
ERP Client <span class="font-normal text-slate-400">(optional)</span>
|
||||
<select name="client_id" id="bank-analyzer-client" class="mt-1 w-full rounded-xl border border-slate-300 bg-white px-3 py-2">
|
||||
<option value="">Standalone analysis — keep current Work-folder storage</option>
|
||||
{% for client in analyzer_clients %}
|
||||
<option value="{{ client.id }}" {% if selected_client_id == client.id %}selected{% endif %}>{{ client.client_code }} · {{ client.client_name }}{% if client.trade_name %} · {{ client.trade_name }}{% endif %}</option>
|
||||
{% endfor %}
|
||||
</select>
|
||||
</label>
|
||||
<label class="text-sm font-semibold text-slate-700">
|
||||
Engagement document destination
|
||||
<select name="engagement_id" id="bank-analyzer-engagement" class="mt-1 w-full rounded-xl border border-slate-300 bg-white px-3 py-2">
|
||||
<option value="">Select engagement after selecting client</option>
|
||||
{% for engagement in analyzer_engagements %}
|
||||
<option value="{{ engagement.id }}" data-client-id="{{ engagement.client_id }}" {% if selected_engagement_id == engagement.id %}selected{% endif %}>
|
||||
{{ engagement.financial_year }} · {{ engagement.catalogue.service_name if engagement.catalogue else ('Engagement #' ~ engagement.id) }}{% if engagement.period_label %} · {{ engagement.period_label }}{% endif %}
|
||||
</option>
|
||||
{% endfor %}
|
||||
</select>
|
||||
</label>
|
||||
</div>
|
||||
<label class="mt-3 flex items-start gap-2 text-sm text-slate-700">
|
||||
<input type="checkbox" name="confirm_same_client" value="1" class="mt-1 rounded border-slate-300">
|
||||
<span>I confirm that all statements uploaded in this analysis belong to the selected client or one of its ERP-recorded legal/trade/business names.</span>
|
||||
</label>
|
||||
<p class="mt-2 text-xs text-indigo-700">When a client is selected, the analyzer validates every statement owner against the client name, trade name, business-unit names, registration legal/trade names and branch names. On successful analysis, the source PDFs and generated workbook are archived using the existing Engagement Documents storage pipeline.</p>
|
||||
</div>
|
||||
{% endif %}
|
||||
<div><label class="mb-1 block text-sm font-semibold text-slate-700">Bank</label><select name="bank_selection" class="w-full rounded-xl border border-slate-300 bg-white px-3 py-2 text-sm">{% for value, label in bank_options %}<option value="{{ value }}" {% if selected_bank == value %}selected{% endif %}>{{ label }}</option>{% endfor %}</select><p class="mt-1 text-xs text-slate-500">Keep Auto Detect or select a bank for direct parser validation.</p></div>
|
||||
<div><label class="mb-1 block text-sm font-semibold text-slate-700">Financial year <span class="font-normal text-slate-400">(optional)</span></label><input name="financial_year" value="{{ financial_year or '' }}" class="w-full rounded-xl border border-slate-300 px-3 py-2 text-sm" placeholder="Example: 2025-26"></div>
|
||||
<div><label class="mb-1 block text-sm font-semibold text-slate-700">Account holder override <span class="font-normal text-slate-400">(optional)</span></label><input name="customer_name" class="w-full rounded-xl border border-slate-300 px-3 py-2 text-sm" placeholder="Use only when statement extraction needs correction"></div>
|
||||
<div><label class="mb-1 block text-sm font-semibold text-slate-700">Account number override <span class="font-normal text-slate-400">(optional)</span></label><input name="account_number" class="w-full rounded-xl border border-slate-300 px-3 py-2 text-sm" placeholder="Use only when statement extraction needs correction"></div>
|
||||
</div>
|
||||
<div class="mt-5 rounded-xl border border-slate-200 bg-slate-50 p-4"><label class="flex items-start gap-3"><input type="checkbox" name="enable_classification" value="1" {% if classification_enabled %}checked{% endif %} class="mt-1 h-4 w-4 rounded border-slate-300"><span><span class="block text-sm font-semibold text-slate-800">Enable narration-based transaction classification</span><span class="mt-1 block text-xs text-slate-500">Adds category, party, review and draft financial helper sheets. Final classification must be verified with books and supporting records.</span></span></label></div>
|
||||
<div class="mt-5 rounded-xl border border-brand-200 bg-brand-50 p-4 text-sm text-slate-700"><strong>Multi-bank contra detection:</strong> when the client has more than one bank account, upload all relevant statements in the same analysis job. The analyzer pairs only conservative equal-and-opposite transfers across different account numbers and keeps every pair reviewable.</div><div class="mt-5"><label class="mb-1 block text-sm font-semibold text-slate-700">PDF bank statements</label><input type="file" name="statements" accept="application/pdf,.pdf" multiple required class="block w-full rounded-xl border border-slate-300 bg-white px-3 py-3 text-sm"></div>
|
||||
<div class="mt-5 rounded-xl border border-brand-200 bg-brand-50 p-4 text-sm text-slate-700"><strong>Multi-bank contra detection:</strong> when the client has more than one bank account, upload all relevant statements in the same analysis job. There is no fixed statement-count limit unless your administrator configures one. The analyzer pairs only conservative equal-and-opposite transfers across different account numbers and keeps every pair reviewable.</div><div class="mt-5"><label class="mb-1 block text-sm font-semibold text-slate-700">PDF bank statements — select all accounts/statements for this client</label><input type="file" name="statements" accept="application/pdf,.pdf" multiple required class="block w-full rounded-xl border border-slate-300 bg-white px-3 py-3 text-sm"></div>
|
||||
<div class="mt-5 rounded-xl bg-slate-50 p-4 text-sm text-slate-600"><div class="font-semibold text-slate-800">Queue limits</div><div class="mt-1">Maximum three processing jobs across all users. Each user may have up to three queued or processing jobs. Completed workbooks and original uploaded statements remain available for 24 hours. Failed-job statements are also retained for 24 hours for debugging.</div></div>
|
||||
<div class="mt-6 flex flex-wrap gap-3"><button class="rounded-xl bg-brand-600 px-5 py-2.5 text-sm font-semibold text-white hover:bg-brand-700">Submit Analysis</button></div>
|
||||
</form>
|
||||
@@ -39,4 +70,29 @@
|
||||
{% if recent_jobs %}<section class="rounded-2xl border border-slate-200 bg-white p-6 shadow-soft"><div class="flex items-center justify-between"><h2 class="text-lg font-bold text-slate-900">Recent Analyses</h2><a href="/tools/bank-statement-analyzer/jobs" class="text-sm font-semibold text-brand-700">View all</a></div><div class="mt-4 overflow-x-auto"><table class="min-w-full text-sm"><thead><tr class="border-b text-left text-slate-500"><th class="py-2 pr-4">Submitted</th><th class="py-2 pr-4">Bank</th><th class="py-2 pr-4">Files</th><th class="py-2 pr-4">Status</th><th class="py-2">Action</th></tr></thead><tbody>{% for item in recent_jobs %}<tr class="border-b border-slate-100"><td class="py-3 pr-4">{{ item.submitted_at }}</td><td class="py-3 pr-4">{{ item.selected_bank|replace('_',' ')|title }}</td><td class="py-3 pr-4">{{ item.file_count }}</td><td class="py-3 pr-4">{{ item.status|title }}{% if item.queue_position %} · Position {{ item.queue_position }}{% endif %}</td><td class="py-3"><a class="font-semibold text-brand-700" href="/tools/bank-statement-analyzer?job={{ item.id }}#analysis-status">View</a></td></tr>{% endfor %}</tbody></table></div></section>{% endif %}
|
||||
</div>
|
||||
{% if active_job and active_job.status in ['queued','processing'] %}<script>(function(){const box=document.getElementById('analysis-status');const id=box.dataset.jobId;async function poll(){try{const r=await fetch(`/tools/bank-statement-analyzer/jobs/${id}/status`,{headers:{'Accept':'application/json'}});if(!r.ok)return;const d=await r.json();document.getElementById('job-status-badge').textContent=d.status.charAt(0).toUpperCase()+d.status.slice(1);document.getElementById('queue-position').textContent=d.queue_position||'—';document.getElementById('estimated-wait').textContent=d.estimated_wait||'—';document.getElementById('progress-text').textContent=d.progress_percent+'%';document.getElementById('progress-bar').style.width=d.progress_percent+'%';if(d.status==='completed'||d.status==='failed'){window.location.reload();return;}}catch(e){}setTimeout(poll,5000)}setTimeout(poll,5000)})();</script>{% endif %}
|
||||
|
||||
<script>
|
||||
(function () {
|
||||
const client = document.getElementById("bank-analyzer-client");
|
||||
const engagement = document.getElementById("bank-analyzer-engagement");
|
||||
if (!client || !engagement) return;
|
||||
|
||||
const options = Array.from(engagement.querySelectorAll("option[data-client-id]"));
|
||||
function filterEngagements() {
|
||||
const selectedClient = client.value;
|
||||
let selectedStillVisible = false;
|
||||
options.forEach((option) => {
|
||||
const visible = !!selectedClient && option.dataset.clientId === selectedClient;
|
||||
option.hidden = !visible;
|
||||
option.disabled = !visible;
|
||||
if (visible && option.selected) selectedStillVisible = true;
|
||||
});
|
||||
if (!selectedStillVisible) engagement.value = "";
|
||||
engagement.required = !!selectedClient;
|
||||
}
|
||||
client.addEventListener("change", filterEngagements);
|
||||
filterEngagements();
|
||||
})();
|
||||
</script>
|
||||
|
||||
{% endblock %}
|
||||
|
||||
@@ -16,6 +16,7 @@ from app.core.templating import templates
|
||||
from app.modules.core.rbac.deps import get_user_permissions, get_user_roles
|
||||
|
||||
from .parsers.registry import BANK_OPTIONS
|
||||
from .client_context import analyzer_client_context, validate_selected_client_and_engagement
|
||||
from .service import (
|
||||
can_use,
|
||||
create_job_folder,
|
||||
@@ -85,6 +86,10 @@ def _ctx(request, db, user, **extra):
|
||||
"selected_bank": "auto",
|
||||
"financial_year": "",
|
||||
"classification_enabled": True,
|
||||
"analyzer_clients": [],
|
||||
"analyzer_engagements": [],
|
||||
"selected_client_id": None,
|
||||
"selected_engagement_id": None,
|
||||
}
|
||||
data.update(extra)
|
||||
return data
|
||||
@@ -112,13 +117,27 @@ def index(request: Request, job: str | None = None):
|
||||
selected_job = get_owned_job(user.id, job) if job else None
|
||||
recent = [_localised_job_view(item, timezone_name) for item in list_user_jobs(user.id, limit=8)]
|
||||
active_job = _localised_job_view(selected_job, timezone_name) if selected_job else None
|
||||
return templates.TemplateResponse("modules/bank_statement_analyzer/templates/bank_statement_analyzer/index.html", _ctx(request, db, user, error="", active_job=active_job, recent_jobs=recent, display_timezone=timezone_name))
|
||||
client_context = analyzer_client_context(db, request=request, user=user, roles=roles)
|
||||
return templates.TemplateResponse(
|
||||
"modules/bank_statement_analyzer/templates/bank_statement_analyzer/index.html",
|
||||
_ctx(
|
||||
request,
|
||||
db,
|
||||
user,
|
||||
error="",
|
||||
active_job=active_job,
|
||||
recent_jobs=recent,
|
||||
display_timezone=timezone_name,
|
||||
analyzer_clients=client_context["clients"],
|
||||
analyzer_engagements=client_context["engagements"],
|
||||
),
|
||||
)
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
@router.post("/analyze")
|
||||
async def analyze(request: Request, csrf_token: str = Form(...), bank_selection: str = Form("auto"), financial_year: str = Form(""), customer_name: str = Form(""), account_number: str = Form(""), enable_classification: str | None = Form(None), statements: list[UploadFile] = File(...)):
|
||||
async def analyze(request: Request, csrf_token: str = Form(...), bank_selection: str = Form("auto"), financial_year: str = Form(""), customer_name: str = Form(""), account_number: str = Form(""), client_id: str = Form(""), engagement_id: str = Form(""), confirm_same_client: str | None = Form(None), enable_classification: str | None = Form(None), statements: list[UploadFile] = File(...)):
|
||||
db = CommonSessionLocal()
|
||||
selected_bank = bank_selection if bank_selection in dict(BANK_OPTIONS) else "auto"
|
||||
classification_enabled = enable_classification == "1"
|
||||
@@ -128,10 +147,35 @@ async def analyze(request: Request, csrf_token: str = Form(...), bank_selection:
|
||||
if denied:
|
||||
return denied
|
||||
validate_csrf(request, csrf_token)
|
||||
resolved_client_id = int(client_id) if client_id.strip() else None
|
||||
resolved_engagement_id = int(engagement_id) if engagement_id.strip() else None
|
||||
validate_selected_client_and_engagement(
|
||||
db,
|
||||
request=request,
|
||||
user=user,
|
||||
roles=roles,
|
||||
client_id=resolved_client_id,
|
||||
engagement_id=resolved_engagement_id,
|
||||
)
|
||||
|
||||
job_id, input_dir, _output_dir = create_job_folder(user, roles)
|
||||
job_dir = input_dir.parent
|
||||
paths = await save_uploads(statements, input_dir)
|
||||
enqueue_job(user=user, roles=roles, job_id=job_id, paths=paths, job_dir=job_dir, bank_selection=selected_bank, financial_year=financial_year, customer_override=customer_name, account_override=account_number, classification_enabled=classification_enabled)
|
||||
enqueue_job(
|
||||
user=user,
|
||||
roles=roles,
|
||||
job_id=job_id,
|
||||
paths=paths,
|
||||
job_dir=job_dir,
|
||||
bank_selection=selected_bank,
|
||||
financial_year=financial_year,
|
||||
customer_override=customer_name,
|
||||
account_override=account_number,
|
||||
classification_enabled=classification_enabled,
|
||||
client_id=resolved_client_id,
|
||||
engagement_id=resolved_engagement_id,
|
||||
ownership_confirmation=(confirm_same_client == "1"),
|
||||
)
|
||||
return RedirectResponse(f"/tools/bank-statement-analyzer?job={job_id}#analysis-status", status_code=303)
|
||||
except Exception as exc:
|
||||
if job_dir:
|
||||
@@ -142,6 +186,7 @@ async def analyze(request: Request, csrf_token: str = Form(...), bank_selection:
|
||||
return RedirectResponse("/login", status_code=303)
|
||||
timezone_name = _display_timezone_name(request, db, user)
|
||||
recent = [_localised_job_view(item, timezone_name) for item in list_user_jobs(user.id, limit=8)]
|
||||
client_context = analyzer_client_context(db, request=request, user=user, roles=get_user_roles(db, user.id))
|
||||
return templates.TemplateResponse("modules/bank_statement_analyzer/templates/bank_statement_analyzer/index.html", _ctx(request, db, user, error=str(exc), active_job=None, recent_jobs=recent, selected_bank=selected_bank, financial_year=financial_year, classification_enabled=classification_enabled, display_timezone=timezone_name), status_code=400)
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
Reference in New Issue
Block a user