Add Phase 11B client aware multi bank ownership and engagement archive

This commit is contained in:
A R R R Associates
2026-08-22 20:45:36 +05:30
parent 8748ee9539
commit 016299cf0c
9 changed files with 714 additions and 8 deletions
@@ -0,0 +1,80 @@
"""Phase 11B client-aware Bank Analyzer ownership and engagement archive.
Revision ID: 20260822_bank_client_archive_p11b
Revises: 20260822_bank_to_tally_p11
"""
from alembic import op
import sqlalchemy as sa
revision = "20260822_bank_client_archive_p11b"
down_revision = "20260822_bank_to_tally_p11"
branch_labels = None
depends_on = None
def upgrade():
op.add_column(
"bank_statement_analysis_jobs",
sa.Column("client_id", sa.Integer(), sa.ForeignKey("clients.id", ondelete="SET NULL"), nullable=True),
)
op.add_column(
"bank_statement_analysis_jobs",
sa.Column("engagement_id", sa.Integer(), sa.ForeignKey("client_service_subscriptions.id", ondelete="SET NULL"), nullable=True),
)
op.add_column(
"bank_statement_analysis_jobs",
sa.Column("ownership_confirmation", sa.Boolean(), nullable=False, server_default=sa.false()),
)
op.add_column(
"bank_statement_analysis_jobs",
sa.Column("ownership_status", sa.String(30), nullable=False, server_default="not_checked"),
)
op.add_column(
"bank_statement_analysis_jobs",
sa.Column("ownership_validation_json", sa.Text(), nullable=True),
)
op.add_column(
"bank_statement_analysis_jobs",
sa.Column("engagement_archive_status", sa.String(30), nullable=False, server_default="not_requested"),
)
op.add_column(
"bank_statement_analysis_jobs",
sa.Column("engagement_archive_json", sa.Text(), nullable=True),
)
op.create_index(
"ix_bank_statement_analysis_jobs_client_id",
"bank_statement_analysis_jobs",
["client_id"],
)
op.create_index(
"ix_bank_statement_analysis_jobs_engagement_id",
"bank_statement_analysis_jobs",
["engagement_id"],
)
op.create_index(
"ix_bank_statement_analysis_jobs_ownership_status",
"bank_statement_analysis_jobs",
["ownership_status"],
)
op.create_index(
"ix_bank_statement_analysis_jobs_engagement_archive_status",
"bank_statement_analysis_jobs",
["engagement_archive_status"],
)
def downgrade():
op.drop_index("ix_bank_statement_analysis_jobs_engagement_archive_status", table_name="bank_statement_analysis_jobs")
op.drop_index("ix_bank_statement_analysis_jobs_ownership_status", table_name="bank_statement_analysis_jobs")
op.drop_index("ix_bank_statement_analysis_jobs_engagement_id", table_name="bank_statement_analysis_jobs")
op.drop_index("ix_bank_statement_analysis_jobs_client_id", table_name="bank_statement_analysis_jobs")
op.drop_column("bank_statement_analysis_jobs", "engagement_archive_json")
op.drop_column("bank_statement_analysis_jobs", "engagement_archive_status")
op.drop_column("bank_statement_analysis_jobs", "ownership_validation_json")
op.drop_column("bank_statement_analysis_jobs", "ownership_status")
op.drop_column("bank_statement_analysis_jobs", "ownership_confirmation")
op.drop_column("bank_statement_analysis_jobs", "engagement_id")
op.drop_column("bank_statement_analysis_jobs", "client_id")
+7
View File
@@ -79,6 +79,13 @@ def import_completed_job(db, *, tenant_id, client_id, job_id, user_id):
raise ValueError("Completed Bank Analyzer workbook was not found.") raise ValueError("Completed Bank Analyzer workbook was not found.")
if job.tenant_id not in (None, tenant_id): if job.tenant_id not in (None, tenant_id):
raise ValueError("Bank Analyzer job belongs to a different firm.") raise ValueError("Bank Analyzer job belongs to a different firm.")
if getattr(job, "client_id", None) and int(job.client_id) != int(client_id):
raise ValueError(
"This Bank Analyzer job is already bound to a different ERP client. "
"Import it using the client confirmed during statement ownership validation."
)
if getattr(job, "client_id", None) and getattr(job, "ownership_status", "") != "confirmed":
raise ValueError("Client-bound Bank Analyzer job has not passed ownership validation.")
frame = pd.read_excel(job.output_file, sheet_name="Transaction Classification") frame = pd.read_excel(job.output_file, sheet_name="Transaction Classification")
tally_options = available_tally_guids(db, tenant_id, client_id) tally_options = available_tally_guids(db, tenant_id, client_id)
@@ -12,7 +12,7 @@
<form method="post" action="/tools/accounting/bank-posting/import" class="grid gap-4 md:grid-cols-3"> <form method="post" action="/tools/accounting/bank-posting/import" class="grid gap-4 md:grid-cols-3">
<input type="hidden" name="csrf_token" value="{{ csrf_token }}"> <input type="hidden" name="csrf_token" value="{{ csrf_token }}">
<label class="text-sm font-medium">Client<select name="client_id" required class="mt-1 w-full rounded-xl border border-slate-300 px-3 py-2"><option value="">Select client</option>{% for c in clients %}<option value="{{ c.id }}" {% if selected_client and c.id==selected_client.id %}selected{% endif %}>{{ c.client_name }}</option>{% endfor %}</select></label> <label class="text-sm font-medium">Client<select name="client_id" required class="mt-1 w-full rounded-xl border border-slate-300 px-3 py-2"><option value="">Select client</option>{% for c in clients %}<option value="{{ c.id }}" {% if selected_client and c.id==selected_client.id %}selected{% endif %}>{{ c.client_name }}</option>{% endfor %}</select></label>
<label class="text-sm font-medium">Completed Bank Analyzer job<select name="job_id" required class="mt-1 w-full rounded-xl border border-slate-300 px-3 py-2"><option value="">Select analysis</option>{% for j in jobs %}<option value="{{ j.id }}">{{ j.completed_at_utc or j.submitted_at_utc }} · {{ j.selected_bank|replace('_',' ')|title }} · {{ j.file_count }} file(s)</option>{% endfor %}</select></label> <label class="text-sm font-medium">Completed Bank Analyzer job<select name="job_id" required class="mt-1 w-full rounded-xl border border-slate-300 px-3 py-2"><option value="">Select analysis</option>{% for j in jobs %}<option value="{{ j.id }}" data-client-id="{{ j.client_id or '' }}">{{ j.completed_at_utc or j.submitted_at_utc }} · {{ j.selected_bank|replace('_',' ')|title }} · {{ j.file_count }} file(s){% if j.client_id %} · Client #{{ j.client_id }}{% endif %}</option>{% endfor %}</select></label>
<div class="flex items-end"><button class="w-full rounded-xl bg-brand-600 px-4 py-2 text-sm font-semibold text-white">Import to Accounting Engine</button></div> <div class="flex items-end"><button class="w-full rounded-xl bg-brand-600 px-4 py-2 text-sm font-semibold text-white">Import to Accounting Engine</button></div>
</form> </form>
</section> </section>
@@ -73,4 +73,20 @@
</section> </section>
{% endif %} {% endif %}
</div> </div>
<script>
(function () {
const job = document.querySelector('select[name="job_id"]');
const client = document.querySelector('select[name="client_id"]');
if (!job || !client) return;
job.addEventListener("change", function () {
const option = job.options[job.selectedIndex];
const clientId = option ? option.dataset.clientId : "";
if (clientId) {
client.value = clientId;
}
});
})();
</script>
{% endblock %} {% endblock %}
@@ -615,6 +615,10 @@ def analyze_files(paths, customer_override="", account_override="", bank_hint="a
frames = [] frames = []
for path in paths: for path in paths:
meta, df = parse_pdf(path, bank_hint=bank_hint) meta, df = parse_pdf(path, bank_hint=bank_hint)
# Preserve the bank-extracted owner before an optional manual correction.
# Phase 11B ownership validation always prefers this original value.
meta.extracted_customer_name = getattr(meta, "customer_name", "") or ""
df["extracted_customer_name"] = meta.extracted_customer_name
if customer_override: if customer_override:
meta.customer_name = customer_override meta.customer_name = customer_override
df["customer_name"] = customer_override df["customer_name"] = customer_override
@@ -0,0 +1,438 @@
from __future__ import annotations
import json
import re
from difflib import SequenceMatcher
from pathlib import Path
from types import SimpleNamespace
from sqlalchemy import or_, select
from app.modules.clients.models import Client, ClientBranch, ClientBusinessUnit
from app.modules.consultants.models import ClientConsultantLink
from app.modules.consultants.service import get_consultant_by_user
from app.modules.documents.services import (
build_document_scope,
list_visible_engagements,
save_uploaded_revision,
user_can_upload_to_engagement,
)
from app.modules.registrations.models import ClientRegistration
from app.modules.services.models import ClientServiceSubscription
_LEGAL_NOISE = {
"M", "S", "MS", "MR", "MRS", "MISS",
"PRIVATE", "PVT", "LIMITED", "LTD", "LLP",
"PROPRIETOR", "PROPRIETORSHIP", "PROP",
"THE", "INDIA",
}
def _text(value) -> str:
return str(value or "").strip()
def _normalise_name(value) -> str:
raw = re.sub(r"[^A-Z0-9]+", " ", _text(value).upper()).strip()
tokens = [token for token in raw.split() if token and token not in _LEGAL_NOISE]
return " ".join(tokens)
def _name_match(extracted: str, alias: str) -> tuple[bool, int, str]:
left = _normalise_name(extracted)
right = _normalise_name(alias)
if not left or not right:
return False, 0, ""
if left == right:
return True, 100, "exact normalized account-holder match"
# Conservative containment is allowed only for reasonably descriptive names.
if min(len(left), len(right)) >= 8 and (left in right or right in left):
score = int(round(100 * min(len(left), len(right)) / max(len(left), len(right))))
if score >= 82:
return True, max(90, score), "legal/trade-name containment match"
ratio = int(round(SequenceMatcher(None, left, right).ratio() * 100))
if ratio >= 92:
return True, ratio, "high-confidence legal/trade-name similarity match"
return False, ratio, ""
def client_owner_aliases(db, *, tenant_id: int, client_id: int) -> list[str]:
client = db.get(Client, int(client_id))
if not client or client.tenant_id != int(tenant_id) or client.is_archived:
raise ValueError("Selected client was not found.")
values = [client.client_name, client.trade_name]
for unit in db.execute(
select(ClientBusinessUnit).where(
ClientBusinessUnit.tenant_id == tenant_id,
ClientBusinessUnit.client_id == client.id,
ClientBusinessUnit.is_active.is_(True),
)
).scalars().all():
values.extend([unit.business_name, unit.trade_name])
for branch in db.execute(
select(ClientBranch).where(
ClientBranch.tenant_id == tenant_id,
ClientBranch.client_id == client.id,
ClientBranch.is_active.is_(True),
)
).scalars().all():
values.append(branch.branch_name)
for registration in db.execute(
select(ClientRegistration).where(
ClientRegistration.tenant_id == tenant_id,
ClientRegistration.client_id == client.id,
ClientRegistration.status.in_(["active", "valid", "registered"]),
)
).scalars().all():
values.extend([registration.legal_name, registration.trade_name])
result = []
seen = set()
for value in values:
value = _text(value)
key = _normalise_name(value)
if value and key and key not in seen:
seen.add(key)
result.append(value)
return result
def _consultant_context(db, user, tenant_id: int):
consultant = get_consultant_by_user(
db,
tenant_id=tenant_id,
user_id=int(user.id),
)
return consultant
def analyzer_client_context(db, *, request, user, roles: list[str]) -> dict:
"""Return role-scoped clients and archive-capable engagements.
Partner/Manager/Staff reuse the existing Document module visibility rules.
Consultant reuses the existing ClientConsultantLink permissions.
"""
tenant_id = int(
request.session.get("active_tenant_id")
or getattr(user, "tenant_id", 0)
or 0
)
role_set = set(roles or [])
clients: dict[int, Client] = {}
engagements: dict[int, ClientServiceSubscription] = {}
if "Consultant" in role_set:
consultant = _consultant_context(db, user, tenant_id)
if not consultant:
return {"clients": [], "engagements": [], "consultant": None}
links = list(db.execute(
select(ClientConsultantLink).where(
ClientConsultantLink.tenant_id == tenant_id,
ClientConsultantLink.consultant_id == consultant.id,
ClientConsultantLink.is_active.is_(True),
ClientConsultantLink.can_view_client.is_(True),
)
).scalars().all())
client_ids = {int(link.client_id) for link in links}
upload_ids = {
int(link.client_id)
for link in links
if bool(link.can_upload_documents)
}
if client_ids:
for client in db.execute(
select(Client).where(
Client.tenant_id == tenant_id,
Client.id.in_(client_ids),
Client.is_archived.is_(False),
Client.is_active.is_(True),
).order_by(Client.client_name.asc())
).scalars().all():
clients[int(client.id)] = client
if upload_ids:
for engagement in db.execute(
select(ClientServiceSubscription).where(
ClientServiceSubscription.tenant_id == tenant_id,
ClientServiceSubscription.client_id.in_(upload_ids),
ClientServiceSubscription.is_active.is_(True),
).order_by(
ClientServiceSubscription.financial_year.desc(),
ClientServiceSubscription.id.desc(),
)
).scalars().all():
engagements[int(engagement.id)] = engagement
return {
"clients": list(clients.values()),
"engagements": list(engagements.values()),
"consultant": consultant,
}
# Firm Partner / Manager / Staff use the existing engagement-document scope.
scope = build_document_scope(request, db, user)
visible = list_visible_engagements(db, user, scope, limit=1000)
for engagement in visible:
client = getattr(engagement, "client", None) or db.get(Client, engagement.client_id)
if client and client.is_active and not client.is_archived:
clients[int(client.id)] = client
if user_can_upload_to_engagement(db, user, engagement, scope):
engagements[int(engagement.id)] = engagement
return {
"clients": sorted(clients.values(), key=lambda c: (c.client_name or "").casefold()),
"engagements": list(engagements.values()),
"consultant": None,
}
def validate_selected_client_and_engagement(
db,
*,
request,
user,
roles: list[str],
client_id: int | None,
engagement_id: int | None,
):
if not client_id:
if engagement_id:
raise ValueError("Select a client before selecting an engagement.")
return None, None
context = analyzer_client_context(db, request=request, user=user, roles=roles)
clients = {int(row.id): row for row in context["clients"]}
engagements = {int(row.id): row for row in context["engagements"]}
client = clients.get(int(client_id))
if not client:
raise ValueError("Selected client is not available in your Bank Analyzer scope.")
if not engagement_id:
raise ValueError(
"Select an engagement document destination when a client is selected."
)
engagement = engagements.get(int(engagement_id))
if not engagement or int(engagement.client_id) != int(client.id):
raise ValueError(
"Selected engagement is not available for document upload for this client."
)
return client, engagement
def validate_statement_ownership(
db,
*,
tenant_id: int,
client_id: int | None,
metas: list,
owner_override: str = "",
ownership_confirmation: bool = False,
) -> dict:
"""Validate that all statements belong to one ERP client/legal/trade-name family.
With an ERP client selected the check is strict. Every statement must expose
an owner name or use a manually confirmed owner override, and that name must
match one of the ERP legal/trade aliases.
"""
extracted = []
for position, meta in enumerate(metas, start=1):
raw = _text(
getattr(meta, "extracted_customer_name", None)
or getattr(meta, "customer_name", None)
)
extracted.append({
"statement_id": getattr(meta, "statement_id", f"STMT-{position:03d}"),
"source_file": _text(getattr(meta, "source_file", "")),
"owner_name": raw,
})
if not client_id:
# Preserve standalone behavior. We record the owner names for review but
# do not change current storage or reject mixed standalone analyses.
return {
"status": "standalone",
"client_id": None,
"aliases": [],
"statements": extracted,
"message": "Standalone analysis; no ERP client ownership binding requested.",
}
aliases = client_owner_aliases(
db,
tenant_id=int(tenant_id),
client_id=int(client_id),
)
if not aliases:
raise ValueError(
"Selected client does not have a usable legal/trade name for bank ownership validation."
)
override = _text(owner_override)
if override:
matched_override = any(_name_match(override, alias)[0] for alias in aliases)
if not matched_override:
raise ValueError(
f"Account-holder override '{override}' does not match the selected client's "
"legal/trade names."
)
results = []
failures = []
for statement in extracted:
owner = statement["owner_name"] or override
if not owner:
failures.append(
f"{statement['statement_id']}: account-holder name could not be extracted. "
"Use Account holder override only after verifying the statement manually."
)
results.append({**statement, "matched": False, "score": 0, "matched_alias": ""})
continue
best = (False, 0, "", "")
for alias in aliases:
matched, score, reason = _name_match(owner, alias)
if score > best[1]:
best = (matched, score, alias, reason)
matched, score, alias, reason = best
results.append({
**statement,
"effective_owner_name": owner,
"matched": bool(matched),
"score": int(score),
"matched_alias": alias,
"reason": reason,
})
if not matched:
failures.append(
f"{statement['statement_id']}: '{owner}' does not match the selected client's "
"legal/trade names."
)
if failures:
raise ValueError(
"Bank account ownership validation failed. " + " ".join(failures)
)
if len(metas) > 1 and not ownership_confirmation:
raise ValueError(
"Confirm that all uploaded bank statements belong to the selected client "
"or its approved trade/business names."
)
return {
"status": "confirmed",
"client_id": int(client_id),
"aliases": aliases,
"statements": results,
"message": f"Ownership confirmed for {len(results)} statement(s).",
}
def _upload_proxy(path: Path, content_type: str):
handle = path.open("rb")
return SimpleNamespace(
filename=path.name,
content_type=content_type,
file=handle,
)
def archive_analysis_to_engagement(
db,
*,
job,
engagement: ClientServiceSubscription | None,
metas: list,
source_paths: list[Path],
output_path: Path,
user,
) -> dict:
if not job.client_id or not job.engagement_id or engagement is None:
return {"status": "not_requested", "documents": []}
archived = []
meta_by_source = {
Path(_text(getattr(meta, "source_file", ""))).name: meta
for meta in metas
}
for source in source_paths:
upload = _upload_proxy(source, "application/pdf")
try:
meta = meta_by_source.get(source.name)
bank = _text(getattr(meta, "bank_name", "")) if meta else ""
account = _text(getattr(meta, "account_number", "")) if meta else ""
title_bits = ["Bank Statement"]
if bank:
title_bits.append(bank)
if account:
title_bits.append(account[-6:])
document = save_uploaded_revision(
db,
engagement=engagement,
upload_file=upload,
title=" - ".join(title_bits),
document_type="BANK_STATEMENT",
description=(
f"Bank Statement Analyzer source. Job {job.id}. "
f"Ownership status: {job.ownership_status}."
),
remarks="Automatically archived from Bank Statement Analyzer.",
user=user,
udin_required=False,
)
archived.append({
"document_id": int(document.id),
"filename": source.name,
"type": "BANK_STATEMENT",
})
finally:
upload.file.close()
workbook_upload = _upload_proxy(
output_path,
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
)
try:
document = save_uploaded_revision(
db,
engagement=engagement,
upload_file=workbook_upload,
title="Bank Statement Analyzer - Analysis Workbook",
document_type="WORKING_PAPER",
description=(
f"Generated Bank Statement Analyzer workbook for job {job.id}; "
f"includes multi-bank contra review where applicable."
),
remarks="Automatically archived from Bank Statement Analyzer.",
user=user,
udin_required=False,
)
archived.append({
"document_id": int(document.id),
"filename": output_path.name,
"type": "WORKING_PAPER",
})
finally:
workbook_upload.file.close()
return {
"status": "archived",
"client_id": int(job.client_id),
"engagement_id": int(job.engagement_id),
"documents": archived,
}
@@ -17,6 +17,16 @@ class BankStatementAnalysisJob(CommonBase):
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True) user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True)
role_bucket: Mapped[str] = mapped_column(String(30), nullable=False) role_bucket: Mapped[str] = mapped_column(String(30), nullable=False)
# Phase 11B: optional client-aware analysis. Null preserves the existing
# standalone Work-folder behavior exactly.
client_id: Mapped[int | None] = mapped_column(ForeignKey("clients.id", ondelete="SET NULL"), nullable=True, index=True)
engagement_id: Mapped[int | None] = mapped_column(ForeignKey("client_service_subscriptions.id", ondelete="SET NULL"), nullable=True, index=True)
ownership_confirmation: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
ownership_status: Mapped[str] = mapped_column(String(30), nullable=False, default="not_checked", index=True)
ownership_validation_json: Mapped[str | None] = mapped_column(Text, nullable=True)
engagement_archive_status: Mapped[str] = mapped_column(String(30), nullable=False, default="not_requested", index=True)
engagement_archive_json: Mapped[str | None] = mapped_column(Text, nullable=True)
selected_bank: Mapped[str] = mapped_column(String(40), nullable=False, default="auto") selected_bank: Mapped[str] = mapped_column(String(40), nullable=False, default="auto")
financial_year: Mapped[str | None] = mapped_column(String(20), nullable=True) financial_year: Mapped[str | None] = mapped_column(String(20), nullable=True)
customer_override: Mapped[str | None] = mapped_column(String(255), nullable=True) customer_override: Mapped[str | None] = mapped_column(String(255), nullable=True)
+53 -3
View File
@@ -20,9 +20,16 @@ from app.core.db.common import CommonSessionLocal
from .analyzer import analyze_files, export_excel from .analyzer import analyze_files, export_excel
from .models import BankStatementAnalysisJob from .models import BankStatementAnalysisJob
from .client_context import (
archive_analysis_to_engagement,
validate_statement_ownership,
)
from app.modules.core.iam.models import User
from app.modules.services.models import ClientServiceSubscription
ALLOWED_ROLES = {"Partner", "Manager", "Branch Manager", "Staff", "Employee", "Consultant"} ALLOWED_ROLES = {"Partner", "Manager", "Branch Manager", "Staff", "Employee", "Consultant"}
MAX_FILES = int(os.getenv("BANK_ANALYZER_MAX_FILES", "24")) # 0 means no fixed statement-count ceiling; per-file size and queue controls remain.
MAX_FILES = int(os.getenv("BANK_ANALYZER_MAX_FILES", "0"))
MAX_FILE_BYTES = int(os.getenv("BANK_ANALYZER_MAX_FILE_MB", "50")) * 1024 * 1024 MAX_FILE_BYTES = int(os.getenv("BANK_ANALYZER_MAX_FILE_MB", "50")) * 1024 * 1024
MAX_GLOBAL_PROCESSING = 3 MAX_GLOBAL_PROCESSING = 3
MAX_PENDING_PER_USER = 3 MAX_PENDING_PER_USER = 3
@@ -117,7 +124,7 @@ async def save_uploads(files: list[UploadFile], input_dir: Path) -> list[Path]:
usable = [item for item in files if item and (item.filename or "").strip()] usable = [item for item in files if item and (item.filename or "").strip()]
if not usable: if not usable:
raise ValueError("Please select at least one PDF bank statement.") raise ValueError("Please select at least one PDF bank statement.")
if len(usable) > MAX_FILES: if MAX_FILES > 0 and len(usable) > MAX_FILES:
raise ValueError(f"A maximum of {MAX_FILES} PDF files can be analyzed in one job.") raise ValueError(f"A maximum of {MAX_FILES} PDF files can be analyzed in one job.")
saved: list[Path] = [] saved: list[Path] = []
try: try:
@@ -155,7 +162,7 @@ def pending_count_for_user(user_id: int) -> int:
db.close() db.close()
def enqueue_job(*, user, roles: Iterable[str], job_id: str, paths: list[Path], job_dir: Path, bank_selection: str, financial_year: str, customer_override: str, account_override: str, classification_enabled: bool) -> BankStatementAnalysisJob: def enqueue_job(*, user, roles: Iterable[str], job_id: str, paths: list[Path], job_dir: Path, bank_selection: str, financial_year: str, customer_override: str, account_override: str, classification_enabled: bool, client_id: int | None = None, engagement_id: int | None = None, ownership_confirmation: bool = False) -> BankStatementAnalysisJob:
if pending_count_for_user(int(user.id)) >= MAX_PENDING_PER_USER: if pending_count_for_user(int(user.id)) >= MAX_PENDING_PER_USER:
shutil.rmtree(job_dir, ignore_errors=True) shutil.rmtree(job_dir, ignore_errors=True)
raise ValueError("You already have three queued or processing analyses. Please wait for one to complete before submitting another.") raise ValueError("You already have three queued or processing analyses. Please wait for one to complete before submitting another.")
@@ -169,6 +176,11 @@ def enqueue_job(*, user, roles: Iterable[str], job_id: str, paths: list[Path], j
branch_id=getattr(user, "branch_id", None), branch_id=getattr(user, "branch_id", None),
user_id=int(user.id), user_id=int(user.id),
role_bucket=bucket, role_bucket=bucket,
client_id=int(client_id) if client_id else None,
engagement_id=int(engagement_id) if engagement_id else None,
ownership_confirmation=bool(ownership_confirmation),
ownership_status="pending" if client_id else "not_checked",
engagement_archive_status="pending" if engagement_id else "not_requested",
selected_bank=bank_selection, selected_bank=bank_selection,
financial_year=(financial_year or "").strip() or None, financial_year=(financial_year or "").strip() or None,
customer_override=(customer_override or "").strip() or None, customer_override=(customer_override or "").strip() or None,
@@ -245,6 +257,18 @@ def _process_job(job_id: str) -> None:
bank_hint=job.selected_bank, bank_hint=job.selected_bank,
classification_enabled=job.classification_enabled, classification_enabled=job.classification_enabled,
) )
ownership = validate_statement_ownership(
db,
tenant_id=int(job.tenant_id or 0),
client_id=job.client_id,
metas=metas,
owner_override=job.customer_override or "",
ownership_confirmation=bool(job.ownership_confirmation),
)
job.ownership_status = str(ownership.get("status") or "not_checked")
job.ownership_validation_json = json.dumps(ownership, ensure_ascii=False)
job.progress_percent = 75 job.progress_percent = 75
db.commit() db.commit()
output = output_dir / _workbook_filename(metas) output = output_dir / _workbook_filename(metas)
@@ -257,6 +281,28 @@ def _process_job(job_id: str) -> None:
selected_bank=job.selected_bank, selected_bank=job.selected_bank,
classification_enabled=job.classification_enabled, classification_enabled=job.classification_enabled,
) )
archive_result = {"status": "not_requested", "documents": []}
if job.client_id and job.engagement_id:
engagement = db.get(ClientServiceSubscription, int(job.engagement_id))
uploader = db.get(User, int(job.user_id))
if not engagement or int(engagement.client_id) != int(job.client_id):
raise ValueError("Selected engagement is no longer valid for the selected client.")
if not uploader:
raise ValueError("Bank Analyzer uploader could not be resolved for document archive.")
archive_result = archive_analysis_to_engagement(
db,
job=job,
engagement=engagement,
metas=metas,
source_paths=paths,
output_path=output,
user=uploader,
)
job.engagement_archive_status = str(archive_result.get("status") or "failed")
job.engagement_archive_json = json.dumps(archive_result, ensure_ascii=False)
db.commit()
summary = { summary = {
"job_id": job.id, "job_id": job.id,
"statement_count": len(metas), "statement_count": len(metas),
@@ -272,6 +318,10 @@ def _process_job(job_id: str) -> None:
"account_number": next((meta.account_number for meta in metas if meta.account_number), ""), "account_number": next((meta.account_number for meta in metas if meta.account_number), ""),
"financial_year": job.financial_year or "", "financial_year": job.financial_year or "",
"classification_enabled": job.classification_enabled, "classification_enabled": job.classification_enabled,
"client_id": job.client_id,
"engagement_id": job.engagement_id,
"ownership_status": job.ownership_status,
"engagement_archive_status": job.engagement_archive_status,
} }
# Keep the original uploaded statements until the job expiry time. # Keep the original uploaded statements until the job expiry time.
# This applies equally to completed and failed jobs and allows the # This applies equally to completed and failed jobs and allows the
@@ -13,13 +13,44 @@
<form action="/tools/bank-statement-analyzer/analyze" method="post" enctype="multipart/form-data" class="rounded-2xl border border-slate-200 bg-white p-6 shadow-soft"> <form action="/tools/bank-statement-analyzer/analyze" method="post" enctype="multipart/form-data" class="rounded-2xl border border-slate-200 bg-white p-6 shadow-soft">
<input type="hidden" name="csrf_token" value="{{ csrf_token }}"> <input type="hidden" name="csrf_token" value="{{ csrf_token }}">
<div class="grid gap-5 md:grid-cols-2"> <div class="grid gap-5 md:grid-cols-2">
{% if analyzer_clients %}
<div class="md:col-span-2 rounded-xl border border-indigo-200 bg-indigo-50 p-4">
<div class="grid gap-4 md:grid-cols-2">
<label class="text-sm font-semibold text-slate-700">
ERP Client <span class="font-normal text-slate-400">(optional)</span>
<select name="client_id" id="bank-analyzer-client" class="mt-1 w-full rounded-xl border border-slate-300 bg-white px-3 py-2">
<option value="">Standalone analysis — keep current Work-folder storage</option>
{% for client in analyzer_clients %}
<option value="{{ client.id }}" {% if selected_client_id == client.id %}selected{% endif %}>{{ client.client_code }} · {{ client.client_name }}{% if client.trade_name %} · {{ client.trade_name }}{% endif %}</option>
{% endfor %}
</select>
</label>
<label class="text-sm font-semibold text-slate-700">
Engagement document destination
<select name="engagement_id" id="bank-analyzer-engagement" class="mt-1 w-full rounded-xl border border-slate-300 bg-white px-3 py-2">
<option value="">Select engagement after selecting client</option>
{% for engagement in analyzer_engagements %}
<option value="{{ engagement.id }}" data-client-id="{{ engagement.client_id }}" {% if selected_engagement_id == engagement.id %}selected{% endif %}>
{{ engagement.financial_year }} · {{ engagement.catalogue.service_name if engagement.catalogue else ('Engagement #' ~ engagement.id) }}{% if engagement.period_label %} · {{ engagement.period_label }}{% endif %}
</option>
{% endfor %}
</select>
</label>
</div>
<label class="mt-3 flex items-start gap-2 text-sm text-slate-700">
<input type="checkbox" name="confirm_same_client" value="1" class="mt-1 rounded border-slate-300">
<span>I confirm that all statements uploaded in this analysis belong to the selected client or one of its ERP-recorded legal/trade/business names.</span>
</label>
<p class="mt-2 text-xs text-indigo-700">When a client is selected, the analyzer validates every statement owner against the client name, trade name, business-unit names, registration legal/trade names and branch names. On successful analysis, the source PDFs and generated workbook are archived using the existing Engagement Documents storage pipeline.</p>
</div>
{% endif %}
<div><label class="mb-1 block text-sm font-semibold text-slate-700">Bank</label><select name="bank_selection" class="w-full rounded-xl border border-slate-300 bg-white px-3 py-2 text-sm">{% for value, label in bank_options %}<option value="{{ value }}" {% if selected_bank == value %}selected{% endif %}>{{ label }}</option>{% endfor %}</select><p class="mt-1 text-xs text-slate-500">Keep Auto Detect or select a bank for direct parser validation.</p></div> <div><label class="mb-1 block text-sm font-semibold text-slate-700">Bank</label><select name="bank_selection" class="w-full rounded-xl border border-slate-300 bg-white px-3 py-2 text-sm">{% for value, label in bank_options %}<option value="{{ value }}" {% if selected_bank == value %}selected{% endif %}>{{ label }}</option>{% endfor %}</select><p class="mt-1 text-xs text-slate-500">Keep Auto Detect or select a bank for direct parser validation.</p></div>
<div><label class="mb-1 block text-sm font-semibold text-slate-700">Financial year <span class="font-normal text-slate-400">(optional)</span></label><input name="financial_year" value="{{ financial_year or '' }}" class="w-full rounded-xl border border-slate-300 px-3 py-2 text-sm" placeholder="Example: 2025-26"></div> <div><label class="mb-1 block text-sm font-semibold text-slate-700">Financial year <span class="font-normal text-slate-400">(optional)</span></label><input name="financial_year" value="{{ financial_year or '' }}" class="w-full rounded-xl border border-slate-300 px-3 py-2 text-sm" placeholder="Example: 2025-26"></div>
<div><label class="mb-1 block text-sm font-semibold text-slate-700">Account holder override <span class="font-normal text-slate-400">(optional)</span></label><input name="customer_name" class="w-full rounded-xl border border-slate-300 px-3 py-2 text-sm" placeholder="Use only when statement extraction needs correction"></div> <div><label class="mb-1 block text-sm font-semibold text-slate-700">Account holder override <span class="font-normal text-slate-400">(optional)</span></label><input name="customer_name" class="w-full rounded-xl border border-slate-300 px-3 py-2 text-sm" placeholder="Use only when statement extraction needs correction"></div>
<div><label class="mb-1 block text-sm font-semibold text-slate-700">Account number override <span class="font-normal text-slate-400">(optional)</span></label><input name="account_number" class="w-full rounded-xl border border-slate-300 px-3 py-2 text-sm" placeholder="Use only when statement extraction needs correction"></div> <div><label class="mb-1 block text-sm font-semibold text-slate-700">Account number override <span class="font-normal text-slate-400">(optional)</span></label><input name="account_number" class="w-full rounded-xl border border-slate-300 px-3 py-2 text-sm" placeholder="Use only when statement extraction needs correction"></div>
</div> </div>
<div class="mt-5 rounded-xl border border-slate-200 bg-slate-50 p-4"><label class="flex items-start gap-3"><input type="checkbox" name="enable_classification" value="1" {% if classification_enabled %}checked{% endif %} class="mt-1 h-4 w-4 rounded border-slate-300"><span><span class="block text-sm font-semibold text-slate-800">Enable narration-based transaction classification</span><span class="mt-1 block text-xs text-slate-500">Adds category, party, review and draft financial helper sheets. Final classification must be verified with books and supporting records.</span></span></label></div> <div class="mt-5 rounded-xl border border-slate-200 bg-slate-50 p-4"><label class="flex items-start gap-3"><input type="checkbox" name="enable_classification" value="1" {% if classification_enabled %}checked{% endif %} class="mt-1 h-4 w-4 rounded border-slate-300"><span><span class="block text-sm font-semibold text-slate-800">Enable narration-based transaction classification</span><span class="mt-1 block text-xs text-slate-500">Adds category, party, review and draft financial helper sheets. Final classification must be verified with books and supporting records.</span></span></label></div>
<div class="mt-5 rounded-xl border border-brand-200 bg-brand-50 p-4 text-sm text-slate-700"><strong>Multi-bank contra detection:</strong> when the client has more than one bank account, upload all relevant statements in the same analysis job. The analyzer pairs only conservative equal-and-opposite transfers across different account numbers and keeps every pair reviewable.</div><div class="mt-5"><label class="mb-1 block text-sm font-semibold text-slate-700">PDF bank statements</label><input type="file" name="statements" accept="application/pdf,.pdf" multiple required class="block w-full rounded-xl border border-slate-300 bg-white px-3 py-3 text-sm"></div> <div class="mt-5 rounded-xl border border-brand-200 bg-brand-50 p-4 text-sm text-slate-700"><strong>Multi-bank contra detection:</strong> when the client has more than one bank account, upload all relevant statements in the same analysis job. There is no fixed statement-count limit unless your administrator configures one. The analyzer pairs only conservative equal-and-opposite transfers across different account numbers and keeps every pair reviewable.</div><div class="mt-5"><label class="mb-1 block text-sm font-semibold text-slate-700">PDF bank statements — select all accounts/statements for this client</label><input type="file" name="statements" accept="application/pdf,.pdf" multiple required class="block w-full rounded-xl border border-slate-300 bg-white px-3 py-3 text-sm"></div>
<div class="mt-5 rounded-xl bg-slate-50 p-4 text-sm text-slate-600"><div class="font-semibold text-slate-800">Queue limits</div><div class="mt-1">Maximum three processing jobs across all users. Each user may have up to three queued or processing jobs. Completed workbooks and original uploaded statements remain available for 24 hours. Failed-job statements are also retained for 24 hours for debugging.</div></div> <div class="mt-5 rounded-xl bg-slate-50 p-4 text-sm text-slate-600"><div class="font-semibold text-slate-800">Queue limits</div><div class="mt-1">Maximum three processing jobs across all users. Each user may have up to three queued or processing jobs. Completed workbooks and original uploaded statements remain available for 24 hours. Failed-job statements are also retained for 24 hours for debugging.</div></div>
<div class="mt-6 flex flex-wrap gap-3"><button class="rounded-xl bg-brand-600 px-5 py-2.5 text-sm font-semibold text-white hover:bg-brand-700">Submit Analysis</button></div> <div class="mt-6 flex flex-wrap gap-3"><button class="rounded-xl bg-brand-600 px-5 py-2.5 text-sm font-semibold text-white hover:bg-brand-700">Submit Analysis</button></div>
</form> </form>
@@ -39,4 +70,29 @@
{% if recent_jobs %}<section class="rounded-2xl border border-slate-200 bg-white p-6 shadow-soft"><div class="flex items-center justify-between"><h2 class="text-lg font-bold text-slate-900">Recent Analyses</h2><a href="/tools/bank-statement-analyzer/jobs" class="text-sm font-semibold text-brand-700">View all</a></div><div class="mt-4 overflow-x-auto"><table class="min-w-full text-sm"><thead><tr class="border-b text-left text-slate-500"><th class="py-2 pr-4">Submitted</th><th class="py-2 pr-4">Bank</th><th class="py-2 pr-4">Files</th><th class="py-2 pr-4">Status</th><th class="py-2">Action</th></tr></thead><tbody>{% for item in recent_jobs %}<tr class="border-b border-slate-100"><td class="py-3 pr-4">{{ item.submitted_at }}</td><td class="py-3 pr-4">{{ item.selected_bank|replace('_',' ')|title }}</td><td class="py-3 pr-4">{{ item.file_count }}</td><td class="py-3 pr-4">{{ item.status|title }}{% if item.queue_position %} · Position {{ item.queue_position }}{% endif %}</td><td class="py-3"><a class="font-semibold text-brand-700" href="/tools/bank-statement-analyzer?job={{ item.id }}#analysis-status">View</a></td></tr>{% endfor %}</tbody></table></div></section>{% endif %} {% if recent_jobs %}<section class="rounded-2xl border border-slate-200 bg-white p-6 shadow-soft"><div class="flex items-center justify-between"><h2 class="text-lg font-bold text-slate-900">Recent Analyses</h2><a href="/tools/bank-statement-analyzer/jobs" class="text-sm font-semibold text-brand-700">View all</a></div><div class="mt-4 overflow-x-auto"><table class="min-w-full text-sm"><thead><tr class="border-b text-left text-slate-500"><th class="py-2 pr-4">Submitted</th><th class="py-2 pr-4">Bank</th><th class="py-2 pr-4">Files</th><th class="py-2 pr-4">Status</th><th class="py-2">Action</th></tr></thead><tbody>{% for item in recent_jobs %}<tr class="border-b border-slate-100"><td class="py-3 pr-4">{{ item.submitted_at }}</td><td class="py-3 pr-4">{{ item.selected_bank|replace('_',' ')|title }}</td><td class="py-3 pr-4">{{ item.file_count }}</td><td class="py-3 pr-4">{{ item.status|title }}{% if item.queue_position %} · Position {{ item.queue_position }}{% endif %}</td><td class="py-3"><a class="font-semibold text-brand-700" href="/tools/bank-statement-analyzer?job={{ item.id }}#analysis-status">View</a></td></tr>{% endfor %}</tbody></table></div></section>{% endif %}
</div> </div>
{% if active_job and active_job.status in ['queued','processing'] %}<script>(function(){const box=document.getElementById('analysis-status');const id=box.dataset.jobId;async function poll(){try{const r=await fetch(`/tools/bank-statement-analyzer/jobs/${id}/status`,{headers:{'Accept':'application/json'}});if(!r.ok)return;const d=await r.json();document.getElementById('job-status-badge').textContent=d.status.charAt(0).toUpperCase()+d.status.slice(1);document.getElementById('queue-position').textContent=d.queue_position||'—';document.getElementById('estimated-wait').textContent=d.estimated_wait||'—';document.getElementById('progress-text').textContent=d.progress_percent+'%';document.getElementById('progress-bar').style.width=d.progress_percent+'%';if(d.status==='completed'||d.status==='failed'){window.location.reload();return;}}catch(e){}setTimeout(poll,5000)}setTimeout(poll,5000)})();</script>{% endif %} {% if active_job and active_job.status in ['queued','processing'] %}<script>(function(){const box=document.getElementById('analysis-status');const id=box.dataset.jobId;async function poll(){try{const r=await fetch(`/tools/bank-statement-analyzer/jobs/${id}/status`,{headers:{'Accept':'application/json'}});if(!r.ok)return;const d=await r.json();document.getElementById('job-status-badge').textContent=d.status.charAt(0).toUpperCase()+d.status.slice(1);document.getElementById('queue-position').textContent=d.queue_position||'—';document.getElementById('estimated-wait').textContent=d.estimated_wait||'—';document.getElementById('progress-text').textContent=d.progress_percent+'%';document.getElementById('progress-bar').style.width=d.progress_percent+'%';if(d.status==='completed'||d.status==='failed'){window.location.reload();return;}}catch(e){}setTimeout(poll,5000)}setTimeout(poll,5000)})();</script>{% endif %}
<script>
(function () {
const client = document.getElementById("bank-analyzer-client");
const engagement = document.getElementById("bank-analyzer-engagement");
if (!client || !engagement) return;
const options = Array.from(engagement.querySelectorAll("option[data-client-id]"));
function filterEngagements() {
const selectedClient = client.value;
let selectedStillVisible = false;
options.forEach((option) => {
const visible = !!selectedClient && option.dataset.clientId === selectedClient;
option.hidden = !visible;
option.disabled = !visible;
if (visible && option.selected) selectedStillVisible = true;
});
if (!selectedStillVisible) engagement.value = "";
engagement.required = !!selectedClient;
}
client.addEventListener("change", filterEngagements);
filterEngagements();
})();
</script>
{% endblock %} {% endblock %}
+48 -3
View File
@@ -16,6 +16,7 @@ from app.core.templating import templates
from app.modules.core.rbac.deps import get_user_permissions, get_user_roles from app.modules.core.rbac.deps import get_user_permissions, get_user_roles
from .parsers.registry import BANK_OPTIONS from .parsers.registry import BANK_OPTIONS
from .client_context import analyzer_client_context, validate_selected_client_and_engagement
from .service import ( from .service import (
can_use, can_use,
create_job_folder, create_job_folder,
@@ -85,6 +86,10 @@ def _ctx(request, db, user, **extra):
"selected_bank": "auto", "selected_bank": "auto",
"financial_year": "", "financial_year": "",
"classification_enabled": True, "classification_enabled": True,
"analyzer_clients": [],
"analyzer_engagements": [],
"selected_client_id": None,
"selected_engagement_id": None,
} }
data.update(extra) data.update(extra)
return data return data
@@ -112,13 +117,27 @@ def index(request: Request, job: str | None = None):
selected_job = get_owned_job(user.id, job) if job else None selected_job = get_owned_job(user.id, job) if job else None
recent = [_localised_job_view(item, timezone_name) for item in list_user_jobs(user.id, limit=8)] recent = [_localised_job_view(item, timezone_name) for item in list_user_jobs(user.id, limit=8)]
active_job = _localised_job_view(selected_job, timezone_name) if selected_job else None active_job = _localised_job_view(selected_job, timezone_name) if selected_job else None
return templates.TemplateResponse("modules/bank_statement_analyzer/templates/bank_statement_analyzer/index.html", _ctx(request, db, user, error="", active_job=active_job, recent_jobs=recent, display_timezone=timezone_name)) client_context = analyzer_client_context(db, request=request, user=user, roles=roles)
return templates.TemplateResponse(
"modules/bank_statement_analyzer/templates/bank_statement_analyzer/index.html",
_ctx(
request,
db,
user,
error="",
active_job=active_job,
recent_jobs=recent,
display_timezone=timezone_name,
analyzer_clients=client_context["clients"],
analyzer_engagements=client_context["engagements"],
),
)
finally: finally:
db.close() db.close()
@router.post("/analyze") @router.post("/analyze")
async def analyze(request: Request, csrf_token: str = Form(...), bank_selection: str = Form("auto"), financial_year: str = Form(""), customer_name: str = Form(""), account_number: str = Form(""), enable_classification: str | None = Form(None), statements: list[UploadFile] = File(...)): async def analyze(request: Request, csrf_token: str = Form(...), bank_selection: str = Form("auto"), financial_year: str = Form(""), customer_name: str = Form(""), account_number: str = Form(""), client_id: str = Form(""), engagement_id: str = Form(""), confirm_same_client: str | None = Form(None), enable_classification: str | None = Form(None), statements: list[UploadFile] = File(...)):
db = CommonSessionLocal() db = CommonSessionLocal()
selected_bank = bank_selection if bank_selection in dict(BANK_OPTIONS) else "auto" selected_bank = bank_selection if bank_selection in dict(BANK_OPTIONS) else "auto"
classification_enabled = enable_classification == "1" classification_enabled = enable_classification == "1"
@@ -128,10 +147,35 @@ async def analyze(request: Request, csrf_token: str = Form(...), bank_selection:
if denied: if denied:
return denied return denied
validate_csrf(request, csrf_token) validate_csrf(request, csrf_token)
resolved_client_id = int(client_id) if client_id.strip() else None
resolved_engagement_id = int(engagement_id) if engagement_id.strip() else None
validate_selected_client_and_engagement(
db,
request=request,
user=user,
roles=roles,
client_id=resolved_client_id,
engagement_id=resolved_engagement_id,
)
job_id, input_dir, _output_dir = create_job_folder(user, roles) job_id, input_dir, _output_dir = create_job_folder(user, roles)
job_dir = input_dir.parent job_dir = input_dir.parent
paths = await save_uploads(statements, input_dir) paths = await save_uploads(statements, input_dir)
enqueue_job(user=user, roles=roles, job_id=job_id, paths=paths, job_dir=job_dir, bank_selection=selected_bank, financial_year=financial_year, customer_override=customer_name, account_override=account_number, classification_enabled=classification_enabled) enqueue_job(
user=user,
roles=roles,
job_id=job_id,
paths=paths,
job_dir=job_dir,
bank_selection=selected_bank,
financial_year=financial_year,
customer_override=customer_name,
account_override=account_number,
classification_enabled=classification_enabled,
client_id=resolved_client_id,
engagement_id=resolved_engagement_id,
ownership_confirmation=(confirm_same_client == "1"),
)
return RedirectResponse(f"/tools/bank-statement-analyzer?job={job_id}#analysis-status", status_code=303) return RedirectResponse(f"/tools/bank-statement-analyzer?job={job_id}#analysis-status", status_code=303)
except Exception as exc: except Exception as exc:
if job_dir: if job_dir:
@@ -142,6 +186,7 @@ async def analyze(request: Request, csrf_token: str = Form(...), bank_selection:
return RedirectResponse("/login", status_code=303) return RedirectResponse("/login", status_code=303)
timezone_name = _display_timezone_name(request, db, user) timezone_name = _display_timezone_name(request, db, user)
recent = [_localised_job_view(item, timezone_name) for item in list_user_jobs(user.id, limit=8)] recent = [_localised_job_view(item, timezone_name) for item in list_user_jobs(user.id, limit=8)]
client_context = analyzer_client_context(db, request=request, user=user, roles=get_user_roles(db, user.id))
return templates.TemplateResponse("modules/bank_statement_analyzer/templates/bank_statement_analyzer/index.html", _ctx(request, db, user, error=str(exc), active_job=None, recent_jobs=recent, selected_bank=selected_bank, financial_year=financial_year, classification_enabled=classification_enabled, display_timezone=timezone_name), status_code=400) return templates.TemplateResponse("modules/bank_statement_analyzer/templates/bank_statement_analyzer/index.html", _ctx(request, db, user, error=str(exc), active_job=None, recent_jobs=recent, selected_bank=selected_bank, financial_year=financial_year, classification_enabled=classification_enabled, display_timezone=timezone_name), status_code=400)
finally: finally:
db.close() db.close()