Add Phase 20 stored bank reuse and richer reconciliation

app/modules/bank_statement_analyzer/client_context.py

@@ -365,12 +365,37 @@ def archive_analysis_to_engagement(
         return {"status": "not_requested", "documents": []}
 
     archived = []
+
+    reused = []
+    try:
+        reused = json.loads(getattr(job, "stored_source_versions_json", None) or "[]")
+    except Exception:
+        reused = []
+    reused_by_copied_name = {
+        _text(row.get("copied_filename")): row
+        for row in reused
+        if isinstance(row, dict) and _text(row.get("copied_filename"))
+    }
+
     meta_by_source = {
         Path(_text(getattr(meta, "source_file", ""))).name: meta
         for meta in metas
     }
 
     for source in source_paths:
+        reused_row = reused_by_copied_name.get(source.name)
+        if reused_row:
+            archived.append(
+                {
+                    "document_id": int(reused_row.get("document_id") or 0),
+                    "version_id": int(reused_row.get("version_id") or 0),
+                    "filename": reused_row.get("original_filename") or source.name,
+                    "type": "BANK_STATEMENT",
+                    "reused_existing_document": True,
+                }
+            )
+            continue
+
         upload = _upload_proxy(source, "application/pdf")
         try:
             meta = meta_by_source.get(source.name)

app/modules/bank_statement_analyzer/models.py

@@ -39,6 +39,11 @@ class BankStatementAnalysisJob(CommonBase):
     accounting_import_status: Mapped[str] = mapped_column(String(30), nullable=False, default="not_requested", index=True)
     accounting_import_json: Mapped[str | None] = mapped_column(Text, nullable=True)
 
+    # Phase 20: stored-statement reuse. These fields only record provenance;
+    # the physical files remain in the existing Engagement Documents storage.
+    stored_source_versions_json: Mapped[str | None] = mapped_column(Text, nullable=True)
+    source_hashes_json: Mapped[str | None] = mapped_column(Text, nullable=True)
+
     status: Mapped[str] = mapped_column(String(20), nullable=False, default="queued", index=True)
     progress_percent: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
     file_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)

app/modules/bank_statement_analyzer/service.py

@@ -68,6 +68,80 @@ def _segment(value: object, default: str = "NA") -> str:
 
 
 
+def _statement_period(value):
+    text_value = str(value or "").strip()
+    if not text_value:
+        return None
+    for fmt in (
+        "%Y-%m-%d",
+        "%d/%m/%Y",
+        "%d-%m-%Y",
+        "%d.%m.%Y",
+        "%d-%b-%Y",
+        "%d %b %Y",
+        "%d %B %Y",
+        "%Y/%m/%d",
+    ):
+        try:
+            return datetime.strptime(text_value[:10], fmt).date()
+        except Exception:
+            pass
+    try:
+        return datetime.fromisoformat(text_value.replace("Z", "+00:00")).date()
+    except Exception:
+        return None
+
+
+def _statement_overlap_warnings(metas: list) -> list[dict]:
+    warnings = []
+    groups = {}
+    for meta in metas:
+        account = str(getattr(meta, "account_number", "") or "").strip()
+        bank = str(getattr(meta, "bank_name", "") or "").strip()
+        start = _statement_period(getattr(meta, "period_from", ""))
+        end = _statement_period(getattr(meta, "period_to", ""))
+        if not account or not start or not end:
+            continue
+        if end < start:
+            start, end = end, start
+        groups.setdefault((bank.casefold(), account), []).append(
+            {
+                "bank_name": bank,
+                "account_number": account,
+                "period_from": start,
+                "period_to": end,
+                "source_file": Path(str(getattr(meta, "source_file", "") or "")).name,
+            }
+        )
+
+    for (_bank_key, _account), rows in groups.items():
+        rows.sort(key=lambda row: (row["period_from"], row["period_to"]))
+        for idx, left in enumerate(rows):
+            for right in rows[idx + 1:]:
+                if right["period_from"] > left["period_to"]:
+                    break
+                overlap_from = max(left["period_from"], right["period_from"])
+                overlap_to = min(left["period_to"], right["period_to"])
+                if overlap_from <= overlap_to:
+                    warnings.append(
+                        {
+                            "bank_name": left["bank_name"],
+                            "account_number": left["account_number"],
+                            "first_file": left["source_file"],
+                            "second_file": right["source_file"],
+                            "overlap_from": overlap_from.isoformat(),
+                            "overlap_to": overlap_to.isoformat(),
+                            "message": (
+                                f"Overlapping statement periods for account "
+                                f"{left['account_number']}: {left['source_file']} and "
+                                f"{right['source_file']} overlap from "
+                                f"{overlap_from.isoformat()} to {overlap_to.isoformat()}."
+                            ),
+                        }
+                    )
+    return warnings
+
+
 def _workbook_filename(metas: list) -> str:
     """Build a safe BankName_ClientName.xlsx filename for every bank."""
     bank_names = [str(getattr(meta, "bank_name", "") or "").strip() for meta in metas]
@@ -162,7 +236,7 @@ def pending_count_for_user(user_id: int) -> int:
         db.close()
 
 
-def enqueue_job(*, user, roles: Iterable[str], job_id: str, paths: list[Path], job_dir: Path, bank_selection: str, financial_year: str, customer_override: str, account_override: str, classification_enabled: bool, client_id: int | None = None, engagement_id: int | None = None, ownership_confirmation: bool = False, purpose: str = "analyze_only") -> BankStatementAnalysisJob:
+def enqueue_job(*, user, roles: Iterable[str], job_id: str, paths: list[Path], job_dir: Path, bank_selection: str, financial_year: str, customer_override: str, account_override: str, classification_enabled: bool, client_id: int | None = None, engagement_id: int | None = None, ownership_confirmation: bool = False, purpose: str = "analyze_only", stored_source_versions: list[dict] | None = None, source_hashes: list[dict] | None = None) -> BankStatementAnalysisJob:
     if pending_count_for_user(int(user.id)) >= MAX_PENDING_PER_USER:
         shutil.rmtree(job_dir, ignore_errors=True)
         raise ValueError("You already have three queued or processing analyses. Please wait for one to complete before submitting another.")
@@ -192,6 +266,16 @@ def enqueue_job(*, user, roles: Iterable[str], job_id: str, paths: list[Path], j
             if purpose in {"accounting_entries", "bank_reconciliation"}
             else "not_requested"
         ),
+        stored_source_versions_json=(
+            json.dumps(stored_source_versions, ensure_ascii=False)
+            if stored_source_versions
+            else None
+        ),
+        source_hashes_json=(
+            json.dumps(source_hashes, ensure_ascii=False)
+            if source_hashes
+            else None
+        ),
         status="queued",
         progress_percent=0,
         file_count=len(paths),
@@ -328,6 +412,13 @@ def _process_job(job_id: str) -> None:
             "engagement_id": job.engagement_id,
             "ownership_status": job.ownership_status,
             "engagement_archive_status": job.engagement_archive_status,
+            "stored_statement_reuse_count": len(
+                json.loads(job.stored_source_versions_json or "[]")
+            ) if getattr(job, "stored_source_versions_json", None) else 0,
+            "source_hash_count": len(
+                json.loads(job.source_hashes_json or "[]")
+            ) if getattr(job, "source_hashes_json", None) else 0,
+            "statement_overlap_warnings": _statement_overlap_warnings(metas),
         }
         # Keep the original uploaded statements until the job expiry time.
         # This applies equally to completed and failed jobs and allows the
@@ -530,6 +621,8 @@ def job_view(job: BankStatementAnalysisJob) -> dict:
         "purpose": getattr(job, "purpose", "analyze_only"),
         "accounting_import_status": getattr(job, "accounting_import_status", "not_requested"),
         "accounting_import": json.loads(job.accounting_import_json) if getattr(job, "accounting_import_json", None) else {},
+        "stored_source_versions": json.loads(job.stored_source_versions_json) if getattr(job, "stored_source_versions_json", None) else [],
+        "source_hashes": json.loads(job.source_hashes_json) if getattr(job, "source_hashes_json", None) else [],
         "submitted_at": job.submitted_at_utc,
         "completed_at": job.completed_at_utc,
         "expires_at": job.expires_at_utc,

app/modules/bank_statement_analyzer/stored_statement_service.py

@@ -0,0 +1,254 @@
+from __future__ import annotations
+
+import hashlib
+import json
+import shutil
+from pathlib import Path
+
+from sqlalchemy import select
+from sqlalchemy.orm import selectinload
+
+from app.modules.documents.models import (
+    DocumentDownloadRequest,
+    EngagementDocument,
+    EngagementDocumentVersion,
+)
+from app.modules.documents.services import (
+    create_download_request_for_version,
+    download_request_cache_path,
+    version_absolute_path,
+)
+
+from .client_context import analyzer_client_context
+
+
+def _sha256(path: Path) -> str:
+    hasher = hashlib.sha256()
+    with path.open("rb") as handle:
+        while True:
+            chunk = handle.read(1024 * 1024)
+            if not chunk:
+                break
+            hasher.update(chunk)
+    return hasher.hexdigest()
+
+
+def _allowed_engagement_ids(db, *, request, user, roles, client_id: int) -> set[int]:
+    context = analyzer_client_context(db, request=request, user=user, roles=roles)
+    visible_clients = {int(row.id) for row in context["clients"]}
+    if int(client_id) not in visible_clients:
+        raise PermissionError("Selected client is not visible for Bank Statement Analyzer.")
+    return {
+        int(row.id)
+        for row in context["engagements"]
+        if int(row.client_id) == int(client_id)
+    }
+
+
+def list_stored_bank_statements(
+    db,
+    *,
+    request,
+    user,
+    roles,
+    client_id: int,
+    limit: int = 200,
+):
+    allowed_engagements = _allowed_engagement_ids(
+        db,
+        request=request,
+        user=user,
+        roles=roles,
+        client_id=client_id,
+    )
+    if not allowed_engagements:
+        return []
+
+    docs = list(
+        db.execute(
+            select(EngagementDocument)
+            .options(selectinload(EngagementDocument.versions))
+            .where(
+                EngagementDocument.client_id == int(client_id),
+                EngagementDocument.engagement_id.in_(allowed_engagements),
+                EngagementDocument.document_type == "BANK_STATEMENT",
+                EngagementDocument.is_deleted.is_(False),
+            )
+            .order_by(EngagementDocument.updated_at_utc.desc(), EngagementDocument.id.desc())
+            .limit(max(1, min(500, int(limit))))
+        ).scalars().all()
+    )
+
+    result = []
+    for doc in docs:
+        version = doc.versions[0] if doc.versions else None
+        if not version:
+            continue
+        result.append(
+            {
+                "document_id": int(doc.id),
+                "version_id": int(version.id),
+                "engagement_id": int(doc.engagement_id),
+                "title": doc.title,
+                "filename": version.original_filename,
+                "size_bytes": int(version.file_size_bytes or 0),
+                "sha256": version.file_hash_sha256,
+                "uploaded_at": version.uploaded_at_utc.isoformat() if version.uploaded_at_utc else "",
+                "storage_status": version.storage_status,
+            }
+        )
+    return result
+
+
+def _ready_cached_path(db, *, version_id: int, user_id: int):
+    req = db.execute(
+        select(DocumentDownloadRequest)
+        .where(
+            DocumentDownloadRequest.version_id == int(version_id),
+            DocumentDownloadRequest.requested_by_user_id == int(user_id),
+            DocumentDownloadRequest.request_status == "ready",
+        )
+        .order_by(DocumentDownloadRequest.fulfilled_at_utc.desc(), DocumentDownloadRequest.id.desc())
+        .limit(1)
+    ).scalar_one_or_none()
+    if not req:
+        return None
+    path = download_request_cache_path(req)
+    return path if path and path.exists() else None
+
+
+def prepare_stored_bank_statements(
+    db,
+    *,
+    request,
+    user,
+    roles,
+    client_id: int,
+    version_ids: list[int],
+    input_dir: Path,
+):
+    selected_ids = list(dict.fromkeys(int(value) for value in version_ids if int(value) > 0))
+    if not selected_ids:
+        return [], [], []
+
+    allowed_engagements = _allowed_engagement_ids(
+        db,
+        request=request,
+        user=user,
+        roles=roles,
+        client_id=client_id,
+    )
+
+    versions = list(
+        db.execute(
+            select(EngagementDocumentVersion)
+            .join(
+                EngagementDocument,
+                EngagementDocument.id == EngagementDocumentVersion.document_id,
+            )
+            .where(
+                EngagementDocumentVersion.id.in_(selected_ids),
+                EngagementDocumentVersion.client_id == int(client_id),
+                EngagementDocumentVersion.engagement_id.in_(allowed_engagements),
+                EngagementDocument.document_type == "BANK_STATEMENT",
+                EngagementDocument.is_deleted.is_(False),
+            )
+        ).scalars().all()
+    )
+    by_id = {int(row.id): row for row in versions}
+
+    if set(selected_ids) != set(by_id):
+        raise PermissionError("One or more stored bank statements are not available to this user/client.")
+
+    prepared = []
+    provenance = []
+    pending = []
+
+    for order, version_id in enumerate(selected_ids, start=1):
+        version = by_id[version_id]
+        source = version_absolute_path(version)
+        if not source.exists():
+            source = _ready_cached_path(
+                db,
+                version_id=version.id,
+                user_id=user.id,
+            )
+
+        if source is None or not source.exists():
+            req = create_download_request_for_version(
+                db,
+                version=version,
+                user=user,
+                request=request,
+            )
+            if req is not None and req.id is None:
+                db.flush()
+            if req is not None and req.request_status == "ready":
+                cached = download_request_cache_path(req)
+                if cached is None or not cached.exists():
+                    # A stale ready request must be made retrievable again.
+                    req.request_status = "retry"
+                    req.cached_relative_path = None
+                    req.cached_hash_sha256 = None
+                    req.last_error = "Cached retrieval copy was missing; queued again for branch Local Agent."
+                    db.add(req)
+                    db.flush()
+            if not req:
+                raise FileNotFoundError(
+                    f"{version.original_filename}: the stored file is not currently available "
+                    "on ERP or a registered branch storage node."
+                )
+            pending.append(
+                {
+                    "version_id": int(version.id),
+                    "filename": version.original_filename,
+                    "request_id": int(req.id) if req.id else None,
+                    "status": req.request_status,
+                }
+            )
+            continue
+
+        actual_hash = _sha256(source)
+        expected = (version.file_hash_sha256 or "").strip().lower()
+        if expected and actual_hash.lower() != expected:
+            raise ValueError(
+                f"{version.original_filename}: stored file hash verification failed."
+            )
+
+        safe_name = Path(version.original_filename or f"statement_{version.id}.pdf").name
+        if Path(safe_name).suffix.lower() != ".pdf":
+            raise ValueError(f"{safe_name}: only stored PDF bank statements can be reused.")
+
+        target = input_dir / f"stored_{order:03d}_v{version.id}_{safe_name}"
+        shutil.copy2(source, target)
+        prepared.append(target)
+        provenance.append(
+            {
+                "version_id": int(version.id),
+                "document_id": int(version.document_id),
+                "engagement_id": int(version.engagement_id),
+                "original_filename": version.original_filename,
+                "copied_filename": target.name,
+                "sha256": actual_hash,
+            }
+        )
+
+    db.flush()
+    return prepared, provenance, pending
+
+
+def deduplicate_source_paths(paths: list[Path]):
+    unique = []
+    hashes = []
+    seen = set()
+
+    for path in paths:
+        digest = _sha256(path)
+        if digest in seen:
+            path.unlink(missing_ok=True)
+            continue
+        seen.add(digest)
+        unique.append(path)
+        hashes.append({"filename": path.name, "sha256": digest})
+
+    return unique, hashes

app/modules/bank_statement_analyzer/templates/bank_statement_analyzer/index.html

@@ -65,7 +65,7 @@
               <input type="checkbox" name="confirm_same_client" value="1" class="mt-1 rounded border-slate-300">
               <span>I confirm that all statements uploaded in this analysis belong to the selected client or one of its ERP-recorded legal/trade/business names.</span>
             </label>
-            <p class="mt-2 text-xs text-indigo-700">When a client is selected, the analyzer validates every statement owner against the client name, trade name, business-unit names, registration legal/trade names and branch names. On successful analysis, the source PDFs and generated workbook are archived using the existing Engagement Documents storage pipeline.</p>
+            <p class="mt-2 text-xs text-indigo-700">When a client is selected, the analyzer validates every statement owner against the client name, trade name, business-unit names, registration legal/trade names and branch names. On successful analysis, newly uploaded source PDFs and the generated workbook are archived using the existing Engagement Documents storage pipeline. Previously stored PDFs selected for reuse remain linked to their existing document/version and are not duplicated.</p>
           </div>
           {% endif %}
       <div><label class="mb-1 block text-sm font-semibold text-slate-700">Bank</label><select name="bank_selection" class="w-full rounded-xl border border-slate-300 bg-white px-3 py-2 text-sm">{% for value, label in bank_options %}<option value="{{ value }}" {% if selected_bank == value %}selected{% endif %}>{{ label }}</option>{% endfor %}</select><p class="mt-1 text-xs text-slate-500">Keep Auto Detect or select a bank for direct parser validation.</p></div>
@@ -74,7 +74,36 @@
       <div><label class="mb-1 block text-sm font-semibold text-slate-700">Account number override <span class="font-normal text-slate-400">(optional)</span></label><input name="account_number" class="w-full rounded-xl border border-slate-300 px-3 py-2 text-sm" placeholder="Use only when statement extraction needs correction"></div>
     </div>
     <div class="mt-5 rounded-xl border border-slate-200 bg-slate-50 p-4"><label class="flex items-start gap-3"><input type="checkbox" name="enable_classification" value="1" {% if classification_enabled %}checked{% endif %} class="mt-1 h-4 w-4 rounded border-slate-300"><span><span class="block text-sm font-semibold text-slate-800">Enable narration-based transaction classification</span><span class="mt-1 block text-xs text-slate-500">Adds category, party, review and draft financial helper sheets. Final classification must be verified with books and supporting records.</span></span></label></div>
-    <div class="mt-5 rounded-xl border border-brand-200 bg-brand-50 p-4 text-sm text-slate-700"><strong>Multi-bank contra detection:</strong> when the client has more than one bank account, upload all relevant statements in the same analysis job. There is no fixed statement-count limit unless your administrator configures one. The analyzer pairs only conservative equal-and-opposite transfers across different account numbers and keeps every pair reviewable.</div><div class="mt-5"><label class="mb-1 block text-sm font-semibold text-slate-700">PDF bank statements — select all accounts/statements for this client</label><input type="file" name="statements" accept="application/pdf,.pdf" multiple required class="block w-full rounded-xl border border-slate-300 bg-white px-3 py-3 text-sm"></div>
+    <div class="mt-5 rounded-xl border border-brand-200 bg-brand-50 p-4 text-sm text-slate-700"><strong>Multi-bank contra detection:</strong> when the client has more than one bank account, upload all relevant statements in the same analysis job. There is no fixed statement-count limit unless your administrator configures one. The analyzer pairs only conservative equal-and-opposite transfers across different account numbers and keeps every pair reviewable.</div><div class="mt-5 grid gap-4 lg:grid-cols-2">
+      <div class="rounded-xl border border-slate-200 bg-white p-4">
+        <label class="mb-1 block text-sm font-semibold text-slate-700">Upload new PDF bank statements</label>
+        <input type="file" name="statements" accept="application/pdf,.pdf" multiple class="block w-full rounded-xl border border-slate-300 bg-white px-3 py-3 text-sm">
+        <p class="mt-2 text-xs text-slate-500">Select any number of new statements. They can be combined with previously stored statements below.</p>
+      </div>
+
+      <div class="rounded-xl border border-indigo-200 bg-indigo-50/40 p-4">
+        <div class="flex items-center justify-between gap-2">
+          <div>
+            <div class="text-sm font-semibold text-slate-800">Reuse bank statements already stored for this client</div>
+            <p class="mt-1 text-xs text-slate-500">Existing BANK_STATEMENT documents are reused from Engagement Documents/local storage and are not archived again as duplicates.</p>
+          </div>
+          <button type="button" id="refresh-stored-bank-statements" class="rounded-lg border border-indigo-300 bg-white px-3 py-2 text-xs font-semibold text-indigo-800">Refresh</button>
+        </div>
+        <div id="stored-bank-statements" class="mt-3 max-h-64 space-y-2 overflow-y-auto">
+          {% for row in stored_bank_statements or [] %}
+          <label class="flex items-start gap-2 rounded-lg border border-indigo-100 bg-white p-3 text-xs">
+            <input type="checkbox" name="stored_version_ids" value="{{ row.version_id }}" class="mt-1">
+            <span>
+              <span class="block font-semibold text-slate-800">{{ row.title }}</span>
+              <span class="block text-slate-500">{{ row.filename }} · v{{ row.version_id }} · {{ row.storage_status|replace('_',' ')|title }}</span>
+            </span>
+          </label>
+          {% else %}
+          <div class="text-xs text-slate-500">Select an ERP client to load previously stored bank statements.</div>
+          {% endfor %}
+        </div>
+      </div>
+    </div>
     <div class="mt-5 rounded-xl bg-slate-50 p-4 text-sm text-slate-600"><div class="font-semibold text-slate-800">Queue limits</div><div class="mt-1">Maximum three processing jobs across all users. Each user may have up to three queued or processing jobs. Completed workbooks and original uploaded statements remain available for 24 hours. Failed-job statements are also retained for 24 hours for debugging.</div></div>
     <div class="mt-6 flex flex-wrap gap-3"><button class="rounded-xl bg-brand-600 px-5 py-2.5 text-sm font-semibold text-white hover:bg-brand-700">Submit Analysis</button></div>
   </form>
@@ -85,7 +114,7 @@
     <div class="mt-5 grid gap-4 sm:grid-cols-2 lg:grid-cols-4"><div class="rounded-xl bg-slate-50 p-4"><div class="text-xs text-slate-500">Files</div><div class="mt-1 text-lg font-bold">{{ active_job.file_count }}</div></div><div class="rounded-xl bg-slate-50 p-4"><div class="text-xs text-slate-500">Queue position</div><div id="queue-position" class="mt-1 text-lg font-bold">{{ active_job.queue_position or '—' }}</div></div><div class="rounded-xl bg-slate-50 p-4"><div class="text-xs text-slate-500">Estimated wait</div><div id="estimated-wait" class="mt-1 text-sm font-bold">{{ active_job.estimated_wait or '—' }}</div></div><div class="rounded-xl bg-slate-50 p-4"><div class="text-xs text-slate-500">Progress</div><div id="progress-text" class="mt-1 text-lg font-bold">{{ active_job.progress_percent }}%</div></div></div>
     <div class="mt-4 h-2 overflow-hidden rounded-full bg-slate-200"><div id="progress-bar" class="h-full bg-brand-600 transition-all" style="width: {{ active_job.progress_percent }}%"></div></div>
     {% if active_job.status == 'completed' %}
-      <div class="mt-6 rounded-xl border border-emerald-200 bg-emerald-50 p-5"><h3 class="font-bold text-emerald-900">Analysis completed</h3><div class="mt-4 grid gap-3 sm:grid-cols-2 lg:grid-cols-4"><div><div class="text-xs text-emerald-700">Statements</div><div class="font-bold">{{ active_job.summary.statement_count or 0 }}</div></div><div><div class="text-xs text-emerald-700">Transactions extracted</div><div class="font-bold">{{ active_job.summary.rows_extracted or 0 }}</div></div><div><div class="text-xs text-emerald-700">Exact duplicates</div><div class="font-bold">{{ active_job.summary.exact_duplicate_rows or 0 }}</div></div><div><div class="text-xs text-emerald-700">Review items</div><div class="font-bold">{{ active_job.summary.review_items or 0 }}</div></div><div><div class="text-xs text-emerald-700">Inter-bank contra pairs</div><div class="font-bold">{{ active_job.summary.contra_pairs or 0 }}</div></div></div><div class="mt-5 flex flex-wrap gap-3"><a href="/tools/bank-statement-analyzer/jobs/{{ active_job.id }}/download" class="rounded-xl bg-emerald-700 px-5 py-2.5 text-sm font-semibold text-white">Download Excel</a>{% for file in active_job.original_files %}<a href="/tools/bank-statement-analyzer/jobs/{{ active_job.id }}/statements/{{ file.index }}/download" class="rounded-xl border border-emerald-300 bg-white px-5 py-2.5 text-sm font-semibold text-emerald-800">Download Statement {{ file.index }}</a>{% endfor %}<a href="/tools/bank-statement-analyzer" class="rounded-xl border border-emerald-300 bg-white px-5 py-2.5 text-sm font-semibold text-emerald-800">Analyze Another Bank</a></div><p class="mt-3 text-xs text-emerald-700">The workbook and original statement{{ 's' if active_job.file_count != 1 else '' }} remain available until {{ active_job.expires_at or '24 hours after completion' }}.</p></div>
+      <div class="mt-6 rounded-xl border border-emerald-200 bg-emerald-50 p-5"><h3 class="font-bold text-emerald-900">Analysis completed</h3><div class="mt-4 grid gap-3 sm:grid-cols-2 lg:grid-cols-4"><div><div class="text-xs text-emerald-700">Statements</div><div class="font-bold">{{ active_job.summary.statement_count or 0 }}</div></div><div><div class="text-xs text-emerald-700">Transactions extracted</div><div class="font-bold">{{ active_job.summary.rows_extracted or 0 }}</div></div><div><div class="text-xs text-emerald-700">Exact duplicates</div><div class="font-bold">{{ active_job.summary.exact_duplicate_rows or 0 }}</div></div><div><div class="text-xs text-emerald-700">Review items</div><div class="font-bold">{{ active_job.summary.review_items or 0 }}</div></div><div><div class="text-xs text-emerald-700">Inter-bank contra pairs</div><div class="font-bold">{{ active_job.summary.contra_pairs or 0 }}</div></div></div>{% if active_job.summary.stored_statement_reuse_count %}<div class="mt-3 rounded-lg border border-indigo-200 bg-indigo-50 p-3 text-xs text-indigo-800">Reused {{ active_job.summary.stored_statement_reuse_count }} statement(s) from existing Engagement Documents without creating duplicate source documents.</div>{% endif %}{% if active_job.summary.statement_overlap_warnings %}<div class="mt-3 rounded-lg border border-amber-200 bg-amber-50 p-3 text-xs text-amber-900"><div class="font-semibold">Overlapping statement periods detected</div>{% for warning in active_job.summary.statement_overlap_warnings %}<div class="mt-1">{{ warning.message }}</div>{% endfor %}<div class="mt-1 text-amber-700">Transactions are still deduplicated by the existing analyzer rules; verify overlapping-period coverage before reconciliation.</div></div>{% endif %}<div class="mt-5 flex flex-wrap gap-3"><a href="/tools/bank-statement-analyzer/jobs/{{ active_job.id }}/download" class="rounded-xl bg-emerald-700 px-5 py-2.5 text-sm font-semibold text-white">Download Excel</a>{% for file in active_job.original_files %}<a href="/tools/bank-statement-analyzer/jobs/{{ active_job.id }}/statements/{{ file.index }}/download" class="rounded-xl border border-emerald-300 bg-white px-5 py-2.5 text-sm font-semibold text-emerald-800">Download Statement {{ file.index }}</a>{% endfor %}<a href="/tools/bank-statement-analyzer" class="rounded-xl border border-emerald-300 bg-white px-5 py-2.5 text-sm font-semibold text-emerald-800">Analyze Another Bank</a></div><p class="mt-3 text-xs text-emerald-700">The workbook and original statement{{ 's' if active_job.file_count != 1 else '' }} remain available until {{ active_job.expires_at or '24 hours after completion' }}.</p></div>
     {% elif active_job.status == 'failed' %}<div class="mt-5 rounded-xl border border-red-200 bg-red-50 p-4 text-sm text-red-800"><strong>Analysis failed.</strong><div class="mt-1">{{ active_job.error_message }}</div>{% if active_job.original_files %}<div class="mt-4 flex flex-wrap gap-3">{% for file in active_job.original_files %}<a href="/tools/bank-statement-analyzer/jobs/{{ active_job.id }}/statements/{{ file.index }}/download" class="rounded-xl border border-red-300 bg-white px-4 py-2 text-sm font-semibold text-red-800">Download Statement {{ file.index }}</a>{% endfor %}</div><p class="mt-3 text-xs text-red-700">Original statement{{ 's are' if active_job.file_count != 1 else ' is' }} retained until {{ active_job.expires_at or '24 hours after failure' }} for debugging.</p>{% endif %}<a href="/tools/bank-statement-analyzer" class="mt-3 inline-block font-semibold underline">Analyze another statement</a></div>
     {% else %}<div id="live-message" class="mt-5 rounded-xl border border-blue-200 bg-blue-50 p-4 text-sm text-blue-800">{% if active_job.status == 'queued' %}Your job is queued. You may safely leave this page and return through My Analysis Jobs.{% else %}Your statements are being processed.{% endif %}</div>{% endif %}
   {% if active_job.status == 'completed' and active_job.purpose in ['accounting_entries','bank_reconciliation'] %}<div class="mt-4 rounded-xl border border-blue-200 bg-blue-50 p-4 text-sm text-blue-900">Accounting import: {{ active_job.accounting_import_status|replace('_',' ')|title }}{% if active_job.summary.client_id %} · <a class="font-semibold underline" href="/tools/accounting/bank-posting?client_id={{ active_job.summary.client_id }}">Open accounting queue</a>{% endif %}</div>{% endif %}</section>
@@ -114,8 +143,46 @@
     if (!selectedStillVisible) engagement.value = "";
     engagement.required = !!selectedClient;
   }
-  client.addEventListener("change", filterEngagements);
+  async function loadStoredStatements() {
+    const box = document.getElementById("stored-bank-statements");
+    if (!box) return;
+    const selectedClient = client.value;
+    if (!selectedClient) {
+      box.innerHTML = '<div class="text-xs text-slate-500">Select an ERP client to load previously stored bank statements.</div>';
+      return;
+    }
+    box.innerHTML = '<div class="text-xs text-slate-500">Loading stored statements…</div>';
+    try {
+      const response = await fetch(`/tools/bank-statement-analyzer/stored-statements?client_id=${encodeURIComponent(selectedClient)}`, {headers: {"Accept":"application/json"}});
+      const data = await response.json();
+      if (!response.ok) throw new Error(data.detail || "Unable to load stored statements.");
+      const items = data.items || [];
+      if (!items.length) {
+        box.innerHTML = '<div class="text-xs text-slate-500">No stored BANK_STATEMENT documents are available for this client.</div>';
+        return;
+      }
+      box.innerHTML = "";
+      items.forEach((row) => {
+        const label = document.createElement("label");
+        label.className = "flex items-start gap-2 rounded-lg border border-indigo-100 bg-white p-3 text-xs";
+        label.innerHTML = `<input type="checkbox" name="stored_version_ids" value="${row.version_id}" class="mt-1"><span><span class="block font-semibold text-slate-800"></span><span class="block text-slate-500"></span></span>`;
+        label.querySelector(".font-semibold").textContent = row.title || row.filename;
+        label.querySelector(".text-slate-500").textContent = `${row.filename} · version ${row.version_id} · ${(row.storage_status || "").replaceAll("_"," ")}`;
+        box.appendChild(label);
+      });
+    } catch (error) {
+      box.innerHTML = `<div class="text-xs text-red-700">${error.message}</div>`;
+    }
+  }
+
+  client.addEventListener("change", () => {
+    filterEngagements();
+    loadStoredStatements();
+  });
+  const refreshButton = document.getElementById("refresh-stored-bank-statements");
+  if (refreshButton) refreshButton.addEventListener("click", loadStoredStatements);
   filterEngagements();
+  if (client.value) loadStoredStatements();
 })();
 </script>
 

app/modules/bank_statement_analyzer/ui.py

@@ -17,6 +17,11 @@ from app.modules.core.rbac.deps import get_user_permissions, get_user_roles
 
 from .parsers.registry import BANK_OPTIONS
 from .client_context import analyzer_client_context, validate_selected_client_and_engagement
+from .stored_statement_service import (
+    deduplicate_source_paths,
+    list_stored_bank_statements,
+    prepare_stored_bank_statements,
+)
 from .service import (
     can_use,
     create_job_folder,
@@ -107,7 +112,7 @@ def _auth(request, db):
 
 
 @router.get("")
-def index(request: Request, job: str | None = None):
+def index(request: Request, job: str | None = None, client_id: int | None = None):
     ensure_worker_started()
     db = CommonSessionLocal()
     try:
@@ -119,6 +124,19 @@ def index(request: Request, job: str | None = None):
         recent = [_localised_job_view(item, timezone_name) for item in list_user_jobs(user.id, limit=8)]
         active_job = _localised_job_view(selected_job, timezone_name) if selected_job else None
         client_context = analyzer_client_context(db, request=request, user=user, roles=roles)
+        visible_client_ids = {int(row.id) for row in client_context["clients"]}
+        selected_client_id = int(client_id) if client_id and int(client_id) in visible_client_ids else None
+        stored_statements = (
+            list_stored_bank_statements(
+                db,
+                request=request,
+                user=user,
+                roles=roles,
+                client_id=selected_client_id,
+            )
+            if selected_client_id
+            else []
+        )
         return templates.TemplateResponse(
             "modules/bank_statement_analyzer/templates/bank_statement_analyzer/index.html",
             _ctx(
@@ -131,14 +149,38 @@ def index(request: Request, job: str | None = None):
                 display_timezone=timezone_name,
                 analyzer_clients=client_context["clients"],
                 analyzer_engagements=client_context["engagements"],
+                selected_client_id=selected_client_id,
+                stored_bank_statements=stored_statements,
             ),
         )
     finally:
         db.close()
 
 
+@router.get("/stored-statements")
+def stored_statements(request: Request, client_id: int):
+    db = CommonSessionLocal()
+    try:
+        user, roles, denied = _auth(request, db)
+        if denied:
+            return JSONResponse({"detail": "Access denied"}, status_code=403)
+        try:
+            rows = list_stored_bank_statements(
+                db,
+                request=request,
+                user=user,
+                roles=roles,
+                client_id=int(client_id),
+            )
+            return JSONResponse({"items": rows})
+        except Exception as exc:
+            return JSONResponse({"detail": str(exc)}, status_code=400)
+    finally:
+        db.close()
+
+
 @router.post("/analyze")
-async def analyze(request: Request, csrf_token: str = Form(...), bank_selection: str = Form("auto"), financial_year: str = Form(""), customer_name: str = Form(""), account_number: str = Form(""), client_id: str = Form(""), engagement_id: str = Form(""), confirm_same_client: str | None = Form(None), enable_classification: str | None = Form(None), purpose: str = Form("analyze_only"), statements: list[UploadFile] = File(...)):
+async def analyze(request: Request, csrf_token: str = Form(...), bank_selection: str = Form("auto"), financial_year: str = Form(""), customer_name: str = Form(""), account_number: str = Form(""), client_id: str = Form(""), engagement_id: str = Form(""), confirm_same_client: str | None = Form(None), enable_classification: str | None = Form(None), purpose: str = Form("analyze_only"), stored_version_ids: list[int] = Form([]), statements: list[UploadFile] = File(default=[])):
     db = CommonSessionLocal()
     selected_bank = bank_selection if bank_selection in dict(BANK_OPTIONS) else "auto"
     classification_enabled = enable_classification == "1"
@@ -169,7 +211,50 @@ async def analyze(request: Request, csrf_token: str = Form(...), bank_selection:
 
         job_id, input_dir, _output_dir = create_job_folder(user, roles)
         job_dir = input_dir.parent
-        paths = await save_uploads(statements, input_dir)
+
+        uploaded = []
+        usable_uploads = [item for item in (statements or []) if item and (item.filename or "").strip()]
+        if usable_uploads:
+            uploaded = await save_uploads(usable_uploads, input_dir)
+
+        stored_paths = []
+        stored_provenance = []
+        pending_retrieval = []
+        if stored_version_ids:
+            if not resolved_client_id:
+                raise ValueError("Select an ERP client before reusing stored bank statements.")
+            stored_paths, stored_provenance, pending_retrieval = prepare_stored_bank_statements(
+                db,
+                request=request,
+                user=user,
+                roles=roles,
+                client_id=resolved_client_id,
+                version_ids=stored_version_ids,
+                input_dir=input_dir,
+            )
+
+        if pending_retrieval:
+            db.commit()
+            details = ", ".join(
+                f"{row['filename']} (retrieval #{row.get('request_id') or 'pending'})"
+                for row in pending_retrieval
+            )
+            raise ValueError(
+                "Stored statement retrieval has been queued from branch local storage: "
+                + details
+                + ". Wait for the Local Agent to return the file, then submit the analysis again."
+            )
+
+        paths, source_hashes = deduplicate_source_paths(uploaded + stored_paths)
+        retained_names = {path.name for path in paths}
+        stored_provenance = [
+            row
+            for row in stored_provenance
+            if row.get("copied_filename") in retained_names
+        ]
+        if not paths:
+            raise ValueError("Select at least one new PDF or one stored bank statement.")
+
         enqueue_job(
             user=user,
             roles=roles,
@@ -185,6 +270,8 @@ async def analyze(request: Request, csrf_token: str = Form(...), bank_selection:
             engagement_id=resolved_engagement_id,
             ownership_confirmation=(confirm_same_client == "1"),
             purpose=purpose,
+            stored_source_versions=stored_provenance,
+            source_hashes=source_hashes,
         )
         return RedirectResponse(f"/tools/bank-statement-analyzer?job={job_id}#analysis-status", status_code=303)
     except Exception as exc:
@@ -215,6 +302,17 @@ async def analyze(request: Request, csrf_token: str = Form(...), bank_selection:
                 analyzer_engagements=client_context["engagements"],
                 selected_client_id=(int(client_id) if str(client_id).strip().isdigit() else None),
                 selected_engagement_id=(int(engagement_id) if str(engagement_id).strip().isdigit() else None),
+                stored_bank_statements=(
+                    list_stored_bank_statements(
+                        db,
+                        request=request,
+                        user=user,
+                        roles=get_user_roles(db, user.id),
+                        client_id=int(client_id),
+                    )
+                    if str(client_id).strip().isdigit()
+                    else []
+                ),
             ),
             status_code=400,
         )