Fix bank analyzer duplicate column handling

app/modules/bank_statement_analyzer/analyzer.py

@@ -11,6 +11,46 @@ from .parsers.common import infer_mode
 HIGH_VALUE_THRESHOLD = 50000.0
 LOW_BALANCE_THRESHOLD = 1000.0
 
+ANALYSIS_COLUMNS = [
+    "transaction_date", "value_date", "narration", "debit", "credit", "balance",
+    "reference_no", "bank_name", "customer_name", "account_number", "source_file",
+    "source_page", "parser_name", "mode", "amount", "direction", "narration_key",
+    "exact_key", "exact_duplicate", "possible_key", "possible_duplicate", "category",
+    "counterparty", "review_note",
+]
+
+
+def _ensure_analysis_columns(df: pd.DataFrame | None) -> pd.DataFrame:
+    """Return a DataFrame with every analyzer-owned column present.
+
+    Bank parsers may legitimately return an empty frame (for example when a PDF
+    layout is detected but no transaction rows can be extracted). Downstream
+    dashboard, duplicate and workbook code must still see a stable schema.
+    """
+    x = df.copy() if isinstance(df, pd.DataFrame) else pd.DataFrame()
+    bool_columns = {"exact_duplicate", "possible_duplicate"}
+    numeric_columns = {"debit", "credit", "balance", "amount"}
+    for column in ANALYSIS_COLUMNS:
+        if column in x.columns:
+            continue
+        if column in bool_columns:
+            x[column] = pd.Series(False, index=x.index, dtype="bool")
+        elif column in numeric_columns:
+            x[column] = pd.Series(index=x.index, dtype="float64")
+        else:
+            x[column] = pd.Series(index=x.index, dtype="object")
+    if "exact_duplicate" in x.columns:
+        x["exact_duplicate"] = x["exact_duplicate"].fillna(False).astype(bool)
+    if "possible_duplicate" in x.columns:
+        x["possible_duplicate"] = x["possible_duplicate"].fillna(False).astype(bool)
+    return x
+
+
+def _duplicate_count(df: pd.DataFrame, column: str) -> int:
+    if df is None or column not in df.columns or df.empty:
+        return 0
+    return int(df[column].fillna(False).astype(bool).sum())
+
 
 def clean_key(value):
     value = re.sub(r"\s+", " ", str(value or "").upper()).strip()
@@ -137,9 +177,9 @@ def _review_note(row) -> str:
 
 
 def enrich(df, classification_enabled: bool = True):
-    if df.empty:
+    x = _ensure_analysis_columns(df)
-        return df
+    if x.empty:
-    x = df.copy()
+        return x
     x["mode"] = x["narration"].map(infer_mode)
     x["amount"] = x["debit"].fillna(0) + x["credit"].fillna(0)
     x["direction"] = x.apply(_direction, axis=1)
@@ -162,7 +202,7 @@ def enrich(df, classification_enabled: bool = True):
         x["category"] = "Classification disabled"
         x["counterparty"] = x["narration"].map(lambda value: _text(value)[:120] or "Unidentified")
         x["review_note"] = x.apply(_review_note, axis=1)
-    return x
+    return _ensure_analysis_columns(x)
 
 
 def analyze_files(paths, customer_override="", account_override="", bank_hint="auto", classification_enabled=True):
@@ -178,8 +218,19 @@ def analyze_files(paths, customer_override="", account_override="", bank_hint="a
             df["account_number"] = account_override
         metas.append(meta)
         frames.append(df)
-    all_df = enrich(pd.concat(frames, ignore_index=True) if frames else pd.DataFrame(), classification_enabled)
+    combined = pd.concat(frames, ignore_index=True) if frames else pd.DataFrame()
-    unique_df = all_df.drop_duplicates("exact_key", keep="first").copy() if not all_df.empty else all_df.copy()
+    all_df = _ensure_analysis_columns(enrich(combined, classification_enabled))
+    unique_df = (
+        all_df.drop_duplicates("exact_key", keep="first").copy()
+        if not all_df.empty
+        else _ensure_analysis_columns(all_df)
+    )
+    unique_df = _ensure_analysis_columns(unique_df)
+    if metas and all_df.empty:
+        raise ValueError(
+            "The bank statement format was identified, but no transaction rows could be extracted. "
+            "Please verify that the PDF is text-readable and that this statement layout is supported."
+        )
     return metas, all_df, unique_df
 
 
@@ -255,9 +306,9 @@ def duplicate_summary(all_df):
     return pd.DataFrame(
         [
             {"check": "All extracted rows", "count": len(all_df)},
-            {"check": "Exact duplicate rows", "count": int(all_df.exact_duplicate.sum()) if not all_df.empty else 0},
+            {"check": "Exact duplicate rows", "count": _duplicate_count(all_df, "exact_duplicate")},
-            {"check": "Possible duplicate rows", "count": int(all_df.possible_duplicate.sum()) if not all_df.empty else 0},
+            {"check": "Possible duplicate rows", "count": _duplicate_count(all_df, "possible_duplicate")},
-            {"check": "Unique rows after exact deduplication", "count": int(all_df.exact_key.nunique()) if not all_df.empty else 0},
+            {"check": "Unique rows after exact deduplication", "count": int(all_df["exact_key"].nunique()) if not all_df.empty and "exact_key" in all_df.columns else 0},
         ]
     )
 
@@ -297,12 +348,14 @@ def _workbook_columns(df):
 
 
 def export_excel(output, metas, all_df, unique_df, financial_year="", selected_bank="auto", classification_enabled=True):
+    all_df = _ensure_analysis_columns(all_df)
+    unique_df = _ensure_analysis_columns(unique_df)
     recon = reconcile(metas, all_df)
     customer = next((meta.customer_name for meta in metas if meta.customer_name), "")
     account = next((meta.account_number for meta in metas if meta.account_number), "")
     banks = ", ".join(sorted({meta.bank_name for meta in metas}))
-    exact_count = int(all_df.exact_duplicate.sum()) if not all_df.empty else 0
+    exact_count = _duplicate_count(all_df, "exact_duplicate")
-    possible_count = int(all_df.possible_duplicate.sum()) if not all_df.empty else 0
+    possible_count = _duplicate_count(all_df, "possible_duplicate")
     opening = next((meta.opening_balance for meta in metas if meta.opening_balance is not None), None)
     closing = next((meta.closing_balance for meta in reversed(metas) if meta.closing_balance is not None), None)
     dashboard = pd.DataFrame(
@@ -357,8 +410,8 @@ def export_excel(output, metas, all_df, unique_df, financial_year="", selected_b
         recon.to_excel(writer, sheet_name="Statement Reconciliation", index=False)
         all_export.to_excel(writer, sheet_name="All Extracted Rows", index=False)
         unique_export.to_excel(writer, sheet_name="Unique Transactions", index=False)
-        all_export[all_df.exact_duplicate.to_numpy()].to_excel(writer, sheet_name="Exact Duplicates", index=False)
+        all_export[all_df["exact_duplicate"].fillna(False).astype(bool).to_numpy()].to_excel(writer, sheet_name="Exact Duplicates", index=False)
-        all_export[all_df.possible_duplicate.to_numpy()].to_excel(writer, sheet_name="Possible Duplicates", index=False)
+        all_export[all_df["possible_duplicate"].fillna(False).astype(bool).to_numpy()].to_excel(writer, sheet_name="Possible Duplicates", index=False)
         monthly_summary(unique_df).to_excel(writer, sheet_name="Monthly Summary", index=False)
         mode_summary(unique_df).to_excel(writer, sheet_name="Mode Summary", index=False)
         category_summary(unique_df).to_excel(writer, sheet_name="Category Summary", index=False)

app/modules/bank_statement_analyzer/service.py

@@ -244,8 +244,8 @@ def _process_job(job_id: str) -> None:
             "statement_count": len(metas),
             "rows_extracted": int(len(all_df)),
             "unique_transactions": int(len(unique_df)),
-            "exact_duplicate_rows": int(all_df.exact_duplicate.sum()) if not all_df.empty else 0,
+            "exact_duplicate_rows": int(all_df["exact_duplicate"].fillna(False).astype(bool).sum()) if not all_df.empty and "exact_duplicate" in all_df.columns else 0,
-            "possible_duplicate_rows": int(all_df.possible_duplicate.sum()) if not all_df.empty else 0,
+            "possible_duplicate_rows": int(all_df["possible_duplicate"].fillna(False).astype(bool).sum()) if not all_df.empty and "possible_duplicate" in all_df.columns else 0,
             "review_items": int(unique_df.review_note.fillna("").ne("").sum()) if not unique_df.empty and "review_note" in unique_df.columns else 0,
             "categories": int(unique_df.category.nunique()) if not unique_df.empty and "category" in unique_df.columns else 0,
             "banks": sorted({meta.bank_name for meta in metas}),