diff --git a/app/modules/bank_statement_analyzer/analyzer.py b/app/modules/bank_statement_analyzer/analyzer.py index 5ea1601..97838eb 100644 --- a/app/modules/bank_statement_analyzer/analyzer.py +++ b/app/modules/bank_statement_analyzer/analyzer.py @@ -11,6 +11,46 @@ from .parsers.common import infer_mode HIGH_VALUE_THRESHOLD = 50000.0 LOW_BALANCE_THRESHOLD = 1000.0 +ANALYSIS_COLUMNS = [ + "transaction_date", "value_date", "narration", "debit", "credit", "balance", + "reference_no", "bank_name", "customer_name", "account_number", "source_file", + "source_page", "parser_name", "mode", "amount", "direction", "narration_key", + "exact_key", "exact_duplicate", "possible_key", "possible_duplicate", "category", + "counterparty", "review_note", +] + + +def _ensure_analysis_columns(df: pd.DataFrame | None) -> pd.DataFrame: + """Return a DataFrame with every analyzer-owned column present. + + Bank parsers may legitimately return an empty frame (for example when a PDF + layout is detected but no transaction rows can be extracted). Downstream + dashboard, duplicate and workbook code must still see a stable schema. + """ + x = df.copy() if isinstance(df, pd.DataFrame) else pd.DataFrame() + bool_columns = {"exact_duplicate", "possible_duplicate"} + numeric_columns = {"debit", "credit", "balance", "amount"} + for column in ANALYSIS_COLUMNS: + if column in x.columns: + continue + if column in bool_columns: + x[column] = pd.Series(False, index=x.index, dtype="bool") + elif column in numeric_columns: + x[column] = pd.Series(index=x.index, dtype="float64") + else: + x[column] = pd.Series(index=x.index, dtype="object") + if "exact_duplicate" in x.columns: + x["exact_duplicate"] = x["exact_duplicate"].fillna(False).astype(bool) + if "possible_duplicate" in x.columns: + x["possible_duplicate"] = x["possible_duplicate"].fillna(False).astype(bool) + return x + + +def _duplicate_count(df: pd.DataFrame, column: str) -> int: + if df is None or column not in df.columns or df.empty: + return 0 + return int(df[column].fillna(False).astype(bool).sum()) + def clean_key(value): value = re.sub(r"\s+", " ", str(value or "").upper()).strip() @@ -137,9 +177,9 @@ def _review_note(row) -> str: def enrich(df, classification_enabled: bool = True): - if df.empty: - return df - x = df.copy() + x = _ensure_analysis_columns(df) + if x.empty: + return x x["mode"] = x["narration"].map(infer_mode) x["amount"] = x["debit"].fillna(0) + x["credit"].fillna(0) x["direction"] = x.apply(_direction, axis=1) @@ -162,7 +202,7 @@ def enrich(df, classification_enabled: bool = True): x["category"] = "Classification disabled" x["counterparty"] = x["narration"].map(lambda value: _text(value)[:120] or "Unidentified") x["review_note"] = x.apply(_review_note, axis=1) - return x + return _ensure_analysis_columns(x) def analyze_files(paths, customer_override="", account_override="", bank_hint="auto", classification_enabled=True): @@ -178,8 +218,19 @@ def analyze_files(paths, customer_override="", account_override="", bank_hint="a df["account_number"] = account_override metas.append(meta) frames.append(df) - all_df = enrich(pd.concat(frames, ignore_index=True) if frames else pd.DataFrame(), classification_enabled) - unique_df = all_df.drop_duplicates("exact_key", keep="first").copy() if not all_df.empty else all_df.copy() + combined = pd.concat(frames, ignore_index=True) if frames else pd.DataFrame() + all_df = _ensure_analysis_columns(enrich(combined, classification_enabled)) + unique_df = ( + all_df.drop_duplicates("exact_key", keep="first").copy() + if not all_df.empty + else _ensure_analysis_columns(all_df) + ) + unique_df = _ensure_analysis_columns(unique_df) + if metas and all_df.empty: + raise ValueError( + "The bank statement format was identified, but no transaction rows could be extracted. " + "Please verify that the PDF is text-readable and that this statement layout is supported." + ) return metas, all_df, unique_df @@ -255,9 +306,9 @@ def duplicate_summary(all_df): return pd.DataFrame( [ {"check": "All extracted rows", "count": len(all_df)}, - {"check": "Exact duplicate rows", "count": int(all_df.exact_duplicate.sum()) if not all_df.empty else 0}, - {"check": "Possible duplicate rows", "count": int(all_df.possible_duplicate.sum()) if not all_df.empty else 0}, - {"check": "Unique rows after exact deduplication", "count": int(all_df.exact_key.nunique()) if not all_df.empty else 0}, + {"check": "Exact duplicate rows", "count": _duplicate_count(all_df, "exact_duplicate")}, + {"check": "Possible duplicate rows", "count": _duplicate_count(all_df, "possible_duplicate")}, + {"check": "Unique rows after exact deduplication", "count": int(all_df["exact_key"].nunique()) if not all_df.empty and "exact_key" in all_df.columns else 0}, ] ) @@ -297,12 +348,14 @@ def _workbook_columns(df): def export_excel(output, metas, all_df, unique_df, financial_year="", selected_bank="auto", classification_enabled=True): + all_df = _ensure_analysis_columns(all_df) + unique_df = _ensure_analysis_columns(unique_df) recon = reconcile(metas, all_df) customer = next((meta.customer_name for meta in metas if meta.customer_name), "") account = next((meta.account_number for meta in metas if meta.account_number), "") banks = ", ".join(sorted({meta.bank_name for meta in metas})) - exact_count = int(all_df.exact_duplicate.sum()) if not all_df.empty else 0 - possible_count = int(all_df.possible_duplicate.sum()) if not all_df.empty else 0 + exact_count = _duplicate_count(all_df, "exact_duplicate") + possible_count = _duplicate_count(all_df, "possible_duplicate") opening = next((meta.opening_balance for meta in metas if meta.opening_balance is not None), None) closing = next((meta.closing_balance for meta in reversed(metas) if meta.closing_balance is not None), None) dashboard = pd.DataFrame( @@ -357,8 +410,8 @@ def export_excel(output, metas, all_df, unique_df, financial_year="", selected_b recon.to_excel(writer, sheet_name="Statement Reconciliation", index=False) all_export.to_excel(writer, sheet_name="All Extracted Rows", index=False) unique_export.to_excel(writer, sheet_name="Unique Transactions", index=False) - all_export[all_df.exact_duplicate.to_numpy()].to_excel(writer, sheet_name="Exact Duplicates", index=False) - all_export[all_df.possible_duplicate.to_numpy()].to_excel(writer, sheet_name="Possible Duplicates", index=False) + all_export[all_df["exact_duplicate"].fillna(False).astype(bool).to_numpy()].to_excel(writer, sheet_name="Exact Duplicates", index=False) + all_export[all_df["possible_duplicate"].fillna(False).astype(bool).to_numpy()].to_excel(writer, sheet_name="Possible Duplicates", index=False) monthly_summary(unique_df).to_excel(writer, sheet_name="Monthly Summary", index=False) mode_summary(unique_df).to_excel(writer, sheet_name="Mode Summary", index=False) category_summary(unique_df).to_excel(writer, sheet_name="Category Summary", index=False) diff --git a/app/modules/bank_statement_analyzer/service.py b/app/modules/bank_statement_analyzer/service.py index 0b2feef..6a8c69c 100644 --- a/app/modules/bank_statement_analyzer/service.py +++ b/app/modules/bank_statement_analyzer/service.py @@ -244,8 +244,8 @@ def _process_job(job_id: str) -> None: "statement_count": len(metas), "rows_extracted": int(len(all_df)), "unique_transactions": int(len(unique_df)), - "exact_duplicate_rows": int(all_df.exact_duplicate.sum()) if not all_df.empty else 0, - "possible_duplicate_rows": int(all_df.possible_duplicate.sum()) if not all_df.empty else 0, + "exact_duplicate_rows": int(all_df["exact_duplicate"].fillna(False).astype(bool).sum()) if not all_df.empty and "exact_duplicate" in all_df.columns else 0, + "possible_duplicate_rows": int(all_df["possible_duplicate"].fillna(False).astype(bool).sum()) if not all_df.empty and "possible_duplicate" in all_df.columns else 0, "review_items": int(unique_df.review_note.fillna("").ne("").sum()) if not unique_df.empty and "review_note" in unique_df.columns else 0, "categories": int(unique_df.category.nunique()) if not unique_df.empty and "category" in unique_df.columns else 0, "banks": sorted({meta.bank_name for meta in metas}),