Rakseeibra/arrr-erp
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix bank analyzer duplicate column handling

Browse Source
This commit is contained in:
A R R R Associates
2026-07-13 22:04:19 +05:30
parent 897c7dad38
commit 06aca4adc9
2 changed files with 68 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/modules/bank_statement_analyzer/analyzer.py
+66 -13
View File
@@ -11,6 +11,46 @@ from .parsers.common import infer_mode
HIGH_VALUE_THRESHOLD = 50000.0
LOW_BALANCE_THRESHOLD = 1000.0
ANALYSIS_COLUMNS = [
"transaction_date", "value_date", "narration", "debit", "credit", "balance",
"reference_no", "bank_name", "customer_name", "account_number", "source_file",
"source_page", "parser_name", "mode", "amount", "direction", "narration_key",
"exact_key", "exact_duplicate", "possible_key", "possible_duplicate", "category",
"counterparty", "review_note",
]
def _ensure_analysis_columns(df: pd.DataFrame | None) -> pd.DataFrame:
"""Return a DataFrame with every analyzer-owned column present.
Bank parsers may legitimately return an empty frame (for example when a PDF
layout is detected but no transaction rows can be extracted). Downstream
dashboard, duplicate and workbook code must still see a stable schema.
"""
x = df.copy() if isinstance(df, pd.DataFrame) else pd.DataFrame()
bool_columns = {"exact_duplicate", "possible_duplicate"}
numeric_columns = {"debit", "credit", "balance", "amount"}
for column in ANALYSIS_COLUMNS:
if column in x.columns:
continue
if column in bool_columns:
x[column] = pd.Series(False, index=x.index, dtype="bool")
elif column in numeric_columns:
x[column] = pd.Series(index=x.index, dtype="float64")
else:
x[column] = pd.Series(index=x.index, dtype="object")
if "exact_duplicate" in x.columns:
x["exact_duplicate"] = x["exact_duplicate"].fillna(False).astype(bool)
if "possible_duplicate" in x.columns:
x["possible_duplicate"] = x["possible_duplicate"].fillna(False).astype(bool)
return x
def _duplicate_count(df: pd.DataFrame, column: str) -> int:
if df is None or column not in df.columns or df.empty:
return 0
return int(df[column].fillna(False).astype(bool).sum())
def clean_key(value):
value = re.sub(r"\s+", " ", str(value or "").upper()).strip()
@@ -137,9 +177,9 @@ def _review_note(row) -> str:
def enrich(df, classification_enabled: bool = True):
if df.empty:
return df
x = df.copy()
x = _ensure_analysis_columns(df)
if x.empty:
return x
x["mode"] = x["narration"].map(infer_mode)
x["amount"] = x["debit"].fillna(0) + x["credit"].fillna(0)
x["direction"] = x.apply(_direction, axis=1)
@@ -162,7 +202,7 @@ def enrich(df, classification_enabled: bool = True):
x["category"] = "Classification disabled"
x["counterparty"] = x["narration"].map(lambda value: _text(value)[:120] or "Unidentified")
x["review_note"] = x.apply(_review_note, axis=1)
return x
return _ensure_analysis_columns(x)
def analyze_files(paths, customer_override="", account_override="", bank_hint="auto", classification_enabled=True):
@@ -178,8 +218,19 @@ def analyze_files(paths, customer_override="", account_override="", bank_hint="a
df["account_number"] = account_override
metas.append(meta)
frames.append(df)
all_df = enrich(pd.concat(frames, ignore_index=True) if frames else pd.DataFrame(), classification_enabled)
unique_df = all_df.drop_duplicates("exact_key", keep="first").copy() if not all_df.empty else all_df.copy()
combined = pd.concat(frames, ignore_index=True) if frames else pd.DataFrame()
all_df = _ensure_analysis_columns(enrich(combined, classification_enabled))
unique_df = (
all_df.drop_duplicates("exact_key", keep="first").copy()
if not all_df.empty
else _ensure_analysis_columns(all_df)
)
unique_df = _ensure_analysis_columns(unique_df)
if metas and all_df.empty:
raise ValueError(
"The bank statement format was identified, but no transaction rows could be extracted. "
"Please verify that the PDF is text-readable and that this statement layout is supported."
)
return metas, all_df, unique_df
@@ -255,9 +306,9 @@ def duplicate_summary(all_df):
return pd.DataFrame(
[
{"check": "All extracted rows", "count": len(all_df)},
{"check": "Exact duplicate rows", "count": int(all_df.exact_duplicate.sum()) if not all_df.empty else 0},
{"check": "Possible duplicate rows", "count": int(all_df.possible_duplicate.sum()) if not all_df.empty else 0},
{"check": "Unique rows after exact deduplication", "count": int(all_df.exact_key.nunique()) if not all_df.empty else 0},
{"check": "Exact duplicate rows", "count": _duplicate_count(all_df, "exact_duplicate")},
{"check": "Possible duplicate rows", "count": _duplicate_count(all_df, "possible_duplicate")},
{"check": "Unique rows after exact deduplication", "count": int(all_df["exact_key"].nunique()) if not all_df.empty and "exact_key" in all_df.columns else 0},
]
)
@@ -297,12 +348,14 @@ def _workbook_columns(df):
def export_excel(output, metas, all_df, unique_df, financial_year="", selected_bank="auto", classification_enabled=True):
all_df = _ensure_analysis_columns(all_df)
unique_df = _ensure_analysis_columns(unique_df)
recon = reconcile(metas, all_df)
customer = next((meta.customer_name for meta in metas if meta.customer_name), "")
account = next((meta.account_number for meta in metas if meta.account_number), "")
banks = ", ".join(sorted({meta.bank_name for meta in metas}))
exact_count = int(all_df.exact_duplicate.sum()) if not all_df.empty else 0
possible_count = int(all_df.possible_duplicate.sum()) if not all_df.empty else 0
exact_count = _duplicate_count(all_df, "exact_duplicate")
possible_count = _duplicate_count(all_df, "possible_duplicate")
opening = next((meta.opening_balance for meta in metas if meta.opening_balance is not None), None)
closing = next((meta.closing_balance for meta in reversed(metas) if meta.closing_balance is not None), None)
dashboard = pd.DataFrame(
@@ -357,8 +410,8 @@ def export_excel(output, metas, all_df, unique_df, financial_year="", selected_b
recon.to_excel(writer, sheet_name="Statement Reconciliation", index=False)
all_export.to_excel(writer, sheet_name="All Extracted Rows", index=False)
unique_export.to_excel(writer, sheet_name="Unique Transactions", index=False)
all_export[all_df.exact_duplicate.to_numpy()].to_excel(writer, sheet_name="Exact Duplicates", index=False)
all_export[all_df.possible_duplicate.to_numpy()].to_excel(writer, sheet_name="Possible Duplicates", index=False)
all_export[all_df["exact_duplicate"].fillna(False).astype(bool).to_numpy()].to_excel(writer, sheet_name="Exact Duplicates", index=False)
all_export[all_df["possible_duplicate"].fillna(False).astype(bool).to_numpy()].to_excel(writer, sheet_name="Possible Duplicates", index=False)
monthly_summary(unique_df).to_excel(writer, sheet_name="Monthly Summary", index=False)
mode_summary(unique_df).to_excel(writer, sheet_name="Mode Summary", index=False)
category_summary(unique_df).to_excel(writer, sheet_name="Category Summary", index=False)
app/modules/bank_statement_analyzer/service.py
+2 -2
View File
@@ -244,8 +244,8 @@ def _process_job(job_id: str) -> None:
"statement_count": len(metas),
"rows_extracted": int(len(all_df)),
"unique_transactions": int(len(unique_df)),
"exact_duplicate_rows": int(all_df.exact_duplicate.sum()) if not all_df.empty else 0,
"possible_duplicate_rows": int(all_df.possible_duplicate.sum()) if not all_df.empty else 0,
"exact_duplicate_rows": int(all_df["exact_duplicate"].fillna(False).astype(bool).sum()) if not all_df.empty and "exact_duplicate" in all_df.columns else 0,
"possible_duplicate_rows": int(all_df["possible_duplicate"].fillna(False).astype(bool).sum()) if not all_df.empty and "possible_duplicate" in all_df.columns else 0,
"review_items": int(unique_df.review_note.fillna("").ne("").sum()) if not unique_df.empty and "review_note" in unique_df.columns else 0,
"categories": int(unique_df.category.nunique()) if not unique_df.empty and "category" in unique_df.columns else 0,
"banks": sorted({meta.bank_name for meta in metas}),