Add delta-based bank extraction and summary reconciliations

app/modules/bank_statement_analyzer/parsers/base.py

@@ -29,7 +29,11 @@ class StatementMeta:
 STANDARD_COLUMNS = [
     "transaction_date", "value_date", "narration", "reference_no",
     "debit", "credit", "balance", "bank_name", "customer_name",
-    "account_number", "source_file", "source_page", "parser_name"
+    "account_number", "source_file", "source_page", "parser_name",
+    # Internal extraction-audit fields. These are retained for diagnostics but
+    # are intentionally omitted from the client-facing workbook.
+    "printed_debit", "printed_credit", "balance_delta", "movement_difference",
+    "correction_applied", "correction_reason", "extraction_confidence",
 ]
 
 def amount(v):
@@ -64,6 +68,76 @@ def extract_text(path: str|Path) -> str:
 def page_of_line(text: str, position: int) -> int:
     return text[:position].count('\f')+1
 
+def _apply_balance_delta_validation(df: pd.DataFrame, meta: StatementMeta) -> pd.DataFrame:
+    """Validate and, when necessary, correct debit/credit using balance movement.
+
+    Printed PDF columns remain the first extraction source.  The running-balance
+    delta is the independent accounting control.  Where the printed movement and
+    the balance delta disagree, the delta determines the corrected side and
+    amount.  This common routine is used by every bank parser through ``finalize``.
+    """
+    if df.empty:
+        return df
+
+    tolerance = 0.01
+    x = df.copy()
+    x["printed_debit"] = pd.to_numeric(x.get("debit"), errors="coerce")
+    x["printed_credit"] = pd.to_numeric(x.get("credit"), errors="coerce")
+    x["balance_delta"] = pd.NA
+    x["movement_difference"] = pd.NA
+    x["correction_applied"] = False
+    x["correction_reason"] = ""
+    x["extraction_confidence"] = "Printed columns"
+
+    dated = pd.to_datetime(x.get("transaction_date"), errors="coerce")
+    valid_dates = dated.dropna()
+    descending = len(valid_dates) >= 2 and valid_dates.iloc[0] > valid_dates.iloc[-1]
+    order = list(reversed(x.index.tolist())) if descending else x.index.tolist()
+
+    previous_balance = meta.opening_balance
+    for idx in order:
+        current_balance = x.at[idx, "balance"]
+        if pd.isna(current_balance):
+            x.at[idx, "extraction_confidence"] = "Review - balance unavailable"
+            continue
+
+        current_balance = float(current_balance)
+        if previous_balance is None or pd.isna(previous_balance):
+            previous_balance = current_balance
+            x.at[idx, "extraction_confidence"] = "Printed columns - no opening delta"
+            continue
+
+        delta = round(current_balance - float(previous_balance), 2)
+        debit = float(x.at[idx, "debit"]) if pd.notna(x.at[idx, "debit"]) else 0.0
+        credit = float(x.at[idx, "credit"]) if pd.notna(x.at[idx, "credit"]) else 0.0
+        printed_movement = round(credit - debit, 2)
+        movement_difference = round(delta - printed_movement, 2)
+        x.at[idx, "balance_delta"] = delta
+        x.at[idx, "movement_difference"] = movement_difference
+
+        if abs(movement_difference) <= tolerance:
+            x.at[idx, "extraction_confidence"] = "100% - printed movement matches delta"
+        elif abs(delta) > tolerance:
+            corrected_debit = round(abs(delta), 2) if delta < 0 else 0.0
+            corrected_credit = round(delta, 2) if delta > 0 else 0.0
+            x.at[idx, "debit"] = corrected_debit
+            x.at[idx, "credit"] = corrected_credit
+            x.at[idx, "correction_applied"] = True
+            x.at[idx, "correction_reason"] = "Debit/credit corrected from running-balance delta"
+            if abs(abs(printed_movement) - abs(delta)) <= tolerance:
+                x.at[idx, "extraction_confidence"] = "99% - amount matched, side corrected by delta"
+            elif debit == 0.0 and credit == 0.0:
+                x.at[idx, "extraction_confidence"] = "98% - missing movement derived from delta"
+            else:
+                x.at[idx, "extraction_confidence"] = "Review - printed movement replaced by delta"
+        else:
+            x.at[idx, "extraction_confidence"] = "Review - zero balance movement"
+
+        previous_balance = current_balance
+
+    return x
+
+
 def finalize(df: pd.DataFrame, meta: StatementMeta) -> pd.DataFrame:
     if df is None or df.empty:
         return pd.DataFrame(columns=STANDARD_COLUMNS)
@@ -76,8 +150,11 @@ def finalize(df: pd.DataFrame, meta: StatementMeta) -> pd.DataFrame:
             df[c] = pd.NaT
         else:
             df[c] = values.map(parse_flexible_date)
-    df['narration']=df.get('narration','').fillna('').map(norm)
-    df['reference_no']=df.get('reference_no','').fillna('').map(norm)
+    df = _apply_balance_delta_validation(df, meta)
+    narration_values = df['narration'] if 'narration' in df.columns else pd.Series('', index=df.index)
+    reference_values = df['reference_no'] if 'reference_no' in df.columns else pd.Series('', index=df.index)
+    df['narration']=narration_values.fillna('').map(norm)
+    df['reference_no']=reference_values.fillna('').map(norm)
     df['bank_name']=meta.bank_name
     df['customer_name']=meta.customer_name
     df['account_number']=meta.account_number