From bbc5afe1c0ff39a152603187eacdf182c8e0857f Mon Sep 17 00:00:00 2001 From: A R R R Associates Date: Mon, 13 Jul 2026 10:12:19 +0530 Subject: [PATCH] Add bank statement analyzer with automatic work storage --- .../bank_statement_analyzer/__init__.py | 1 + .../bank_statement_analyzer/analyzer.py | 107 ++++++++++++ .../parsers/__init__.py | 2 + .../bank_statement_analyzer/parsers/axis.py | 37 ++++ .../bank_statement_analyzer/parsers/base.py | 91 ++++++++++ .../bank_statement_analyzer/parsers/common.py | 20 +++ .../bank_statement_analyzer/parsers/hdfc.py | 58 +++++++ .../bank_statement_analyzer/parsers/idfc.py | 56 ++++++ .../parsers/indian_bank.py | 102 +++++++++++ .../parsers/indusind.py | 53 ++++++ .../bank_statement_analyzer/parsers/kotak.py | 54 ++++++ .../parsers/registry.py | 25 +++ .../bank_statement_analyzer/parsers/sbi.py | 100 +++++++++++ .../bank_statement_analyzer/service.py | 161 ++++++++++++++++++ .../bank_statement_analyzer/index.html | 20 +++ .../bank_statement_analyzer/result.html | 15 ++ app/modules/bank_statement_analyzer/ui.py | 116 +++++++++++++ app/ui/app.py | 2 + .../components/consultant_navigation_v2.html | 6 + .../components/manager_navigation_v2.html | 6 + .../components/partner_navigation_v2.html | 6 + .../components/staff_navigation_v2.html | 6 + requirements.txt | 3 + 23 files changed, 1047 insertions(+) create mode 100644 app/modules/bank_statement_analyzer/__init__.py create mode 100644 app/modules/bank_statement_analyzer/analyzer.py create mode 100644 app/modules/bank_statement_analyzer/parsers/__init__.py create mode 100644 app/modules/bank_statement_analyzer/parsers/axis.py create mode 100644 app/modules/bank_statement_analyzer/parsers/base.py create mode 100644 app/modules/bank_statement_analyzer/parsers/common.py create mode 100644 app/modules/bank_statement_analyzer/parsers/hdfc.py create mode 100644 app/modules/bank_statement_analyzer/parsers/idfc.py create mode 100644 app/modules/bank_statement_analyzer/parsers/indian_bank.py create mode 100644 app/modules/bank_statement_analyzer/parsers/indusind.py create mode 100644 app/modules/bank_statement_analyzer/parsers/kotak.py create mode 100644 app/modules/bank_statement_analyzer/parsers/registry.py create mode 100644 app/modules/bank_statement_analyzer/parsers/sbi.py create mode 100644 app/modules/bank_statement_analyzer/service.py create mode 100644 app/modules/bank_statement_analyzer/templates/bank_statement_analyzer/index.html create mode 100644 app/modules/bank_statement_analyzer/templates/bank_statement_analyzer/result.html create mode 100644 app/modules/bank_statement_analyzer/ui.py diff --git a/app/modules/bank_statement_analyzer/__init__.py b/app/modules/bank_statement_analyzer/__init__.py new file mode 100644 index 0000000..12fe0b7 --- /dev/null +++ b/app/modules/bank_statement_analyzer/__init__.py @@ -0,0 +1 @@ +"""Bank statement analyzer ERP module.""" diff --git a/app/modules/bank_statement_analyzer/analyzer.py b/app/modules/bank_statement_analyzer/analyzer.py new file mode 100644 index 0000000..29d4b56 --- /dev/null +++ b/app/modules/bank_statement_analyzer/analyzer.py @@ -0,0 +1,107 @@ +from __future__ import annotations +from pathlib import Path +import re +import pandas as pd +from .parsers import parse_pdf +from parsers.common import infer_mode + + +def clean_key(s): + s=re.sub(r'\s+',' ',str(s or '').upper()).strip() + return re.sub(r'\b\d{8,}\b','',s) + +def enrich(df): + if df.empty:return df + x=df.copy() + x['mode']=x['narration'].map(infer_mode) + x['amount']=x['debit'].fillna(0)+x['credit'].fillna(0) + x['direction']=x['debit'].notna().map({True:'Debit',False:'Credit'}) + x['narration_key']=x['narration'].map(clean_key) + x['exact_key']=x.apply(lambda r:f"{r.transaction_date}|{r.value_date}|{r.debit}|{r.credit}|{r.balance}|{clean_key(r.narration)}",axis=1) + x['exact_duplicate']=x.duplicated('exact_key',keep=False) + # possible duplicate: same date, direction, amount and normalized narration across different source files + x['possible_key']=x.apply(lambda r:f"{r.transaction_date}|{r.direction}|{r.amount:.2f}|{r.narration_key}",axis=1) + x['possible_duplicate']=x.duplicated('possible_key',keep=False) & ~x['exact_duplicate'] + return x + +def analyze_files(paths, customer_override='', account_override=''): + metas=[]; dfs=[] + for p in paths: + meta,df=parse_pdf(p) + if customer_override: meta.customer_name=customer_override; df['customer_name']=customer_override + if account_override: meta.account_number=account_override; df['account_number']=account_override + metas.append(meta); dfs.append(df) + all_df=enrich(pd.concat(dfs,ignore_index=True) if dfs else pd.DataFrame()) + # remove exact overlap duplicates, retaining first source occurrence + unique_df=all_df.drop_duplicates('exact_key',keep='first').copy() if not all_df.empty else all_df.copy() + return metas,all_df,unique_df + +def reconcile(metas,all_df): + rows=[] + for m in metas: + d=all_df[all_df.source_file.eq(m.source_file)] if not all_df.empty else pd.DataFrame() + ed=float(d.debit.sum()) if not d.empty else 0 + ec=float(d.credit.sum()) if not d.empty else 0 + last=float(d.balance.dropna().iloc[-1]) if not d.empty and d.balance.notna().any() else None + rows.append({**m.to_dict(),'extracted_transactions':len(d),'extracted_debit':ed,'extracted_credit':ec,'extracted_closing_balance':last, + 'debit_difference':None if m.total_debit is None else round(ed-m.total_debit,2), + 'credit_difference':None if m.total_credit is None else round(ec-m.total_credit,2), + 'closing_difference':None if m.closing_balance is None or last is None else round(last-m.closing_balance,2)}) + return pd.DataFrame(rows) + +def monthly_summary(df): + if df.empty:return pd.DataFrame() + x=df.copy(); x['month']=x.transaction_date.dt.to_period('M').astype(str) + return x.groupby('month',dropna=False).agg(transaction_count=('amount','size'),total_debit=('debit','sum'),total_credit=('credit','sum'),net_movement=('credit','sum')).reset_index().assign(net_movement=lambda z:z.total_credit-z.total_debit) + +def mode_summary(df): + if df.empty:return pd.DataFrame() + return df.groupby(['mode','direction']).agg(transaction_count=('amount','size'),amount=('amount','sum')).reset_index() + +def duplicate_summary(all_df): + return pd.DataFrame([ + {'check':'All extracted rows','count':len(all_df)}, + {'check':'Exact duplicate rows','count':int(all_df.exact_duplicate.sum()) if not all_df.empty else 0}, + {'check':'Possible duplicate rows','count':int(all_df.possible_duplicate.sum()) if not all_df.empty else 0}, + {'check':'Unique rows after exact deduplication','count':int(all_df.exact_key.nunique()) if not all_df.empty else 0}, + ]) + +def export_excel(output,metas,all_df,unique_df): + recon=reconcile(metas,all_df) + cust=next((m.customer_name for m in metas if m.customer_name),'') + acct=next((m.account_number for m in metas if m.account_number),'') + banks=', '.join(sorted({m.bank_name for m in metas})) + dashboard=pd.DataFrame([ + ['Customer / Account Holder',cust],['Account Number',acct],['Bank(s)',banks], + ['Statements Uploaded',len(metas)],['Rows Extracted',len(all_df)],['Unique Transactions',len(unique_df)], + ['Exact Duplicate Rows',int(all_df.exact_duplicate.sum()) if not all_df.empty else 0], + ['Possible Duplicate Rows',int(all_df.possible_duplicate.sum()) if not all_df.empty else 0], + ['Total Debit (Unique)',float(unique_df.debit.sum()) if not unique_df.empty else 0], + ['Total Credit (Unique)',float(unique_df.credit.sum()) if not unique_df.empty else 0], + ],columns=['Metric','Value']) + with pd.ExcelWriter(output,engine='xlsxwriter',datetime_format='dd-mmm-yyyy') as w: + dashboard.to_excel(w,'Dashboard',index=False) + recon.to_excel(w,'Statement Reconciliation',index=False) + all_df.to_excel(w,'All Extracted Rows',index=False) + unique_df.to_excel(w,'Unique Transactions',index=False) + all_df[all_df.exact_duplicate].to_excel(w,'Exact Duplicates',index=False) + all_df[all_df.possible_duplicate].to_excel(w,'Possible Duplicates',index=False) + monthly_summary(unique_df).to_excel(w,'Monthly Summary',index=False) + mode_summary(unique_df).to_excel(w,'Mode Summary',index=False) + duplicate_summary(all_df).to_excel(w,'Duplicate Summary',index=False) + notes=pd.DataFrame({'Notes':[ + 'Exact duplicates use transaction date, value date, debit, credit, balance and normalized narration.', + 'Possible duplicates use same date, direction, amount and normalized narration; review before deletion.', + 'Bank-specific parsers are selected automatically. Customer name/account number can be manually overridden in the app.', + 'The workbook is a bank-statement analysis aid, not a substitute for ledger, GST, inventory, receivable/payable and cash-book records.' + ]}) + notes.to_excel(w,'Notes',index=False) + wb=w.book + head=wb.add_format({'bold':True,'bg_color':'#1F4E78','font_color':'white','border':1}) + money=wb.add_format({'num_format':'#,##0.00'}) + for name,ws in w.sheets.items(): + ws.freeze_panes(1,0); ws.autofilter(0,0,0,max(0,ws.dim_colmax)) + ws.set_row(0,22,head) + ws.set_column(0,max(0,ws.dim_colmax),18) + w.sheets['Dashboard'].set_column('A:A',32); w.sheets['Dashboard'].set_column('B:B',28) + return output diff --git a/app/modules/bank_statement_analyzer/parsers/__init__.py b/app/modules/bank_statement_analyzer/parsers/__init__.py new file mode 100644 index 0000000..22a8dc3 --- /dev/null +++ b/app/modules/bank_statement_analyzer/parsers/__init__.py @@ -0,0 +1,2 @@ +from .registry import parse_pdf, detect_parser, PARSERS +from .base import StatementMeta, STANDARD_COLUMNS diff --git a/app/modules/bank_statement_analyzer/parsers/axis.py b/app/modules/bank_statement_analyzer/parsers/axis.py new file mode 100644 index 0000000..f2256a5 --- /dev/null +++ b/app/modules/bank_statement_analyzer/parsers/axis.py @@ -0,0 +1,37 @@ +from __future__ import annotations +import re, pandas as pd +from pathlib import Path +from .base import * +from .common import find + +class AxisParser(BaseParser): + bank_name='Axis Bank'; parser_name='AxisParser' + @classmethod + def detect(cls,text): return 0.98 if 'SMART STATEMENT REPORT' in text.upper() and 'UTIB' in text.upper() else 0 + def parse(self,path,text=None): + text=text or extract_text(path); meta=StatementMeta(bank_name=self.bank_name,source_file=Path(path).name,parser_name=self.parser_name,confidence='High') + # name is first meaningful line after report title + m=re.search(r'Smart Statement Report\s*\n\s*([^\n]+)',text,re.I); meta.customer_name=norm(m.group(1)) if m else '' + meta.account_number=find(r'Statement of Account No\s*-\s*([^\s]*)',text) + meta.ifsc=find(r'IFSC:\s*([A-Z0-9]+)',text) + m=re.search(r'for period\s*\((\d{2}/\d{2}/\d{4})\s+to\s+(\d{2}/\d{2}/\d{4})\)',text,re.I) + if m: meta.period_from=pd.to_datetime(m.group(1),dayfirst=True).strftime('%Y-%m-%d'); meta.period_to=pd.to_datetime(m.group(2),dayfirst=True).strftime('%Y-%m-%d') + meta.opening_balance=amount(find(r'Opening Balance:\s*INR\s*([\d,]+\.\d{2})',text)) + pat=re.compile(r'^\s*(\d+)\s+(\d{2}/\d{2}/\d{4})\s+(\d{2}/\d{2}/\d{4})\s+(.*)$') + rows=[]; cur=None; page=1 + for line in text.splitlines(): + if '\f' in line: page+=line.count('\f') + m=pat.match(line) + if m: + if cur: rows.append(cur) + cur={'transaction_date':m.group(2),'value_date':m.group(3),'body':m.group(4),'source_page':page} + elif cur and line.strip() and not re.match(r'^(S\. No\.|Smart Statement|Page )',line.strip(),re.I): cur['body']+=' '+line.strip() + if cur: rows.append(cur) + out=[] + for r in rows: + b=norm(r['body']); ma=re.search(r'INR\s*([\d,]+\.\d{2})\s+(CR|DR)\s+INR\s*([\d,]+\.\d{2})',b,re.I) + if not ma: continue + txn=amount(ma.group(1)); typ=ma.group(2).upper(); bal=amount(ma.group(3)); narr=b[:ma.start()].strip(); ref='' + z=re.search(r'([A-Z0-9/-]{8,})',narr); ref=z.group(1) if z else '' + out.append({**r,'narration':narr,'reference_no':ref,'debit':txn if typ=='DR' else None,'credit':txn if typ=='CR' else None,'balance':bal}) + return meta,finalize(pd.DataFrame(out),meta) diff --git a/app/modules/bank_statement_analyzer/parsers/base.py b/app/modules/bank_statement_analyzer/parsers/base.py new file mode 100644 index 0000000..6574f8a --- /dev/null +++ b/app/modules/bank_statement_analyzer/parsers/base.py @@ -0,0 +1,91 @@ +from __future__ import annotations +from dataclasses import dataclass, asdict +from pathlib import Path +from typing import Optional +import re, subprocess, tempfile +import pandas as pd +import pdfplumber + +@dataclass +class StatementMeta: + bank_name: str = "" + customer_name: str = "" + account_number: str = "" + customer_id: str = "" + ifsc: str = "" + period_from: str = "" + period_to: str = "" + opening_balance: Optional[float] = None + total_debit: Optional[float] = None + total_credit: Optional[float] = None + closing_balance: Optional[float] = None + source_file: str = "" + parser_name: str = "" + confidence: str = "Medium" + + def to_dict(self): + return asdict(self) + +STANDARD_COLUMNS = [ + "transaction_date", "value_date", "narration", "reference_no", + "debit", "credit", "balance", "bank_name", "customer_name", + "account_number", "source_file", "source_page", "parser_name" +] + +def amount(v): + if v is None: return None + s=str(v).strip().replace('INR','').replace('Rs.','').replace('₹','').replace(',','').replace('+','') + s=s.replace('CR','').replace('DR','').strip() + if s in ('','-'): return None + neg=s.startswith('-') + s=s.lstrip('-') + try: + x=float(s) + return -x if neg else x + except: return None + +def norm(s): return re.sub(r'\s+',' ',str(s or '')).strip() + +def extract_text(path: str|Path) -> str: + """Prefer pdftotext layout output; fall back to pdfplumber.""" + path=str(path) + try: + p=subprocess.run(['pdftotext','-layout',path,'-'], capture_output=True, text=True, timeout=120) + if p.returncode==0 and len(p.stdout.strip())>50: + return p.stdout + except Exception: + pass + parts=[] + with pdfplumber.open(path) as pdf: + for page in pdf.pages: + parts.append(page.extract_text(x_tolerance=1,y_tolerance=3,layout=True) or '') + return '\n\f\n'.join(parts) + +def page_of_line(text: str, position: int) -> int: + return text[:position].count('\f')+1 + +def finalize(df: pd.DataFrame, meta: StatementMeta) -> pd.DataFrame: + if df is None or df.empty: + return pd.DataFrame(columns=STANDARD_COLUMNS) + for c in ['debit','credit','balance']: + df[c]=pd.to_numeric(df.get(c),errors='coerce') + for c in ['transaction_date','value_date']: + df[c]=pd.to_datetime(df.get(c),errors='coerce',dayfirst=True) + df['narration']=df.get('narration','').fillna('').map(norm) + df['reference_no']=df.get('reference_no','').fillna('').map(norm) + df['bank_name']=meta.bank_name + df['customer_name']=meta.customer_name + df['account_number']=meta.account_number + df['source_file']=meta.source_file + df['parser_name']=meta.parser_name + if 'source_page' not in df: df['source_page']=None + for c in STANDARD_COLUMNS: + if c not in df: df[c]=None + return df[STANDARD_COLUMNS] + +class BaseParser: + bank_name='Unknown' + parser_name='BaseParser' + @classmethod + def detect(cls,text:str)->float: return 0.0 + def parse(self,path:str|Path,text:str|None=None): raise NotImplementedError diff --git a/app/modules/bank_statement_analyzer/parsers/common.py b/app/modules/bank_statement_analyzer/parsers/common.py new file mode 100644 index 0000000..162c35b --- /dev/null +++ b/app/modules/bank_statement_analyzer/parsers/common.py @@ -0,0 +1,20 @@ +from __future__ import annotations +import re +from .base import amount, norm + +def find(pattern,text,group=1,flags=re.I|re.M): + m=re.search(pattern,text,flags) + return norm(m.group(group)) if m else '' + +def date_iso(s): + import pandas as pd + x=pd.to_datetime(s,errors='coerce',dayfirst=True) + return '' if pd.isna(x) else x.strftime('%Y-%m-%d') + +def split_pages(text): return text.split('\f') + +def infer_mode(n): + u=(n or '').upper() + for k,v in [('UPI','UPI'),('NEFT','NEFT'),('IMPS','IMPS'),('RTGS','RTGS'),('CASH DEPOSIT','Cash Deposit'),('CASH WITHDRAWAL','Cash Withdrawal'),('ATM','ATM'),('CHEQUE','Cheque'),('CHQ','Cheque'),('POS','POS'),('EDC','Card Settlement'),('ACH','ACH')]: + if k in u:return v + return 'Other' diff --git a/app/modules/bank_statement_analyzer/parsers/hdfc.py b/app/modules/bank_statement_analyzer/parsers/hdfc.py new file mode 100644 index 0000000..19375f6 --- /dev/null +++ b/app/modules/bank_statement_analyzer/parsers/hdfc.py @@ -0,0 +1,58 @@ +from __future__ import annotations +import re, pandas as pd +from pathlib import Path +from .base import * +from .common import find + +class HDFCParser(BaseParser): + bank_name='HDFC Bank'; parser_name='HDFCParser' + @classmethod + def detect(cls,text): + u=text.upper(); return 0.97 if 'HDFC' in u and 'WITHDRAWAL AMT.' in u and 'DEPOSIT AMT.' in u else 0 + def parse(self,path,text=None): + text=text or extract_text(path); meta=StatementMeta(bank_name=self.bank_name,source_file=Path(path).name,parser_name=self.parser_name,confidence='High') + meta.account_number=find(r'Account No\s*:?\s*([0-9X*]+)',text) + meta.customer_id=find(r'Cust ID\s*:?\s*([0-9X*]+)',text) + meta.ifsc=find(r'(?:RTGS/\s*NEFT IFSC|IFSC)\s*:?\s*([A-Z0-9]+)',text) + m=re.search(r'From\s*:\s*(\d{2}/\d{2}/\d{4})\s+To\s*:\s*(\d{2}/\d{2}/\d{4})',text,re.I) + if m: + meta.period_from=pd.to_datetime(m.group(1),dayfirst=True).strftime('%Y-%m-%d'); meta.period_to=pd.to_datetime(m.group(2),dayfirst=True).strftime('%Y-%m-%d') + # first non-empty line before address usually contains customer name; allow manual override in UI + m=re.search(r'\n\s{2,}([^\n:]{3,60})\s*\n.*?Address\s*:',text,re.S|re.I) + if m: meta.customer_name=norm(m.group(1).splitlines()[-1]) + lines=text.splitlines(); rows=[]; cur=None; page=1 + wd_pos,dep_pos,bal_pos=130,165,190 + date_re=re.compile(r'^\s*(\d{2}/\d{2}/\d{2})\s+(.*)$') + for line in lines: + if '\f' in line: page += line.count('\f') + if 'Withdrawal Amt.' in line and 'Deposit Amt.' in line: + wd_pos=line.find('Withdrawal Amt.'); dep_pos=line.find('Deposit Amt.'); bal_pos=line.find('Closing Balance'); continue + m=date_re.match(line) + if m: + if cur: rows.append(cur) + cur={'transaction_date':m.group(1),'value_date':'','raw_lines':[line],'source_page':page,'wd_pos':wd_pos,'dep_pos':dep_pos,'bal_pos':bal_pos} + elif cur and line.strip() and not re.match(r'^(Page No|Statement of account|Date\s+Narration|This is a computer)',line.strip(),re.I): cur['raw_lines'].append(line) + if cur: rows.append(cur) + out=[] + for r in rows: + first=r['raw_lines'][0]; body=' '.join(x.strip() for x in r['raw_lines']) + # identify value date and ref on first line + dts=list(re.finditer(r'\d{2}/\d{2}/\d{2}',first)) + if len(dts)>1: r['value_date']=dts[-1].group() + else: r['value_date']=r['transaction_date'] + nums=list(re.finditer(r'(?=r['bal_pos']-5: bal=x + elif p>=r['dep_pos']-5: credit=x + elif p>=r['wd_pos']-5: debit=x + if bal is None and nums: bal=amount(nums[-1].group()) + # narration is text between date and likely ref/value-date region, plus continuation lines + narr=first[dts[0].end():] + if len(dts)>1: narr=narr[:dts[-1].start()-dts[0].end()] + narr=norm(narr+' '+' '.join(x.strip() for x in r['raw_lines'][1:])) + ref=''; z=re.search(r'\b([A-Z0-9]{10,})\b',body); ref=z.group(1) if z else '' + if debit is None and credit is None: continue + out.append({**r,'narration':narr,'reference_no':ref,'debit':debit,'credit':credit,'balance':bal}) + return meta,finalize(pd.DataFrame(out),meta) diff --git a/app/modules/bank_statement_analyzer/parsers/idfc.py b/app/modules/bank_statement_analyzer/parsers/idfc.py new file mode 100644 index 0000000..f9aa580 --- /dev/null +++ b/app/modules/bank_statement_analyzer/parsers/idfc.py @@ -0,0 +1,56 @@ +from __future__ import annotations +import re +import pandas as pd +from pathlib import Path +from .base import BaseParser, StatementMeta, extract_text, amount, norm, finalize, page_of_line +from .common import find, date_iso + +class IDFCFirstParser(BaseParser): + bank_name='IDFC FIRST Bank'; parser_name='IDFCFirstParser' + @classmethod + def detect(cls,text): + u=text.upper(); return 0.99 if 'IDFC FIRST BANK' in u and 'STATEMENT PERIOD' in u else 0 + def parse(self,path,text=None): + text=text or extract_text(path) + meta=StatementMeta(bank_name=self.bank_name,source_file=Path(path).name,parser_name=self.parser_name,confidence='High') + meta.customer_name=find(r'CUSTOMER NAME\s*:\s*([^\n]+)',text) + meta.account_number=find(r'ACCOUNT NO\s*:\s*([0-9X*]+)',text) + meta.customer_id=find(r'CUSTOMER ID\s*:\s*([0-9X*]+)',text) + meta.ifsc=find(r'IFSC\s*:\s*([A-Z0-9]+)',text) + m=re.search(r'STATEMENT PERIOD\s*:\s*(\d{4}-\d{2}-\d{2})\s+TO\s+(\d{4}-\d{2}-\d{2})',text,re.I) + if m: meta.period_from,meta.period_to=m.groups() + m=re.search(r'Opening Balance\s+Total Debit\s+Total Credit\s+Closing Balance\s*\n\s*([\d,.]+)\s+([\d,.]+)\s+([\d,.]+)\s+([\d,.]+)',text,re.I) + if m: meta.opening_balance,meta.total_debit,meta.total_credit,meta.closing_balance=map(amount,m.groups()) + lines=text.splitlines(); rows=[]; current=None; pos=0 + pat=re.compile(r'^\s*(\d{2}-[A-Za-z]{3}-\d{4})\s+(\d{2}-[A-Za-z]{3}-\d{4})\s+(.*)$') + for line in lines: + mm=pat.match(line) + if mm: + if current: rows.append(current) + current={'transaction_date':mm.group(1),'value_date':mm.group(2),'body':mm.group(3),'source_page':page_of_line(text,pos)} + elif current and line.strip() and not re.match(r'^(STATEMENT|Opening Balance|REGISTERED OFFICE|Page \d+)',line.strip(),re.I): + current['body']+=' '+line.strip() + pos+=len(line)+1 + if current: rows.append(current) + out=[]; prev=meta.opening_balance + for r in rows: + body=norm(r['body']); nums=list(re.finditer(r'(?=prev else None; debit=txn if bal=2: + txn=vals[-2][1] + before=first[:re.search(r'INR\s*[\d,]+\.\d{2}',first,re.I).start()] if re.search(r'INR\s*[\d,]+\.\d{2}',first,re.I) else first + # presence of '-' before first amount often means no debit; inspect spacing/position + p=vals[-2][0] + if prev is not None and bal is not None: + if abs((prev-txn)-bal)<0.05: debit=txn + elif abs((prev+txn)-bal)<0.05: credit=txn + if debit is None and credit is None: + if p<52: debit=txn + else: credit=txn + narr=re.split(r'\s+INR\s*[\d,]+\.\d{2}',alltxt,1,flags=re.I)[0] + ref=''; z=re.search(r'(?:NEFT|IMPS|UPI|RTGS)[/A-Z0-9-]{6,}',alltxt,re.I); ref=z.group(0) if z else '' + if debit is None and credit is None: continue + out.append({**r,'narration':narr,'reference_no':ref,'debit':debit,'credit':credit,'balance':bal}) + if bal is not None: prev=bal + return meta,finalize(pd.DataFrame(out),meta) + +class IndianBankLegacyParser(BaseParser): + bank_name='Indian Bank'; parser_name='IndianBankLegacyParser' + @classmethod + def detect(cls,text): + u=text.upper(); return 0.97 if 'STATEMENT OF ACCOUNT FROM' in u and 'REMITTER' in u and 'CHEQUE NO' in u else 0 + def parse(self,path,text=None): + text=text or extract_text(path); meta=StatementMeta(bank_name=self.bank_name,source_file=Path(path).name,parser_name=self.parser_name,confidence='Medium') + meta.account_number=find(r'for Account Number\s*\.?\s*([0-9X*]+)',text) + m=re.search(r'STATEMENT OF ACCOUNT from\s*(\d{2}/\d{2}/\d{4})\s*to\s*(\d{2}/\d{2}/\d{4})',text,re.I) + if m: + meta.period_from=pd.to_datetime(m.group(1),dayfirst=True).strftime('%Y-%m-%d'); meta.period_to=pd.to_datetime(m.group(2),dayfirst=True).strftime('%Y-%m-%d') + date_re=re.compile(r'^\s*(\d{2}/\d{2})(?:/\d{4})?\s+(\d{2}/\d{2})(?:/\d{4})?\s+(.*)$') + rows=[]; cur=None; page=1 + for line in text.splitlines(): + if '\f' in line: page+=line.count('\f') + m=date_re.match(line) + if m: + if cur: rows.append(cur) + year=(meta.period_from[:4] if meta.period_from else '2024') + td=m.group(1).replace(' ','')+'/'+year; vd=m.group(2).replace(' ','')+'/'+year + cur={'transaction_date':td,'value_date':vd,'lines':[m.group(3)],'source_page':page} + elif cur and line.strip() and not re.match(r'^(Value Post|Date Date|STATEMENT OF ACCOUNT|Page No)',line.strip(),re.I): cur['lines'].append(line) + if cur: rows.append(cur) + out=[]; prev=None + for r in rows: + first=r['lines'][0]; alltxt=norm(' '.join(r['lines'])) + # balance is amount followed by CR/DR at far right + mb=re.search(r'([\d,]+\.\d{2})(CR|DR)\s*$',first,re.I) + if not mb: continue + bal=amount(mb.group(1)); prefix=first[:mb.start()] + nums=list(re.finditer(r'(?70 else None; debit=txn if nums[-1].start()<=70 else None + narr=alltxt + ref=''; z=re.search(r'(?:UPI|NEFT|IMPS|RTGS)[/A-Z0-9-]{6,}',alltxt,re.I); ref=z.group(0) if z else '' + out.append({**r,'narration':narr,'reference_no':ref,'debit':debit,'credit':credit,'balance':bal}) + prev=bal + if out and meta.opening_balance is None: + first=out[0]; txn=(first.get('debit') or 0)-(first.get('credit') or 0); meta.opening_balance=round((first['balance'] or 0)+txn,2) + return meta,finalize(pd.DataFrame(out),meta) diff --git a/app/modules/bank_statement_analyzer/parsers/indusind.py b/app/modules/bank_statement_analyzer/parsers/indusind.py new file mode 100644 index 0000000..7e5333f --- /dev/null +++ b/app/modules/bank_statement_analyzer/parsers/indusind.py @@ -0,0 +1,53 @@ +from __future__ import annotations +import re, pandas as pd +from pathlib import Path +from .base import * +from .common import find + +class IndusIndParser(BaseParser): + bank_name='IndusInd Bank'; parser_name='IndusIndParser' + @classmethod + def detect(cls,text): return 0.98 if 'INDUSIND' in text.upper() and 'STATEMENT OF ACCOUNT' in text.upper() else 0 + def parse(self,path,text=None): + text=text or extract_text(path); meta=StatementMeta(bank_name=self.bank_name,source_file=Path(path).name,parser_name=self.parser_name,confidence='Medium') + meta.account_number=find(r'Account Number\s*:\s*([0-9X*]+)',text) + meta.customer_id=find(r'Cust\.Reln\.No\s*:\s*([0-9X*]+)',text) + meta.ifsc=find(r'IFSC Code\s*:\s*([A-Z0-9]+)',text) + m=re.search(r'Period\s*:\s*(\d{2}[/-][A-Za-z0-9]{2,3}[/-]\d{2,4})\s*(?:to|TO|-)\s*(\d{2}[/-][A-Za-z0-9]{2,3}[/-]\d{2,4})',text,re.I) + if m: + meta.period_from=pd.to_datetime(m.group(1),dayfirst=True).strftime('%Y-%m-%d'); meta.period_to=pd.to_datetime(m.group(2),dayfirst=True).strftime('%Y-%m-%d') + meta.total_debit=amount(find(r'Total Withdrawal Amount\s*:\s*([\d,]+\.\d{2})',text)) + meta.total_credit=amount(find(r'Total Deposit Amount\s*:\s*([\d,]+\.\d{2})',text)) + # rows start with date and often end with amount Dr/Cr and balance Cr + date_re=re.compile(r'^\s*(\d{2}[-/]?[A-Za-z]{3}[-/]?\d{2,4}|\d{2}-\d{2}-\d{4})\s+(.*)$') + rows=[]; cur=None; page=1 + for line in text.splitlines(): + if '\f' in line: page+=line.count('\f') + m=date_re.match(line) + if m: + if cur: rows.append(cur) + cur={'transaction_date':m.group(1),'value_date':m.group(1),'lines':[m.group(2)],'source_page':page} + elif cur and line.strip() and not re.match(r'^(Statement Summary|Opening Balance|Total Withdrawal|Branch Address|The limits)',line.strip(),re.I): cur['lines'].append(line) + if cur: rows.append(cur) + out=[]; prev=None + for r in rows: + first=r['lines'][0]; alltxt=norm(' '.join(r['lines'])) + # Typical: narration ref debit Dr credit Cr balance Cr + tokens=list(re.finditer(r'([\d,]+\.\d{2})\s*(Dr|Cr)',first,re.I)) + if not tokens: continue + debit=credit=bal=None + if len(tokens)>=2: + bal=amount(tokens[-1].group(1)) + txn=amount(tokens[-2].group(1)); typ=tokens[-2].group(2).upper() + if typ=='DR': debit=txn + else: credit=txn + elif len(tokens)==1: + # carried forward / balance-only line; skip + continue + narr=first[:tokens[-2].start()].strip()+' '+' '.join(x.strip() for x in r['lines'][1:]) + ref=''; z=re.search(r'\b([A-Z0-9]{8,})\b',narr); ref=z.group(1) if z else '' + out.append({**r,'narration':norm(narr),'reference_no':ref,'debit':debit,'credit':credit,'balance':bal}) + if out: + f=out[0]; meta.opening_balance=round((f['balance'] or 0)+(f.get('debit') or 0)-(f.get('credit') or 0),2) + meta.closing_balance=out[-1]['balance'] + return meta,finalize(pd.DataFrame(out),meta) diff --git a/app/modules/bank_statement_analyzer/parsers/kotak.py b/app/modules/bank_statement_analyzer/parsers/kotak.py new file mode 100644 index 0000000..69704e7 --- /dev/null +++ b/app/modules/bank_statement_analyzer/parsers/kotak.py @@ -0,0 +1,54 @@ +from __future__ import annotations +import re, pandas as pd +from pathlib import Path +from .base import * +from .common import find + +class KotakParser(BaseParser): + bank_name='Kotak Mahindra Bank'; parser_name='KotakParser' + @classmethod + def detect(cls,text): + u=text.upper(); return 0.98 if 'KOTAK' in u and 'CHEQUE/REFERENCE#' in u and 'TRANSACTION DETAILS' in u else 0 + def parse(self,path,text=None): + text=text or extract_text(path); meta=StatementMeta(bank_name=self.bank_name,source_file=Path(path).name,parser_name=self.parser_name,confidence='High') + meta.account_number=find(r'Account\s*#\s*Variant\s*KOTAK\s*\n.*?([0-9X*]{6,})',text,flags=re.I|re.S) + meta.ifsc=find(r'IFSC\s+([A-Z0-9]+)',text) + m=re.search(r'(\d{2}\s+[A-Za-z]{3},\s*\d{4})\s*-\s*(\d{2}\s+[A-Za-z]{3},\s*\d{4})',text) + if m: + meta.period_from=pd.to_datetime(m.group(1),dayfirst=True).strftime('%Y-%m-%d'); meta.period_to=pd.to_datetime(m.group(2),dayfirst=True).strftime('%Y-%m-%d') + date_re=re.compile(r'^\s*(\d{2}\s+[A-Za-z]{3},\s*\d{4})\s+(.*)$') + rows=[]; cur=None; page=1 + for line in text.splitlines(): + if '\f' in line: page+=line.count('\f') + m=date_re.match(line) + if m: + if ' - ' in m.group(2) and re.match(r'\d{2}\s+[A-Za-z]{3},',m.group(2).strip()): continue + if cur: rows.append(cur) + cur={'transaction_date':m.group(1),'value_date':m.group(1),'lines':[m.group(2)],'source_page':page} + elif cur and line.strip() and not re.match(r'^(DATE\s+TRANSACTION|Need help|Page \d+)',line.strip(),re.I): cur['lines'].append(line) + if cur: rows.append(cur) + out=[] + for r in rows: + first=r['lines'][0]; alltxt=norm(' '.join(r['lines'])) + # amounts are signed in debit/credit columns, final unsigned balance + nums=list(re.finditer(r'([+-]?\d{1,3}(?:,\d{3})*\.\d{2})',first)) + if not nums: continue + bal=amount(nums[-1].group(1)); debit=credit=None + if len(nums)>=2: + raw=nums[-2].group(1); txn=abs(amount(raw) or 0) + if raw.strip().startswith('-'): debit=txn + elif raw.strip().startswith('+'): credit=txn + else: + # position fallback: debit column before credit + credit=txn if nums[-2].start()>95 else None; debit=txn if nums[-2].start()<=95 else None + elif 'OPENING BALANCE' in first.upper(): + meta.opening_balance=bal; continue + narr=first[:nums[-2].start() if len(nums)>=2 else nums[-1].start()].strip()+' '+' '.join(x.strip() for x in r['lines'][1:]) + ref=''; z=re.search(r'\b(?:UPI|NEFT|IMPS|RTGS)-[A-Z0-9]+\b',alltxt,re.I); ref=z.group(0) if z else '' + if debit is None and credit is None: continue + out.append({**r,'narration':norm(narr),'reference_no':ref,'debit':debit,'credit':credit,'balance':bal}) + if out: + if meta.opening_balance is None: + f=out[0]; meta.opening_balance=round((f['balance'] or 0)+(f.get('debit') or 0)-(f.get('credit') or 0),2) + meta.closing_balance=out[-1]['balance'] + return meta,finalize(pd.DataFrame(out),meta) diff --git a/app/modules/bank_statement_analyzer/parsers/registry.py b/app/modules/bank_statement_analyzer/parsers/registry.py new file mode 100644 index 0000000..ebc091f --- /dev/null +++ b/app/modules/bank_statement_analyzer/parsers/registry.py @@ -0,0 +1,25 @@ +from .idfc import IDFCFirstParser +from .axis import AxisParser +from .hdfc import HDFCParser +from .indian_bank import IndianBankModernParser, IndianBankLegacyParser +from .indusind import IndusIndParser +from .kotak import KotakParser +from .sbi import SBIModernParser, SBIOtherParser +from .base import extract_text + +PARSERS=[IDFCFirstParser,AxisParser,HDFCParser,IndianBankModernParser,IndianBankLegacyParser,IndusIndParser,KotakParser,SBIOtherParser,SBIModernParser] + +def detect_parser(text): + scored=sorted(((p.detect(text),p) for p in PARSERS),key=lambda x:x[0],reverse=True) + if not scored or scored[0][0] <= 0: return None,0 + return scored[0][1](),scored[0][0] + +def parse_pdf(path, bank_hint=None): + text=extract_text(path) + if bank_hint: + for p in PARSERS: + if bank_hint.lower() in p.bank_name.lower() or bank_hint.lower() in p.__name__.lower(): + return p().parse(path,text) + parser,score=detect_parser(text) + if parser is None: raise ValueError('Unsupported statement format. Add a bank-specific parser or use a supported sample format.') + return parser.parse(path,text) diff --git a/app/modules/bank_statement_analyzer/parsers/sbi.py b/app/modules/bank_statement_analyzer/parsers/sbi.py new file mode 100644 index 0000000..058b423 --- /dev/null +++ b/app/modules/bank_statement_analyzer/parsers/sbi.py @@ -0,0 +1,100 @@ +from __future__ import annotations +import re, pandas as pd +from pathlib import Path +from .base import * +from .common import find + +class SBIModernParser(BaseParser): + bank_name='State Bank of India'; parser_name='SBIModernParser' + @classmethod + def detect(cls,text): + u=text.upper(); return 0.98 if 'STATE BANK OF INDIA' in u and 'REF NO./CHEQUE' in u and 'DETAILS' in u else 0 + def parse(self,path,text=None): + text=text or extract_text(path); meta=StatementMeta(bank_name=self.bank_name,source_file=Path(path).name,parser_name=self.parser_name,confidence='High') + meta.customer_name=find(r'Account Name\s*:?\s*([^\n]+)',text) + meta.account_number=find(r'Account Number\s*:?\s*([0-9X*]+)',text) + meta.ifsc=find(r'IFS Code\s*:?\s*([A-Z0-9]+)',text) + m=re.search(r'Account Statement from\s*(\d{1,2}\s+[A-Za-z]{3}\s+\d{4})\s+to\s+(\d{1,2}\s+[A-Za-z]{3}\s+\d{4})',text,re.I) + if m: + meta.period_from=pd.to_datetime(m.group(1),dayfirst=True).strftime('%Y-%m-%d'); meta.period_to=pd.to_datetime(m.group(2),dayfirst=True).strftime('%Y-%m-%d') + meta.opening_balance=amount(find(r'Balance as on[^\n]*\n\s*([\d,]+\.\d{2})',text)) + # Format A: date details ref debit credit balance + date_re=re.compile(r'^\s*(\d{1,2}\s+[A-Za-z]{3}\s+\d{4})\s+(.*)$') + rows=[]; cur=None; page=1 + for line in text.splitlines(): + if '\f' in line: page += line.count('\f') + m=date_re.match(line) + if m: + if cur: rows.append(cur) + cur={'transaction_date':m.group(1),'value_date':m.group(1),'lines':[m.group(2)],'source_page':page} + elif cur and line.strip() and not re.match(r'^(Date\s+Details|Txn Date|Account Statement|State Bank)',line.strip(),re.I): + cur['lines'].append(line) + if cur: rows.append(cur) + out=[]; prev=meta.opening_balance + for r in rows: + first=r['lines'][0]; alltxt=norm(' '.join(r['lines'])) + nums=list(re.finditer(r'(?=2: + txn=amount(nums[-2].group(1)); p=nums[-2].start() + if prev is not None and bal is not None: + if abs((prev-txn)-bal)<0.05: debit=txn + elif abs((prev+txn)-bal)<0.05: credit=txn + if debit is None and credit is None: + if p >= 78: credit=txn + else: debit=txn + narr=first[:nums[-2].start() if len(nums)>=2 else nums[-1].start()].strip()+' '+' '.join(x.strip() for x in r['lines'][1:]) + ref=''; z=re.search(r'(?:UPI|NEFT|IMPS|RTGS)[/A-Z0-9-]{6,}',alltxt,re.I); ref=z.group(0) if z else '' + if debit is None and credit is None: continue + out.append({**r,'narration':norm(narr),'reference_no':ref,'debit':debit,'credit':credit,'balance':bal}) + if bal is not None: prev=bal + if out: + if meta.opening_balance is None: + f=out[0]; meta.opening_balance=round((f['balance'] or 0)+(f.get('debit') or 0)-(f.get('credit') or 0),2) + meta.closing_balance=out[-1]['balance'] + return meta,finalize(pd.DataFrame(out),meta) + +class SBIOtherParser(BaseParser): + bank_name='State Bank of India'; parser_name='SBIOtherParser' + @classmethod + def detect(cls,text): + u=text.upper(); return 0.97 if 'TXN DATE' in u and 'VALUE' in u and 'REF NO./CHEQUE' in u and 'BY TRANSFER' in u else 0 + def parse(self,path,text=None): + text=text or extract_text(path); meta=StatementMeta(bank_name=self.bank_name,source_file=Path(path).name,parser_name=self.parser_name,confidence='High') + meta.customer_name=find(r'Account Name\s*:\s*([^\n]+)',text) + meta.account_number=find(r'Account Number\s*:?\s*([0-9X*]+)',text) + meta.ifsc=find(r'IFS Code\s*:?\s*([A-Z0-9]+)',text) + m=re.search(r'Account Statement from\s*(\d{1,2}\s+[A-Za-z]{3}\s+\d{4})\s+to\s+(\d{1,2}\s+[A-Za-z]{3}\s+\d{4})',text,re.I) + if m: + meta.period_from=pd.to_datetime(m.group(1),dayfirst=True).strftime('%Y-%m-%d'); meta.period_to=pd.to_datetime(m.group(2),dayfirst=True).strftime('%Y-%m-%d') + m=re.search(r'Balance as on\s+[^\n]+\n',text,re.I) + date_re=re.compile(r'^\s*(\d{1,2}\s+[A-Za-z]{3}\s+\d{4})\s+(\d{1,2}\s+[A-Za-z]{3}\s+\d{4})\s+(.*)$') + rows=[]; cur=None; page=1 + for line in text.splitlines(): + if '\f' in line: page += line.count('\f') + mm=date_re.match(line) + if mm: + if cur: rows.append(cur) + cur={'transaction_date':mm.group(1),'value_date':mm.group(2),'lines':[mm.group(3)],'source_page':page} + elif cur and line.strip() and not re.match(r'^(Txn Date|Account Statement|Account Name)',line.strip(),re.I): cur['lines'].append(line) + if cur: rows.append(cur) + out=[] + for r in rows: + first=r['lines'][0]; alltxt=norm(' '.join(r['lines'])) + nums=list(re.finditer(r'(?=2: + txn=amount(nums[-2].group(1)); p=nums[-2].start() + # Based on header layout: Debit starts before Credit + if p < 55: debit=txn + else: credit=txn + narr=first[:nums[-2].start() if len(nums)>=2 else nums[-1].start()].strip()+' '+' '.join(x.strip() for x in r['lines'][1:]) + ref=''; z=re.search(r'(?:UPI|NEFT|IMPS|RTGS)[/A-Z0-9-]{6,}',alltxt,re.I); ref=z.group(0) if z else '' + if debit is None and credit is None: continue + out.append({**r,'narration':norm(narr),'reference_no':ref,'debit':debit,'credit':credit,'balance':bal}) + if out: + f=out[0]; meta.opening_balance=round((f['balance'] or 0)+(f.get('debit') or 0)-(f.get('credit') or 0),2); meta.closing_balance=out[-1]['balance'] + return meta,finalize(pd.DataFrame(out),meta) diff --git a/app/modules/bank_statement_analyzer/service.py b/app/modules/bank_statement_analyzer/service.py new file mode 100644 index 0000000..ac90d2b --- /dev/null +++ b/app/modules/bank_statement_analyzer/service.py @@ -0,0 +1,161 @@ +from __future__ import annotations + +import json +import os +import re +import shutil +import uuid +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Iterable + +from fastapi import UploadFile + +from app.modules.documents.services import DEFAULT_STORAGE_ROOT + +from .analyzer import analyze_files, export_excel + +ALLOWED_ROLES = {"Partner", "Manager", "Branch Manager", "Staff", "Employee", "Consultant"} +MAX_FILES = int(os.getenv("BANK_ANALYZER_MAX_FILES", "24")) +MAX_FILE_BYTES = int(os.getenv("BANK_ANALYZER_MAX_FILE_MB", "50")) * 1024 * 1024 +RETENTION_HOURS = int(os.getenv("BANK_ANALYZER_FAILED_RETENTION_HOURS", "24")) +_SAFE = re.compile(r"[^A-Za-z0-9._-]+") + + +def _now() -> datetime: + return datetime.now(timezone.utc) + + +def _root() -> Path: + """Return the automatic ERP work-storage root. + + The document module already resolves its storage location for the current + deployment. Bank-statement jobs use a sibling ``Work`` folder so no new + environment variable or separate path configuration is required. + """ + return DEFAULT_STORAGE_ROOT.parent / "Work" + + +def _segment(value: object, default: str = "NA") -> str: + text = _SAFE.sub("_", str(value or default).strip()).strip("._-") + return text[:80] or default + + +def role_bucket(roles: Iterable[str]) -> str | None: + role_set = set(roles) + for role, bucket in (("Partner", "Partner"), ("Manager", "Manager"), ("Branch Manager", "Manager"), ("Staff", "Staff"), ("Employee", "Staff"), ("Consultant", "Consultant")): + if role in role_set: + return bucket + return None + + +def can_use(roles: Iterable[str]) -> bool: + return bool(set(roles) & ALLOWED_ROLES) + + +def _user_root(user, roles: Iterable[str]) -> Path: + bucket = role_bucket(roles) + if not bucket: + raise PermissionError("Bank Statement Analyzer is available only to Partner, Manager, Staff and Consultant roles.") + label = _segment(getattr(user, "full_name", None) or getattr(user, "email", None) or f"user_{user.id}") + return _root() / bucket / f"{int(user.id)}_{label}" / "Bank_Statement_Analyzer" + + +def cleanup_expired(user, roles: Iterable[str]) -> None: + base = _user_root(user, roles) + if not base.exists(): + return + cutoff = _now() - timedelta(hours=RETENTION_HOURS) + for child in base.iterdir(): + try: + modified = datetime.fromtimestamp(child.stat().st_mtime, tz=timezone.utc) + if child.is_dir() and modified < cutoff: + shutil.rmtree(child, ignore_errors=True) + except OSError: + continue + + +def create_job(user, roles: Iterable[str]) -> tuple[str, Path, Path]: + cleanup_expired(user, roles) + job_id = uuid.uuid4().hex + job = _user_root(user, roles) / job_id + input_dir = job / "Input" + output_dir = job / "Output" + input_dir.mkdir(parents=True, exist_ok=False) + output_dir.mkdir(parents=True, exist_ok=False) + return job_id, input_dir, output_dir + + +def _validate_pdf_header(data: bytes) -> None: + if not data.startswith(b"%PDF-"): + raise ValueError("Only genuine PDF files are allowed.") + + +async def save_uploads(files: list[UploadFile], input_dir: Path) -> list[Path]: + usable = [f for f in files if f and (f.filename or "").strip()] + if not usable: + raise ValueError("Please select at least one PDF bank statement.") + if len(usable) > MAX_FILES: + raise ValueError(f"A maximum of {MAX_FILES} PDF files can be analyzed in one job.") + saved: list[Path] = [] + for index, upload in enumerate(usable, start=1): + name = Path(upload.filename or f"statement_{index}.pdf").name + if Path(name).suffix.lower() != ".pdf": + raise ValueError(f"{name}: only PDF files are allowed.") + safe_name = f"{index:02d}_{_segment(Path(name).stem, f'statement_{index}')}.pdf" + target = input_dir / safe_name + size = 0 + first = b"" + with target.open("wb") as handle: + while True: + chunk = await upload.read(1024 * 1024) + if not chunk: + break + if not first: + first = chunk[:8] + size += len(chunk) + if size > MAX_FILE_BYTES: + raise ValueError(f"{name}: file exceeds the {MAX_FILE_BYTES // (1024 * 1024)} MB limit.") + handle.write(chunk) + _validate_pdf_header(first) + saved.append(target) + return saved + + +def analyze_job(*, user, roles: Iterable[str], job_id: str, paths: list[Path], output_dir: Path, customer_override: str = "", account_override: str = "") -> dict: + metas, all_df, unique_df = analyze_files(paths, customer_override.strip(), account_override.strip()) + output = output_dir / "Bank_Statement_Analysis.xlsx" + export_excel(output, metas, all_df, unique_df) + summary = { + "job_id": job_id, + "owner_user_id": int(user.id), + "created_at": _now().isoformat(), + "statement_count": len(metas), + "rows_extracted": int(len(all_df)), + "unique_transactions": int(len(unique_df)), + "exact_duplicate_rows": int(all_df.exact_duplicate.sum()) if not all_df.empty else 0, + "possible_duplicate_rows": int(all_df.possible_duplicate.sum()) if not all_df.empty else 0, + "banks": sorted({m.bank_name for m in metas}), + "customer_name": next((m.customer_name for m in metas if m.customer_name), ""), + "account_number": next((m.account_number for m in metas if m.account_number), ""), + "output_file": output.name, + } + (output_dir.parent / "job.json").write_text(json.dumps(summary, indent=2), encoding="utf-8") + return summary + + +def resolve_owned_job(user, roles: Iterable[str], job_id: str) -> tuple[Path, dict]: + if not re.fullmatch(r"[a-f0-9]{32}", job_id or ""): + raise FileNotFoundError("Analysis job not found.") + job = _user_root(user, roles) / job_id + meta_path = job / "job.json" + if not meta_path.is_file(): + raise FileNotFoundError("Analysis job not found.") + meta = json.loads(meta_path.read_text(encoding="utf-8")) + if int(meta.get("owner_user_id", 0)) != int(user.id): + raise PermissionError("You cannot access another user's analysis job.") + return job, meta + + +def delete_job(job: Path) -> None: + shutil.rmtree(job, ignore_errors=True) diff --git a/app/modules/bank_statement_analyzer/templates/bank_statement_analyzer/index.html b/app/modules/bank_statement_analyzer/templates/bank_statement_analyzer/index.html new file mode 100644 index 0000000..b8ba4ee --- /dev/null +++ b/app/modules/bank_statement_analyzer/templates/bank_statement_analyzer/index.html @@ -0,0 +1,20 @@ +{% extends "ui/templates/base/layout.html" %} +{% block content %} +
+
+

Bank Statement Analyzer

+

Upload one or more supported PDF bank statements. The analyzer prepares a reconciled Excel workbook with transaction summaries and duplicate checks.

+
+ {% if error %}
Analysis could not be completed.
{{ error }}
{% endif %} +
+ +
+
+
+
+
+
Supported parsers
Axis Bank, HDFC Bank, IDFC FIRST Bank, Indian Bank, IndusInd Bank, Kotak Mahindra Bank and State Bank of India.
Successful jobs are deleted automatically after the Excel response is sent. Failed or abandoned jobs are cleaned after the configured retention period.
+
+
+
+{% endblock %} diff --git a/app/modules/bank_statement_analyzer/templates/bank_statement_analyzer/result.html b/app/modules/bank_statement_analyzer/templates/bank_statement_analyzer/result.html new file mode 100644 index 0000000..79bd796 --- /dev/null +++ b/app/modules/bank_statement_analyzer/templates/bank_statement_analyzer/result.html @@ -0,0 +1,15 @@ +{% extends "ui/templates/base/layout.html" %} +{% block content %} +
+

Analysis completed

Review the summary below and download the Excel workbook. The uploaded PDFs and temporary workbook are removed automatically after the download response completes.

+
+ {% for label, value in [('Statements', summary.statement_count), ('Rows extracted', summary.rows_extracted), ('Unique transactions', summary.unique_transactions), ('Exact duplicate rows', summary.exact_duplicate_rows), ('Possible duplicate rows', summary.possible_duplicate_rows)] %} +
{{ label }}
{{ value }}
+ {% endfor %} +
+
+
Bank(s)
{{ summary.banks|join(', ') }}
Account holder
{{ summary.customer_name or '-' }}
+
Download Analysis Excel
+
+
+{% endblock %} diff --git a/app/modules/bank_statement_analyzer/ui.py b/app/modules/bank_statement_analyzer/ui.py new file mode 100644 index 0000000..f877ba2 --- /dev/null +++ b/app/modules/bank_statement_analyzer/ui.py @@ -0,0 +1,116 @@ +from __future__ import annotations + +import shutil +from pathlib import Path + +from fastapi import APIRouter, File, Form, Request, UploadFile +from fastapi.responses import FileResponse, RedirectResponse +from starlette.background import BackgroundTask + +from app.core.db.common import CommonSessionLocal +from app.core.http_responses import ui_access_denied, not_found_response +from app.core.security.csrf import get_or_create_csrf_token, validate_csrf +from app.core.security.session_auth import get_current_user +from app.core.templating import templates +from app.modules.core.rbac.deps import get_user_permissions, get_user_roles +from .service import can_use, create_job, save_uploads, analyze_job, resolve_owned_job, delete_job + +router = APIRouter(prefix="/tools/bank-statement-analyzer", tags=["bank-statement-analyzer-ui"]) + + +def _ctx(request, db, user, **extra): + data = { + "request": request, + "current_user": user, + "current_user_roles": get_user_roles(db, user.id), + "current_user_permissions": get_user_permissions(db, user.id), + "csrf_token": get_or_create_csrf_token(request), + "title": "Bank Statement Analyzer", + } + data.update(extra) + return data + + +def _auth(request, db): + user = get_current_user(request, db=db) + if not user: + return None, None, RedirectResponse("/login", status_code=303) + roles = get_user_roles(db, user.id) + if not can_use(roles): + return user, roles, ui_access_denied("Bank Statement Analyzer is available only to Partner, Manager, Staff and Consultant roles.") + return user, roles, None + + +@router.get("") +def index(request: Request): + db = CommonSessionLocal() + try: + user, roles, denied = _auth(request, db) + if denied: + return denied + return templates.TemplateResponse("modules/bank_statement_analyzer/templates/bank_statement_analyzer/index.html", _ctx(request, db, user, error="")) + finally: + db.close() + + +@router.post("/analyze") +async def analyze(request: Request, csrf_token: str = Form(...), customer_name: str = Form(""), account_number: str = Form(""), statements: list[UploadFile] = File(...)): + db = CommonSessionLocal() + job_dir: Path | None = None + try: + user, roles, denied = _auth(request, db) + if denied: + return denied + validate_csrf(request, csrf_token) + job_id, input_dir, output_dir = create_job(user, roles) + job_dir = input_dir.parent + paths = await save_uploads(statements, input_dir) + summary = analyze_job(user=user, roles=roles, job_id=job_id, paths=paths, output_dir=output_dir, customer_override=customer_name, account_override=account_number) + return templates.TemplateResponse("modules/bank_statement_analyzer/templates/bank_statement_analyzer/result.html", _ctx(request, db, user, summary=summary)) + except Exception as exc: + if job_dir and job_dir.exists(): + # Failed jobs are retained for the configured short retention period for troubleshooting/retry. + pass + user = get_current_user(request, db=db) + if not user: + return RedirectResponse("/login", status_code=303) + return templates.TemplateResponse("modules/bank_statement_analyzer/templates/bank_statement_analyzer/index.html", _ctx(request, db, user, error=str(exc)), status_code=400) + finally: + db.close() + + +@router.get("/{job_id}/download") +def download(job_id: str, request: Request): + db = CommonSessionLocal() + try: + user, roles, denied = _auth(request, db) + if denied: + return denied + try: + job, meta = resolve_owned_job(user, roles, job_id) + except FileNotFoundError: + return not_found_response(request, "Analysis job not found or already cleaned up.") + output = job / "Output" / meta["output_file"] + if not output.is_file(): + return not_found_response(request, "Analysis workbook not found.") + return FileResponse(path=output, filename="Bank_Statement_Analysis.xlsx", media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", background=BackgroundTask(delete_job, job)) + finally: + db.close() + + +@router.post("/{job_id}/delete") +def delete(job_id: str, request: Request, csrf_token: str = Form(...)): + db = CommonSessionLocal() + try: + user, roles, denied = _auth(request, db) + if denied: + return denied + validate_csrf(request, csrf_token) + try: + job, _ = resolve_owned_job(user, roles, job_id) + delete_job(job) + except FileNotFoundError: + pass + return RedirectResponse("/tools/bank-statement-analyzer", status_code=303) + finally: + db.close() diff --git a/app/ui/app.py b/app/ui/app.py index c61e61d..50872b1 100644 --- a/app/ui/app.py +++ b/app/ui/app.py @@ -32,6 +32,7 @@ from app.modules.workspace_navigation.ui import router as workspace_navigation_r from app.modules.firm_admin_dashboard.ui import router as firm_admin_dashboard_router from app.modules.aqmm_dashboard.ui import router as aqmm_dashboard_router from app.modules.peer_review_export.ui import router as peer_review_export_router +from app.modules.bank_statement_analyzer.ui import router as bank_statement_analyzer_router def mount_ui(app: FastAPI) -> None: @@ -50,6 +51,7 @@ def mount_ui(app: FastAPI) -> None: app.include_router(services_ui_router) app.include_router(aqmm_dashboard_router) app.include_router(peer_review_export_router) + app.include_router(bank_statement_analyzer_router) app.include_router(work_tracker_ui_router) app.include_router(billing_ui_router) app.include_router(platform_billing_ui_router) diff --git a/app/ui/templates/components/consultant_navigation_v2.html b/app/ui/templates/components/consultant_navigation_v2.html index d6d600e..79fcd09 100644 --- a/app/ui/templates/components/consultant_navigation_v2.html +++ b/app/ui/templates/components/consultant_navigation_v2.html @@ -61,6 +61,12 @@ 'visible': true, 'active': _consultant_path.startswith('/consultant/profile') }, + { + 'label': 'Bank Analyzer', + 'url': '/tools/bank-statement-analyzer', + 'visible': true, + 'active': _consultant_path.startswith('/tools/bank-statement-analyzer') + }, { 'label': 'Alerts', 'url': '/alerts', diff --git a/app/ui/templates/components/manager_navigation_v2.html b/app/ui/templates/components/manager_navigation_v2.html index 96a619d..beac61e 100644 --- a/app/ui/templates/components/manager_navigation_v2.html +++ b/app/ui/templates/components/manager_navigation_v2.html @@ -53,6 +53,12 @@ 'visible': true, 'active': _manager_path.startswith('/employees/leave') }, + { + 'label': 'Bank Analyzer', + 'url': '/tools/bank-statement-analyzer', + 'visible': true, + 'active': _manager_path.startswith('/tools/bank-statement-analyzer') + }, { 'label': 'Alerts', 'url': '/alerts', diff --git a/app/ui/templates/components/partner_navigation_v2.html b/app/ui/templates/components/partner_navigation_v2.html index 57e8e6d..d041955 100644 --- a/app/ui/templates/components/partner_navigation_v2.html +++ b/app/ui/templates/components/partner_navigation_v2.html @@ -58,6 +58,12 @@ 'visible': true, 'active': _partner_path.startswith('/billing') }, + { + 'label': 'Bank Analyzer', + 'url': '/tools/bank-statement-analyzer', + 'visible': true, + 'active': _partner_path.startswith('/tools/bank-statement-analyzer') + }, { 'label': 'Alerts', 'url': '/alerts', diff --git a/app/ui/templates/components/staff_navigation_v2.html b/app/ui/templates/components/staff_navigation_v2.html index 9df7c28..aa0e637 100644 --- a/app/ui/templates/components/staff_navigation_v2.html +++ b/app/ui/templates/components/staff_navigation_v2.html @@ -62,6 +62,12 @@ 'visible': can_view_employee_portal(current_user, current_user_permissions, current_user_roles), 'active': _staff_path.startswith('/employee/profile') }, + { + 'label': 'Bank Analyzer', + 'url': '/tools/bank-statement-analyzer', + 'visible': true, + 'active': _staff_path.startswith('/tools/bank-statement-analyzer') + }, { 'label': 'Alerts', 'url': '/alerts', diff --git a/requirements.txt b/requirements.txt index bd3c841..8c8b244 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,3 +15,6 @@ email-validator dnspython==2.7.0 openpyxl itsdangerous +pandas +xlsxwriter +pdfplumber