Add bank statement analyzer with automatic work storage
This commit is contained in:
A R R R Associates
@@ -0,0 +1,2 @@
|
||||
from .registry import parse_pdf, detect_parser, PARSERS
|
||||
from .base import StatementMeta, STANDARD_COLUMNS
|
||||
@@ -0,0 +1,37 @@
|
||||
from __future__ import annotations
|
||||
import re, pandas as pd
|
||||
from pathlib import Path
|
||||
from .base import *
|
||||
from .common import find
|
||||
|
||||
class AxisParser(BaseParser):
|
||||
bank_name='Axis Bank'; parser_name='AxisParser'
|
||||
@classmethod
|
||||
def detect(cls,text): return 0.98 if 'SMART STATEMENT REPORT' in text.upper() and 'UTIB' in text.upper() else 0
|
||||
def parse(self,path,text=None):
|
||||
text=text or extract_text(path); meta=StatementMeta(bank_name=self.bank_name,source_file=Path(path).name,parser_name=self.parser_name,confidence='High')
|
||||
# name is first meaningful line after report title
|
||||
m=re.search(r'Smart Statement Report\s*\n\s*([^\n]+)',text,re.I); meta.customer_name=norm(m.group(1)) if m else ''
|
||||
meta.account_number=find(r'Statement of Account No\s*-\s*([^\s]*)',text)
|
||||
meta.ifsc=find(r'IFSC:\s*([A-Z0-9]+)',text)
|
||||
m=re.search(r'for period\s*\((\d{2}/\d{2}/\d{4})\s+to\s+(\d{2}/\d{2}/\d{4})\)',text,re.I)
|
||||
if m: meta.period_from=pd.to_datetime(m.group(1),dayfirst=True).strftime('%Y-%m-%d'); meta.period_to=pd.to_datetime(m.group(2),dayfirst=True).strftime('%Y-%m-%d')
|
||||
meta.opening_balance=amount(find(r'Opening Balance:\s*INR\s*([\d,]+\.\d{2})',text))
|
||||
pat=re.compile(r'^\s*(\d+)\s+(\d{2}/\d{2}/\d{4})\s+(\d{2}/\d{2}/\d{4})\s+(.*)$')
|
||||
rows=[]; cur=None; page=1
|
||||
for line in text.splitlines():
|
||||
if '\f' in line: page+=line.count('\f')
|
||||
m=pat.match(line)
|
||||
if m:
|
||||
if cur: rows.append(cur)
|
||||
cur={'transaction_date':m.group(2),'value_date':m.group(3),'body':m.group(4),'source_page':page}
|
||||
elif cur and line.strip() and not re.match(r'^(S\. No\.|Smart Statement|Page )',line.strip(),re.I): cur['body']+=' '+line.strip()
|
||||
if cur: rows.append(cur)
|
||||
out=[]
|
||||
for r in rows:
|
||||
b=norm(r['body']); ma=re.search(r'INR\s*([\d,]+\.\d{2})\s+(CR|DR)\s+INR\s*([\d,]+\.\d{2})',b,re.I)
|
||||
if not ma: continue
|
||||
txn=amount(ma.group(1)); typ=ma.group(2).upper(); bal=amount(ma.group(3)); narr=b[:ma.start()].strip(); ref=''
|
||||
z=re.search(r'([A-Z0-9/-]{8,})',narr); ref=z.group(1) if z else ''
|
||||
out.append({**r,'narration':narr,'reference_no':ref,'debit':txn if typ=='DR' else None,'credit':txn if typ=='CR' else None,'balance':bal})
|
||||
return meta,finalize(pd.DataFrame(out),meta)
|
||||
@@ -0,0 +1,91 @@
|
||||
from __future__ import annotations
|
||||
from dataclasses import dataclass, asdict
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
import re, subprocess, tempfile
|
||||
import pandas as pd
|
||||
import pdfplumber
|
||||
|
||||
@dataclass
|
||||
class StatementMeta:
|
||||
bank_name: str = ""
|
||||
customer_name: str = ""
|
||||
account_number: str = ""
|
||||
customer_id: str = ""
|
||||
ifsc: str = ""
|
||||
period_from: str = ""
|
||||
period_to: str = ""
|
||||
opening_balance: Optional[float] = None
|
||||
total_debit: Optional[float] = None
|
||||
total_credit: Optional[float] = None
|
||||
closing_balance: Optional[float] = None
|
||||
source_file: str = ""
|
||||
parser_name: str = ""
|
||||
confidence: str = "Medium"
|
||||
|
||||
def to_dict(self):
|
||||
return asdict(self)
|
||||
|
||||
STANDARD_COLUMNS = [
|
||||
"transaction_date", "value_date", "narration", "reference_no",
|
||||
"debit", "credit", "balance", "bank_name", "customer_name",
|
||||
"account_number", "source_file", "source_page", "parser_name"
|
||||
]
|
||||
|
||||
def amount(v):
|
||||
if v is None: return None
|
||||
s=str(v).strip().replace('INR','').replace('Rs.','').replace('₹','').replace(',','').replace('+','')
|
||||
s=s.replace('CR','').replace('DR','').strip()
|
||||
if s in ('','-'): return None
|
||||
neg=s.startswith('-')
|
||||
s=s.lstrip('-')
|
||||
try:
|
||||
x=float(s)
|
||||
return -x if neg else x
|
||||
except: return None
|
||||
|
||||
def norm(s): return re.sub(r'\s+',' ',str(s or '')).strip()
|
||||
|
||||
def extract_text(path: str|Path) -> str:
|
||||
"""Prefer pdftotext layout output; fall back to pdfplumber."""
|
||||
path=str(path)
|
||||
try:
|
||||
p=subprocess.run(['pdftotext','-layout',path,'-'], capture_output=True, text=True, timeout=120)
|
||||
if p.returncode==0 and len(p.stdout.strip())>50:
|
||||
return p.stdout
|
||||
except Exception:
|
||||
pass
|
||||
parts=[]
|
||||
with pdfplumber.open(path) as pdf:
|
||||
for page in pdf.pages:
|
||||
parts.append(page.extract_text(x_tolerance=1,y_tolerance=3,layout=True) or '')
|
||||
return '\n\f\n'.join(parts)
|
||||
|
||||
def page_of_line(text: str, position: int) -> int:
|
||||
return text[:position].count('\f')+1
|
||||
|
||||
def finalize(df: pd.DataFrame, meta: StatementMeta) -> pd.DataFrame:
|
||||
if df is None or df.empty:
|
||||
return pd.DataFrame(columns=STANDARD_COLUMNS)
|
||||
for c in ['debit','credit','balance']:
|
||||
df[c]=pd.to_numeric(df.get(c),errors='coerce')
|
||||
for c in ['transaction_date','value_date']:
|
||||
df[c]=pd.to_datetime(df.get(c),errors='coerce',dayfirst=True)
|
||||
df['narration']=df.get('narration','').fillna('').map(norm)
|
||||
df['reference_no']=df.get('reference_no','').fillna('').map(norm)
|
||||
df['bank_name']=meta.bank_name
|
||||
df['customer_name']=meta.customer_name
|
||||
df['account_number']=meta.account_number
|
||||
df['source_file']=meta.source_file
|
||||
df['parser_name']=meta.parser_name
|
||||
if 'source_page' not in df: df['source_page']=None
|
||||
for c in STANDARD_COLUMNS:
|
||||
if c not in df: df[c]=None
|
||||
return df[STANDARD_COLUMNS]
|
||||
|
||||
class BaseParser:
|
||||
bank_name='Unknown'
|
||||
parser_name='BaseParser'
|
||||
@classmethod
|
||||
def detect(cls,text:str)->float: return 0.0
|
||||
def parse(self,path:str|Path,text:str|None=None): raise NotImplementedError
|
||||
@@ -0,0 +1,20 @@
|
||||
from __future__ import annotations
|
||||
import re
|
||||
from .base import amount, norm
|
||||
|
||||
def find(pattern,text,group=1,flags=re.I|re.M):
|
||||
m=re.search(pattern,text,flags)
|
||||
return norm(m.group(group)) if m else ''
|
||||
|
||||
def date_iso(s):
|
||||
import pandas as pd
|
||||
x=pd.to_datetime(s,errors='coerce',dayfirst=True)
|
||||
return '' if pd.isna(x) else x.strftime('%Y-%m-%d')
|
||||
|
||||
def split_pages(text): return text.split('\f')
|
||||
|
||||
def infer_mode(n):
|
||||
u=(n or '').upper()
|
||||
for k,v in [('UPI','UPI'),('NEFT','NEFT'),('IMPS','IMPS'),('RTGS','RTGS'),('CASH DEPOSIT','Cash Deposit'),('CASH WITHDRAWAL','Cash Withdrawal'),('ATM','ATM'),('CHEQUE','Cheque'),('CHQ','Cheque'),('POS','POS'),('EDC','Card Settlement'),('ACH','ACH')]:
|
||||
if k in u:return v
|
||||
return 'Other'
|
||||
@@ -0,0 +1,58 @@
|
||||
from __future__ import annotations
|
||||
import re, pandas as pd
|
||||
from pathlib import Path
|
||||
from .base import *
|
||||
from .common import find
|
||||
|
||||
class HDFCParser(BaseParser):
|
||||
bank_name='HDFC Bank'; parser_name='HDFCParser'
|
||||
@classmethod
|
||||
def detect(cls,text):
|
||||
u=text.upper(); return 0.97 if 'HDFC' in u and 'WITHDRAWAL AMT.' in u and 'DEPOSIT AMT.' in u else 0
|
||||
def parse(self,path,text=None):
|
||||
text=text or extract_text(path); meta=StatementMeta(bank_name=self.bank_name,source_file=Path(path).name,parser_name=self.parser_name,confidence='High')
|
||||
meta.account_number=find(r'Account No\s*:?\s*([0-9X*]+)',text)
|
||||
meta.customer_id=find(r'Cust ID\s*:?\s*([0-9X*]+)',text)
|
||||
meta.ifsc=find(r'(?:RTGS/\s*NEFT IFSC|IFSC)\s*:?\s*([A-Z0-9]+)',text)
|
||||
m=re.search(r'From\s*:\s*(\d{2}/\d{2}/\d{4})\s+To\s*:\s*(\d{2}/\d{2}/\d{4})',text,re.I)
|
||||
if m:
|
||||
meta.period_from=pd.to_datetime(m.group(1),dayfirst=True).strftime('%Y-%m-%d'); meta.period_to=pd.to_datetime(m.group(2),dayfirst=True).strftime('%Y-%m-%d')
|
||||
# first non-empty line before address usually contains customer name; allow manual override in UI
|
||||
m=re.search(r'\n\s{2,}([^\n:]{3,60})\s*\n.*?Address\s*:',text,re.S|re.I)
|
||||
if m: meta.customer_name=norm(m.group(1).splitlines()[-1])
|
||||
lines=text.splitlines(); rows=[]; cur=None; page=1
|
||||
wd_pos,dep_pos,bal_pos=130,165,190
|
||||
date_re=re.compile(r'^\s*(\d{2}/\d{2}/\d{2})\s+(.*)$')
|
||||
for line in lines:
|
||||
if '\f' in line: page += line.count('\f')
|
||||
if 'Withdrawal Amt.' in line and 'Deposit Amt.' in line:
|
||||
wd_pos=line.find('Withdrawal Amt.'); dep_pos=line.find('Deposit Amt.'); bal_pos=line.find('Closing Balance'); continue
|
||||
m=date_re.match(line)
|
||||
if m:
|
||||
if cur: rows.append(cur)
|
||||
cur={'transaction_date':m.group(1),'value_date':'','raw_lines':[line],'source_page':page,'wd_pos':wd_pos,'dep_pos':dep_pos,'bal_pos':bal_pos}
|
||||
elif cur and line.strip() and not re.match(r'^(Page No|Statement of account|Date\s+Narration|This is a computer)',line.strip(),re.I): cur['raw_lines'].append(line)
|
||||
if cur: rows.append(cur)
|
||||
out=[]
|
||||
for r in rows:
|
||||
first=r['raw_lines'][0]; body=' '.join(x.strip() for x in r['raw_lines'])
|
||||
# identify value date and ref on first line
|
||||
dts=list(re.finditer(r'\d{2}/\d{2}/\d{2}',first))
|
||||
if len(dts)>1: r['value_date']=dts[-1].group()
|
||||
else: r['value_date']=r['transaction_date']
|
||||
nums=list(re.finditer(r'(?<!\d)(?:\d{1,3}(?:,\d{3})+|\d+)\.\d{2}(?!\d)',first))
|
||||
debit=credit=bal=None
|
||||
for n in nums:
|
||||
x=amount(n.group()); p=n.start()
|
||||
if p>=r['bal_pos']-5: bal=x
|
||||
elif p>=r['dep_pos']-5: credit=x
|
||||
elif p>=r['wd_pos']-5: debit=x
|
||||
if bal is None and nums: bal=amount(nums[-1].group())
|
||||
# narration is text between date and likely ref/value-date region, plus continuation lines
|
||||
narr=first[dts[0].end():]
|
||||
if len(dts)>1: narr=narr[:dts[-1].start()-dts[0].end()]
|
||||
narr=norm(narr+' '+' '.join(x.strip() for x in r['raw_lines'][1:]))
|
||||
ref=''; z=re.search(r'\b([A-Z0-9]{10,})\b',body); ref=z.group(1) if z else ''
|
||||
if debit is None and credit is None: continue
|
||||
out.append({**r,'narration':narr,'reference_no':ref,'debit':debit,'credit':credit,'balance':bal})
|
||||
return meta,finalize(pd.DataFrame(out),meta)
|
||||
@@ -0,0 +1,56 @@
|
||||
from __future__ import annotations
|
||||
import re
|
||||
import pandas as pd
|
||||
from pathlib import Path
|
||||
from .base import BaseParser, StatementMeta, extract_text, amount, norm, finalize, page_of_line
|
||||
from .common import find, date_iso
|
||||
|
||||
class IDFCFirstParser(BaseParser):
|
||||
bank_name='IDFC FIRST Bank'; parser_name='IDFCFirstParser'
|
||||
@classmethod
|
||||
def detect(cls,text):
|
||||
u=text.upper(); return 0.99 if 'IDFC FIRST BANK' in u and 'STATEMENT PERIOD' in u else 0
|
||||
def parse(self,path,text=None):
|
||||
text=text or extract_text(path)
|
||||
meta=StatementMeta(bank_name=self.bank_name,source_file=Path(path).name,parser_name=self.parser_name,confidence='High')
|
||||
meta.customer_name=find(r'CUSTOMER NAME\s*:\s*([^\n]+)',text)
|
||||
meta.account_number=find(r'ACCOUNT NO\s*:\s*([0-9X*]+)',text)
|
||||
meta.customer_id=find(r'CUSTOMER ID\s*:\s*([0-9X*]+)',text)
|
||||
meta.ifsc=find(r'IFSC\s*:\s*([A-Z0-9]+)',text)
|
||||
m=re.search(r'STATEMENT PERIOD\s*:\s*(\d{4}-\d{2}-\d{2})\s+TO\s+(\d{4}-\d{2}-\d{2})',text,re.I)
|
||||
if m: meta.period_from,meta.period_to=m.groups()
|
||||
m=re.search(r'Opening Balance\s+Total Debit\s+Total Credit\s+Closing Balance\s*\n\s*([\d,.]+)\s+([\d,.]+)\s+([\d,.]+)\s+([\d,.]+)',text,re.I)
|
||||
if m: meta.opening_balance,meta.total_debit,meta.total_credit,meta.closing_balance=map(amount,m.groups())
|
||||
lines=text.splitlines(); rows=[]; current=None; pos=0
|
||||
pat=re.compile(r'^\s*(\d{2}-[A-Za-z]{3}-\d{4})\s+(\d{2}-[A-Za-z]{3}-\d{4})\s+(.*)$')
|
||||
for line in lines:
|
||||
mm=pat.match(line)
|
||||
if mm:
|
||||
if current: rows.append(current)
|
||||
current={'transaction_date':mm.group(1),'value_date':mm.group(2),'body':mm.group(3),'source_page':page_of_line(text,pos)}
|
||||
elif current and line.strip() and not re.match(r'^(STATEMENT|Opening Balance|REGISTERED OFFICE|Page \d+)',line.strip(),re.I):
|
||||
current['body']+=' '+line.strip()
|
||||
pos+=len(line)+1
|
||||
if current: rows.append(current)
|
||||
out=[]; prev=meta.opening_balance
|
||||
for r in rows:
|
||||
body=norm(r['body']); nums=list(re.finditer(r'(?<!\d)(?:\d{1,3}(?:,\d{2,3})+|\d+)\.\d{2}(?!\d)',body))
|
||||
if not nums: continue
|
||||
bal=amount(nums[-1].group()); rem=body[:nums[-1].start()].strip(); vals=[amount(x.group()) for x in nums[:-1]]
|
||||
debit=credit=None
|
||||
if vals:
|
||||
txn=vals[-1]
|
||||
if prev is not None and bal is not None:
|
||||
d1=round(prev-txn,2); c1=round(prev+txn,2)
|
||||
if abs(d1-bal)<0.02: debit=txn
|
||||
elif abs(c1-bal)<0.02: credit=txn
|
||||
else:
|
||||
# explicit columns often leave one amount only; infer from change
|
||||
credit=txn if bal>=prev else None; debit=txn if bal<prev else None
|
||||
else: credit=txn
|
||||
ref=''
|
||||
z=re.search(r'(?:UPI|NEFT|IMPS|RTGS)[/-](?:MOB/|OPM/|INET/|DR/|CR/)?([A-Z0-9]{8,})',rem,re.I)
|
||||
if z: ref=z.group(1)
|
||||
out.append({**r,'narration':rem,'reference_no':ref,'debit':debit,'credit':credit,'balance':bal})
|
||||
if bal is not None: prev=bal
|
||||
return meta,finalize(pd.DataFrame(out),meta)
|
||||
@@ -0,0 +1,102 @@
|
||||
from __future__ import annotations
|
||||
import re, pandas as pd
|
||||
from pathlib import Path
|
||||
from .base import *
|
||||
from .common import find
|
||||
|
||||
class IndianBankModernParser(BaseParser):
|
||||
bank_name='Indian Bank'; parser_name='IndianBankModernParser'
|
||||
@classmethod
|
||||
def detect(cls,text):
|
||||
u=text.upper(); return 0.98 if 'ACCOUNT STATEMENT' in u and 'TRANSACTION DETAILS' in u and 'TOTAL CREDITS' in u else 0
|
||||
def parse(self,path,text=None):
|
||||
text=text or extract_text(path); meta=StatementMeta(bank_name=self.bank_name,source_file=Path(path).name,parser_name=self.parser_name,confidence='High')
|
||||
meta.customer_name=find(r'Account Holder Name\s*\n\s*([^\n]+)',text)
|
||||
meta.account_number=find(r'Account Number\s*\n?\s*([0-9X*]+)',text)
|
||||
m=re.search(r'For period:\s*(\d{2}\s+[A-Za-z]{3}\s+\d{4})\s*-\s*(\d{2}\s+[A-Za-z]{3}\s+\d{4})',text,re.I)
|
||||
if m:
|
||||
meta.period_from=pd.to_datetime(m.group(1),dayfirst=True).strftime('%Y-%m-%d'); meta.period_to=pd.to_datetime(m.group(2),dayfirst=True).strftime('%Y-%m-%d')
|
||||
meta.opening_balance=amount(find(r'Opening Balance\s+INR\s*([\d,]+\.\d{2})',text))
|
||||
meta.total_credit=amount(find(r'Total Credits\s+\+\s*INR\s*([\d,]+\.\d{2})',text))
|
||||
meta.total_debit=amount(find(r'Total Debits\s+-\s*INR\s*([\d,]+\.\d{2})',text))
|
||||
meta.closing_balance=amount(find(r'Ending Balance\s+INR\s*([\d,]+\.\d{2})',text))
|
||||
date_re=re.compile(r'^\s*([A-Za-z]{3}\s+\d{2}\s+\d{4})\s+(.*)$')
|
||||
rows=[]; cur=None; page=1
|
||||
for line in text.splitlines():
|
||||
if '\f' in line: page+=line.count('\f')
|
||||
m=date_re.match(line)
|
||||
if m:
|
||||
if cur: rows.append(cur)
|
||||
cur={'transaction_date':m.group(1),'value_date':m.group(1),'lines':[m.group(2)],'source_page':page}
|
||||
elif cur and line.strip() and not re.match(r'^(Date\s+Transaction|ACCOUNT STATEMENT|Page)',line.strip(),re.I): cur['lines'].append(line)
|
||||
if cur: rows.append(cur)
|
||||
out=[]; prev=meta.opening_balance
|
||||
for r in rows:
|
||||
first=r['lines'][0]; alltxt=norm(' '.join(r['lines']))
|
||||
vals=[(m.start(),amount(m.group(1))) for m in re.finditer(r'INR\s*([\d,]+\.\d{2})',first,re.I)]
|
||||
# columns in sample: debit ~45, credit ~65, balance ~85. Use signs/placeholders and running balance.
|
||||
bal=vals[-1][1] if vals else None; debit=credit=None
|
||||
if len(vals)>=2:
|
||||
txn=vals[-2][1]
|
||||
before=first[:re.search(r'INR\s*[\d,]+\.\d{2}',first,re.I).start()] if re.search(r'INR\s*[\d,]+\.\d{2}',first,re.I) else first
|
||||
# presence of '-' before first amount often means no debit; inspect spacing/position
|
||||
p=vals[-2][0]
|
||||
if prev is not None and bal is not None:
|
||||
if abs((prev-txn)-bal)<0.05: debit=txn
|
||||
elif abs((prev+txn)-bal)<0.05: credit=txn
|
||||
if debit is None and credit is None:
|
||||
if p<52: debit=txn
|
||||
else: credit=txn
|
||||
narr=re.split(r'\s+INR\s*[\d,]+\.\d{2}',alltxt,1,flags=re.I)[0]
|
||||
ref=''; z=re.search(r'(?:NEFT|IMPS|UPI|RTGS)[/A-Z0-9-]{6,}',alltxt,re.I); ref=z.group(0) if z else ''
|
||||
if debit is None and credit is None: continue
|
||||
out.append({**r,'narration':narr,'reference_no':ref,'debit':debit,'credit':credit,'balance':bal})
|
||||
if bal is not None: prev=bal
|
||||
return meta,finalize(pd.DataFrame(out),meta)
|
||||
|
||||
class IndianBankLegacyParser(BaseParser):
|
||||
bank_name='Indian Bank'; parser_name='IndianBankLegacyParser'
|
||||
@classmethod
|
||||
def detect(cls,text):
|
||||
u=text.upper(); return 0.97 if 'STATEMENT OF ACCOUNT FROM' in u and 'REMITTER' in u and 'CHEQUE NO' in u else 0
|
||||
def parse(self,path,text=None):
|
||||
text=text or extract_text(path); meta=StatementMeta(bank_name=self.bank_name,source_file=Path(path).name,parser_name=self.parser_name,confidence='Medium')
|
||||
meta.account_number=find(r'for Account Number\s*\.?\s*([0-9X*]+)',text)
|
||||
m=re.search(r'STATEMENT OF ACCOUNT from\s*(\d{2}/\d{2}/\d{4})\s*to\s*(\d{2}/\d{2}/\d{4})',text,re.I)
|
||||
if m:
|
||||
meta.period_from=pd.to_datetime(m.group(1),dayfirst=True).strftime('%Y-%m-%d'); meta.period_to=pd.to_datetime(m.group(2),dayfirst=True).strftime('%Y-%m-%d')
|
||||
date_re=re.compile(r'^\s*(\d{2}/\d{2})(?:/\d{4})?\s+(\d{2}/\d{2})(?:/\d{4})?\s+(.*)$')
|
||||
rows=[]; cur=None; page=1
|
||||
for line in text.splitlines():
|
||||
if '\f' in line: page+=line.count('\f')
|
||||
m=date_re.match(line)
|
||||
if m:
|
||||
if cur: rows.append(cur)
|
||||
year=(meta.period_from[:4] if meta.period_from else '2024')
|
||||
td=m.group(1).replace(' ','')+'/'+year; vd=m.group(2).replace(' ','')+'/'+year
|
||||
cur={'transaction_date':td,'value_date':vd,'lines':[m.group(3)],'source_page':page}
|
||||
elif cur and line.strip() and not re.match(r'^(Value Post|Date Date|STATEMENT OF ACCOUNT|Page No)',line.strip(),re.I): cur['lines'].append(line)
|
||||
if cur: rows.append(cur)
|
||||
out=[]; prev=None
|
||||
for r in rows:
|
||||
first=r['lines'][0]; alltxt=norm(' '.join(r['lines']))
|
||||
# balance is amount followed by CR/DR at far right
|
||||
mb=re.search(r'([\d,]+\.\d{2})(CR|DR)\s*$',first,re.I)
|
||||
if not mb: continue
|
||||
bal=amount(mb.group(1)); prefix=first[:mb.start()]
|
||||
nums=list(re.finditer(r'(?<!\d)([\d,]+\.\d{2})(?!\d)',prefix))
|
||||
txn=amount(nums[-1].group(1)) if nums else None
|
||||
debit=credit=None
|
||||
if txn is not None and prev is not None:
|
||||
if abs((prev-txn)-bal)<0.05: debit=txn
|
||||
elif abs((prev+txn)-bal)<0.05: credit=txn
|
||||
if txn is not None and debit is None and credit is None:
|
||||
# column location: DR before CR in legacy format
|
||||
credit=txn if nums[-1].start()>70 else None; debit=txn if nums[-1].start()<=70 else None
|
||||
narr=alltxt
|
||||
ref=''; z=re.search(r'(?:UPI|NEFT|IMPS|RTGS)[/A-Z0-9-]{6,}',alltxt,re.I); ref=z.group(0) if z else ''
|
||||
out.append({**r,'narration':narr,'reference_no':ref,'debit':debit,'credit':credit,'balance':bal})
|
||||
prev=bal
|
||||
if out and meta.opening_balance is None:
|
||||
first=out[0]; txn=(first.get('debit') or 0)-(first.get('credit') or 0); meta.opening_balance=round((first['balance'] or 0)+txn,2)
|
||||
return meta,finalize(pd.DataFrame(out),meta)
|
||||
@@ -0,0 +1,53 @@
|
||||
from __future__ import annotations
|
||||
import re, pandas as pd
|
||||
from pathlib import Path
|
||||
from .base import *
|
||||
from .common import find
|
||||
|
||||
class IndusIndParser(BaseParser):
|
||||
bank_name='IndusInd Bank'; parser_name='IndusIndParser'
|
||||
@classmethod
|
||||
def detect(cls,text): return 0.98 if 'INDUSIND' in text.upper() and 'STATEMENT OF ACCOUNT' in text.upper() else 0
|
||||
def parse(self,path,text=None):
|
||||
text=text or extract_text(path); meta=StatementMeta(bank_name=self.bank_name,source_file=Path(path).name,parser_name=self.parser_name,confidence='Medium')
|
||||
meta.account_number=find(r'Account Number\s*:\s*([0-9X*]+)',text)
|
||||
meta.customer_id=find(r'Cust\.Reln\.No\s*:\s*([0-9X*]+)',text)
|
||||
meta.ifsc=find(r'IFSC Code\s*:\s*([A-Z0-9]+)',text)
|
||||
m=re.search(r'Period\s*:\s*(\d{2}[/-][A-Za-z0-9]{2,3}[/-]\d{2,4})\s*(?:to|TO|-)\s*(\d{2}[/-][A-Za-z0-9]{2,3}[/-]\d{2,4})',text,re.I)
|
||||
if m:
|
||||
meta.period_from=pd.to_datetime(m.group(1),dayfirst=True).strftime('%Y-%m-%d'); meta.period_to=pd.to_datetime(m.group(2),dayfirst=True).strftime('%Y-%m-%d')
|
||||
meta.total_debit=amount(find(r'Total Withdrawal Amount\s*:\s*([\d,]+\.\d{2})',text))
|
||||
meta.total_credit=amount(find(r'Total Deposit Amount\s*:\s*([\d,]+\.\d{2})',text))
|
||||
# rows start with date and often end with amount Dr/Cr and balance Cr
|
||||
date_re=re.compile(r'^\s*(\d{2}[-/]?[A-Za-z]{3}[-/]?\d{2,4}|\d{2}-\d{2}-\d{4})\s+(.*)$')
|
||||
rows=[]; cur=None; page=1
|
||||
for line in text.splitlines():
|
||||
if '\f' in line: page+=line.count('\f')
|
||||
m=date_re.match(line)
|
||||
if m:
|
||||
if cur: rows.append(cur)
|
||||
cur={'transaction_date':m.group(1),'value_date':m.group(1),'lines':[m.group(2)],'source_page':page}
|
||||
elif cur and line.strip() and not re.match(r'^(Statement Summary|Opening Balance|Total Withdrawal|Branch Address|The limits)',line.strip(),re.I): cur['lines'].append(line)
|
||||
if cur: rows.append(cur)
|
||||
out=[]; prev=None
|
||||
for r in rows:
|
||||
first=r['lines'][0]; alltxt=norm(' '.join(r['lines']))
|
||||
# Typical: narration ref debit Dr credit Cr balance Cr
|
||||
tokens=list(re.finditer(r'([\d,]+\.\d{2})\s*(Dr|Cr)',first,re.I))
|
||||
if not tokens: continue
|
||||
debit=credit=bal=None
|
||||
if len(tokens)>=2:
|
||||
bal=amount(tokens[-1].group(1))
|
||||
txn=amount(tokens[-2].group(1)); typ=tokens[-2].group(2).upper()
|
||||
if typ=='DR': debit=txn
|
||||
else: credit=txn
|
||||
elif len(tokens)==1:
|
||||
# carried forward / balance-only line; skip
|
||||
continue
|
||||
narr=first[:tokens[-2].start()].strip()+' '+' '.join(x.strip() for x in r['lines'][1:])
|
||||
ref=''; z=re.search(r'\b([A-Z0-9]{8,})\b',narr); ref=z.group(1) if z else ''
|
||||
out.append({**r,'narration':norm(narr),'reference_no':ref,'debit':debit,'credit':credit,'balance':bal})
|
||||
if out:
|
||||
f=out[0]; meta.opening_balance=round((f['balance'] or 0)+(f.get('debit') or 0)-(f.get('credit') or 0),2)
|
||||
meta.closing_balance=out[-1]['balance']
|
||||
return meta,finalize(pd.DataFrame(out),meta)
|
||||
@@ -0,0 +1,54 @@
|
||||
from __future__ import annotations
|
||||
import re, pandas as pd
|
||||
from pathlib import Path
|
||||
from .base import *
|
||||
from .common import find
|
||||
|
||||
class KotakParser(BaseParser):
|
||||
bank_name='Kotak Mahindra Bank'; parser_name='KotakParser'
|
||||
@classmethod
|
||||
def detect(cls,text):
|
||||
u=text.upper(); return 0.98 if 'KOTAK' in u and 'CHEQUE/REFERENCE#' in u and 'TRANSACTION DETAILS' in u else 0
|
||||
def parse(self,path,text=None):
|
||||
text=text or extract_text(path); meta=StatementMeta(bank_name=self.bank_name,source_file=Path(path).name,parser_name=self.parser_name,confidence='High')
|
||||
meta.account_number=find(r'Account\s*#\s*Variant\s*KOTAK\s*\n.*?([0-9X*]{6,})',text,flags=re.I|re.S)
|
||||
meta.ifsc=find(r'IFSC\s+([A-Z0-9]+)',text)
|
||||
m=re.search(r'(\d{2}\s+[A-Za-z]{3},\s*\d{4})\s*-\s*(\d{2}\s+[A-Za-z]{3},\s*\d{4})',text)
|
||||
if m:
|
||||
meta.period_from=pd.to_datetime(m.group(1),dayfirst=True).strftime('%Y-%m-%d'); meta.period_to=pd.to_datetime(m.group(2),dayfirst=True).strftime('%Y-%m-%d')
|
||||
date_re=re.compile(r'^\s*(\d{2}\s+[A-Za-z]{3},\s*\d{4})\s+(.*)$')
|
||||
rows=[]; cur=None; page=1
|
||||
for line in text.splitlines():
|
||||
if '\f' in line: page+=line.count('\f')
|
||||
m=date_re.match(line)
|
||||
if m:
|
||||
if ' - ' in m.group(2) and re.match(r'\d{2}\s+[A-Za-z]{3},',m.group(2).strip()): continue
|
||||
if cur: rows.append(cur)
|
||||
cur={'transaction_date':m.group(1),'value_date':m.group(1),'lines':[m.group(2)],'source_page':page}
|
||||
elif cur and line.strip() and not re.match(r'^(DATE\s+TRANSACTION|Need help|Page \d+)',line.strip(),re.I): cur['lines'].append(line)
|
||||
if cur: rows.append(cur)
|
||||
out=[]
|
||||
for r in rows:
|
||||
first=r['lines'][0]; alltxt=norm(' '.join(r['lines']))
|
||||
# amounts are signed in debit/credit columns, final unsigned balance
|
||||
nums=list(re.finditer(r'([+-]?\d{1,3}(?:,\d{3})*\.\d{2})',first))
|
||||
if not nums: continue
|
||||
bal=amount(nums[-1].group(1)); debit=credit=None
|
||||
if len(nums)>=2:
|
||||
raw=nums[-2].group(1); txn=abs(amount(raw) or 0)
|
||||
if raw.strip().startswith('-'): debit=txn
|
||||
elif raw.strip().startswith('+'): credit=txn
|
||||
else:
|
||||
# position fallback: debit column before credit
|
||||
credit=txn if nums[-2].start()>95 else None; debit=txn if nums[-2].start()<=95 else None
|
||||
elif 'OPENING BALANCE' in first.upper():
|
||||
meta.opening_balance=bal; continue
|
||||
narr=first[:nums[-2].start() if len(nums)>=2 else nums[-1].start()].strip()+' '+' '.join(x.strip() for x in r['lines'][1:])
|
||||
ref=''; z=re.search(r'\b(?:UPI|NEFT|IMPS|RTGS)-[A-Z0-9]+\b',alltxt,re.I); ref=z.group(0) if z else ''
|
||||
if debit is None and credit is None: continue
|
||||
out.append({**r,'narration':norm(narr),'reference_no':ref,'debit':debit,'credit':credit,'balance':bal})
|
||||
if out:
|
||||
if meta.opening_balance is None:
|
||||
f=out[0]; meta.opening_balance=round((f['balance'] or 0)+(f.get('debit') or 0)-(f.get('credit') or 0),2)
|
||||
meta.closing_balance=out[-1]['balance']
|
||||
return meta,finalize(pd.DataFrame(out),meta)
|
||||
@@ -0,0 +1,25 @@
|
||||
from .idfc import IDFCFirstParser
|
||||
from .axis import AxisParser
|
||||
from .hdfc import HDFCParser
|
||||
from .indian_bank import IndianBankModernParser, IndianBankLegacyParser
|
||||
from .indusind import IndusIndParser
|
||||
from .kotak import KotakParser
|
||||
from .sbi import SBIModernParser, SBIOtherParser
|
||||
from .base import extract_text
|
||||
|
||||
PARSERS=[IDFCFirstParser,AxisParser,HDFCParser,IndianBankModernParser,IndianBankLegacyParser,IndusIndParser,KotakParser,SBIOtherParser,SBIModernParser]
|
||||
|
||||
def detect_parser(text):
|
||||
scored=sorted(((p.detect(text),p) for p in PARSERS),key=lambda x:x[0],reverse=True)
|
||||
if not scored or scored[0][0] <= 0: return None,0
|
||||
return scored[0][1](),scored[0][0]
|
||||
|
||||
def parse_pdf(path, bank_hint=None):
|
||||
text=extract_text(path)
|
||||
if bank_hint:
|
||||
for p in PARSERS:
|
||||
if bank_hint.lower() in p.bank_name.lower() or bank_hint.lower() in p.__name__.lower():
|
||||
return p().parse(path,text)
|
||||
parser,score=detect_parser(text)
|
||||
if parser is None: raise ValueError('Unsupported statement format. Add a bank-specific parser or use a supported sample format.')
|
||||
return parser.parse(path,text)
|
||||
@@ -0,0 +1,100 @@
|
||||
from __future__ import annotations
|
||||
import re, pandas as pd
|
||||
from pathlib import Path
|
||||
from .base import *
|
||||
from .common import find
|
||||
|
||||
class SBIModernParser(BaseParser):
|
||||
bank_name='State Bank of India'; parser_name='SBIModernParser'
|
||||
@classmethod
|
||||
def detect(cls,text):
|
||||
u=text.upper(); return 0.98 if 'STATE BANK OF INDIA' in u and 'REF NO./CHEQUE' in u and 'DETAILS' in u else 0
|
||||
def parse(self,path,text=None):
|
||||
text=text or extract_text(path); meta=StatementMeta(bank_name=self.bank_name,source_file=Path(path).name,parser_name=self.parser_name,confidence='High')
|
||||
meta.customer_name=find(r'Account Name\s*:?\s*([^\n]+)',text)
|
||||
meta.account_number=find(r'Account Number\s*:?\s*([0-9X*]+)',text)
|
||||
meta.ifsc=find(r'IFS Code\s*:?\s*([A-Z0-9]+)',text)
|
||||
m=re.search(r'Account Statement from\s*(\d{1,2}\s+[A-Za-z]{3}\s+\d{4})\s+to\s+(\d{1,2}\s+[A-Za-z]{3}\s+\d{4})',text,re.I)
|
||||
if m:
|
||||
meta.period_from=pd.to_datetime(m.group(1),dayfirst=True).strftime('%Y-%m-%d'); meta.period_to=pd.to_datetime(m.group(2),dayfirst=True).strftime('%Y-%m-%d')
|
||||
meta.opening_balance=amount(find(r'Balance as on[^\n]*\n\s*([\d,]+\.\d{2})',text))
|
||||
# Format A: date details ref debit credit balance
|
||||
date_re=re.compile(r'^\s*(\d{1,2}\s+[A-Za-z]{3}\s+\d{4})\s+(.*)$')
|
||||
rows=[]; cur=None; page=1
|
||||
for line in text.splitlines():
|
||||
if '\f' in line: page += line.count('\f')
|
||||
m=date_re.match(line)
|
||||
if m:
|
||||
if cur: rows.append(cur)
|
||||
cur={'transaction_date':m.group(1),'value_date':m.group(1),'lines':[m.group(2)],'source_page':page}
|
||||
elif cur and line.strip() and not re.match(r'^(Date\s+Details|Txn Date|Account Statement|State Bank)',line.strip(),re.I):
|
||||
cur['lines'].append(line)
|
||||
if cur: rows.append(cur)
|
||||
out=[]; prev=meta.opening_balance
|
||||
for r in rows:
|
||||
first=r['lines'][0]; alltxt=norm(' '.join(r['lines']))
|
||||
nums=list(re.finditer(r'(?<!\d)([\d,]+\.\d{2})(?!\d)',first))
|
||||
if not nums: continue
|
||||
bal=amount(nums[-1].group(1)); debit=credit=None
|
||||
# use dashes and positions from SBI layout: debit then credit then balance
|
||||
if len(nums)>=2:
|
||||
txn=amount(nums[-2].group(1)); p=nums[-2].start()
|
||||
if prev is not None and bal is not None:
|
||||
if abs((prev-txn)-bal)<0.05: debit=txn
|
||||
elif abs((prev+txn)-bal)<0.05: credit=txn
|
||||
if debit is None and credit is None:
|
||||
if p >= 78: credit=txn
|
||||
else: debit=txn
|
||||
narr=first[:nums[-2].start() if len(nums)>=2 else nums[-1].start()].strip()+' '+' '.join(x.strip() for x in r['lines'][1:])
|
||||
ref=''; z=re.search(r'(?:UPI|NEFT|IMPS|RTGS)[/A-Z0-9-]{6,}',alltxt,re.I); ref=z.group(0) if z else ''
|
||||
if debit is None and credit is None: continue
|
||||
out.append({**r,'narration':norm(narr),'reference_no':ref,'debit':debit,'credit':credit,'balance':bal})
|
||||
if bal is not None: prev=bal
|
||||
if out:
|
||||
if meta.opening_balance is None:
|
||||
f=out[0]; meta.opening_balance=round((f['balance'] or 0)+(f.get('debit') or 0)-(f.get('credit') or 0),2)
|
||||
meta.closing_balance=out[-1]['balance']
|
||||
return meta,finalize(pd.DataFrame(out),meta)
|
||||
|
||||
class SBIOtherParser(BaseParser):
|
||||
bank_name='State Bank of India'; parser_name='SBIOtherParser'
|
||||
@classmethod
|
||||
def detect(cls,text):
|
||||
u=text.upper(); return 0.97 if 'TXN DATE' in u and 'VALUE' in u and 'REF NO./CHEQUE' in u and 'BY TRANSFER' in u else 0
|
||||
def parse(self,path,text=None):
|
||||
text=text or extract_text(path); meta=StatementMeta(bank_name=self.bank_name,source_file=Path(path).name,parser_name=self.parser_name,confidence='High')
|
||||
meta.customer_name=find(r'Account Name\s*:\s*([^\n]+)',text)
|
||||
meta.account_number=find(r'Account Number\s*:?\s*([0-9X*]+)',text)
|
||||
meta.ifsc=find(r'IFS Code\s*:?\s*([A-Z0-9]+)',text)
|
||||
m=re.search(r'Account Statement from\s*(\d{1,2}\s+[A-Za-z]{3}\s+\d{4})\s+to\s+(\d{1,2}\s+[A-Za-z]{3}\s+\d{4})',text,re.I)
|
||||
if m:
|
||||
meta.period_from=pd.to_datetime(m.group(1),dayfirst=True).strftime('%Y-%m-%d'); meta.period_to=pd.to_datetime(m.group(2),dayfirst=True).strftime('%Y-%m-%d')
|
||||
m=re.search(r'Balance as on\s+[^\n]+\n',text,re.I)
|
||||
date_re=re.compile(r'^\s*(\d{1,2}\s+[A-Za-z]{3}\s+\d{4})\s+(\d{1,2}\s+[A-Za-z]{3}\s+\d{4})\s+(.*)$')
|
||||
rows=[]; cur=None; page=1
|
||||
for line in text.splitlines():
|
||||
if '\f' in line: page += line.count('\f')
|
||||
mm=date_re.match(line)
|
||||
if mm:
|
||||
if cur: rows.append(cur)
|
||||
cur={'transaction_date':mm.group(1),'value_date':mm.group(2),'lines':[mm.group(3)],'source_page':page}
|
||||
elif cur and line.strip() and not re.match(r'^(Txn Date|Account Statement|Account Name)',line.strip(),re.I): cur['lines'].append(line)
|
||||
if cur: rows.append(cur)
|
||||
out=[]
|
||||
for r in rows:
|
||||
first=r['lines'][0]; alltxt=norm(' '.join(r['lines']))
|
||||
nums=list(re.finditer(r'(?<!\d)([\d,]+\.\d{2})(?!\d)',first))
|
||||
if not nums: continue
|
||||
bal=amount(nums[-1].group(1)); debit=credit=None
|
||||
if len(nums)>=2:
|
||||
txn=amount(nums[-2].group(1)); p=nums[-2].start()
|
||||
# Based on header layout: Debit starts before Credit
|
||||
if p < 55: debit=txn
|
||||
else: credit=txn
|
||||
narr=first[:nums[-2].start() if len(nums)>=2 else nums[-1].start()].strip()+' '+' '.join(x.strip() for x in r['lines'][1:])
|
||||
ref=''; z=re.search(r'(?:UPI|NEFT|IMPS|RTGS)[/A-Z0-9-]{6,}',alltxt,re.I); ref=z.group(0) if z else ''
|
||||
if debit is None and credit is None: continue
|
||||
out.append({**r,'narration':norm(narr),'reference_no':ref,'debit':debit,'credit':credit,'balance':bal})
|
||||
if out:
|
||||
f=out[0]; meta.opening_balance=round((f['balance'] or 0)+(f.get('debit') or 0)-(f.get('credit') or 0),2); meta.closing_balance=out[-1]['balance']
|
||||
return meta,finalize(pd.DataFrame(out),meta)
|
||||
Reference in New Issue
Block a user