diff --git a/Dockerfile b/Dockerfile index f389406..62de574 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,6 +9,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ build-essential \ libpq-dev \ curl \ + poppler-utils \ + tesseract-ocr \ && rm -rf /var/lib/apt/lists/* COPY requirements.txt . diff --git a/app/modules/bank_statement_analyzer/parsers/base.py b/app/modules/bank_statement_analyzer/parsers/base.py index 0a44b0c..870ce60 100644 --- a/app/modules/bank_statement_analyzer/parsers/base.py +++ b/app/modules/bank_statement_analyzer/parsers/base.py @@ -2,7 +2,8 @@ from __future__ import annotations from dataclasses import dataclass, asdict from pathlib import Path from typing import Optional -import re, subprocess, tempfile +from concurrent.futures import ThreadPoolExecutor, as_completed +import os, re, subprocess, tempfile import pandas as pd import pdfplumber @@ -51,20 +52,153 @@ def amount(v): def norm(s): return re.sub(r'\s+',' ',str(s or '')).strip() -def extract_text(path: str|Path) -> str: - """Prefer pdftotext layout output; fall back to pdfplumber.""" - path=str(path) +_MIN_DIGITAL_TEXT_CHARS = 120 +_MIN_DIGITAL_ALPHA_CHARS = 30 +_OCR_DPI = max(120, min(int(os.getenv("BANK_ANALYZER_OCR_DPI", "180")), 300)) +_OCR_WORKERS = max(1, min(int(os.getenv("BANK_ANALYZER_OCR_WORKERS", "2")), 4)) + + +def _has_meaningful_text(value: str) -> bool: + text = str(value or "").strip() + if len(text) < _MIN_DIGITAL_TEXT_CHARS: + return False + return sum(character.isalpha() for character in text) >= _MIN_DIGITAL_ALPHA_CHARS + + +def _run_command(arguments: list[str], timeout: int) -> subprocess.CompletedProcess[str]: try: - p=subprocess.run(['pdftotext','-layout',path,'-'], capture_output=True, text=True, timeout=120) - if p.returncode==0 and len(p.stdout.strip())>50: - return p.stdout - except Exception: + return subprocess.run( + arguments, + capture_output=True, + text=True, + timeout=timeout, + check=False, + ) + except FileNotFoundError as exc: + raise RuntimeError( + f"Required bank-statement extraction command is unavailable: {arguments[0]}" + ) from exc + + +def _ocr_pdf_page(path: Path, page_number: int, work_dir: Path) -> tuple[int, str]: + prefix = work_dir / f"page_{page_number:05d}" + rendered_image = prefix.with_suffix(".png") + + render = _run_command( + [ + "pdftoppm", + "-f", str(page_number), + "-l", str(page_number), + "-r", str(_OCR_DPI), + "-png", + "-singlefile", + str(path), + str(prefix), + ], + timeout=180, + ) + if render.returncode != 0 or not rendered_image.exists(): + message = render.stderr.strip() or f"Unable to render page {page_number} for OCR." + raise RuntimeError(message) + + try: + environment = os.environ.copy() + environment.setdefault("OMP_THREAD_LIMIT", "1") + ocr = subprocess.run( + [ + "tesseract", + str(rendered_image), + "stdout", + "-l", "eng", + "--psm", "4", + "-c", "preserve_interword_spaces=1", + ], + capture_output=True, + text=True, + timeout=180, + check=False, + env=environment, + ) + if ocr.returncode != 0: + raise RuntimeError(ocr.stderr.strip() or f"OCR failed for page {page_number}.") + return page_number, ocr.stdout or "" + finally: + rendered_image.unlink(missing_ok=True) + + +def _ocr_selected_pages(path: Path, page_numbers: list[int]) -> dict[int, str]: + if not page_numbers: + return {} + + extracted: dict[int, str] = {} + with tempfile.TemporaryDirectory(prefix="bank_statement_ocr_") as temporary: + work_dir = Path(temporary) + with ThreadPoolExecutor(max_workers=min(_OCR_WORKERS, len(page_numbers))) as executor: + futures = { + executor.submit(_ocr_pdf_page, path, page_number, work_dir): page_number + for page_number in page_numbers + } + for future in as_completed(futures): + page_number, text = future.result() + extracted[page_number] = text + return extracted + + +def extract_text(path: str | Path) -> str: + """Extract PDF text and transparently OCR image-only or mixed pages. + + Existing digitally-generated statements continue to use the original fast + ``pdftotext``/``pdfplumber`` path. OCR is invoked only when a page has no + meaningful embedded text, so existing parsers and workbook behaviour remain + unchanged for supported digital statements. + """ + pdf_path = Path(path) + + try: + result = _run_command( + ["pdftotext", "-layout", str(pdf_path), "-"], + timeout=120, + ) + if result.returncode == 0 and _has_meaningful_text(result.stdout): + return result.stdout + except RuntimeError: + # pdfplumber remains the normal fallback when pdftotext is unavailable. pass - parts=[] - with pdfplumber.open(path) as pdf: - for page in pdf.pages: - parts.append(page.extract_text(x_tolerance=1,y_tolerance=3,layout=True) or '') - return '\n\f\n'.join(parts) + + digital_pages: list[str] = [] + pages_requiring_ocr: list[int] = [] + with pdfplumber.open(str(pdf_path)) as pdf: + for page_number, page in enumerate(pdf.pages, start=1): + page_text = page.extract_text( + x_tolerance=1, + y_tolerance=3, + layout=True, + ) or "" + digital_pages.append(page_text) + if not _has_meaningful_text(page_text): + pages_requiring_ocr.append(page_number) + + if not pages_requiring_ocr: + return "\n\f\n".join(digital_pages) + + try: + ocr_pages = _ocr_selected_pages(pdf_path, pages_requiring_ocr) + except RuntimeError as exc: + if any(_has_meaningful_text(page) for page in digital_pages): + # Preserve the former mixed-PDF behaviour when OCR tooling is absent. + return "\n\f\n".join(digital_pages) + raise ValueError( + "This bank statement is image-only and OCR processing is unavailable. " + "Install poppler-utils and tesseract-ocr in the application image." + ) from exc + + merged_pages = [ + ocr_pages.get(page_number, digital_pages[page_number - 1]) + if page_number in pages_requiring_ocr + else digital_pages[page_number - 1] + for page_number in range(1, len(digital_pages) + 1) + ] + return "\n\f\n".join(merged_pages) def page_of_line(text: str, position: int) -> int: return text[:position].count('\f')+1 diff --git a/app/modules/bank_statement_analyzer/parsers/sbi.py b/app/modules/bank_statement_analyzer/parsers/sbi.py index 9ee1c0b..dde3f81 100644 --- a/app/modules/bank_statement_analyzer/parsers/sbi.py +++ b/app/modules/bank_statement_analyzer/parsers/sbi.py @@ -8,13 +8,26 @@ class SBIModernParser(BaseParser): bank_name='State Bank of India'; parser_name='SBIModernParser' @classmethod def detect(cls,text): - u=text.upper(); return 0.98 if 'STATE BANK OF INDIA' in u and 'REF NO./CHEQUE' in u and 'DETAILS' in u else 0 + u=norm(text).upper() + bank_marker='STATE BANK OF INDIA' in u + digital_layout='REF NO./CHEQUE' in u and 'DETAILS' in u + yono_print_layout=( + 'STATEMENT OF ACCOUNT' in u + and 'VALUE DATE' in u + and 'POST DATE' in u + and 'DEBIT' in u + and 'CREDIT' in u + and 'BALANCE' in u + ) + return 0.98 if bank_marker and (digital_layout or yono_print_layout) else 0 def parse(self,path,text=None): text=text or extract_text(path); meta=StatementMeta(bank_name=self.bank_name,source_file=Path(path).name,parser_name=self.parser_name,confidence='High') meta.customer_name=find(r'Account Name\s*:?\s*([^\n]+)',text) + if not meta.customer_name: + meta.customer_name=find(r'\bMs\.?\s+([A-Z][A-Z .]{3,})',text) meta.account_number=find(r'Account Number\s*:?\s*([0-9X*]+)',text) - meta.ifsc=find(r'IFS Code\s*:?\s*([A-Z0-9]+)',text) - m=re.search(rf'Account Statement from\s*({DATE_TOKEN_PATTERN})\s+to\s+({DATE_TOKEN_PATTERN})', text, re.I) + meta.ifsc=find(r'(?:IFS|IFSC) Code\s*:?\s*([A-Z0-9]+)',text) + m=re.search(rf'(?:Account Statement from|Statement From\s*:?)\s*({DATE_TOKEN_PATTERN})\s+to\s+({DATE_TOKEN_PATTERN})', text, re.I) if m: meta.period_from=date_iso(m.group(1)); meta.period_to=date_iso(m.group(2)) meta.opening_balance=amount(find(r'Balance as on[^\n]*\n\s*([\d,]+\.\d{2})',text)) @@ -43,8 +56,15 @@ class SBIModernParser(BaseParser): if abs((prev-txn)-bal)<0.05: debit=txn elif abs((prev+txn)-bal)<0.05: credit=txn if debit is None and credit is None: - if p >= 78: credit=txn - else: debit=txn + upper=alltxt.upper() + if '/CR/' in upper or upper.startswith('DEP TFR') or 'INTEREST CREDIT' in upper: + credit=txn + elif '/DR/' in upper or upper.startswith('WDL TFR') or upper.startswith('DEBIT '): + debit=txn + elif p >= 78: + credit=txn + else: + debit=txn narr=first[:nums[-2].start() if len(nums)>=2 else nums[-1].start()].strip()+' '+' '.join(x.strip() for x in r['lines'][1:]) ref=''; z=re.search(r'(?:UPI|NEFT|IMPS|RTGS)[/A-Z0-9-]{6,}',alltxt,re.I); ref=z.group(0) if z else '' if debit is None and credit is None: continue