Add common OCR fallback for image-only bank statements

app/modules/bank_statement_analyzer/parsers/base.py

@@ -2,7 +2,8 @@ from __future__ import annotations
 from dataclasses import dataclass, asdict
 from pathlib import Path
 from typing import Optional
-import re, subprocess, tempfile
+from concurrent.futures import ThreadPoolExecutor, as_completed
+import os, re, subprocess, tempfile
 import pandas as pd
 import pdfplumber
 
@@ -51,20 +52,153 @@ def amount(v):
 
 def norm(s): return re.sub(r'\s+',' ',str(s or '')).strip()
 
-def extract_text(path: str|Path) -> str:
-    """Prefer pdftotext layout output; fall back to pdfplumber."""
-    path=str(path)
+_MIN_DIGITAL_TEXT_CHARS = 120
+_MIN_DIGITAL_ALPHA_CHARS = 30
+_OCR_DPI = max(120, min(int(os.getenv("BANK_ANALYZER_OCR_DPI", "180")), 300))
+_OCR_WORKERS = max(1, min(int(os.getenv("BANK_ANALYZER_OCR_WORKERS", "2")), 4))
+
+
+def _has_meaningful_text(value: str) -> bool:
+    text = str(value or "").strip()
+    if len(text) < _MIN_DIGITAL_TEXT_CHARS:
+        return False
+    return sum(character.isalpha() for character in text) >= _MIN_DIGITAL_ALPHA_CHARS
+
+
+def _run_command(arguments: list[str], timeout: int) -> subprocess.CompletedProcess[str]:
     try:
-        p=subprocess.run(['pdftotext','-layout',path,'-'], capture_output=True, text=True, timeout=120)
-        if p.returncode==0 and len(p.stdout.strip())>50:
-            return p.stdout
-    except Exception:
+        return subprocess.run(
+            arguments,
+            capture_output=True,
+            text=True,
+            timeout=timeout,
+            check=False,
+        )
+    except FileNotFoundError as exc:
+        raise RuntimeError(
+            f"Required bank-statement extraction command is unavailable: {arguments[0]}"
+        ) from exc
+
+
+def _ocr_pdf_page(path: Path, page_number: int, work_dir: Path) -> tuple[int, str]:
+    prefix = work_dir / f"page_{page_number:05d}"
+    rendered_image = prefix.with_suffix(".png")
+
+    render = _run_command(
+        [
+            "pdftoppm",
+            "-f", str(page_number),
+            "-l", str(page_number),
+            "-r", str(_OCR_DPI),
+            "-png",
+            "-singlefile",
+            str(path),
+            str(prefix),
+        ],
+        timeout=180,
+    )
+    if render.returncode != 0 or not rendered_image.exists():
+        message = render.stderr.strip() or f"Unable to render page {page_number} for OCR."
+        raise RuntimeError(message)
+
+    try:
+        environment = os.environ.copy()
+        environment.setdefault("OMP_THREAD_LIMIT", "1")
+        ocr = subprocess.run(
+            [
+                "tesseract",
+                str(rendered_image),
+                "stdout",
+                "-l", "eng",
+                "--psm", "4",
+                "-c", "preserve_interword_spaces=1",
+            ],
+            capture_output=True,
+            text=True,
+            timeout=180,
+            check=False,
+            env=environment,
+        )
+        if ocr.returncode != 0:
+            raise RuntimeError(ocr.stderr.strip() or f"OCR failed for page {page_number}.")
+        return page_number, ocr.stdout or ""
+    finally:
+        rendered_image.unlink(missing_ok=True)
+
+
+def _ocr_selected_pages(path: Path, page_numbers: list[int]) -> dict[int, str]:
+    if not page_numbers:
+        return {}
+
+    extracted: dict[int, str] = {}
+    with tempfile.TemporaryDirectory(prefix="bank_statement_ocr_") as temporary:
+        work_dir = Path(temporary)
+        with ThreadPoolExecutor(max_workers=min(_OCR_WORKERS, len(page_numbers))) as executor:
+            futures = {
+                executor.submit(_ocr_pdf_page, path, page_number, work_dir): page_number
+                for page_number in page_numbers
+            }
+            for future in as_completed(futures):
+                page_number, text = future.result()
+                extracted[page_number] = text
+    return extracted
+
+
+def extract_text(path: str | Path) -> str:
+    """Extract PDF text and transparently OCR image-only or mixed pages.
+
+    Existing digitally-generated statements continue to use the original fast
+    ``pdftotext``/``pdfplumber`` path. OCR is invoked only when a page has no
+    meaningful embedded text, so existing parsers and workbook behaviour remain
+    unchanged for supported digital statements.
+    """
+    pdf_path = Path(path)
+
+    try:
+        result = _run_command(
+            ["pdftotext", "-layout", str(pdf_path), "-"],
+            timeout=120,
+        )
+        if result.returncode == 0 and _has_meaningful_text(result.stdout):
+            return result.stdout
+    except RuntimeError:
+        # pdfplumber remains the normal fallback when pdftotext is unavailable.
         pass
-    parts=[]
-    with pdfplumber.open(path) as pdf:
-        for page in pdf.pages:
-            parts.append(page.extract_text(x_tolerance=1,y_tolerance=3,layout=True) or '')
-    return '\n\f\n'.join(parts)
+
+    digital_pages: list[str] = []
+    pages_requiring_ocr: list[int] = []
+    with pdfplumber.open(str(pdf_path)) as pdf:
+        for page_number, page in enumerate(pdf.pages, start=1):
+            page_text = page.extract_text(
+                x_tolerance=1,
+                y_tolerance=3,
+                layout=True,
+            ) or ""
+            digital_pages.append(page_text)
+            if not _has_meaningful_text(page_text):
+                pages_requiring_ocr.append(page_number)
+
+    if not pages_requiring_ocr:
+        return "\n\f\n".join(digital_pages)
+
+    try:
+        ocr_pages = _ocr_selected_pages(pdf_path, pages_requiring_ocr)
+    except RuntimeError as exc:
+        if any(_has_meaningful_text(page) for page in digital_pages):
+            # Preserve the former mixed-PDF behaviour when OCR tooling is absent.
+            return "\n\f\n".join(digital_pages)
+        raise ValueError(
+            "This bank statement is image-only and OCR processing is unavailable. "
+            "Install poppler-utils and tesseract-ocr in the application image."
+        ) from exc
+
+    merged_pages = [
+        ocr_pages.get(page_number, digital_pages[page_number - 1])
+        if page_number in pages_requiring_ocr
+        else digital_pages[page_number - 1]
+        for page_number in range(1, len(digital_pages) + 1)
+    ]
+    return "\n\f\n".join(merged_pages)
 
 def page_of_line(text: str, position: int) -> int:
     return text[:position].count('\f')+1