from __future__ import annotations import email import imaplib import re import ssl from dataclasses import dataclass from datetime import datetime, timezone from email.header import decode_header, make_header from email.message import Message from email.utils import getaddresses, parsedate_to_datetime from pathlib import Path from typing import Iterable from uuid import uuid4 from sqlalchemy import or_, select from sqlalchemy.orm import Session from app.modules.clients.models import Client from app.modules.consultants.models import ConsultantProfile from app.modules.core.iam.models import User from app.modules.email_integration.models import EmailIncomingAttachment, EmailIncomingMessage, EmailSetting from app.modules.email_integration.mapping_service import apply_email_mapping UPLOAD_ROOT = Path("app/ui/static/uploads/incoming_emails") MAX_ATTACHMENT_BYTES = 15 * 1024 * 1024 @dataclass class ImapFetchResult: fetched: int = 0 imported: int = 0 skipped_existing: int = 0 failed: int = 0 error: str | None = None def _decode_mime(value: str | None) -> str: if not value: return "" try: return str(make_header(decode_header(value))) except Exception: return value def _normalise_email(value: str | None) -> str: return (value or "").strip().lower() def _addresses(header_value: str | None) -> list[tuple[str, str]]: decoded = _decode_mime(header_value) return [(name, addr.lower()) for name, addr in getaddresses([decoded]) if addr] def _joined_addresses(header_value: str | None) -> str: return ", ".join(addr for _name, addr in _addresses(header_value)) def _received_at(msg: Message) -> datetime | None: raw_date = msg.get("Date") if not raw_date: return None try: parsed = parsedate_to_datetime(raw_date) if parsed.tzinfo is None: parsed = parsed.replace(tzinfo=timezone.utc) return parsed.astimezone(timezone.utc) except Exception: return None def _extract_bodies(msg: Message) -> tuple[str | None, str | None]: text_parts: list[str] = [] html_parts: list[str] = [] def decode_part(part: Message) -> str: payload = part.get_payload(decode=True) if payload is None: raw = part.get_payload() return raw if isinstance(raw, str) else "" charset = part.get_content_charset() or "utf-8" try: return payload.decode(charset, errors="replace") except Exception: return payload.decode("utf-8", errors="replace") if msg.is_multipart(): for part in msg.walk(): if part.get_content_maintype() == "multipart": continue if part.get_filename(): continue ctype = part.get_content_type().lower() content = decode_part(part).strip() if not content: continue if ctype == "text/plain": text_parts.append(content) elif ctype == "text/html": html_parts.append(content) else: ctype = msg.get_content_type().lower() content = decode_part(msg).strip() if ctype == "text/html": html_parts.append(content) else: text_parts.append(content) text = "\n\n".join(text_parts).strip() or None html = "\n\n".join(html_parts).strip() or None return text, html def _safe_filename(name: str | None) -> str: decoded = _decode_mime(name or "attachment") or "attachment" cleaned = re.sub(r"[^A-Za-z0-9._ -]", "_", decoded).strip(" .") return cleaned[:180] or "attachment" def _save_attachments(msg: Message, tenant_id: int | None, branch_id: int | None, incoming_id: int) -> list[EmailIncomingAttachment]: saved: list[EmailIncomingAttachment] = [] base = UPLOAD_ROOT / str(tenant_id or "system") / str(branch_id or "all") / str(incoming_id) base.mkdir(parents=True, exist_ok=True) for part in msg.walk() if msg.is_multipart() else []: if part.get_content_maintype() == "multipart": continue filename = part.get_filename() disposition = (part.get("Content-Disposition") or "").lower() if not filename and "attachment" not in disposition: continue payload = part.get_payload(decode=True) or b"" if not payload: continue if len(payload) > MAX_ATTACHMENT_BYTES: continue safe = _safe_filename(filename) target_name = f"{uuid4().hex}_{safe}" target = base / target_name target.write_bytes(payload) saved.append( EmailIncomingAttachment( incoming_message_id=incoming_id, tenant_id=tenant_id, branch_id=branch_id, filename=safe, content_type=part.get_content_type(), size_bytes=len(payload), storage_path=str(target).replace("\\", "/"), ) ) return saved def _match_sender(db: Session, tenant_id: int | None, branch_id: int | None, sender_email: str | None) -> tuple[int | None, int | None, int | None, str]: email_value = _normalise_email(sender_email) if not email_value: return None, None, None, "NEW" user_id = None client_id = None consultant_id = None user = db.execute(select(User).where(User.email == email_value).limit(1)).scalar_one_or_none() if user: user_id = int(user.id) if tenant_id: client_filters = [Client.tenant_id == tenant_id, or_(Client.email == email_value, Client.alternate_email == email_value)] if branch_id: client_filters.append(Client.branch_id == branch_id) client = db.execute(select(Client).where(*client_filters).limit(1)).scalar_one_or_none() if client: client_id = int(client.id) consultant_filters = [ConsultantProfile.tenant_id == tenant_id, ConsultantProfile.email == email_value] if branch_id: consultant_filters.append(or_(ConsultantProfile.branch_id == branch_id, ConsultantProfile.branch_id.is_(None))) consultant = db.execute(select(ConsultantProfile).where(*consultant_filters).limit(1)).scalar_one_or_none() if consultant: consultant_id = int(consultant.id) status = "MATCHED" if any([user_id, client_id, consultant_id]) else "NEW" return user_id, client_id, consultant_id, status def _connect(setting: EmailSetting): host = (setting.imap_host or "").strip() port = int(setting.imap_port or 993) security = (setting.imap_security or "SSL").upper() if not host or not setting.imap_username or not setting.imap_password: raise RuntimeError("IMAP settings are incomplete. Please configure IMAP host, username and password.") if security == "SSL": conn = imaplib.IMAP4_SSL(host, port, ssl_context=ssl.create_default_context()) else: conn = imaplib.IMAP4(host, port) if security == "STARTTLS": conn.starttls(ssl_context=ssl.create_default_context()) conn.login(setting.imap_username, setting.imap_password) return conn def fetch_incoming_emails( db: Session, *, setting: EmailSetting, tenant_id: int | None, branch_id: int | None, folder: str = "INBOX", unread_only: bool = True, limit: int = 25, mark_seen: bool = False, ) -> ImapFetchResult: result = ImapFetchResult() mailbox_email = _normalise_email(setting.imap_username or setting.from_email or "mailbox") conn = None try: conn = _connect(setting) typ, _ = conn.select(folder, readonly=not mark_seen) if typ != "OK": raise RuntimeError(f"Unable to open IMAP folder: {folder}") criteria = "UNSEEN" if unread_only else "ALL" typ, data = conn.search(None, criteria) if typ != "OK": raise RuntimeError("IMAP search failed.") ids = (data[0] or b"").split() ids = ids[-max(1, min(int(limit or 25), 100)):] result.fetched = len(ids) for msg_seq in ids: try: typ, uid_data = conn.fetch(msg_seq, "(UID)") uid_text = uid_data[0].decode(errors="ignore") if uid_data and uid_data[0] else msg_seq.decode() uid_match = re.search(r"UID (\d+)", uid_text) provider_uid = uid_match.group(1) if uid_match else msg_seq.decode() exists = db.execute( select(EmailIncomingMessage).where( EmailIncomingMessage.tenant_id == tenant_id, EmailIncomingMessage.branch_id == branch_id, EmailIncomingMessage.mailbox_email == mailbox_email, EmailIncomingMessage.folder_name == folder, EmailIncomingMessage.provider_uid == provider_uid, ) ).scalar_one_or_none() if exists: result.skipped_existing += 1 continue typ, msg_data = conn.fetch(msg_seq, "(RFC822)") if typ != "OK" or not msg_data: result.failed += 1 continue raw = None for item in msg_data: if isinstance(item, tuple): raw = item[1] break if not raw: result.failed += 1 continue msg = email.message_from_bytes(raw) from_rows = _addresses(msg.get("From")) sender_name, sender_email = from_rows[0] if from_rows else ("", "") body_text, body_html = _extract_bodies(msg) user_id, client_id, consultant_id, status = _match_sender(db, tenant_id, branch_id, sender_email) incoming = EmailIncomingMessage( tenant_id=tenant_id, branch_id=branch_id, mailbox_email=mailbox_email, folder_name=folder, provider_uid=provider_uid, provider_message_id=_decode_mime(msg.get("Message-ID")) or None, sender_email=sender_email or None, sender_name=sender_name or None, recipient_emails=_joined_addresses(msg.get("To")) or None, cc_emails=_joined_addresses(msg.get("Cc")) or None, subject=_decode_mime(msg.get("Subject"))[:500] or None, body_text=body_text, body_html=body_html, raw_headers="\n".join(f"{k}: {v}" for k, v in msg.items()), received_at_utc=_received_at(msg), status=status, matched_user_id=user_id, matched_client_id=client_id, matched_consultant_id=consultant_id, ) db.add(incoming) db.flush() attachments = _save_attachments(msg, tenant_id, branch_id, int(incoming.id)) for attachment in attachments: db.add(attachment) incoming.has_attachments = bool(attachments) incoming.attachment_count = len(attachments) # Phase 7S.3: immediately try tracking-code mapping after fetch. # Failure to map should never fail the IMAP import. try: apply_email_mapping(db, incoming) except Exception as map_exc: incoming.mapping_status = "ERROR" incoming.mapping_notes = str(map_exc)[:1000] result.imported += 1 except Exception: result.failed += 1 if mark_seen: for msg_seq in ids: try: conn.store(msg_seq, "+FLAGS", "\\Seen") except Exception: pass return result except Exception as exc: result.error = str(exc) return result finally: if conn is not None: try: conn.close() except Exception: pass try: conn.logout() except Exception: pass