A R R R Associates
328 lines
12 KiB
Python
328 lines
12 KiB
Python
from __future__ import annotations
|
|
|
|
import email
|
|
import imaplib
|
|
import re
|
|
import ssl
|
|
from dataclasses import dataclass
|
|
from datetime import datetime, timezone
|
|
from email.header import decode_header, make_header
|
|
from email.message import Message
|
|
from email.utils import getaddresses, parsedate_to_datetime
|
|
from pathlib import Path
|
|
from typing import Iterable
|
|
from uuid import uuid4
|
|
|
|
from sqlalchemy import or_, select
|
|
from sqlalchemy.orm import Session
|
|
|
|
from app.modules.clients.models import Client
|
|
from app.modules.consultants.models import ConsultantProfile
|
|
from app.modules.core.iam.models import User
|
|
from app.modules.email_integration.models import EmailIncomingAttachment, EmailIncomingMessage, EmailSetting
|
|
from app.modules.email_integration.mapping_service import apply_email_mapping
|
|
|
|
UPLOAD_ROOT = Path("app/ui/static/uploads/incoming_emails")
|
|
MAX_ATTACHMENT_BYTES = 15 * 1024 * 1024
|
|
|
|
|
|
@dataclass
|
|
class ImapFetchResult:
|
|
fetched: int = 0
|
|
imported: int = 0
|
|
skipped_existing: int = 0
|
|
failed: int = 0
|
|
error: str | None = None
|
|
|
|
|
|
def _decode_mime(value: str | None) -> str:
|
|
if not value:
|
|
return ""
|
|
try:
|
|
return str(make_header(decode_header(value)))
|
|
except Exception:
|
|
return value
|
|
|
|
|
|
def _normalise_email(value: str | None) -> str:
|
|
return (value or "").strip().lower()
|
|
|
|
|
|
def _addresses(header_value: str | None) -> list[tuple[str, str]]:
|
|
decoded = _decode_mime(header_value)
|
|
return [(name, addr.lower()) for name, addr in getaddresses([decoded]) if addr]
|
|
|
|
|
|
def _joined_addresses(header_value: str | None) -> str:
|
|
return ", ".join(addr for _name, addr in _addresses(header_value))
|
|
|
|
|
|
def _received_at(msg: Message) -> datetime | None:
|
|
raw_date = msg.get("Date")
|
|
if not raw_date:
|
|
return None
|
|
try:
|
|
parsed = parsedate_to_datetime(raw_date)
|
|
if parsed.tzinfo is None:
|
|
parsed = parsed.replace(tzinfo=timezone.utc)
|
|
return parsed.astimezone(timezone.utc)
|
|
except Exception:
|
|
return None
|
|
|
|
|
|
def _extract_bodies(msg: Message) -> tuple[str | None, str | None]:
|
|
text_parts: list[str] = []
|
|
html_parts: list[str] = []
|
|
|
|
def decode_part(part: Message) -> str:
|
|
payload = part.get_payload(decode=True)
|
|
if payload is None:
|
|
raw = part.get_payload()
|
|
return raw if isinstance(raw, str) else ""
|
|
charset = part.get_content_charset() or "utf-8"
|
|
try:
|
|
return payload.decode(charset, errors="replace")
|
|
except Exception:
|
|
return payload.decode("utf-8", errors="replace")
|
|
|
|
if msg.is_multipart():
|
|
for part in msg.walk():
|
|
if part.get_content_maintype() == "multipart":
|
|
continue
|
|
if part.get_filename():
|
|
continue
|
|
ctype = part.get_content_type().lower()
|
|
content = decode_part(part).strip()
|
|
if not content:
|
|
continue
|
|
if ctype == "text/plain":
|
|
text_parts.append(content)
|
|
elif ctype == "text/html":
|
|
html_parts.append(content)
|
|
else:
|
|
ctype = msg.get_content_type().lower()
|
|
content = decode_part(msg).strip()
|
|
if ctype == "text/html":
|
|
html_parts.append(content)
|
|
else:
|
|
text_parts.append(content)
|
|
|
|
text = "\n\n".join(text_parts).strip() or None
|
|
html = "\n\n".join(html_parts).strip() or None
|
|
return text, html
|
|
|
|
|
|
def _safe_filename(name: str | None) -> str:
|
|
decoded = _decode_mime(name or "attachment") or "attachment"
|
|
cleaned = re.sub(r"[^A-Za-z0-9._ -]", "_", decoded).strip(" .")
|
|
return cleaned[:180] or "attachment"
|
|
|
|
|
|
def _save_attachments(msg: Message, tenant_id: int | None, branch_id: int | None, incoming_id: int) -> list[EmailIncomingAttachment]:
|
|
saved: list[EmailIncomingAttachment] = []
|
|
base = UPLOAD_ROOT / str(tenant_id or "system") / str(branch_id or "all") / str(incoming_id)
|
|
base.mkdir(parents=True, exist_ok=True)
|
|
|
|
for part in msg.walk() if msg.is_multipart() else []:
|
|
if part.get_content_maintype() == "multipart":
|
|
continue
|
|
filename = part.get_filename()
|
|
disposition = (part.get("Content-Disposition") or "").lower()
|
|
if not filename and "attachment" not in disposition:
|
|
continue
|
|
payload = part.get_payload(decode=True) or b""
|
|
if not payload:
|
|
continue
|
|
if len(payload) > MAX_ATTACHMENT_BYTES:
|
|
continue
|
|
safe = _safe_filename(filename)
|
|
target_name = f"{uuid4().hex}_{safe}"
|
|
target = base / target_name
|
|
target.write_bytes(payload)
|
|
saved.append(
|
|
EmailIncomingAttachment(
|
|
incoming_message_id=incoming_id,
|
|
tenant_id=tenant_id,
|
|
branch_id=branch_id,
|
|
filename=safe,
|
|
content_type=part.get_content_type(),
|
|
size_bytes=len(payload),
|
|
storage_path=str(target).replace("\\", "/"),
|
|
)
|
|
)
|
|
return saved
|
|
|
|
|
|
def _match_sender(db: Session, tenant_id: int | None, branch_id: int | None, sender_email: str | None) -> tuple[int | None, int | None, int | None, str]:
|
|
email_value = _normalise_email(sender_email)
|
|
if not email_value:
|
|
return None, None, None, "NEW"
|
|
|
|
user_id = None
|
|
client_id = None
|
|
consultant_id = None
|
|
|
|
user = db.execute(select(User).where(User.email == email_value).limit(1)).scalar_one_or_none()
|
|
if user:
|
|
user_id = int(user.id)
|
|
|
|
if tenant_id:
|
|
client_filters = [Client.tenant_id == tenant_id, or_(Client.email == email_value, Client.alternate_email == email_value)]
|
|
if branch_id:
|
|
client_filters.append(Client.branch_id == branch_id)
|
|
client = db.execute(select(Client).where(*client_filters).limit(1)).scalar_one_or_none()
|
|
if client:
|
|
client_id = int(client.id)
|
|
|
|
consultant_filters = [ConsultantProfile.tenant_id == tenant_id, ConsultantProfile.email == email_value]
|
|
if branch_id:
|
|
consultant_filters.append(or_(ConsultantProfile.branch_id == branch_id, ConsultantProfile.branch_id.is_(None)))
|
|
consultant = db.execute(select(ConsultantProfile).where(*consultant_filters).limit(1)).scalar_one_or_none()
|
|
if consultant:
|
|
consultant_id = int(consultant.id)
|
|
|
|
status = "MATCHED" if any([user_id, client_id, consultant_id]) else "NEW"
|
|
return user_id, client_id, consultant_id, status
|
|
|
|
|
|
def _connect(setting: EmailSetting):
|
|
host = (setting.imap_host or "").strip()
|
|
port = int(setting.imap_port or 993)
|
|
security = (setting.imap_security or "SSL").upper()
|
|
if not host or not setting.imap_username or not setting.imap_password:
|
|
raise RuntimeError("IMAP settings are incomplete. Please configure IMAP host, username and password.")
|
|
|
|
if security == "SSL":
|
|
conn = imaplib.IMAP4_SSL(host, port, ssl_context=ssl.create_default_context())
|
|
else:
|
|
conn = imaplib.IMAP4(host, port)
|
|
if security == "STARTTLS":
|
|
conn.starttls(ssl_context=ssl.create_default_context())
|
|
conn.login(setting.imap_username, setting.imap_password)
|
|
return conn
|
|
|
|
|
|
def fetch_incoming_emails(
|
|
db: Session,
|
|
*,
|
|
setting: EmailSetting,
|
|
tenant_id: int | None,
|
|
branch_id: int | None,
|
|
folder: str = "INBOX",
|
|
unread_only: bool = True,
|
|
limit: int = 25,
|
|
mark_seen: bool = False,
|
|
) -> ImapFetchResult:
|
|
result = ImapFetchResult()
|
|
mailbox_email = _normalise_email(setting.imap_username or setting.from_email or "mailbox")
|
|
conn = None
|
|
try:
|
|
conn = _connect(setting)
|
|
typ, _ = conn.select(folder, readonly=not mark_seen)
|
|
if typ != "OK":
|
|
raise RuntimeError(f"Unable to open IMAP folder: {folder}")
|
|
criteria = "UNSEEN" if unread_only else "ALL"
|
|
typ, data = conn.search(None, criteria)
|
|
if typ != "OK":
|
|
raise RuntimeError("IMAP search failed.")
|
|
ids = (data[0] or b"").split()
|
|
ids = ids[-max(1, min(int(limit or 25), 100)):]
|
|
result.fetched = len(ids)
|
|
|
|
for msg_seq in ids:
|
|
try:
|
|
typ, uid_data = conn.fetch(msg_seq, "(UID)")
|
|
uid_text = uid_data[0].decode(errors="ignore") if uid_data and uid_data[0] else msg_seq.decode()
|
|
uid_match = re.search(r"UID (\d+)", uid_text)
|
|
provider_uid = uid_match.group(1) if uid_match else msg_seq.decode()
|
|
|
|
exists = db.execute(
|
|
select(EmailIncomingMessage).where(
|
|
EmailIncomingMessage.tenant_id == tenant_id,
|
|
EmailIncomingMessage.branch_id == branch_id,
|
|
EmailIncomingMessage.mailbox_email == mailbox_email,
|
|
EmailIncomingMessage.folder_name == folder,
|
|
EmailIncomingMessage.provider_uid == provider_uid,
|
|
)
|
|
).scalar_one_or_none()
|
|
if exists:
|
|
result.skipped_existing += 1
|
|
continue
|
|
|
|
typ, msg_data = conn.fetch(msg_seq, "(RFC822)")
|
|
if typ != "OK" or not msg_data:
|
|
result.failed += 1
|
|
continue
|
|
raw = None
|
|
for item in msg_data:
|
|
if isinstance(item, tuple):
|
|
raw = item[1]
|
|
break
|
|
if not raw:
|
|
result.failed += 1
|
|
continue
|
|
msg = email.message_from_bytes(raw)
|
|
from_rows = _addresses(msg.get("From"))
|
|
sender_name, sender_email = from_rows[0] if from_rows else ("", "")
|
|
body_text, body_html = _extract_bodies(msg)
|
|
user_id, client_id, consultant_id, status = _match_sender(db, tenant_id, branch_id, sender_email)
|
|
|
|
incoming = EmailIncomingMessage(
|
|
tenant_id=tenant_id,
|
|
branch_id=branch_id,
|
|
mailbox_email=mailbox_email,
|
|
folder_name=folder,
|
|
provider_uid=provider_uid,
|
|
provider_message_id=_decode_mime(msg.get("Message-ID")) or None,
|
|
sender_email=sender_email or None,
|
|
sender_name=sender_name or None,
|
|
recipient_emails=_joined_addresses(msg.get("To")) or None,
|
|
cc_emails=_joined_addresses(msg.get("Cc")) or None,
|
|
subject=_decode_mime(msg.get("Subject"))[:500] or None,
|
|
body_text=body_text,
|
|
body_html=body_html,
|
|
raw_headers="\n".join(f"{k}: {v}" for k, v in msg.items()),
|
|
received_at_utc=_received_at(msg),
|
|
status=status,
|
|
matched_user_id=user_id,
|
|
matched_client_id=client_id,
|
|
matched_consultant_id=consultant_id,
|
|
)
|
|
db.add(incoming)
|
|
db.flush()
|
|
attachments = _save_attachments(msg, tenant_id, branch_id, int(incoming.id))
|
|
for attachment in attachments:
|
|
db.add(attachment)
|
|
incoming.has_attachments = bool(attachments)
|
|
incoming.attachment_count = len(attachments)
|
|
# Phase 7S.3: immediately try tracking-code mapping after fetch.
|
|
# Failure to map should never fail the IMAP import.
|
|
try:
|
|
apply_email_mapping(db, incoming)
|
|
except Exception as map_exc:
|
|
incoming.mapping_status = "ERROR"
|
|
incoming.mapping_notes = str(map_exc)[:1000]
|
|
result.imported += 1
|
|
except Exception:
|
|
result.failed += 1
|
|
if mark_seen:
|
|
for msg_seq in ids:
|
|
try:
|
|
conn.store(msg_seq, "+FLAGS", "\\Seen")
|
|
except Exception:
|
|
pass
|
|
return result
|
|
except Exception as exc:
|
|
result.error = str(exc)
|
|
return result
|
|
finally:
|
|
if conn is not None:
|
|
try:
|
|
conn.close()
|
|
except Exception:
|
|
pass
|
|
try:
|
|
conn.logout()
|
|
except Exception:
|
|
pass
|