Prepare ERP source for Gitea deployment
This commit is contained in:
A R R R Associates
@@ -0,0 +1,327 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import email
|
||||
import imaplib
|
||||
import re
|
||||
import ssl
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from email.header import decode_header, make_header
|
||||
from email.message import Message
|
||||
from email.utils import getaddresses, parsedate_to_datetime
|
||||
from pathlib import Path
|
||||
from typing import Iterable
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import or_, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.modules.clients.models import Client
|
||||
from app.modules.consultants.models import ConsultantProfile
|
||||
from app.modules.core.iam.models import User
|
||||
from app.modules.email_integration.models import EmailIncomingAttachment, EmailIncomingMessage, EmailSetting
|
||||
from app.modules.email_integration.mapping_service import apply_email_mapping
|
||||
|
||||
UPLOAD_ROOT = Path("app/ui/static/uploads/incoming_emails")
|
||||
MAX_ATTACHMENT_BYTES = 15 * 1024 * 1024
|
||||
|
||||
|
||||
@dataclass
|
||||
class ImapFetchResult:
|
||||
fetched: int = 0
|
||||
imported: int = 0
|
||||
skipped_existing: int = 0
|
||||
failed: int = 0
|
||||
error: str | None = None
|
||||
|
||||
|
||||
def _decode_mime(value: str | None) -> str:
|
||||
if not value:
|
||||
return ""
|
||||
try:
|
||||
return str(make_header(decode_header(value)))
|
||||
except Exception:
|
||||
return value
|
||||
|
||||
|
||||
def _normalise_email(value: str | None) -> str:
|
||||
return (value or "").strip().lower()
|
||||
|
||||
|
||||
def _addresses(header_value: str | None) -> list[tuple[str, str]]:
|
||||
decoded = _decode_mime(header_value)
|
||||
return [(name, addr.lower()) for name, addr in getaddresses([decoded]) if addr]
|
||||
|
||||
|
||||
def _joined_addresses(header_value: str | None) -> str:
|
||||
return ", ".join(addr for _name, addr in _addresses(header_value))
|
||||
|
||||
|
||||
def _received_at(msg: Message) -> datetime | None:
|
||||
raw_date = msg.get("Date")
|
||||
if not raw_date:
|
||||
return None
|
||||
try:
|
||||
parsed = parsedate_to_datetime(raw_date)
|
||||
if parsed.tzinfo is None:
|
||||
parsed = parsed.replace(tzinfo=timezone.utc)
|
||||
return parsed.astimezone(timezone.utc)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _extract_bodies(msg: Message) -> tuple[str | None, str | None]:
|
||||
text_parts: list[str] = []
|
||||
html_parts: list[str] = []
|
||||
|
||||
def decode_part(part: Message) -> str:
|
||||
payload = part.get_payload(decode=True)
|
||||
if payload is None:
|
||||
raw = part.get_payload()
|
||||
return raw if isinstance(raw, str) else ""
|
||||
charset = part.get_content_charset() or "utf-8"
|
||||
try:
|
||||
return payload.decode(charset, errors="replace")
|
||||
except Exception:
|
||||
return payload.decode("utf-8", errors="replace")
|
||||
|
||||
if msg.is_multipart():
|
||||
for part in msg.walk():
|
||||
if part.get_content_maintype() == "multipart":
|
||||
continue
|
||||
if part.get_filename():
|
||||
continue
|
||||
ctype = part.get_content_type().lower()
|
||||
content = decode_part(part).strip()
|
||||
if not content:
|
||||
continue
|
||||
if ctype == "text/plain":
|
||||
text_parts.append(content)
|
||||
elif ctype == "text/html":
|
||||
html_parts.append(content)
|
||||
else:
|
||||
ctype = msg.get_content_type().lower()
|
||||
content = decode_part(msg).strip()
|
||||
if ctype == "text/html":
|
||||
html_parts.append(content)
|
||||
else:
|
||||
text_parts.append(content)
|
||||
|
||||
text = "\n\n".join(text_parts).strip() or None
|
||||
html = "\n\n".join(html_parts).strip() or None
|
||||
return text, html
|
||||
|
||||
|
||||
def _safe_filename(name: str | None) -> str:
|
||||
decoded = _decode_mime(name or "attachment") or "attachment"
|
||||
cleaned = re.sub(r"[^A-Za-z0-9._ -]", "_", decoded).strip(" .")
|
||||
return cleaned[:180] or "attachment"
|
||||
|
||||
|
||||
def _save_attachments(msg: Message, tenant_id: int | None, branch_id: int | None, incoming_id: int) -> list[EmailIncomingAttachment]:
|
||||
saved: list[EmailIncomingAttachment] = []
|
||||
base = UPLOAD_ROOT / str(tenant_id or "system") / str(branch_id or "all") / str(incoming_id)
|
||||
base.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
for part in msg.walk() if msg.is_multipart() else []:
|
||||
if part.get_content_maintype() == "multipart":
|
||||
continue
|
||||
filename = part.get_filename()
|
||||
disposition = (part.get("Content-Disposition") or "").lower()
|
||||
if not filename and "attachment" not in disposition:
|
||||
continue
|
||||
payload = part.get_payload(decode=True) or b""
|
||||
if not payload:
|
||||
continue
|
||||
if len(payload) > MAX_ATTACHMENT_BYTES:
|
||||
continue
|
||||
safe = _safe_filename(filename)
|
||||
target_name = f"{uuid4().hex}_{safe}"
|
||||
target = base / target_name
|
||||
target.write_bytes(payload)
|
||||
saved.append(
|
||||
EmailIncomingAttachment(
|
||||
incoming_message_id=incoming_id,
|
||||
tenant_id=tenant_id,
|
||||
branch_id=branch_id,
|
||||
filename=safe,
|
||||
content_type=part.get_content_type(),
|
||||
size_bytes=len(payload),
|
||||
storage_path=str(target).replace("\\", "/"),
|
||||
)
|
||||
)
|
||||
return saved
|
||||
|
||||
|
||||
def _match_sender(db: Session, tenant_id: int | None, branch_id: int | None, sender_email: str | None) -> tuple[int | None, int | None, int | None, str]:
|
||||
email_value = _normalise_email(sender_email)
|
||||
if not email_value:
|
||||
return None, None, None, "NEW"
|
||||
|
||||
user_id = None
|
||||
client_id = None
|
||||
consultant_id = None
|
||||
|
||||
user = db.execute(select(User).where(User.email == email_value).limit(1)).scalar_one_or_none()
|
||||
if user:
|
||||
user_id = int(user.id)
|
||||
|
||||
if tenant_id:
|
||||
client_filters = [Client.tenant_id == tenant_id, or_(Client.email == email_value, Client.alternate_email == email_value)]
|
||||
if branch_id:
|
||||
client_filters.append(Client.branch_id == branch_id)
|
||||
client = db.execute(select(Client).where(*client_filters).limit(1)).scalar_one_or_none()
|
||||
if client:
|
||||
client_id = int(client.id)
|
||||
|
||||
consultant_filters = [ConsultantProfile.tenant_id == tenant_id, ConsultantProfile.email == email_value]
|
||||
if branch_id:
|
||||
consultant_filters.append(or_(ConsultantProfile.branch_id == branch_id, ConsultantProfile.branch_id.is_(None)))
|
||||
consultant = db.execute(select(ConsultantProfile).where(*consultant_filters).limit(1)).scalar_one_or_none()
|
||||
if consultant:
|
||||
consultant_id = int(consultant.id)
|
||||
|
||||
status = "MATCHED" if any([user_id, client_id, consultant_id]) else "NEW"
|
||||
return user_id, client_id, consultant_id, status
|
||||
|
||||
|
||||
def _connect(setting: EmailSetting):
|
||||
host = (setting.imap_host or "").strip()
|
||||
port = int(setting.imap_port or 993)
|
||||
security = (setting.imap_security or "SSL").upper()
|
||||
if not host or not setting.imap_username or not setting.imap_password:
|
||||
raise RuntimeError("IMAP settings are incomplete. Please configure IMAP host, username and password.")
|
||||
|
||||
if security == "SSL":
|
||||
conn = imaplib.IMAP4_SSL(host, port, ssl_context=ssl.create_default_context())
|
||||
else:
|
||||
conn = imaplib.IMAP4(host, port)
|
||||
if security == "STARTTLS":
|
||||
conn.starttls(ssl_context=ssl.create_default_context())
|
||||
conn.login(setting.imap_username, setting.imap_password)
|
||||
return conn
|
||||
|
||||
|
||||
def fetch_incoming_emails(
|
||||
db: Session,
|
||||
*,
|
||||
setting: EmailSetting,
|
||||
tenant_id: int | None,
|
||||
branch_id: int | None,
|
||||
folder: str = "INBOX",
|
||||
unread_only: bool = True,
|
||||
limit: int = 25,
|
||||
mark_seen: bool = False,
|
||||
) -> ImapFetchResult:
|
||||
result = ImapFetchResult()
|
||||
mailbox_email = _normalise_email(setting.imap_username or setting.from_email or "mailbox")
|
||||
conn = None
|
||||
try:
|
||||
conn = _connect(setting)
|
||||
typ, _ = conn.select(folder, readonly=not mark_seen)
|
||||
if typ != "OK":
|
||||
raise RuntimeError(f"Unable to open IMAP folder: {folder}")
|
||||
criteria = "UNSEEN" if unread_only else "ALL"
|
||||
typ, data = conn.search(None, criteria)
|
||||
if typ != "OK":
|
||||
raise RuntimeError("IMAP search failed.")
|
||||
ids = (data[0] or b"").split()
|
||||
ids = ids[-max(1, min(int(limit or 25), 100)):]
|
||||
result.fetched = len(ids)
|
||||
|
||||
for msg_seq in ids:
|
||||
try:
|
||||
typ, uid_data = conn.fetch(msg_seq, "(UID)")
|
||||
uid_text = uid_data[0].decode(errors="ignore") if uid_data and uid_data[0] else msg_seq.decode()
|
||||
uid_match = re.search(r"UID (\d+)", uid_text)
|
||||
provider_uid = uid_match.group(1) if uid_match else msg_seq.decode()
|
||||
|
||||
exists = db.execute(
|
||||
select(EmailIncomingMessage).where(
|
||||
EmailIncomingMessage.tenant_id == tenant_id,
|
||||
EmailIncomingMessage.branch_id == branch_id,
|
||||
EmailIncomingMessage.mailbox_email == mailbox_email,
|
||||
EmailIncomingMessage.folder_name == folder,
|
||||
EmailIncomingMessage.provider_uid == provider_uid,
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if exists:
|
||||
result.skipped_existing += 1
|
||||
continue
|
||||
|
||||
typ, msg_data = conn.fetch(msg_seq, "(RFC822)")
|
||||
if typ != "OK" or not msg_data:
|
||||
result.failed += 1
|
||||
continue
|
||||
raw = None
|
||||
for item in msg_data:
|
||||
if isinstance(item, tuple):
|
||||
raw = item[1]
|
||||
break
|
||||
if not raw:
|
||||
result.failed += 1
|
||||
continue
|
||||
msg = email.message_from_bytes(raw)
|
||||
from_rows = _addresses(msg.get("From"))
|
||||
sender_name, sender_email = from_rows[0] if from_rows else ("", "")
|
||||
body_text, body_html = _extract_bodies(msg)
|
||||
user_id, client_id, consultant_id, status = _match_sender(db, tenant_id, branch_id, sender_email)
|
||||
|
||||
incoming = EmailIncomingMessage(
|
||||
tenant_id=tenant_id,
|
||||
branch_id=branch_id,
|
||||
mailbox_email=mailbox_email,
|
||||
folder_name=folder,
|
||||
provider_uid=provider_uid,
|
||||
provider_message_id=_decode_mime(msg.get("Message-ID")) or None,
|
||||
sender_email=sender_email or None,
|
||||
sender_name=sender_name or None,
|
||||
recipient_emails=_joined_addresses(msg.get("To")) or None,
|
||||
cc_emails=_joined_addresses(msg.get("Cc")) or None,
|
||||
subject=_decode_mime(msg.get("Subject"))[:500] or None,
|
||||
body_text=body_text,
|
||||
body_html=body_html,
|
||||
raw_headers="\n".join(f"{k}: {v}" for k, v in msg.items()),
|
||||
received_at_utc=_received_at(msg),
|
||||
status=status,
|
||||
matched_user_id=user_id,
|
||||
matched_client_id=client_id,
|
||||
matched_consultant_id=consultant_id,
|
||||
)
|
||||
db.add(incoming)
|
||||
db.flush()
|
||||
attachments = _save_attachments(msg, tenant_id, branch_id, int(incoming.id))
|
||||
for attachment in attachments:
|
||||
db.add(attachment)
|
||||
incoming.has_attachments = bool(attachments)
|
||||
incoming.attachment_count = len(attachments)
|
||||
# Phase 7S.3: immediately try tracking-code mapping after fetch.
|
||||
# Failure to map should never fail the IMAP import.
|
||||
try:
|
||||
apply_email_mapping(db, incoming)
|
||||
except Exception as map_exc:
|
||||
incoming.mapping_status = "ERROR"
|
||||
incoming.mapping_notes = str(map_exc)[:1000]
|
||||
result.imported += 1
|
||||
except Exception:
|
||||
result.failed += 1
|
||||
if mark_seen:
|
||||
for msg_seq in ids:
|
||||
try:
|
||||
conn.store(msg_seq, "+FLAGS", "\\Seen")
|
||||
except Exception:
|
||||
pass
|
||||
return result
|
||||
except Exception as exc:
|
||||
result.error = str(exc)
|
||||
return result
|
||||
finally:
|
||||
if conn is not None:
|
||||
try:
|
||||
conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
conn.logout()
|
||||
except Exception:
|
||||
pass
|
||||
Reference in New Issue
Block a user