"""Read-only mail intake. The mailbox is selected read-only and bodies are fetched with BODY.PEEK[], so no flag changes. The UIDVALIDITY/UID cursor lives in the library. Delivery records leave out UIDs, so a rescan after a UIDVALIDITY change reproduces the same records instead of duplicating them. """ import email import email.policy import email.utils import html import imaplib import json import os import re import ssl from datetime import datetime, timezone from pathlib import Path from . import registry as reg from . import store CURSOR = Path('data') / 'state' / 'imap-cursor.json' class ImapMailbox: def __init__(self, host, port, folder, user, password): self.host, self.port, self.folder = host, int(port), folder self.user, self.password = user, password self.name = f'{user}/{folder}' def fetch(self, cursor): """Yield (uidvalidity, uid, internaldate, raw) for UIDs past the cursor.""" context = ssl.create_default_context() with imaplib.IMAP4_SSL(self.host, self.port, ssl_context=context) as imap: imap.login(self.user, self.password) status, _ = imap.select(self.folder, readonly=True) if status != 'OK': raise OSError(f'cannot select {self.folder}') uidvalidity = int(imap.response('UIDVALIDITY')[1][0]) last = cursor.get('last_uid', 0) if cursor.get('uidvalidity') == uidvalidity else 0 status, data = imap.uid('SEARCH', None, f'UID {last + 1}:*') if status != 'OK': raise OSError('UID SEARCH failed') uids = sorted(int(u) for u in data[0].split() if int(u) > last) for uid in uids: status, data = imap.uid('FETCH', str(uid), '(INTERNALDATE BODY.PEEK[])') if status != 'OK': raise OSError(f'UID FETCH {uid} failed') meta, raw = next((part[0], part[1]) for part in data if isinstance(part, tuple)) when = imaplib.Internaldate2tuple(meta) stamp = datetime(*when[:6], tzinfo=timezone.utc).isoformat() if when else None yield uidvalidity, uid, stamp, raw class DirectoryMailbox: """A fixture mailbox: *.eml files in name order, UID = position + 1.""" def __init__(self, path, uidvalidity=1): self.path, self.uidvalidity = Path(path), uidvalidity self.name = f'dir:{self.path.name}' def fetch(self, cursor): last = cursor.get('last_uid', 0) if cursor.get('uidvalidity') == self.uidvalidity else 0 for uid, path in enumerate(sorted(self.path.glob('*.eml')), start=1): if uid > last: stamp = datetime.fromtimestamp(path.stat().st_mtime, timezone.utc).isoformat() yield self.uidvalidity, uid, stamp, path.read_bytes() def mailbox_from_config(config, environ=None): """Build the live mailbox. Credentials come only from the environment.""" environ = os.environ if environ is None else environ spec = config.get('mailbox') or {} user = environ.get(spec.get('user_env', 'INBOUND_LIBRARY_IMAP_USER')) password = environ.get(spec.get('password_env', 'INBOUND_LIBRARY_IMAP_PASSWORD')) if not user or not password: return None return ImapMailbox(spec.get('host', 'imap.ionos.de'), spec.get('port', 993), spec.get('folder', 'INBOX'), user, password) def _text(message): plain = message.get_body(preferencelist=('plain',)) if plain is not None: return plain.get_content() rich = message.get_body(preferencelist=('html',)) if rich is None: return '' stripped = re.sub(r'(?is)<(script|style).*?', ' ', rich.get_content()) stripped = re.sub(r'(?s)|

||', '\n', stripped) return html.unescape(re.sub(r'(?s)<[^>]+>', '', stripped)).strip() def retain_message(root, registry, raw, mailbox, received_at): """Retain one RFC 822 message and its attachments. Returns the delivery id.""" message = email.message_from_bytes(raw, policy=email.policy.default) item, _ = store.put_item(root, raw) sender = email.utils.parseaddr(str(message.get('From', '')))[1].lower() subject = str(message.get('Subject', '')) list_id = str(message.get('List-Id', '')) source = reg.source_for_mail(registry, sender, subject, list_id) provenance = { 'channel': 'imap', 'mailbox': mailbox, 'received_at': received_at, 'message_id': str(message.get('Message-ID', '')), 'sender': sender, 'subject': subject, 'list_id': list_id, 'source': source, 'authentication_results': [str(h) for h in message.get_all('Authentication-Results', [])], } delivery = store.record_delivery(root, item, provenance) attachments = [] for part in message.iter_attachments(): payload = part.get_payload(decode=True) if payload is None: continue child, _ = store.put_item(root, payload) store.record_delivery(root, child, { 'channel': 'attachment', 'parent_item': item, 'received_at': received_at, 'filename': part.get_filename() or '', 'content_type': part.get_content_type(), 'source': source}) attachments.append({'item': child, 'filename': part.get_filename() or '', 'content_type': part.get_content_type()}) store.put_derived(root, item, { 'subject': subject, 'sender': sender, 'date': str(message.get('Date', '')), 'text': _text(message), 'attachments': attachments}) return delivery def scan(root, mailbox, registry=None): """Retain every new message. The cursor advances only after each retain.""" root = Path(root) registry = registry or reg.load(root) cursor_path = root / CURSOR cursor = json.loads(cursor_path.read_text()) if cursor_path.exists() else {} retained = [] for uidvalidity, uid, received_at, raw in mailbox.fetch(cursor): retained.append(retain_message(root, registry, raw, mailbox.name, received_at)) cursor = {'mailbox': mailbox.name, 'uidvalidity': uidvalidity, 'last_uid': uid} store.atomic(cursor_path, json.dumps(cursor, indent=2).encode() + b'\n') state = {**cursor, 'scanned_at': datetime.now(timezone.utc).isoformat()} store.atomic(cursor_path, json.dumps(state, indent=2).encode() + b'\n') return retained def requeue_quarantine(root, registry=None): """Re-route quarantined mail whose sender now matches a registered source.""" registry = registry or reg.load(root) routed = [] for delivery in store.records(root, 'deliveries'): if delivery['status'] != 'quarantined' or delivery.get('channel') != 'imap': continue source = reg.source_for_mail(registry, delivery.get('sender'), delivery.get('subject'), delivery.get('list_id')) if source: raw = store.get_item(root, delivery['item']) routed.append(retain_message(root, registry, raw, delivery['mailbox'], delivery['received_at'])) return routed