inbound-library/inbound_library/intake.py

"""Read-only mail intake.

The mailbox is selected read-only and bodies are fetched with BODY.PEEK[], so no
flag changes. The UIDVALIDITY/UID cursor lives in the library. Delivery records
leave out UIDs, so a rescan after a UIDVALIDITY change reproduces the same
records instead of duplicating them.
"""
import email
import email.policy
import email.utils
import html
import imaplib
import json
import os
import re
import ssl
from datetime import datetime, timezone
from pathlib import Path

from . import registry as reg
from . import store

CURSOR = Path('data') / 'state' / 'imap-cursor.json'


class ImapMailbox:
    def __init__(self, host, port, folder, user, password):
        self.host, self.port, self.folder = host, int(port), folder
        self.user, self.password = user, password
        self.name = f'{user}/{folder}'

    def fetch(self, cursor):
        """Yield (uidvalidity, uid, internaldate, raw) for UIDs past the cursor."""
        context = ssl.create_default_context()
        with imaplib.IMAP4_SSL(self.host, self.port, ssl_context=context) as imap:
            imap.login(self.user, self.password)
            status, _ = imap.select(self.folder, readonly=True)
            if status != 'OK':
                raise OSError(f'cannot select {self.folder}')
            uidvalidity = int(imap.response('UIDVALIDITY')[1][0])
            last = cursor.get('last_uid', 0) if cursor.get('uidvalidity') == uidvalidity else 0
            status, data = imap.uid('SEARCH', None, f'UID {last + 1}:*')
            if status != 'OK':
                raise OSError('UID SEARCH failed')
            uids = sorted(int(u) for u in data[0].split() if int(u) > last)
            for uid in uids:
                status, data = imap.uid('FETCH', str(uid), '(INTERNALDATE BODY.PEEK[])')
                if status != 'OK':
                    raise OSError(f'UID FETCH {uid} failed')
                meta, raw = next((part[0], part[1]) for part in data if isinstance(part, tuple))
                when = imaplib.Internaldate2tuple(meta)
                stamp = datetime(*when[:6], tzinfo=timezone.utc).isoformat() if when else None
                yield uidvalidity, uid, stamp, raw


class DirectoryMailbox:
    """A fixture mailbox: *.eml files in name order, UID = position + 1."""

    def __init__(self, path, uidvalidity=1):
        self.path, self.uidvalidity = Path(path), uidvalidity
        self.name = f'dir:{self.path.name}'

    def fetch(self, cursor):
        last = cursor.get('last_uid', 0) if cursor.get('uidvalidity') == self.uidvalidity else 0
        for uid, path in enumerate(sorted(self.path.glob('*.eml')), start=1):
            if uid > last:
                stamp = datetime.fromtimestamp(path.stat().st_mtime, timezone.utc).isoformat()
                yield self.uidvalidity, uid, stamp, path.read_bytes()


def mailbox_from_config(config, environ=None):
    """Build the live mailbox. Credentials come only from the environment."""
    environ = os.environ if environ is None else environ
    spec = config.get('mailbox') or {}
    user = environ.get(spec.get('user_env', 'INBOUND_LIBRARY_IMAP_USER'))
    password = environ.get(spec.get('password_env', 'INBOUND_LIBRARY_IMAP_PASSWORD'))
    if not user or not password:
        return None
    return ImapMailbox(spec.get('host', 'imap.ionos.de'), spec.get('port', 993),
                       spec.get('folder', 'INBOX'), user, password)


def _text(message):
    plain = message.get_body(preferencelist=('plain',))
    if plain is not None:
        return plain.get_content()
    rich = message.get_body(preferencelist=('html',))
    if rich is None:
        return ''
    stripped = re.sub(r'(?is)<(script|style).*?</\1>', ' ', rich.get_content())
    stripped = re.sub(r'(?s)<br\s*/?>|</p>|</div>|</li>', '\n', stripped)
    return html.unescape(re.sub(r'(?s)<[^>]+>', '', stripped)).strip()


def retain_message(root, registry, raw, mailbox, received_at):
    """Retain one RFC 822 message and its attachments. Returns the delivery id."""
    message = email.message_from_bytes(raw, policy=email.policy.default)
    item, _ = store.put_item(root, raw)
    sender = email.utils.parseaddr(str(message.get('From', '')))[1].lower()
    subject = str(message.get('Subject', ''))
    list_id = str(message.get('List-Id', ''))
    source = reg.source_for_mail(registry, sender, subject, list_id)
    provenance = {
        'channel': 'imap', 'mailbox': mailbox, 'received_at': received_at,
        'message_id': str(message.get('Message-ID', '')), 'sender': sender,
        'subject': subject, 'list_id': list_id, 'source': source,
        'authentication_results': [str(h) for h in message.get_all('Authentication-Results', [])],
    }
    delivery = store.record_delivery(root, item, provenance)
    attachments = []
    for part in message.iter_attachments():
        payload = part.get_payload(decode=True)
        if payload is None:
            continue
        child, _ = store.put_item(root, payload)
        store.record_delivery(root, child, {
            'channel': 'attachment', 'parent_item': item, 'received_at': received_at,
            'filename': part.get_filename() or '', 'content_type': part.get_content_type(),
            'source': source})
        attachments.append({'item': child, 'filename': part.get_filename() or '',
                            'content_type': part.get_content_type()})
    store.put_derived(root, item, {
        'subject': subject, 'sender': sender, 'date': str(message.get('Date', '')),
        'text': _text(message), 'attachments': attachments})
    return delivery


def scan(root, mailbox, registry=None):
    """Retain every new message. The cursor advances only after each retain."""
    root = Path(root)
    registry = registry or reg.load(root)
    cursor_path = root / CURSOR
    cursor = json.loads(cursor_path.read_text()) if cursor_path.exists() else {}
    retained = []
    for uidvalidity, uid, received_at, raw in mailbox.fetch(cursor):
        retained.append(retain_message(root, registry, raw, mailbox.name, received_at))
        cursor = {'mailbox': mailbox.name, 'uidvalidity': uidvalidity, 'last_uid': uid}
        store.atomic(cursor_path, json.dumps(cursor, indent=2).encode() + b'\n')
    state = {**cursor, 'scanned_at': datetime.now(timezone.utc).isoformat()}
    store.atomic(cursor_path, json.dumps(state, indent=2).encode() + b'\n')
    return retained


def requeue_quarantine(root, registry=None):
    """Re-route quarantined mail whose sender now matches a registered source."""
    registry = registry or reg.load(root)
    routed = []
    for delivery in store.records(root, 'deliveries'):
        if delivery['status'] != 'quarantined' or delivery.get('channel') != 'imap':
            continue
        source = reg.source_for_mail(registry, delivery.get('sender'), delivery.get('subject'),
                                     delivery.get('list_id'))
        if source:
            raw = store.get_item(root, delivery['item'])
            routed.append(retain_message(root, registry, raw, delivery['mailbox'],
                                         delivery['received_at']))
    return routed