inbound-library/inbound_library/intake.py

"""Read-only mail intake.

The mailbox is selected read-only and bodies are fetched with BODY.PEEK[], so no
flag changes. The UIDVALIDITY/UID cursor lives in the library. Delivery records
leave out UIDs, so a rescan after a UIDVALIDITY change reproduces the same
records instead of duplicating them.
"""
import email
import email.policy
import email.utils
import html
import imaplib
import json
import os
import re
import ssl
from datetime import datetime, timezone
from pathlib import Path

from . import registry as reg
from . import store

CURSOR = Path('data') / 'state' / 'imap-cursor.json'


class ImapMailbox:
    def __init__(self, host, port, folder, user, password):
        self.host, self.port, self.folder = host, int(port), folder
        self.user, self.password = user, password
        self.name = f'{user}/{folder}'

    def fetch(self, cursor):
        """Yield (uidvalidity, uid, internaldate, raw) for UIDs past the cursor."""
        context = ssl.create_default_context()
        with imaplib.IMAP4_SSL(self.host, self.port, ssl_context=context) as imap:
            imap.login(self.user, self.password)
            status, _ = imap.select(self.folder, readonly=True)
            if status != 'OK':
                raise OSError(f'cannot select {self.folder}')
            uidvalidity = int(imap.response('UIDVALIDITY')[1][0])
            last = cursor.get('last_uid', 0) if cursor.get('uidvalidity') == uidvalidity else 0
            status, data = imap.uid('SEARCH', None, f'UID {last + 1}:*')
            if status != 'OK':
                raise OSError('UID SEARCH failed')
            uids = sorted(int(u) for u in data[0].split() if int(u) > last)
            for uid in uids:
                status, data = imap.uid('FETCH', str(uid), '(INTERNALDATE BODY.PEEK[])')
                if status != 'OK':
                    raise OSError(f'UID FETCH {uid} failed')
                meta, raw = next((part[0], part[1]) for part in data if isinstance(part, tuple))
                when = imaplib.Internaldate2tuple(meta)
                stamp = datetime(*when[:6], tzinfo=timezone.utc).isoformat() if when else None
                yield uidvalidity, uid, stamp, raw


class DirectoryMailbox:
    """A fixture mailbox: *.eml files in name order, UID = position + 1."""

    def __init__(self, path, uidvalidity=1):
        self.path, self.uidvalidity = Path(path), uidvalidity
        self.name = f'dir:{self.path.name}'

    def fetch(self, cursor):
        last = cursor.get('last_uid', 0) if cursor.get('uidvalidity') == self.uidvalidity else 0
        for uid, path in enumerate(sorted(self.path.glob('*.eml')), start=1):
            if uid > last:
                stamp = datetime.fromtimestamp(path.stat().st_mtime, timezone.utc).isoformat()
                yield self.uidvalidity, uid, stamp, path.read_bytes()


def mailbox_from_config(config, environ=None, fetch=None):
    """Build the live mailbox, or None when no credential source is configured.

    The environment variables are an explicit override (tests, one-off runs).
    Otherwise the credential is read from OpenBao on every call through the
    AppRole in config["credential"], and never cached.
    """
    environ = os.environ if environ is None else environ
    spec = config.get('mailbox') or {}
    host, port = spec.get('host', 'imap.ionos.de'), spec.get('port', 993)
    folder = spec.get('folder', 'INBOX')
    user = environ.get(spec.get('user_env', 'INBOUND_LIBRARY_IMAP_USER'))
    password = environ.get(spec.get('password_env', 'INBOUND_LIBRARY_IMAP_PASSWORD'))
    if user and password:
        return ImapMailbox(host, port, folder, user, password)
    credential = config.get('credential')
    if not credential:
        return None
    from . import vault
    fields = (fetch or vault.fetch)(credential)
    if not fields.get('IMAP_USERNAME') or not fields.get('IMAP_PASSWORD'):
        raise vault.CredentialUnavailable('credential lacks IMAP_USERNAME or IMAP_PASSWORD')
    return ImapMailbox(fields.get('IMAP_HOST', host), fields.get('IMAP_PORT', port),
                       fields.get('IMAP_FOLDER', folder), fields['IMAP_USERNAME'],
                       fields['IMAP_PASSWORD'])


def _text(message):
    plain = message.get_body(preferencelist=('plain',))
    if plain is not None:
        return plain.get_content()
    rich = message.get_body(preferencelist=('html',))
    if rich is None:
        return ''
    stripped = re.sub(r'(?is)<(script|style).*?</\1>', ' ', rich.get_content())
    stripped = re.sub(r'(?s)<br\s*/?>|</p>|</div>|</li>', '\n', stripped)
    return html.unescape(re.sub(r'(?s)<[^>]+>', '', stripped)).strip()


# Header line of a forwarded message, as Gmail and common clients write it in the
# body, in English or German.
FORWARDED_FROM = re.compile(r'^\s*(?:From|Von)\s*:\s*(.+)$', re.MULTILINE)
FORWARD_MARKER = re.compile(r'-{3,}\s*(?:Forwarded message|Weitergeleitete Nachricht)\s*-{3,}',
                            re.IGNORECASE)


def forwarded_sender(text):
    """Original sender of a forwarded mail, from the first header block after the marker."""
    marker = FORWARD_MARKER.search(text or '')
    if not marker:
        return ''
    match = FORWARDED_FROM.search(text, marker.end())
    return email.utils.parseaddr(match.group(1))[1].lower() if match else ''


def retain_message(root, registry, raw, mailbox, received_at):
    """Retain one RFC 822 message and its attachments. Returns the delivery id."""
    message = email.message_from_bytes(raw, policy=email.policy.default)
    item, _ = store.put_item(root, raw)
    sender = email.utils.parseaddr(str(message.get('From', '')))[1].lower()
    subject = str(message.get('Subject', ''))
    list_id = str(message.get('List-Id', ''))
    text = _text(message)
    forwarded_from = forwarded_sender(text)
    source = reg.source_for_mail(registry, sender, subject, list_id, forwarded_from)
    provenance = {
        'channel': 'imap', 'mailbox': mailbox, 'received_at': received_at,
        'message_id': str(message.get('Message-ID', '')), 'sender': sender,
        'subject': subject, 'list_id': list_id, 'forwarded_from': forwarded_from,
        'source': source,
        'authentication_results': [str(h) for h in message.get_all('Authentication-Results', [])],
    }
    delivery = store.record_delivery(root, item, provenance)
    attachments = []
    for part in message.iter_attachments():
        payload = part.get_payload(decode=True)
        if payload is None:
            continue
        child, _ = store.put_item(root, payload)
        store.record_delivery(root, child, {
            'channel': 'attachment', 'parent_item': item, 'received_at': received_at,
            'filename': part.get_filename() or '', 'content_type': part.get_content_type(),
            'source': source})
        attachments.append({'item': child, 'filename': part.get_filename() or '',
                            'content_type': part.get_content_type()})
    store.put_derived(root, item, {
        'subject': subject, 'sender': sender, 'date': str(message.get('Date', '')),
        'text': text, 'attachments': attachments})
    return delivery


def scan(root, mailbox, registry=None):
    """Retain every new message. The cursor advances only after each retain."""
    root = Path(root)
    registry = registry or reg.load(root)
    cursor_path = root / CURSOR
    cursor = json.loads(cursor_path.read_text()) if cursor_path.exists() else {}
    retained = []
    for uidvalidity, uid, received_at, raw in mailbox.fetch(cursor):
        retained.append(retain_message(root, registry, raw, mailbox.name, received_at))
        cursor = {'mailbox': mailbox.name, 'uidvalidity': uidvalidity, 'last_uid': uid}
        store.atomic(cursor_path, json.dumps(cursor, indent=2).encode() + b'\n')
    state = {**cursor, 'scanned_at': datetime.now(timezone.utc).isoformat()}
    store.atomic(cursor_path, json.dumps(state, indent=2).encode() + b'\n')
    return retained


def requeue_quarantine(root, registry=None):
    """Re-route quarantined mail whose sender now matches a registered source."""
    registry = registry or reg.load(root)
    routed = []
    for delivery in store.records(root, 'deliveries'):
        if delivery['status'] != 'quarantined' or delivery.get('channel') != 'imap':
            continue
        text = (store.get_derived(root, delivery['item']) or {}).get('text', '')
        source = reg.source_for_mail(registry, delivery.get('sender'), delivery.get('subject'),
                                     delivery.get('list_id'), forwarded_sender(text))
        if source:
            raw = store.get_item(root, delivery['item'])
            routed.append(retain_message(root, registry, raw, delivery['mailbox'],
                                         delivery['received_at']))
    return routed