email-connect/src/email_connect/scanner.py

from __future__ import annotations

import json
from dataclasses import dataclass
from datetime import UTC, datetime
from pathlib import Path
from uuid import uuid4

from .config import AppConfig
from .evidence import endpoint_quality_from_candidate
from .mailbox import source_for_config
from .models import MailboxScan
from .parser import parse_message_bytes
from .recipients import load_expected_recipients
from .reporting import write_evidence_report
from .storage import StateStore


@dataclass(frozen=True)
class ScanResult:
    scan: MailboxScan
    report_path: Path | None
    warnings: tuple[str, ...] = ()


def scan_mailbox(
    config: AppConfig,
    *,
    output_dir: str | None = None,
    full_rescan: bool = False,
    report_only_new: bool = False,
    dry_run: bool = False,
    fixture_dir: str | None = None,
    since: str | None = None,
    range_from: str | None = None,
    range_to: str | None = None,
    expected_recipients_path: str | None = None,
    expected_recipient_column: str | None = None,
) -> ScanResult:
    started_at = datetime.now(UTC)
    scan_id = str(uuid4())
    range_start = _parse_datetime(range_from or since or config.scan.range_from or config.scan.since)
    range_end = _parse_datetime(range_to or config.scan.range_to)
    if range_start and range_end and range_start > range_end:
        raise ValueError("scan datetime range lower bound must be before or equal to upper bound.")
    since_at = range_start
    expected = load_expected_recipients(
        expected_recipients_path or config.expected_recipients.path,
        csv_column=expected_recipient_column or config.expected_recipients.csv_column,
    )
    expected_addresses = set(expected.addresses)
    warnings = tuple(f"invalid expected recipient ignored: {entry}" for entry in expected.invalid_entries)
    source = source_for_config(config, fixture_dir_override=fixture_dir)

    store = StateStore(config.storage.path)
    messages_seen = 0
    messages_new = 0
    messages_parsed = 0
    evidence_created = 0
    new_evidence_keys: list[str] = []
    last_source_uid: str | None = None
    try:
        cursor = None if full_rescan else store.get_scan_cursor(config.mailbox.id, config.mailbox.folder)
        for message in source.iter_messages(
            max_messages=config.scan.max_messages_per_run,
            since_uid=cursor,
            full_rescan=full_rescan,
            include_seen=config.scan.include_seen,
            since=since_at,
        ):
            messages_seen += 1
            last_source_uid = message.source_uid
            inbound, parsed, candidate = parse_message_bytes(
                message.raw_bytes,
                mailbox_id=config.mailbox.id,
                raw_message_ref=message.raw_message_ref,
                imap_uid=message.imap_uid,
                source_uid=message.dedup_uid,
            )
            if not _in_range(inbound.received_at, range_start=range_start, range_end=range_end):
                continue
            if dry_run:
                messages_parsed += 1
                continue
            is_new = store.upsert_message(inbound)
            messages_new += int(is_new)
            store.insert_parsed(parsed)
            messages_parsed += 1
            if candidate is not None:
                candidate = _enrich_candidate(candidate, inbound, parsed)
                inserted = store.insert_evidence(candidate)
                evidence_created += int(inserted)
                if inserted:
                    new_evidence_keys.append(candidate.deduplication_key)
                quality_update = endpoint_quality_from_candidate(candidate)
                if quality_update is not None:
                    store.apply_endpoint_quality_update(
                        config.mailbox.id,
                        quality_update,
                        observed_at=candidate.observed_at,
                    )

        if not dry_run and last_source_uid:
            store.set_scan_cursor(
                config.mailbox.id,
                config.mailbox.folder,
                last_source_uid,
                updated_at=datetime.now(UTC),
            )

        report_path = None
        if not dry_run:
            range_evidence_rows = store.evidence_rows(range_start=range_start, range_end=range_end)
            report_rows = store.evidence_rows(
                deduplication_keys=new_evidence_keys if report_only_new else None,
                range_start=range_start,
                range_end=range_end,
            )
            report_rows = [
                *report_rows,
                *_no_evidence_rows(
                    mailbox_id=config.mailbox.id,
                    expected_addresses=expected_addresses,
                    evidence_rows=range_evidence_rows,
                    observed_at=datetime.now(UTC),
                    range_start=range_start,
                    range_end=range_end,
                ),
            ]
            report_path = write_evidence_report(
                report_rows,
                output_dir=output_dir or config.reports.output_dir,
                scan_id=scan_id,
                mailbox_id=config.mailbox.id,
                expected_recipients=expected_addresses,
            )
        finished_at = datetime.now(UTC)
        scan = MailboxScan(
            scan_id=scan_id,
            mailbox_id=config.mailbox.id,
            started_at=started_at,
            finished_at=finished_at,
            scan_mode="full_rescan" if full_rescan else config.scan.mode,
            folder=config.mailbox.folder,
            status="completed",
            messages_seen=messages_seen,
            messages_new=messages_new,
            messages_parsed=messages_parsed,
            evidence_events_created=evidence_created,
            report_path=str(report_path) if report_path else None,
            since=since_at,
            range_start=range_start,
            range_end=range_end,
        )
        if not dry_run:
            store.insert_scan(scan)
        return ScanResult(scan=scan, report_path=report_path, warnings=warnings)
    finally:
        store.close()


def _enrich_candidate(candidate, inbound, parsed):
    metadata = {
        **candidate.metadata,
        "source_from": inbound.from_address,
        "source_to": ", ".join(inbound.to_addresses),
        "source_subject": inbound.subject,
        "message_id_header": inbound.message_id_header,
        "original_recipient": parsed.original_recipient,
        "first_seen_at": inbound.first_seen_at.isoformat(),
        "last_seen_at": inbound.last_seen_at.isoformat(),
    }
    return type(candidate)(
        **{
            **candidate.__dict__,
            "metadata": metadata,
        }
    )


def _parse_datetime(value: str | None) -> datetime | None:
    if not value:
        return None
    normalized = value.strip()
    if not normalized:
        return None
    if len(normalized) == 10:
        normalized = normalized + "T00:00:00+00:00"
    parsed = datetime.fromisoformat(normalized.replace("Z", "+00:00"))
    if parsed.tzinfo is None:
        return parsed.replace(tzinfo=UTC)
    return parsed.astimezone(UTC)


def _in_range(
    received_at: datetime | None,
    *,
    range_start: datetime | None,
    range_end: datetime | None,
) -> bool:
    if range_start is None and range_end is None:
        return True
    if received_at is None:
        return False
    if range_start is not None and received_at < range_start:
        return False
    if range_end is not None and received_at > range_end:
        return False
    return True


def _no_evidence_rows(
    *,
    mailbox_id: str,
    expected_addresses: set[str],
    evidence_rows: list[dict],
    observed_at: datetime,
    range_start: datetime | None,
    range_end: datetime | None,
) -> list[dict]:
    if not expected_addresses:
        return []
    known_evidence_addresses = {
        str(row.get("affected_email_address") or "").lower()
        for row in evidence_rows
        if row.get("affected_email_address")
    }
    rows = []
    for address in sorted(expected_addresses - known_evidence_addresses):
        rows.append(_no_evidence_row(mailbox_id, address, observed_at, range_start=range_start, range_end=range_end))
    return rows


def _no_evidence_row(
    mailbox_id: str,
    address: str,
    observed_at: datetime,
    *,
    range_start: datetime | None,
    range_end: datetime | None,
) -> dict:
    range_key = "|".join([
        range_start.isoformat() if range_start else "",
        range_end.isoformat() if range_end else "",
    ])
    return {
        "mailbox_message_id": "",
        "event_type": "diagnostic.expected_recipient.no_evidence",
        "assessment_category": "undef",
        "assessment_subclass": "undef.no_signal",
        "affected_email_address": address,
        "original_message_id": "",
        "confidence": "high",
        "evidence_strength": "none",
        "occurred_at": "",
        "observed_at": observed_at.isoformat(),
        "deduplication_key": f"{mailbox_id}|expected_recipient|no_evidence|{address}|{range_key}",
        "raw_message_ref": "",
        "notes_json": json.dumps([
            "Expected recipient was supplied by the operator; no mailbox evidence was found in the inspected range.",
            "This is not evidence of delivery success or delivery failure.",
        ]),
        "metadata_json": json.dumps({
            "message_class": "expected_recipient_no_evidence",
            "original_recipient": address,
            "range_start": range_start.isoformat() if range_start else None,
            "range_end": range_end.isoformat() if range_end else None,
        }),
        "known_recipient": True,
    }