tegwick
86f22c2e65
Adds MaildirMailboxSource reading new/ and cur/, wired through source.maildir_dir and mailbox.protocol: maildir. Message identity is the Maildir unique name without its :2,FLAGS suffix, so it survives the new/ to cur/ move; it is carried on MailboxSourceMessage.dedup_uid and appended to the message dedup key only when set, leaving fixture and IMAP keys unchanged. Cursor ordering parses the Maildir delivery time instead of comparing names lexically. mark_seen and a missing directory are rejected, matching the read-only IMAP contract. Also fixes the parse-failure path, which keyed identity on raw_message_ref. A new/ to cur/ move rewrites that ref, so an unparseable message re-registered as new on every rescan; it now prefers the source uid when one exists. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
270 lines
9.5 KiB
Python
270 lines
9.5 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
from dataclasses import dataclass
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
from uuid import uuid4
|
|
|
|
from .config import AppConfig
|
|
from .evidence import endpoint_quality_from_candidate
|
|
from .mailbox import source_for_config
|
|
from .models import MailboxScan
|
|
from .parser import parse_message_bytes
|
|
from .recipients import load_expected_recipients
|
|
from .reporting import write_evidence_report
|
|
from .storage import StateStore
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ScanResult:
|
|
scan: MailboxScan
|
|
report_path: Path | None
|
|
warnings: tuple[str, ...] = ()
|
|
|
|
|
|
def scan_mailbox(
|
|
config: AppConfig,
|
|
*,
|
|
output_dir: str | None = None,
|
|
full_rescan: bool = False,
|
|
report_only_new: bool = False,
|
|
dry_run: bool = False,
|
|
fixture_dir: str | None = None,
|
|
since: str | None = None,
|
|
range_from: str | None = None,
|
|
range_to: str | None = None,
|
|
expected_recipients_path: str | None = None,
|
|
expected_recipient_column: str | None = None,
|
|
) -> ScanResult:
|
|
started_at = datetime.now(UTC)
|
|
scan_id = str(uuid4())
|
|
range_start = _parse_datetime(range_from or since or config.scan.range_from or config.scan.since)
|
|
range_end = _parse_datetime(range_to or config.scan.range_to)
|
|
if range_start and range_end and range_start > range_end:
|
|
raise ValueError("scan datetime range lower bound must be before or equal to upper bound.")
|
|
since_at = range_start
|
|
expected = load_expected_recipients(
|
|
expected_recipients_path or config.expected_recipients.path,
|
|
csv_column=expected_recipient_column or config.expected_recipients.csv_column,
|
|
)
|
|
expected_addresses = set(expected.addresses)
|
|
warnings = tuple(f"invalid expected recipient ignored: {entry}" for entry in expected.invalid_entries)
|
|
source = source_for_config(config, fixture_dir_override=fixture_dir)
|
|
|
|
store = StateStore(config.storage.path)
|
|
messages_seen = 0
|
|
messages_new = 0
|
|
messages_parsed = 0
|
|
evidence_created = 0
|
|
new_evidence_keys: list[str] = []
|
|
last_source_uid: str | None = None
|
|
try:
|
|
cursor = None if full_rescan else store.get_scan_cursor(config.mailbox.id, config.mailbox.folder)
|
|
for message in source.iter_messages(
|
|
max_messages=config.scan.max_messages_per_run,
|
|
since_uid=cursor,
|
|
full_rescan=full_rescan,
|
|
include_seen=config.scan.include_seen,
|
|
since=since_at,
|
|
):
|
|
messages_seen += 1
|
|
last_source_uid = message.source_uid
|
|
inbound, parsed, candidate = parse_message_bytes(
|
|
message.raw_bytes,
|
|
mailbox_id=config.mailbox.id,
|
|
raw_message_ref=message.raw_message_ref,
|
|
imap_uid=message.imap_uid,
|
|
source_uid=message.dedup_uid,
|
|
)
|
|
if not _in_range(inbound.received_at, range_start=range_start, range_end=range_end):
|
|
continue
|
|
if dry_run:
|
|
messages_parsed += 1
|
|
continue
|
|
is_new = store.upsert_message(inbound)
|
|
messages_new += int(is_new)
|
|
store.insert_parsed(parsed)
|
|
messages_parsed += 1
|
|
if candidate is not None:
|
|
candidate = _enrich_candidate(candidate, inbound, parsed)
|
|
inserted = store.insert_evidence(candidate)
|
|
evidence_created += int(inserted)
|
|
if inserted:
|
|
new_evidence_keys.append(candidate.deduplication_key)
|
|
quality_update = endpoint_quality_from_candidate(candidate)
|
|
if quality_update is not None:
|
|
store.apply_endpoint_quality_update(
|
|
config.mailbox.id,
|
|
quality_update,
|
|
observed_at=candidate.observed_at,
|
|
)
|
|
|
|
if not dry_run and last_source_uid:
|
|
store.set_scan_cursor(
|
|
config.mailbox.id,
|
|
config.mailbox.folder,
|
|
last_source_uid,
|
|
updated_at=datetime.now(UTC),
|
|
)
|
|
|
|
report_path = None
|
|
if not dry_run:
|
|
range_evidence_rows = store.evidence_rows(range_start=range_start, range_end=range_end)
|
|
report_rows = store.evidence_rows(
|
|
deduplication_keys=new_evidence_keys if report_only_new else None,
|
|
range_start=range_start,
|
|
range_end=range_end,
|
|
)
|
|
report_rows = [
|
|
*report_rows,
|
|
*_no_evidence_rows(
|
|
mailbox_id=config.mailbox.id,
|
|
expected_addresses=expected_addresses,
|
|
evidence_rows=range_evidence_rows,
|
|
observed_at=datetime.now(UTC),
|
|
range_start=range_start,
|
|
range_end=range_end,
|
|
),
|
|
]
|
|
report_path = write_evidence_report(
|
|
report_rows,
|
|
output_dir=output_dir or config.reports.output_dir,
|
|
scan_id=scan_id,
|
|
mailbox_id=config.mailbox.id,
|
|
expected_recipients=expected_addresses,
|
|
)
|
|
finished_at = datetime.now(UTC)
|
|
scan = MailboxScan(
|
|
scan_id=scan_id,
|
|
mailbox_id=config.mailbox.id,
|
|
started_at=started_at,
|
|
finished_at=finished_at,
|
|
scan_mode="full_rescan" if full_rescan else config.scan.mode,
|
|
folder=config.mailbox.folder,
|
|
status="completed",
|
|
messages_seen=messages_seen,
|
|
messages_new=messages_new,
|
|
messages_parsed=messages_parsed,
|
|
evidence_events_created=evidence_created,
|
|
report_path=str(report_path) if report_path else None,
|
|
since=since_at,
|
|
range_start=range_start,
|
|
range_end=range_end,
|
|
)
|
|
if not dry_run:
|
|
store.insert_scan(scan)
|
|
return ScanResult(scan=scan, report_path=report_path, warnings=warnings)
|
|
finally:
|
|
store.close()
|
|
|
|
|
|
def _enrich_candidate(candidate, inbound, parsed):
|
|
metadata = {
|
|
**candidate.metadata,
|
|
"source_from": inbound.from_address,
|
|
"source_to": ", ".join(inbound.to_addresses),
|
|
"source_subject": inbound.subject,
|
|
"message_id_header": inbound.message_id_header,
|
|
"original_recipient": parsed.original_recipient,
|
|
"first_seen_at": inbound.first_seen_at.isoformat(),
|
|
"last_seen_at": inbound.last_seen_at.isoformat(),
|
|
}
|
|
return type(candidate)(
|
|
**{
|
|
**candidate.__dict__,
|
|
"metadata": metadata,
|
|
}
|
|
)
|
|
|
|
|
|
def _parse_datetime(value: str | None) -> datetime | None:
|
|
if not value:
|
|
return None
|
|
normalized = value.strip()
|
|
if not normalized:
|
|
return None
|
|
if len(normalized) == 10:
|
|
normalized = normalized + "T00:00:00+00:00"
|
|
parsed = datetime.fromisoformat(normalized.replace("Z", "+00:00"))
|
|
if parsed.tzinfo is None:
|
|
return parsed.replace(tzinfo=UTC)
|
|
return parsed.astimezone(UTC)
|
|
|
|
|
|
def _in_range(
|
|
received_at: datetime | None,
|
|
*,
|
|
range_start: datetime | None,
|
|
range_end: datetime | None,
|
|
) -> bool:
|
|
if range_start is None and range_end is None:
|
|
return True
|
|
if received_at is None:
|
|
return False
|
|
if range_start is not None and received_at < range_start:
|
|
return False
|
|
if range_end is not None and received_at > range_end:
|
|
return False
|
|
return True
|
|
|
|
|
|
def _no_evidence_rows(
|
|
*,
|
|
mailbox_id: str,
|
|
expected_addresses: set[str],
|
|
evidence_rows: list[dict],
|
|
observed_at: datetime,
|
|
range_start: datetime | None,
|
|
range_end: datetime | None,
|
|
) -> list[dict]:
|
|
if not expected_addresses:
|
|
return []
|
|
known_evidence_addresses = {
|
|
str(row.get("affected_email_address") or "").lower()
|
|
for row in evidence_rows
|
|
if row.get("affected_email_address")
|
|
}
|
|
rows = []
|
|
for address in sorted(expected_addresses - known_evidence_addresses):
|
|
rows.append(_no_evidence_row(mailbox_id, address, observed_at, range_start=range_start, range_end=range_end))
|
|
return rows
|
|
|
|
|
|
def _no_evidence_row(
|
|
mailbox_id: str,
|
|
address: str,
|
|
observed_at: datetime,
|
|
*,
|
|
range_start: datetime | None,
|
|
range_end: datetime | None,
|
|
) -> dict:
|
|
range_key = "|".join([
|
|
range_start.isoformat() if range_start else "",
|
|
range_end.isoformat() if range_end else "",
|
|
])
|
|
return {
|
|
"mailbox_message_id": "",
|
|
"event_type": "diagnostic.expected_recipient.no_evidence",
|
|
"assessment_category": "undef",
|
|
"assessment_subclass": "undef.no_signal",
|
|
"affected_email_address": address,
|
|
"original_message_id": "",
|
|
"confidence": "high",
|
|
"evidence_strength": "none",
|
|
"occurred_at": "",
|
|
"observed_at": observed_at.isoformat(),
|
|
"deduplication_key": f"{mailbox_id}|expected_recipient|no_evidence|{address}|{range_key}",
|
|
"raw_message_ref": "",
|
|
"notes_json": json.dumps([
|
|
"Expected recipient was supplied by the operator; no mailbox evidence was found in the inspected range.",
|
|
"This is not evidence of delivery success or delivery failure.",
|
|
]),
|
|
"metadata_json": json.dumps({
|
|
"message_class": "expected_recipient_no_evidence",
|
|
"original_recipient": address,
|
|
"range_start": range_start.isoformat() if range_start else None,
|
|
"range_end": range_end.isoformat() if range_end else None,
|
|
}),
|
|
"known_recipient": True,
|
|
}
|