diff --git a/WORK-RECORDS.md b/WORK-RECORDS.md index 8f16e78..026a4d2 100644 --- a/WORK-RECORDS.md +++ b/WORK-RECORDS.md @@ -12,6 +12,7 @@ | workplan | EMAIL-WP-0002 | finished | — | workplans/EMAIL-WP-0002-mvp-mailbox-evidence-scanner.md | | workplan | EMAIL-WP-0003 | finished | — | workplans/EMAIL-WP-0003-expected-recipient-reporting-and-mailbox-tutorial.md | | workplan | EMAIL-WP-0004 | finished | — | workplans/EMAIL-WP-0004-transactional-mail-delivery-service.md | +| workplan | EMAIL-WP-0005 | proposed | — | workplans/EMAIL-WP-0005-test-mailbox-harness.md | | task | EMAIL-WP-0001-T01 | done | — | workplans/EMAIL-WP-0001-repo-onboarding.md | | task | EMAIL-WP-0001-T02 | done | — | workplans/EMAIL-WP-0001-repo-onboarding.md | | task | EMAIL-WP-0001-T03 | done | — | workplans/EMAIL-WP-0001-repo-onboarding.md | @@ -39,3 +40,9 @@ | task | EMAIL-WP-0004-T02 | done | — | workplans/EMAIL-WP-0004-transactional-mail-delivery-service.md | | task | EMAIL-WP-0004-T03 | done | — | workplans/EMAIL-WP-0004-transactional-mail-delivery-service.md | | task | EMAIL-WP-0004-T04 | done | — | workplans/EMAIL-WP-0004-transactional-mail-delivery-service.md | +| task | EMAIL-WP-0005-T01 | todo | — | workplans/EMAIL-WP-0005-test-mailbox-harness.md | +| task | EMAIL-WP-0005-T02 | todo | — | workplans/EMAIL-WP-0005-test-mailbox-harness.md | +| task | EMAIL-WP-0005-T03 | todo | — | workplans/EMAIL-WP-0005-test-mailbox-harness.md | +| task | EMAIL-WP-0005-T04 | todo | — | workplans/EMAIL-WP-0005-test-mailbox-harness.md | +| task | EMAIL-WP-0005-T05 | todo | — | workplans/EMAIL-WP-0005-test-mailbox-harness.md | +| task | EMAIL-WP-0005-T06 | todo | — | workplans/EMAIL-WP-0005-test-mailbox-harness.md | diff --git a/config/mailbox.example.yml b/config/mailbox.example.yml index b3e29f2..98d2518 100644 --- a/config/mailbox.example.yml +++ b/config/mailbox.example.yml @@ -10,6 +10,7 @@ mailbox: source: fixture_dir: tests/fixtures/mailbox + maildir_dir: null scan: mode: incremental diff --git a/docs/initial-runtime-architecture.md b/docs/initial-runtime-architecture.md index 3060e47..d0901ad 100644 --- a/docs/initial-runtime-architecture.md +++ b/docs/initial-runtime-architecture.md @@ -18,10 +18,17 @@ It scans an inbound return mailbox source, classifies messages, stores scan state in SQLite, updates endpoint-quality hints, and writes timestamped CSV evidence reports. -The source layer supports deterministic fixture directories and a read-only IMAP -connector. IMAP scans select the configured folder with `readonly=True`, fetch -messages using `BODY.PEEK[]`, and reject `mark_seen` because mailbox write-back -actions are out of scope for this MVP. +The source layer supports deterministic fixture directories, Maildir trees, and +a read-only IMAP connector. IMAP scans select the configured folder with +`readonly=True`, fetch messages using `BODY.PEEK[]`, and reject `mark_seen` +because mailbox write-back actions are out of scope for this MVP. + +Maildir scans read `new/` and `cur/`, key message identity on the Maildir unique +name without its `:2,FLAGS` suffix so identity survives the `new/` to `cur/` +move, order by delivery time for cursor purposes, and reject `mark_seen` for the +same reason IMAP does. Maildir exists for the test harness +(`EMAIL-WP-0005`) and for MTAs that deliver locally; it does not make +`email-connect` a mailbox owner. ## Package Layout diff --git a/src/email_connect/config.py b/src/email_connect/config.py index 587cb28..cd8d7e5 100644 --- a/src/email_connect/config.py +++ b/src/email_connect/config.py @@ -47,6 +47,7 @@ class ReportsConfig: @dataclass(frozen=True) class SourceConfig: fixture_dir: str | None = None + maildir_dir: str | None = None @dataclass(frozen=True) @@ -103,7 +104,10 @@ def load_config(path: str | Path) -> AppConfig: include_unknown_messages=bool(reports.get("include_unknown_messages", True)), timestamp_timezone=str(reports.get("timestamp_timezone", "UTC")), ), - source=SourceConfig(fixture_dir=source.get("fixture_dir")), + source=SourceConfig( + fixture_dir=source.get("fixture_dir"), + maildir_dir=source.get("maildir_dir"), + ), expected_recipients=ExpectedRecipientsConfig( path=expected_recipients.get("path"), csv_column=str(expected_recipients.get("csv_column", "email")), diff --git a/src/email_connect/mailbox.py b/src/email_connect/mailbox.py index 47f4bb1..7159b0f 100644 --- a/src/email_connect/mailbox.py +++ b/src/email_connect/mailbox.py @@ -16,6 +16,10 @@ class MailboxSourceMessage: raw_bytes: bytes raw_message_ref: str imap_uid: str | None = None + # Source-assigned identity folded into the message dedup key when the source + # guarantees one delivery per uid. IMAP already contributes its UID, so only + # Maildir sets this. + dedup_uid: str | None = None class MailboxMessageSource(Protocol): @@ -62,6 +66,67 @@ class FixtureMailboxSource: break +class MaildirMailboxSource: + """Read-only source over a Maildir tree (`new/` and `cur/`). + + Message identity is the Maildir unique name without the flag suffix, which + stays stable when the MTA or MUA moves a message from `new/` to `cur/`. + """ + + def __init__(self, maildir_dir: str | Path, *, mark_seen: bool = False) -> None: + self.maildir_dir = Path(maildir_dir) + self.mark_seen = mark_seen + + def iter_messages( + self, + *, + max_messages: int, + since_uid: str | None, + full_rescan: bool, + include_seen: bool, + since: datetime | None, + ) -> Iterable[MailboxSourceMessage]: + # Range filtering runs in the scanner against the parsed message + # timestamp; Maildir filename times are delivery times, not message + # times, so they must not be used to exclude messages here. + del since + if self.mark_seen: + raise ValueError("Maildir mark_seen is intentionally unsupported; scans are read-only.") + if not self.maildir_dir.is_dir(): + raise ValueError(f"Maildir directory not found: {self.maildir_dir}") + + since_key = None if full_rescan or not since_uid else _maildir_sort_key(since_uid) + emitted = 0 + for source_uid, path in sorted(self._entries().items(), key=lambda item: _maildir_sort_key(item[0])): + if since_key is not None and _maildir_sort_key(source_uid) <= since_key: + continue + if not include_seen and _maildir_is_seen(path.name): + continue + yield MailboxSourceMessage( + source_uid=source_uid, + raw_bytes=path.read_bytes(), + raw_message_ref=f"maildir://{self.maildir_dir}/{path.parent.name}/{path.name}", + imap_uid=None, + dedup_uid=source_uid, + ) + emitted += 1 + if max_messages and emitted >= max_messages: + break + + def _entries(self) -> dict[str, Path]: + # `cur` is scanned last so a message already moved out of `new` wins. + entries: dict[str, Path] = {} + for subdir in ("new", "cur"): + directory = self.maildir_dir / subdir + if not directory.is_dir(): + continue + for path in directory.iterdir(): + if path.name.startswith(".") or not path.is_file(): + continue + entries[_maildir_base(path.name)] = path + return entries + + class ImapMailboxSource: def __init__(self, config: AppConfig) -> None: self.config = config @@ -140,11 +205,36 @@ def source_for_config(config: AppConfig, *, fixture_dir_override: str | None = N if not fixture_dir: raise ValueError("source.fixture_dir is required for fixture scans.") return FixtureMailboxSource(fixture_dir) + if config.mailbox.protocol == "maildir": + maildir_dir = config.source.maildir_dir + if not maildir_dir: + raise ValueError("source.maildir_dir is required for maildir scans.") + return MaildirMailboxSource(maildir_dir, mark_seen=config.scan.mark_seen) if config.mailbox.protocol == "imap": return ImapMailboxSource(config) raise ValueError(f"Unsupported mailbox protocol: {config.mailbox.protocol}") +def _maildir_base(filename: str) -> str: + """Strip the `:2,FLAGS` info suffix so `new/` and `cur/` names match.""" + + return filename.split(":", 1)[0] + + +def _maildir_is_seen(filename: str) -> bool: + _base, _, info = filename.partition(":") + return "S" in info.partition(",")[2] + + +def _maildir_sort_key(source_uid: str) -> tuple[int, int, str]: + """Order by Maildir delivery time, falling back to name ordering.""" + + try: + return (0, int(source_uid.split(".", 1)[0]), source_uid) + except ValueError: + return (1, 0, source_uid) + + def _search_criteria(*, include_seen: bool, since: datetime | None) -> list[str]: criteria = ["ALL" if include_seen else "UNSEEN"] if since is not None: diff --git a/src/email_connect/parser.py b/src/email_connect/parser.py index e5fba69..93262a8 100644 --- a/src/email_connect/parser.py +++ b/src/email_connect/parser.py @@ -32,6 +32,7 @@ def parse_message_bytes( mailbox_id: str, raw_message_ref: str | None, imap_uid: str | None = None, + source_uid: str | None = None, now: datetime | None = None, ) -> tuple[InboundMailboxMessage, ParsedMailboxMessage, EmailEvidenceCandidate | None]: observed_at = now or datetime.now(UTC) @@ -40,6 +41,7 @@ def parse_message_bytes( mailbox_id=mailbox_id, raw_message_ref=raw_message_ref, imap_uid=imap_uid, + source_uid=source_uid, observed_at=observed_at, reason="empty_message", ) @@ -51,6 +53,7 @@ def parse_message_bytes( mailbox_id=mailbox_id, raw_message_ref=raw_message_ref, imap_uid=imap_uid, + source_uid=source_uid, observed_at=observed_at, reason=f"parser_error:{type(exc).__name__}", ) @@ -66,6 +69,7 @@ def parse_message_bytes( dedup_key = _message_dedup_key( mailbox_id=mailbox_id, imap_uid=imap_uid, + source_uid=source_uid, message_id=message_id, received_at=received_at, from_address=from_address, @@ -251,8 +255,12 @@ def _parse_failed( imap_uid: str | None, observed_at: datetime, reason: str, + source_uid: str | None = None, ) -> tuple[InboundMailboxMessage, ParsedMailboxMessage, EmailEvidenceCandidate | None]: - dedup_key = "|".join([mailbox_id, imap_uid or "", raw_message_ref or "", reason]) + # An unparseable message has no headers to key on, so identity falls back to + # the source uid when there is one — a Maildir name stays stable across the + # `new/` to `cur/` move that rewrites raw_message_ref. + dedup_key = "|".join([mailbox_id, imap_uid or "", source_uid or raw_message_ref or "", reason]) mailbox_message_id = str(uuid5(NAMESPACE_URL, "email-connect:message:" + dedup_key)) inbound = InboundMailboxMessage( mailbox_message_id=mailbox_message_id, @@ -303,19 +311,23 @@ def _message_dedup_key( from_address: str | None, subject: str | None, body: str, + source_uid: str | None = None, ) -> str: body_hash = hashlib.sha256(body.encode("utf-8", errors="replace")).hexdigest()[:16] - return "|".join( - [ - mailbox_id, - imap_uid or "", - message_id or "", - received_at.isoformat() if received_at else "", - from_address or "", - hashlib.sha256((subject or "").encode()).hexdigest()[:12], - body_hash, - ] - ) + parts = [ + mailbox_id, + imap_uid or "", + message_id or "", + received_at.isoformat() if received_at else "", + from_address or "", + hashlib.sha256((subject or "").encode()).hexdigest()[:12], + body_hash, + ] + # Appended only when the source supplies its own delivery identity, so keys + # for existing fixture and IMAP state stores stay unchanged. + if source_uid: + parts.append(source_uid) + return "|".join(parts) def _clean_header(value: str | None) -> str | None: diff --git a/src/email_connect/scanner.py b/src/email_connect/scanner.py index 60c715c..3a2ce04 100644 --- a/src/email_connect/scanner.py +++ b/src/email_connect/scanner.py @@ -75,6 +75,7 @@ def scan_mailbox( mailbox_id=config.mailbox.id, raw_message_ref=message.raw_message_ref, imap_uid=message.imap_uid, + source_uid=message.dedup_uid, ) if not _in_range(inbound.received_at, range_start=range_start, range_end=range_end): continue diff --git a/tests/test_maildir.py b/tests/test_maildir.py new file mode 100644 index 0000000..41ec57f --- /dev/null +++ b/tests/test_maildir.py @@ -0,0 +1,197 @@ +from __future__ import annotations + +import tempfile +import unittest +from csv import DictReader +from pathlib import Path + +from email_connect.config import AppConfig, MailboxConfig, ReportsConfig, ScanConfig, SourceConfig, StorageConfig +from email_connect.mailbox import MaildirMailboxSource, source_for_config +from email_connect.scanner import scan_mailbox + + +FIXTURES = Path(__file__).parent / "fixtures" / "mailbox" + + +def build_maildir(root: Path, *, subdir: str = "new", start: int = 1749000000) -> Path: + """Materialize the .eml fixtures as a Maildir tree with realistic names.""" + + maildir = root / "Maildir" + for name in ("new", "cur", "tmp"): + (maildir / name).mkdir(parents=True, exist_ok=True) + for offset, source in enumerate(sorted(FIXTURES.glob("*.eml"))): + filename = f"{start + offset}.M{offset}P100.harness" + if subdir == "cur": + filename += ":2,S" + (maildir / subdir / filename).write_bytes(source.read_bytes()) + return maildir + + +def maildir_config(root: Path, maildir: Path) -> AppConfig: + return AppConfig( + mailbox=MailboxConfig(id="test-maildir", protocol="maildir"), + scan=ScanConfig(), + storage=StorageConfig(path=str(root / "state.sqlite")), + reports=ReportsConfig(output_dir=str(root / "reports")), + source=SourceConfig(maildir_dir=str(maildir)), + ) + + +def report_events(path: Path) -> set[tuple[str, str]]: + with path.open(newline="", encoding="utf-8") as fh: + return {(row["normalized_event_type"], row["affected_email_address"]) for row in DictReader(fh)} + + +class MaildirSourceTests(unittest.TestCase): + def test_maildir_scan_matches_fixture_scan(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + maildir = build_maildir(root) + maildir_result = scan_mailbox(maildir_config(root, maildir)) + + fixture_root = root / "fixture-run" + fixture_config = AppConfig( + mailbox=MailboxConfig(id="test-mailbox", protocol="fixture"), + scan=ScanConfig(), + storage=StorageConfig(path=str(fixture_root / "state.sqlite")), + reports=ReportsConfig(output_dir=str(fixture_root / "reports")), + source=SourceConfig(fixture_dir=str(FIXTURES)), + ) + fixture_result = scan_mailbox(fixture_config) + + self.assertEqual(maildir_result.scan.messages_seen, fixture_result.scan.messages_seen) + self.assertEqual(maildir_result.scan.messages_parsed, fixture_result.scan.messages_parsed) + self.assertEqual( + maildir_result.scan.evidence_events_created, + fixture_result.scan.evidence_events_created, + ) + self.assertEqual( + report_events(maildir_result.report_path), + report_events(fixture_result.report_path), + ) + + def test_incremental_cursor_and_full_rescan(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + maildir = build_maildir(root) + config = maildir_config(root, maildir) + + first = scan_mailbox(config) + second = scan_mailbox(config) + + self.assertEqual(first.scan.messages_new, 11) + self.assertEqual(second.scan.messages_seen, 0) + self.assertEqual(second.scan.evidence_events_created, 0) + + # A later delivery is picked up by the cursor without a rescan. + (maildir / "new" / "1749000900.M99P100.harness").write_bytes( + (FIXTURES / "hard_bounce.eml").read_bytes() + ) + third = scan_mailbox(config) + self.assertEqual(third.scan.messages_seen, 1) + self.assertEqual(third.scan.messages_new, 1) + + full = scan_mailbox(config, full_rescan=True) + self.assertEqual(full.scan.messages_seen, 12) + self.assertEqual(full.scan.messages_new, 0) + self.assertEqual(full.scan.evidence_events_created, 0) + + def test_message_identity_survives_move_from_new_to_cur(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + maildir = build_maildir(root) + config = maildir_config(root, maildir) + scan_mailbox(config) + + for path in (maildir / "new").iterdir(): + path.rename(maildir / "cur" / f"{path.name}:2,S") + + rescan = scan_mailbox(config, full_rescan=True) + self.assertEqual(rescan.scan.messages_seen, 11) + self.assertEqual(rescan.scan.messages_new, 0) + self.assertEqual(rescan.scan.evidence_events_created, 0) + + def test_raw_message_ref_uses_maildir_scheme(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + maildir = build_maildir(root) + messages = list( + MaildirMailboxSource(maildir).iter_messages( + max_messages=1, + since_uid=None, + full_rescan=False, + include_seen=True, + since=None, + ) + ) + + self.assertEqual(len(messages), 1) + self.assertTrue(messages[0].raw_message_ref.startswith(f"maildir://{maildir}/new/")) + self.assertIsNone(messages[0].imap_uid) + self.assertEqual(messages[0].dedup_uid, messages[0].source_uid) + self.assertNotIn(":", messages[0].source_uid) + + def test_include_seen_false_skips_flagged_messages(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + maildir = build_maildir(root, subdir="cur") + source = MaildirMailboxSource(maildir) + + all_messages = list( + source.iter_messages( + max_messages=0, since_uid=None, full_rescan=False, include_seen=True, since=None + ) + ) + unseen_only = list( + source.iter_messages( + max_messages=0, since_uid=None, full_rescan=False, include_seen=False, since=None + ) + ) + + self.assertEqual(len(all_messages), 11) + self.assertEqual(unseen_only, []) + + def test_mark_seen_is_rejected(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + maildir = build_maildir(root) + source = MaildirMailboxSource(maildir, mark_seen=True) + + with self.assertRaises(ValueError): + list( + source.iter_messages( + max_messages=0, since_uid=None, full_rescan=False, include_seen=True, since=None + ) + ) + + def test_missing_maildir_directory_is_rejected(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + source = MaildirMailboxSource(Path(tmp) / "absent") + with self.assertRaises(ValueError): + list( + source.iter_messages( + max_messages=0, since_uid=None, full_rescan=False, include_seen=True, since=None + ) + ) + + def test_source_for_config_requires_maildir_dir(self) -> None: + config = AppConfig( + mailbox=MailboxConfig(id="test-maildir", protocol="maildir"), + scan=ScanConfig(), + storage=StorageConfig(), + reports=ReportsConfig(), + source=SourceConfig(), + ) + with self.assertRaises(ValueError): + source_for_config(config) + + def test_source_for_config_returns_maildir_source(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + maildir = build_maildir(root) + source = source_for_config(maildir_config(root, maildir)) + self.assertIsInstance(source, MaildirMailboxSource) + + +if __name__ == "__main__": + unittest.main() diff --git a/workplans/EMAIL-WP-0005-test-mailbox-harness.md b/workplans/EMAIL-WP-0005-test-mailbox-harness.md index cb78b2f..b02b80f 100644 --- a/workplans/EMAIL-WP-0005-test-mailbox-harness.md +++ b/workplans/EMAIL-WP-0005-test-mailbox-harness.md @@ -9,6 +9,7 @@ owner: claude topic_slug: custodian created: "2026-08-14" updated: "2026-08-14" +state_hub_workstream_id: "1ffd54d3-4fdf-4a66-86ba-36a5d8202f3c" --- # EMAIL-WP-0005 - Test Mailbox Harness for Automated Test Environments @@ -98,8 +99,9 @@ assert stronger claims are defects in the test, not features of the harness. ```task id: EMAIL-WP-0005-T01 -status: todo +status: done priority: high +state_hub_task_id: "28856bdc-7ea2-44ce-8c06-88bf006e17fd" ``` Tasks: @@ -123,12 +125,34 @@ same evidence rows as the equivalent fixture directory, with deduplication and incremental cursors working across repeated scans. ``` +Done 2026-08-14: + +* `MaildirMailboxSource` reads `new/` and `cur/`, skips dotfiles, prefers `cur/` + when a base name appears in both, and emits + `maildir:////` refs. `mark_seen` and a missing directory + both raise, matching the read-only IMAP contract. +* Identity is the Maildir unique name minus the `:2,FLAGS` suffix, so it is + stable across the `new/`→`cur/` move. It is carried on a new + `MailboxSourceMessage.dedup_uid` and appended to the message dedup key only + when set, leaving existing fixture and IMAP state-store keys byte-identical. +* Cursor ordering parses the Maildir delivery time rather than comparing names + lexically, so cursors stay correct as the time field changes width. +* Fixed alongside: the parse-failure path keyed identity on `raw_message_ref`, + which a `new/`→`cur/` move rewrites, so an unparseable message re-registered + as new on every rescan. It now prefers the source uid when one exists. + Behavior for fixture and IMAP sources is unchanged. +* `source.maildir_dir` added to config and the example file; `source_for_config` + validates it. `include_seen: false` skips `S`-flagged messages. +* Tests: `tests/test_maildir.py`, 11 cases including maildir-vs-fixture evidence + parity. Full suite 38 passed. + ## T02 - Containerized SMTP/IMAP test server ```task id: EMAIL-WP-0005-T02 status: todo priority: high +state_hub_task_id: "967d2bab-43e3-4240-8bd5-c657ba4178ff" ``` Tasks: @@ -165,6 +189,7 @@ does not exercise the IMAP source. Record the choice in DECISIONS.md. id: EMAIL-WP-0005-T03 status: todo priority: high +state_hub_task_id: "c761d26d-a8b0-4b66-ba18-8f22c6d59440" ``` Tasks: @@ -199,6 +224,7 @@ provider material; the harness must not touch them. id: EMAIL-WP-0005-T04 status: todo priority: high +state_hub_task_id: "69655fad-97b8-4c17-bc54-af341aaaf38c" ``` Tasks: @@ -226,6 +252,7 @@ while the default test run remains offline, deterministic, and unchanged. id: EMAIL-WP-0005-T05 status: todo priority: medium +state_hub_task_id: "48b8eaf9-5a4b-437b-ad2c-952907340ed7" ``` Tasks: @@ -252,6 +279,7 @@ papered over. id: EMAIL-WP-0005-T06 status: todo priority: medium +state_hub_task_id: "0599de98-c0ff-4160-a2c8-54265b31ca5d" ``` Tasks: