feat: expand mailbox evidence scanner

src/email_connect/scanner.py

@@ -2,12 +2,13 @@ from __future__ import annotations
 
 from dataclasses import dataclass
 from datetime import UTC, datetime
-from pathlib import Path
 from uuid import uuid4
 
 from .config import AppConfig
+from .evidence import endpoint_quality_from_candidate
+from .mailbox import source_for_config
 from .models import MailboxScan
-from .parser import parse_message_file
+from .parser import parse_message_bytes
 from .reporting import write_evidence_report
 from .storage import StateStore
 
@@ -26,26 +27,39 @@ def scan_mailbox(
     report_only_new: bool = False,
     dry_run: bool = False,
     fixture_dir: str | None = None,
+    since: str | None = None,
 ) -> ScanResult:
     started_at = datetime.now(UTC)
     scan_id = str(uuid4())
-    source_dir = fixture_dir or config.source.fixture_dir
-    if not source_dir:
-        raise ValueError("No source.fixture_dir configured yet. IMAP connector is planned in EMAIL-WP-0002-T02.")
-
-    files = sorted(Path(source_dir).glob("*.eml"))
-    if config.scan.max_messages_per_run:
-        files = files[: config.scan.max_messages_per_run]
+    since_at = _parse_since(since or config.scan.since)
+    source = source_for_config(config, fixture_dir_override=fixture_dir)
 
     store = StateStore(config.storage.path)
     messages_seen = 0
     messages_new = 0
     messages_parsed = 0
     evidence_created = 0
+    new_evidence_keys: list[str] = []
+    last_source_uid: str | None = None
     try:
-        for path in files:
+        cursor = None if full_rescan else store.get_scan_cursor(config.mailbox.id, config.mailbox.folder)
+        for message in source.iter_messages(
+            max_messages=config.scan.max_messages_per_run,
+            since_uid=cursor,
+            full_rescan=full_rescan,
+            include_seen=config.scan.include_seen,
+            since=since_at,
+        ):
             messages_seen += 1
-            inbound, parsed, candidate = parse_message_file(path, mailbox_id=config.mailbox.id)
+            last_source_uid = message.source_uid
+            inbound, parsed, candidate = parse_message_bytes(
+                message.raw_bytes,
+                mailbox_id=config.mailbox.id,
+                raw_message_ref=message.raw_message_ref,
+                imap_uid=message.imap_uid,
+            )
+            if since_at and inbound.received_at and inbound.received_at < since_at:
+                continue
             if dry_run:
                 messages_parsed += 1
                 continue
@@ -55,11 +69,31 @@ def scan_mailbox(
             messages_parsed += 1
             if candidate is not None:
                 candidate = _enrich_candidate(candidate, inbound, parsed)
-                evidence_created += int(store.insert_evidence(candidate))
+                inserted = store.insert_evidence(candidate)
+                evidence_created += int(inserted)
+                if inserted:
+                    new_evidence_keys.append(candidate.deduplication_key)
+                quality_update = endpoint_quality_from_candidate(candidate)
+                if quality_update is not None:
+                    store.apply_endpoint_quality_update(
+                        config.mailbox.id,
+                        quality_update,
+                        observed_at=candidate.observed_at,
+                    )
+
+        if not dry_run and last_source_uid:
+            store.set_scan_cursor(
+                config.mailbox.id,
+                config.mailbox.folder,
+                last_source_uid,
+                updated_at=datetime.now(UTC),
+            )
 
         report_path = None
         if not dry_run:
-            report_rows = store.evidence_rows()
+            report_rows = store.evidence_rows(
+                deduplication_keys=new_evidence_keys if report_only_new else None,
+            )
             report_path = write_evidence_report(
                 report_rows,
                 output_dir=output_dir or config.reports.output_dir,
@@ -80,6 +114,7 @@ def scan_mailbox(
             messages_parsed=messages_parsed,
             evidence_events_created=evidence_created,
             report_path=str(report_path) if report_path else None,
+            since=since_at,
         )
         if not dry_run:
             store.insert_scan(scan)
@@ -105,3 +140,17 @@ def _enrich_candidate(candidate, inbound, parsed):
             "metadata": metadata,
         }
     )
+
+
+def _parse_since(value: str | None) -> datetime | None:
+    if not value:
+        return None
+    normalized = value.strip()
+    if not normalized:
+        return None
+    if len(normalized) == 10:
+        normalized = normalized + "T00:00:00+00:00"
+    parsed = datetime.fromisoformat(normalized.replace("Z", "+00:00"))
+    if parsed.tzinfo is None:
+        return parsed.replace(tzinfo=UTC)
+    return parsed.astimezone(UTC)