fix: repair chained imports and track blocked evidence dependencies

Assistant: codex
Assistant-Model: gpt-6.1-sol
Assistant-Session: 01a1046e-465c-7d43-8c4c-d10f0a1feadd

audit_core/migrate_store.py

@@ -18,9 +18,13 @@ current time.
 
 from __future__ import annotations
 
+import hashlib
 import json
 import sqlite3
 from dataclasses import dataclass, field
+from pathlib import Path
+
+from audit_core.integrity import CHAIN_LOCK_KEY, chain_link
 
 from audit_core.interface import BackendUnavailableError
 
@@ -56,110 +60,172 @@ class MigrationReport:
         return "\n".join(lines)
 
 
-def migrate(sqlite_path: str, backend, *, verify: bool = True) -> MigrationReport:
+def migrate(
+    sqlite_path: str, backend, *, verify: bool = True, source_id: str | None = None
+) -> MigrationReport:
     """Copy every record from ``sqlite_path`` into ``backend``.
 
-    Idempotent: re-running skips events already present with a matching payload
-    hash. An event present with a *different* hash is reported as a conflict
+    Idempotent: re-running skips events with matching hash, time and record.
+    An event present with different historical evidence is reported as a conflict
     and never overwritten — the destination record wins, and a human decides.
     """
+    import psycopg
+
     report = MigrationReport()
+    # Callers moving a source to a new path must retain its explicit source_id.
+    source_id = source_id or str(Path(sqlite_path).resolve())
+    source_id = hashlib.sha256(source_id.encode()).hexdigest()
     source = sqlite3.connect(f"file:{sqlite_path}?mode=ro", uri=True)
     try:
+        source.execute("BEGIN")  # One consistent source snapshot, including counters.
         rows = source.execute(
             "SELECT event_id, payload_hash, accepted_at, correlation_id, tenant, record "
             "FROM events ORDER BY accepted_at, event_id"
         ).fetchall()
         report.events_read = len(rows)
 
-        for event_id, payload_hash, accepted_at, correlation_id, tenant, record in rows:
-            outcome = _import_event(
-                backend, event_id, payload_hash, accepted_at, correlation_id,
-                tenant, json.loads(record),
-            )
-            if outcome == "written":
-                report.events_written += 1
-            elif outcome == "present":
-                report.events_already_present += 1
-            else:
-                report.conflicts.append(outcome)
+        # Same serialization boundary as native accept(), including side-table
+        # receipts. A retry cannot double-count after an interrupted import.
+        with backend.pool.connection() as conn, conn.transaction():
+            conn.execute("SELECT pg_advisory_xact_lock(%s)", (CHAIN_LOCK_KEY,))
+            for event_id, payload_hash, accepted_at, correlation_id, tenant, record in rows:
+                outcome = _import_event(
+                    backend, conn, event_id, payload_hash, accepted_at, correlation_id,
+                    tenant, json.loads(record),
+                )
+                if outcome == "written":
+                    report.events_written += 1
+                elif outcome == "present":
+                    report.events_already_present += 1
+                else:
+                    report.conflicts.append(outcome)
 
-        report.dead_letters_written = _copy_dead_letters(source, backend)
-        report.findings_written = _copy_findings(source, backend)
+            report.dead_letters_written = _copy_dead_letters(source, backend, conn, source_id, report)
+            report.findings_written = _copy_findings(source, backend, conn, source_id, report)
 
         if verify:
             report.verified = _verify(source, backend, report)
+            if not backend.verify_chain().intact:
+                report.conflicts.append("destination hash chain is not intact")
+                report.verified = False
+    except psycopg.Error as exc:
+        raise BackendUnavailableError("SQLite import transaction failed") from exc
     finally:
         source.close()
     return report
 
 
 def _import_event(
-    backend, event_id, payload_hash, accepted_at, correlation_id, tenant, record
+    backend, conn, event_id, payload_hash, accepted_at, correlation_id, tenant, record
 ) -> str:
-    existing = backend._query(
-        f"SELECT payload_hash FROM {backend._events} WHERE event_id = %s", (event_id,)
-    )
+    existing = conn.execute(
+        f"SELECT payload_hash, accepted_at, record FROM {backend._events} WHERE event_id = %s",
+        (event_id,),
+    ).fetchone()
     if existing:
-        if existing[0][0] != payload_hash:
-            return (
-                f"{event_id}: destination holds a different payload "
-                f"(source {payload_hash[:12]}…, destination {existing[0][0][:12]}…) "
-                "— left untouched"
-            )
+        if existing[0] != payload_hash:
+            return f"{event_id}: destination holds a different payload — left untouched"
+        if not _same_instant(existing[1], accepted_at) or existing[2] != record:
+            return f"{event_id}: destination holds different historical evidence — left untouched"
         return "present"
 
-    backend._execute(
+    previous = backend._chain_head(conn)
+    conn.execute(
         f"INSERT INTO {backend._events} "
         "(event_id, payload_hash, accepted_at, observed_at, tenant, correlation_id, "
-        " source, action, record) "
-        "VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s) "
-        "ON CONFLICT (event_id) DO NOTHING",
+        " source, action, record, chain_prev, chain_hash) "
+        "VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)",
         (
             event_id, payload_hash, accepted_at, record.get("observed_at"),
             tenant, correlation_id, record.get("source", ""), record.get("action", ""),
-            json.dumps(record, sort_keys=True),
+            json.dumps(record, sort_keys=True), previous,
+            chain_link(previous, payload_hash, event_id),
         ),
     )
     return "written"
 
 
-def _copy_dead_letters(source: sqlite3.Connection, backend) -> int:
-    try:
-        rows = source.execute(
-            "SELECT event_id, received_at, sender, reason, payload_hash, payload, "
-            "payload_withheld FROM dead_letters ORDER BY id"
-        ).fetchall()
-    except sqlite3.Error:
+def _fingerprint(value) -> str:
+    return hashlib.sha256(json.dumps(value, sort_keys=True).encode()).hexdigest()
+
+
+def _receipt(backend, conn, source_id, kind, record_id):
+    return conn.execute(
+        f'SELECT content_hash, occurrences FROM "{backend.schema}".import_receipts '
+        "WHERE source_id = %s AND kind = %s AND record_id = %s",
+        (source_id, kind, record_id),
+    ).fetchone()
+
+
+def _save_receipt(backend, conn, source_id, kind, record_id, content_hash, occurrences=0):
+    conn.execute(
+        f'INSERT INTO "{backend.schema}".import_receipts '
+        "(source_id, kind, record_id, content_hash, occurrences) VALUES (%s,%s,%s,%s,%s) "
+        "ON CONFLICT (source_id, kind, record_id) DO UPDATE "
+        "SET content_hash = excluded.content_hash, occurrences = excluded.occurrences",
+        (source_id, kind, record_id, content_hash, occurrences),
+    )
+
+
+def _copy_dead_letters(source, backend, conn, source_id, report) -> int:
+    if not source.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name='dead_letters'").fetchone():
         return 0
-    for event_id, received_at, sender, reason, payload_hash, payload, withheld in rows:
-        backend._execute(
+    rows = source.execute(
+        "SELECT id, event_id, received_at, sender, reason, payload_hash, payload, "
+        "payload_withheld FROM dead_letters ORDER BY id"
+    ).fetchall()
+    written = 0
+    for row_id, *data in rows:
+        record_id = str(row_id)
+        fingerprint = _fingerprint(data)
+        receipt = _receipt(backend, conn, source_id, "dead-letter", record_id)
+        if receipt:
+            if receipt[0] != fingerprint:
+                report.conflicts.append(f"dead-letter {record_id}: source changed after import")
+            continue
+        conn.execute(
             f'INSERT INTO "{backend.schema}".dead_letters '
             "(event_id, received_at, sender, reason, payload_hash, payload, payload_withheld) "
             "VALUES (%s, %s, %s, %s, %s, %s, %s)",
-            (event_id, received_at, sender, reason, payload_hash, payload, bool(withheld)),
+            (*data[:-1], bool(data[-1])),
         )
-    return len(rows)
+        _save_receipt(backend, conn, source_id, "dead-letter", record_id, fingerprint)
+        written += 1
+    return written
 
 
-def _copy_findings(source: sqlite3.Connection, backend) -> int:
-    try:
-        rows = source.execute(
-            "SELECT sender, source, action, field_path, outcome, persisted, "
-            "occurrences, first_seen, last_seen FROM secret_findings"
-        ).fetchall()
-    except sqlite3.Error:
+def _copy_findings(source, backend, conn, source_id, report) -> int:
+    if not source.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name='secret_findings'").fetchone():
         return 0
+    rows = source.execute(
+        "SELECT sender, source, action, field_path, outcome, persisted, "
+        "occurrences, first_seen, last_seen FROM secret_findings"
+    ).fetchall()
+    written = 0
     for r in rows:
-        backend._execute(
+        record_id = _fingerprint(r[:5])
+        fingerprint = _fingerprint(r)
+        receipt = _receipt(backend, conn, source_id, "finding", record_id)
+        if receipt and receipt[0] == fingerprint:
+            continue
+        delta = r[6] - (receipt[1] if receipt else 0)
+        if delta < 0:
+            report.conflicts.append(f"finding {record_id}: source occurrence count decreased")
+            continue
+        conn.execute(
             f'INSERT INTO "{backend.schema}".secret_findings '
             "(sender, source, action, field_path, outcome, persisted, occurrences, "
             " first_seen, last_seen) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s) "
             "ON CONFLICT (sender, source, action, field_path, outcome) DO UPDATE "
-            "SET occurrences = secret_findings.occurrences + excluded.occurrences",
-            (r[0], r[1], r[2], r[3], r[4], bool(r[5]), r[6], r[7], r[8]),
+            "SET occurrences = secret_findings.occurrences + excluded.occurrences, "
+            "persisted = secret_findings.persisted OR excluded.persisted, "
+            "first_seen = LEAST(secret_findings.first_seen, excluded.first_seen), "
+            "last_seen = GREATEST(secret_findings.last_seen, excluded.last_seen)",
+            (r[0], r[1], r[2], r[3], r[4], bool(r[5]), delta, r[7], r[8]),
         )
-    return len(rows)
+        _save_receipt(backend, conn, source_id, "finding", record_id, fingerprint, r[6])
+        written += 1
+    return written
 
 
 def _verify(source: sqlite3.Connection, backend, report: MigrationReport) -> bool:
@@ -200,4 +266,4 @@ def _same_instant(destination, source_iso: str) -> bool:
         return False
     if want.tzinfo is None or destination.tzinfo is None:
         return False
-    return abs(destination.timestamp() - want.timestamp()) < 1.0
+    return destination == want