"""Move audit records from a SQLite store into PostgreSQL (AUDIT-WP-0005-T04). As of 2026-08-10 there are no pre-production records to move: audit-core was never deployed, so the only SQLite stores that ever existed were test fixtures. See the workplan for that finding. The tool exists anyway because the SQLite path is still reachable — the entrypoint falls back to it when ``AUDIT_CORE_DATABASE_URL`` is unset. If that fallback is ever used in anger, the records it accumulates are audit records and cannot simply be dropped. Fidelity is the whole point. Records are transferred with their original ``event_id``, ``payload_hash``, and ``accepted_at`` intact; a migration that rewrote acceptance times would destroy the evidence it was meant to preserve. That is why this bypasses ``accept()``, which stamps ``accepted_at`` with the current time. """ from __future__ import annotations import hashlib import json import sqlite3 from dataclasses import dataclass, field from pathlib import Path from audit_core.integrity import CHAIN_LOCK_KEY, chain_link from audit_core.interface import BackendUnavailableError @dataclass class MigrationReport: """What happened, in enough detail to be evidence.""" events_read: int = 0 events_written: int = 0 events_already_present: int = 0 dead_letters_written: int = 0 findings_written: int = 0 conflicts: list[str] = field(default_factory=list) verified: bool = False @property def ok(self) -> bool: return not self.conflicts and self.verified def summary(self) -> str: lines = [ f"events read: {self.events_read}", f"events written: {self.events_written}", f"already present: {self.events_already_present}", f"dead letters written: {self.dead_letters_written}", f"secret findings written:{self.findings_written}", f"verified: {self.verified}", ] if self.conflicts: lines.append(f"CONFLICTS ({len(self.conflicts)}):") lines += [f" - {c}" for c in self.conflicts] return "\n".join(lines) def migrate( sqlite_path: str, backend, *, verify: bool = True, source_id: str | None = None ) -> MigrationReport: """Copy every record from ``sqlite_path`` into ``backend``. Idempotent: re-running skips events with matching hash, time and record. An event present with different historical evidence is reported as a conflict and never overwritten — the destination record wins, and a human decides. """ import psycopg report = MigrationReport() # Callers moving a source to a new path must retain its explicit source_id. source_id = source_id or str(Path(sqlite_path).resolve()) source_id = hashlib.sha256(source_id.encode()).hexdigest() source = sqlite3.connect(f"file:{sqlite_path}?mode=ro", uri=True) try: source.execute("BEGIN") # One consistent source snapshot, including counters. rows = source.execute( "SELECT event_id, payload_hash, accepted_at, correlation_id, tenant, record " "FROM events ORDER BY accepted_at, event_id" ).fetchall() report.events_read = len(rows) # Same serialization boundary as native accept(), including side-table # receipts. A retry cannot double-count after an interrupted import. with backend.pool.connection() as conn, conn.transaction(): conn.execute("SELECT pg_advisory_xact_lock(%s)", (CHAIN_LOCK_KEY,)) for event_id, payload_hash, accepted_at, correlation_id, tenant, record in rows: outcome = _import_event( backend, conn, event_id, payload_hash, accepted_at, correlation_id, tenant, json.loads(record), ) if outcome == "written": report.events_written += 1 elif outcome == "present": report.events_already_present += 1 else: report.conflicts.append(outcome) report.dead_letters_written = _copy_dead_letters(source, backend, conn, source_id, report) report.findings_written = _copy_findings(source, backend, conn, source_id, report) if verify: report.verified = _verify(source, backend, report) if not backend.verify_chain().intact: report.conflicts.append("destination hash chain is not intact") report.verified = False except psycopg.Error as exc: raise BackendUnavailableError("SQLite import transaction failed") from exc finally: source.close() return report def _import_event( backend, conn, event_id, payload_hash, accepted_at, correlation_id, tenant, record ) -> str: existing = conn.execute( f"SELECT payload_hash, accepted_at, record FROM {backend._events} WHERE event_id = %s", (event_id,), ).fetchone() if existing: if existing[0] != payload_hash: return f"{event_id}: destination holds a different payload — left untouched" if not _same_instant(existing[1], accepted_at) or existing[2] != record: return f"{event_id}: destination holds different historical evidence — left untouched" return "present" previous = backend._chain_head(conn) conn.execute( f"INSERT INTO {backend._events} " "(event_id, payload_hash, accepted_at, observed_at, tenant, correlation_id, " " source, action, record, chain_prev, chain_hash) " "VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)", ( event_id, payload_hash, accepted_at, record.get("observed_at"), tenant, correlation_id, record.get("source", ""), record.get("action", ""), json.dumps(record, sort_keys=True), previous, chain_link(previous, payload_hash, event_id), ), ) return "written" def _fingerprint(value) -> str: return hashlib.sha256(json.dumps(value, sort_keys=True).encode()).hexdigest() def _receipt(backend, conn, source_id, kind, record_id): return conn.execute( f'SELECT content_hash, occurrences FROM "{backend.schema}".import_receipts ' "WHERE source_id = %s AND kind = %s AND record_id = %s", (source_id, kind, record_id), ).fetchone() def _save_receipt(backend, conn, source_id, kind, record_id, content_hash, occurrences=0): conn.execute( f'INSERT INTO "{backend.schema}".import_receipts ' "(source_id, kind, record_id, content_hash, occurrences) VALUES (%s,%s,%s,%s,%s) " "ON CONFLICT (source_id, kind, record_id) DO UPDATE " "SET content_hash = excluded.content_hash, occurrences = excluded.occurrences", (source_id, kind, record_id, content_hash, occurrences), ) def _copy_dead_letters(source, backend, conn, source_id, report) -> int: if not source.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name='dead_letters'").fetchone(): return 0 rows = source.execute( "SELECT id, event_id, received_at, sender, reason, payload_hash, payload, " "payload_withheld FROM dead_letters ORDER BY id" ).fetchall() written = 0 for row_id, *data in rows: record_id = str(row_id) fingerprint = _fingerprint(data) receipt = _receipt(backend, conn, source_id, "dead-letter", record_id) if receipt: if receipt[0] != fingerprint: report.conflicts.append(f"dead-letter {record_id}: source changed after import") continue conn.execute( f'INSERT INTO "{backend.schema}".dead_letters ' "(event_id, received_at, sender, reason, payload_hash, payload, payload_withheld) " "VALUES (%s, %s, %s, %s, %s, %s, %s)", (*data[:-1], bool(data[-1])), ) _save_receipt(backend, conn, source_id, "dead-letter", record_id, fingerprint) written += 1 return written def _copy_findings(source, backend, conn, source_id, report) -> int: if not source.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name='secret_findings'").fetchone(): return 0 rows = source.execute( "SELECT sender, source, action, field_path, outcome, persisted, " "occurrences, first_seen, last_seen FROM secret_findings" ).fetchall() written = 0 for r in rows: record_id = _fingerprint(r[:5]) fingerprint = _fingerprint(r) receipt = _receipt(backend, conn, source_id, "finding", record_id) if receipt and receipt[0] == fingerprint: continue delta = r[6] - (receipt[1] if receipt else 0) if delta < 0: report.conflicts.append(f"finding {record_id}: source occurrence count decreased") continue conn.execute( f'INSERT INTO "{backend.schema}".secret_findings ' "(sender, source, action, field_path, outcome, persisted, occurrences, " " first_seen, last_seen) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s) " "ON CONFLICT (sender, source, action, field_path, outcome) DO UPDATE " "SET occurrences = secret_findings.occurrences + excluded.occurrences, " "persisted = secret_findings.persisted OR excluded.persisted, " "first_seen = LEAST(secret_findings.first_seen, excluded.first_seen), " "last_seen = GREATEST(secret_findings.last_seen, excluded.last_seen)", (r[0], r[1], r[2], r[3], r[4], bool(r[5]), delta, r[7], r[8]), ) _save_receipt(backend, conn, source_id, "finding", record_id, fingerprint, r[6]) written += 1 return written def _verify(source: sqlite3.Connection, backend, report: MigrationReport) -> bool: """Confirm every source event is present with its hash and time intact. Verification reads back rather than trusting the write path: a migration that reports success without checking is the same as no migration at all. """ rows = source.execute("SELECT event_id, payload_hash, accepted_at FROM events").fetchall() for event_id, payload_hash, accepted_at in rows: found = backend._query( f"SELECT payload_hash, accepted_at FROM {backend._events} WHERE event_id = %s", (event_id,), ) if not found: report.conflicts.append(f"{event_id}: missing from destination after migration") return False if found[0][0] != payload_hash: report.conflicts.append(f"{event_id}: payload hash differs after migration") return False if not _same_instant(found[0][1], accepted_at): report.conflicts.append( f"{event_id}: accepted_at not preserved " f"(source {accepted_at}, destination {found[0][1]})" ) return False return True def _same_instant(destination, source_iso: str) -> bool: from datetime import datetime try: want = datetime.fromisoformat(str(source_iso).replace("Z", "+00:00")) except ValueError: return False if not hasattr(destination, "timestamp"): return False if want.tzinfo is None or destination.tzinfo is None: return False return destination == want