audit-core/audit_core/migrate_store.py

"""Move audit records from a SQLite store into PostgreSQL (AUDIT-WP-0005-T04).

As of 2026-08-10 there are no pre-production records to move: audit-core was
never deployed, so the only SQLite stores that ever existed were test
fixtures. See the workplan for that finding.

The tool exists anyway because the SQLite path is still reachable — the
entrypoint falls back to it when ``AUDIT_CORE_DATABASE_URL`` is unset. If that
fallback is ever used in anger, the records it accumulates are audit records
and cannot simply be dropped.

Fidelity is the whole point. Records are transferred with their original
``event_id``, ``payload_hash``, and ``accepted_at`` intact; a migration that
rewrote acceptance times would destroy the evidence it was meant to preserve.
That is why this bypasses ``accept()``, which stamps ``accepted_at`` with the
current time.
"""

from __future__ import annotations

import hashlib
import json
import sqlite3
from dataclasses import dataclass, field
from pathlib import Path

from audit_core.integrity import CHAIN_LOCK_KEY, chain_link

from audit_core.interface import BackendUnavailableError


@dataclass
class MigrationReport:
    """What happened, in enough detail to be evidence."""

    events_read: int = 0
    events_written: int = 0
    events_already_present: int = 0
    dead_letters_written: int = 0
    findings_written: int = 0
    conflicts: list[str] = field(default_factory=list)
    verified: bool = False

    @property
    def ok(self) -> bool:
        return not self.conflicts and self.verified

    def summary(self) -> str:
        lines = [
            f"events read:            {self.events_read}",
            f"events written:         {self.events_written}",
            f"already present:        {self.events_already_present}",
            f"dead letters written:   {self.dead_letters_written}",
            f"secret findings written:{self.findings_written}",
            f"verified:               {self.verified}",
        ]
        if self.conflicts:
            lines.append(f"CONFLICTS ({len(self.conflicts)}):")
            lines += [f"  - {c}" for c in self.conflicts]
        return "\n".join(lines)


def migrate(
    sqlite_path: str, backend, *, verify: bool = True, source_id: str | None = None
) -> MigrationReport:
    """Copy every record from ``sqlite_path`` into ``backend``.

    Idempotent: re-running skips events with matching hash, time and record.
    An event present with different historical evidence is reported as a conflict
    and never overwritten — the destination record wins, and a human decides.
    """
    import psycopg

    report = MigrationReport()
    # Callers moving a source to a new path must retain its explicit source_id.
    source_id = source_id or str(Path(sqlite_path).resolve())
    source_id = hashlib.sha256(source_id.encode()).hexdigest()
    source = sqlite3.connect(f"file:{sqlite_path}?mode=ro", uri=True)
    try:
        source.execute("BEGIN")  # One consistent source snapshot, including counters.
        rows = source.execute(
            "SELECT event_id, payload_hash, accepted_at, correlation_id, tenant, record "
            "FROM events ORDER BY accepted_at, event_id"
        ).fetchall()
        report.events_read = len(rows)

        # Same serialization boundary as native accept(), including side-table
        # receipts. A retry cannot double-count after an interrupted import.
        with backend.pool.connection() as conn, conn.transaction():
            conn.execute("SELECT pg_advisory_xact_lock(%s)", (CHAIN_LOCK_KEY,))
            for event_id, payload_hash, accepted_at, correlation_id, tenant, record in rows:
                outcome = _import_event(
                    backend, conn, event_id, payload_hash, accepted_at, correlation_id,
                    tenant, json.loads(record),
                )
                if outcome == "written":
                    report.events_written += 1
                elif outcome == "present":
                    report.events_already_present += 1
                else:
                    report.conflicts.append(outcome)

            report.dead_letters_written = _copy_dead_letters(source, backend, conn, source_id, report)
            report.findings_written = _copy_findings(source, backend, conn, source_id, report)

        if verify:
            report.verified = _verify(source, backend, report)
            if not backend.verify_chain().intact:
                report.conflicts.append("destination hash chain is not intact")
                report.verified = False
    except psycopg.Error as exc:
        raise BackendUnavailableError("SQLite import transaction failed") from exc
    finally:
        source.close()
    return report


def _import_event(
    backend, conn, event_id, payload_hash, accepted_at, correlation_id, tenant, record
) -> str:
    existing = conn.execute(
        f"SELECT payload_hash, accepted_at, record FROM {backend._events} WHERE event_id = %s",
        (event_id,),
    ).fetchone()
    if existing:
        if existing[0] != payload_hash:
            return f"{event_id}: destination holds a different payload — left untouched"
        if not _same_instant(existing[1], accepted_at) or existing[2] != record:
            return f"{event_id}: destination holds different historical evidence — left untouched"
        return "present"

    previous = backend._chain_head(conn)
    conn.execute(
        f"INSERT INTO {backend._events} "
        "(event_id, payload_hash, accepted_at, observed_at, tenant, correlation_id, "
        " source, action, record, chain_prev, chain_hash) "
        "VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)",
        (
            event_id, payload_hash, accepted_at, record.get("observed_at"),
            tenant, correlation_id, record.get("source", ""), record.get("action", ""),
            json.dumps(record, sort_keys=True), previous,
            chain_link(previous, payload_hash, event_id),
        ),
    )
    return "written"


def _fingerprint(value) -> str:
    return hashlib.sha256(json.dumps(value, sort_keys=True).encode()).hexdigest()


def _receipt(backend, conn, source_id, kind, record_id):
    return conn.execute(
        f'SELECT content_hash, occurrences FROM "{backend.schema}".import_receipts '
        "WHERE source_id = %s AND kind = %s AND record_id = %s",
        (source_id, kind, record_id),
    ).fetchone()


def _save_receipt(backend, conn, source_id, kind, record_id, content_hash, occurrences=0):
    conn.execute(
        f'INSERT INTO "{backend.schema}".import_receipts '
        "(source_id, kind, record_id, content_hash, occurrences) VALUES (%s,%s,%s,%s,%s) "
        "ON CONFLICT (source_id, kind, record_id) DO UPDATE "
        "SET content_hash = excluded.content_hash, occurrences = excluded.occurrences",
        (source_id, kind, record_id, content_hash, occurrences),
    )


def _copy_dead_letters(source, backend, conn, source_id, report) -> int:
    if not source.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name='dead_letters'").fetchone():
        return 0
    rows = source.execute(
        "SELECT id, event_id, received_at, sender, reason, payload_hash, payload, "
        "payload_withheld FROM dead_letters ORDER BY id"
    ).fetchall()
    written = 0
    for row_id, *data in rows:
        record_id = str(row_id)
        fingerprint = _fingerprint(data)
        receipt = _receipt(backend, conn, source_id, "dead-letter", record_id)
        if receipt:
            if receipt[0] != fingerprint:
                report.conflicts.append(f"dead-letter {record_id}: source changed after import")
            continue
        conn.execute(
            f'INSERT INTO "{backend.schema}".dead_letters '
            "(event_id, received_at, sender, reason, payload_hash, payload, payload_withheld) "
            "VALUES (%s, %s, %s, %s, %s, %s, %s)",
            (*data[:-1], bool(data[-1])),
        )
        _save_receipt(backend, conn, source_id, "dead-letter", record_id, fingerprint)
        written += 1
    return written


def _copy_findings(source, backend, conn, source_id, report) -> int:
    if not source.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name='secret_findings'").fetchone():
        return 0
    rows = source.execute(
        "SELECT sender, source, action, field_path, outcome, persisted, "
        "occurrences, first_seen, last_seen FROM secret_findings"
    ).fetchall()
    written = 0
    for r in rows:
        record_id = _fingerprint(r[:5])
        fingerprint = _fingerprint(r)
        receipt = _receipt(backend, conn, source_id, "finding", record_id)
        if receipt and receipt[0] == fingerprint:
            continue
        delta = r[6] - (receipt[1] if receipt else 0)
        if delta < 0:
            report.conflicts.append(f"finding {record_id}: source occurrence count decreased")
            continue
        conn.execute(
            f'INSERT INTO "{backend.schema}".secret_findings '
            "(sender, source, action, field_path, outcome, persisted, occurrences, "
            " first_seen, last_seen) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s) "
            "ON CONFLICT (sender, source, action, field_path, outcome) DO UPDATE "
            "SET occurrences = secret_findings.occurrences + excluded.occurrences, "
            "persisted = secret_findings.persisted OR excluded.persisted, "
            "first_seen = LEAST(secret_findings.first_seen, excluded.first_seen), "
            "last_seen = GREATEST(secret_findings.last_seen, excluded.last_seen)",
            (r[0], r[1], r[2], r[3], r[4], bool(r[5]), delta, r[7], r[8]),
        )
        _save_receipt(backend, conn, source_id, "finding", record_id, fingerprint, r[6])
        written += 1
    return written


def _verify(source: sqlite3.Connection, backend, report: MigrationReport) -> bool:
    """Confirm every source event is present with its hash and time intact.

    Verification reads back rather than trusting the write path: a migration
    that reports success without checking is the same as no migration at all.
    """
    rows = source.execute("SELECT event_id, payload_hash, accepted_at FROM events").fetchall()
    for event_id, payload_hash, accepted_at in rows:
        found = backend._query(
            f"SELECT payload_hash, accepted_at FROM {backend._events} WHERE event_id = %s",
            (event_id,),
        )
        if not found:
            report.conflicts.append(f"{event_id}: missing from destination after migration")
            return False
        if found[0][0] != payload_hash:
            report.conflicts.append(f"{event_id}: payload hash differs after migration")
            return False
        if not _same_instant(found[0][1], accepted_at):
            report.conflicts.append(
                f"{event_id}: accepted_at not preserved "
                f"(source {accepted_at}, destination {found[0][1]})"
            )
            return False
    return True


def _same_instant(destination, source_iso: str) -> bool:
    from datetime import datetime

    try:
        want = datetime.fromisoformat(str(source_iso).replace("Z", "+00:00"))
    except ValueError:
        return False
    if not hasattr(destination, "timestamp"):
        return False
    if want.tzinfo is None or destination.tzinfo is None:
        return False
    return destination == want