tegwick
146f4e1dd3
Assistant: codex Assistant-Model: gpt-6.1-sol Assistant-Session: 01a1046e-465c-7d43-8c4c-d10f0a1feadd
269 lines
11 KiB
Python
269 lines
11 KiB
Python
"""Move audit records from a SQLite store into PostgreSQL (AUDIT-WP-0005-T04).
|
|
|
|
As of 2026-08-10 there are no pre-production records to move: audit-core was
|
|
never deployed, so the only SQLite stores that ever existed were test
|
|
fixtures. See the workplan for that finding.
|
|
|
|
The tool exists anyway because the SQLite path is still reachable — the
|
|
entrypoint falls back to it when ``AUDIT_CORE_DATABASE_URL`` is unset. If that
|
|
fallback is ever used in anger, the records it accumulates are audit records
|
|
and cannot simply be dropped.
|
|
|
|
Fidelity is the whole point. Records are transferred with their original
|
|
``event_id``, ``payload_hash``, and ``accepted_at`` intact; a migration that
|
|
rewrote acceptance times would destroy the evidence it was meant to preserve.
|
|
That is why this bypasses ``accept()``, which stamps ``accepted_at`` with the
|
|
current time.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import sqlite3
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path
|
|
|
|
from audit_core.integrity import CHAIN_LOCK_KEY, chain_link
|
|
|
|
from audit_core.interface import BackendUnavailableError
|
|
|
|
|
|
@dataclass
|
|
class MigrationReport:
|
|
"""What happened, in enough detail to be evidence."""
|
|
|
|
events_read: int = 0
|
|
events_written: int = 0
|
|
events_already_present: int = 0
|
|
dead_letters_written: int = 0
|
|
findings_written: int = 0
|
|
conflicts: list[str] = field(default_factory=list)
|
|
verified: bool = False
|
|
|
|
@property
|
|
def ok(self) -> bool:
|
|
return not self.conflicts and self.verified
|
|
|
|
def summary(self) -> str:
|
|
lines = [
|
|
f"events read: {self.events_read}",
|
|
f"events written: {self.events_written}",
|
|
f"already present: {self.events_already_present}",
|
|
f"dead letters written: {self.dead_letters_written}",
|
|
f"secret findings written:{self.findings_written}",
|
|
f"verified: {self.verified}",
|
|
]
|
|
if self.conflicts:
|
|
lines.append(f"CONFLICTS ({len(self.conflicts)}):")
|
|
lines += [f" - {c}" for c in self.conflicts]
|
|
return "\n".join(lines)
|
|
|
|
|
|
def migrate(
|
|
sqlite_path: str, backend, *, verify: bool = True, source_id: str | None = None
|
|
) -> MigrationReport:
|
|
"""Copy every record from ``sqlite_path`` into ``backend``.
|
|
|
|
Idempotent: re-running skips events with matching hash, time and record.
|
|
An event present with different historical evidence is reported as a conflict
|
|
and never overwritten — the destination record wins, and a human decides.
|
|
"""
|
|
import psycopg
|
|
|
|
report = MigrationReport()
|
|
# Callers moving a source to a new path must retain its explicit source_id.
|
|
source_id = source_id or str(Path(sqlite_path).resolve())
|
|
source_id = hashlib.sha256(source_id.encode()).hexdigest()
|
|
source = sqlite3.connect(f"file:{sqlite_path}?mode=ro", uri=True)
|
|
try:
|
|
source.execute("BEGIN") # One consistent source snapshot, including counters.
|
|
rows = source.execute(
|
|
"SELECT event_id, payload_hash, accepted_at, correlation_id, tenant, record "
|
|
"FROM events ORDER BY accepted_at, event_id"
|
|
).fetchall()
|
|
report.events_read = len(rows)
|
|
|
|
# Same serialization boundary as native accept(), including side-table
|
|
# receipts. A retry cannot double-count after an interrupted import.
|
|
with backend.pool.connection() as conn, conn.transaction():
|
|
conn.execute("SELECT pg_advisory_xact_lock(%s)", (CHAIN_LOCK_KEY,))
|
|
for event_id, payload_hash, accepted_at, correlation_id, tenant, record in rows:
|
|
outcome = _import_event(
|
|
backend, conn, event_id, payload_hash, accepted_at, correlation_id,
|
|
tenant, json.loads(record),
|
|
)
|
|
if outcome == "written":
|
|
report.events_written += 1
|
|
elif outcome == "present":
|
|
report.events_already_present += 1
|
|
else:
|
|
report.conflicts.append(outcome)
|
|
|
|
report.dead_letters_written = _copy_dead_letters(source, backend, conn, source_id, report)
|
|
report.findings_written = _copy_findings(source, backend, conn, source_id, report)
|
|
|
|
if verify:
|
|
report.verified = _verify(source, backend, report)
|
|
if not backend.verify_chain().intact:
|
|
report.conflicts.append("destination hash chain is not intact")
|
|
report.verified = False
|
|
except psycopg.Error as exc:
|
|
raise BackendUnavailableError("SQLite import transaction failed") from exc
|
|
finally:
|
|
source.close()
|
|
return report
|
|
|
|
|
|
def _import_event(
|
|
backend, conn, event_id, payload_hash, accepted_at, correlation_id, tenant, record
|
|
) -> str:
|
|
existing = conn.execute(
|
|
f"SELECT payload_hash, accepted_at, record FROM {backend._events} WHERE event_id = %s",
|
|
(event_id,),
|
|
).fetchone()
|
|
if existing:
|
|
if existing[0] != payload_hash:
|
|
return f"{event_id}: destination holds a different payload — left untouched"
|
|
if not _same_instant(existing[1], accepted_at) or existing[2] != record:
|
|
return f"{event_id}: destination holds different historical evidence — left untouched"
|
|
return "present"
|
|
|
|
previous = backend._chain_head(conn)
|
|
conn.execute(
|
|
f"INSERT INTO {backend._events} "
|
|
"(event_id, payload_hash, accepted_at, observed_at, tenant, correlation_id, "
|
|
" source, action, record, chain_prev, chain_hash) "
|
|
"VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)",
|
|
(
|
|
event_id, payload_hash, accepted_at, record.get("observed_at"),
|
|
tenant, correlation_id, record.get("source", ""), record.get("action", ""),
|
|
json.dumps(record, sort_keys=True), previous,
|
|
chain_link(previous, payload_hash, event_id),
|
|
),
|
|
)
|
|
return "written"
|
|
|
|
|
|
def _fingerprint(value) -> str:
|
|
return hashlib.sha256(json.dumps(value, sort_keys=True).encode()).hexdigest()
|
|
|
|
|
|
def _receipt(backend, conn, source_id, kind, record_id):
|
|
return conn.execute(
|
|
f'SELECT content_hash, occurrences FROM "{backend.schema}".import_receipts '
|
|
"WHERE source_id = %s AND kind = %s AND record_id = %s",
|
|
(source_id, kind, record_id),
|
|
).fetchone()
|
|
|
|
|
|
def _save_receipt(backend, conn, source_id, kind, record_id, content_hash, occurrences=0):
|
|
conn.execute(
|
|
f'INSERT INTO "{backend.schema}".import_receipts '
|
|
"(source_id, kind, record_id, content_hash, occurrences) VALUES (%s,%s,%s,%s,%s) "
|
|
"ON CONFLICT (source_id, kind, record_id) DO UPDATE "
|
|
"SET content_hash = excluded.content_hash, occurrences = excluded.occurrences",
|
|
(source_id, kind, record_id, content_hash, occurrences),
|
|
)
|
|
|
|
|
|
def _copy_dead_letters(source, backend, conn, source_id, report) -> int:
|
|
if not source.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name='dead_letters'").fetchone():
|
|
return 0
|
|
rows = source.execute(
|
|
"SELECT id, event_id, received_at, sender, reason, payload_hash, payload, "
|
|
"payload_withheld FROM dead_letters ORDER BY id"
|
|
).fetchall()
|
|
written = 0
|
|
for row_id, *data in rows:
|
|
record_id = str(row_id)
|
|
fingerprint = _fingerprint(data)
|
|
receipt = _receipt(backend, conn, source_id, "dead-letter", record_id)
|
|
if receipt:
|
|
if receipt[0] != fingerprint:
|
|
report.conflicts.append(f"dead-letter {record_id}: source changed after import")
|
|
continue
|
|
conn.execute(
|
|
f'INSERT INTO "{backend.schema}".dead_letters '
|
|
"(event_id, received_at, sender, reason, payload_hash, payload, payload_withheld) "
|
|
"VALUES (%s, %s, %s, %s, %s, %s, %s)",
|
|
(*data[:-1], bool(data[-1])),
|
|
)
|
|
_save_receipt(backend, conn, source_id, "dead-letter", record_id, fingerprint)
|
|
written += 1
|
|
return written
|
|
|
|
|
|
def _copy_findings(source, backend, conn, source_id, report) -> int:
|
|
if not source.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name='secret_findings'").fetchone():
|
|
return 0
|
|
rows = source.execute(
|
|
"SELECT sender, source, action, field_path, outcome, persisted, "
|
|
"occurrences, first_seen, last_seen FROM secret_findings"
|
|
).fetchall()
|
|
written = 0
|
|
for r in rows:
|
|
record_id = _fingerprint(r[:5])
|
|
fingerprint = _fingerprint(r)
|
|
receipt = _receipt(backend, conn, source_id, "finding", record_id)
|
|
if receipt and receipt[0] == fingerprint:
|
|
continue
|
|
delta = r[6] - (receipt[1] if receipt else 0)
|
|
if delta < 0:
|
|
report.conflicts.append(f"finding {record_id}: source occurrence count decreased")
|
|
continue
|
|
conn.execute(
|
|
f'INSERT INTO "{backend.schema}".secret_findings '
|
|
"(sender, source, action, field_path, outcome, persisted, occurrences, "
|
|
" first_seen, last_seen) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s) "
|
|
"ON CONFLICT (sender, source, action, field_path, outcome) DO UPDATE "
|
|
"SET occurrences = secret_findings.occurrences + excluded.occurrences, "
|
|
"persisted = secret_findings.persisted OR excluded.persisted, "
|
|
"first_seen = LEAST(secret_findings.first_seen, excluded.first_seen), "
|
|
"last_seen = GREATEST(secret_findings.last_seen, excluded.last_seen)",
|
|
(r[0], r[1], r[2], r[3], r[4], bool(r[5]), delta, r[7], r[8]),
|
|
)
|
|
_save_receipt(backend, conn, source_id, "finding", record_id, fingerprint, r[6])
|
|
written += 1
|
|
return written
|
|
|
|
|
|
def _verify(source: sqlite3.Connection, backend, report: MigrationReport) -> bool:
|
|
"""Confirm every source event is present with its hash and time intact.
|
|
|
|
Verification reads back rather than trusting the write path: a migration
|
|
that reports success without checking is the same as no migration at all.
|
|
"""
|
|
rows = source.execute("SELECT event_id, payload_hash, accepted_at FROM events").fetchall()
|
|
for event_id, payload_hash, accepted_at in rows:
|
|
found = backend._query(
|
|
f"SELECT payload_hash, accepted_at FROM {backend._events} WHERE event_id = %s",
|
|
(event_id,),
|
|
)
|
|
if not found:
|
|
report.conflicts.append(f"{event_id}: missing from destination after migration")
|
|
return False
|
|
if found[0][0] != payload_hash:
|
|
report.conflicts.append(f"{event_id}: payload hash differs after migration")
|
|
return False
|
|
if not _same_instant(found[0][1], accepted_at):
|
|
report.conflicts.append(
|
|
f"{event_id}: accepted_at not preserved "
|
|
f"(source {accepted_at}, destination {found[0][1]})"
|
|
)
|
|
return False
|
|
return True
|
|
|
|
|
|
def _same_instant(destination, source_iso: str) -> bool:
|
|
from datetime import datetime
|
|
|
|
try:
|
|
want = datetime.fromisoformat(str(source_iso).replace("Z", "+00:00"))
|
|
except ValueError:
|
|
return False
|
|
if not hasattr(destination, "timestamp"):
|
|
return False
|
|
if want.tzinfo is None or destination.tzinfo is None:
|
|
return False
|
|
return destination == want
|