tegwick
146f4e1dd3
Assistant: codex Assistant-Model: gpt-6.1-sol Assistant-Session: 01a1046e-465c-7d43-8c4c-d10f0a1feadd
282 lines
10 KiB
Python
282 lines
10 KiB
Python
"""Migration fidelity tests (AUDIT-WP-0005-T04).
|
|
|
|
The controlling property is that migrating audit records does not alter them.
|
|
An event that arrives in the destination with a rewritten acceptance time has
|
|
been damaged, not moved.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import uuid
|
|
|
|
import pytest
|
|
|
|
from audit_core.ingestion import IngestionApplication
|
|
from audit_core.migrate_store import migrate
|
|
from audit_core.sqlite_backend import SQLiteAuditBackend
|
|
|
|
from tests.test_backend_conformance import HAVE_PG, PG_URL, make_event
|
|
|
|
pytestmark = pytest.mark.skipif(not HAVE_PG, reason="needs PostgreSQL")
|
|
|
|
|
|
@pytest.fixture
|
|
def destination():
|
|
from audit_core.postgres_backend import PostgresAuditBackend
|
|
|
|
schema = f"mig_{uuid.uuid4().hex[:12]}"
|
|
backend = PostgresAuditBackend(PG_URL, schema=schema)
|
|
try:
|
|
yield backend
|
|
finally:
|
|
with backend.pool.connection() as conn:
|
|
conn.execute(f'DROP SCHEMA IF EXISTS "{schema}" CASCADE')
|
|
backend.close()
|
|
|
|
|
|
@pytest.fixture
|
|
def populated_sqlite(tmp_path):
|
|
"""A SQLite store holding events, a dead letter, and a finding."""
|
|
path = str(tmp_path / "legacy.db")
|
|
source = SQLiteAuditBackend(path)
|
|
for i in range(3):
|
|
event = make_event(f"legacy-{i}")
|
|
source.accept(event, hashlib.sha256(f"h{i}".encode()).hexdigest())
|
|
source.record_rejection(
|
|
event_id="bad-1", reason="source_not_allowed", payload_hash="hh",
|
|
sender="user-engine", payload='{"source":"nope"}',
|
|
)
|
|
from audit_core.redaction import Finding
|
|
|
|
source.count_secret_findings(
|
|
sender="user-engine", source="user-engine", action="membership.added",
|
|
outcome="redacted", findings=[Finding("data.auth_token", True)],
|
|
)
|
|
return path, source
|
|
|
|
|
|
def test_migration_preserves_identity_hash_and_time(populated_sqlite, destination):
|
|
path, source = populated_sqlite
|
|
original = {
|
|
row[0]: (row[1], row[2])
|
|
for row in source.db.execute(
|
|
"SELECT event_id, payload_hash, accepted_at FROM events"
|
|
).fetchall()
|
|
}
|
|
|
|
report = migrate(path, destination)
|
|
|
|
assert report.events_read == 3
|
|
assert report.events_written == 3
|
|
assert report.conflicts == []
|
|
assert report.verified is True
|
|
assert report.ok is True
|
|
|
|
for event_id, (payload_hash, accepted_at) in original.items():
|
|
rows = destination._query(
|
|
f"SELECT payload_hash, accepted_at FROM {destination._events} "
|
|
"WHERE event_id = %s", (event_id,),
|
|
)
|
|
assert rows, f"{event_id} missing"
|
|
assert rows[0][0] == payload_hash
|
|
# Acceptance time is evidence; rewriting it would destroy what the
|
|
# migration exists to preserve.
|
|
assert rows[0][1].isoformat().startswith(accepted_at[:16])
|
|
|
|
|
|
def test_migration_carries_dead_letters_and_findings(populated_sqlite, destination):
|
|
path, _ = populated_sqlite
|
|
report = migrate(path, destination)
|
|
|
|
assert report.dead_letters_written == 1
|
|
assert report.findings_written == 1
|
|
assert destination.dead_letters()[0]["reason"] == "source_not_allowed"
|
|
assert destination.secret_findings()[0]["field_path"] == "data.auth_token"
|
|
|
|
|
|
def test_migration_is_idempotent(populated_sqlite, destination):
|
|
path, _ = populated_sqlite
|
|
first = migrate(path, destination)
|
|
second = migrate(path, destination)
|
|
|
|
assert first.events_written == 3
|
|
assert second.events_written == 0
|
|
assert second.events_already_present == 3
|
|
assert second.ok is True
|
|
|
|
count = destination._query(
|
|
f"SELECT count(*) FROM {destination._events}", ()
|
|
)[0][0]
|
|
assert count == 3
|
|
|
|
|
|
def test_a_diverging_destination_record_is_a_conflict_not_an_overwrite(
|
|
populated_sqlite, destination
|
|
):
|
|
"""The destination wins and a human decides.
|
|
|
|
Silently overwriting a stored audit record with a differing one would be
|
|
the same class of failure as losing it.
|
|
"""
|
|
path, _ = populated_sqlite
|
|
migrate(path, destination)
|
|
|
|
# Simulate divergence: same id, different payload hash in the source.
|
|
source = SQLiteAuditBackend(path)
|
|
source.db.execute("BEGIN IMMEDIATE")
|
|
source.db.execute(
|
|
"UPDATE events SET payload_hash = 'diverged' WHERE event_id = 'legacy-1'"
|
|
)
|
|
source.db.execute("COMMIT")
|
|
|
|
report = migrate(path, destination)
|
|
assert report.conflicts, "divergence must be reported"
|
|
assert "legacy-1" in report.conflicts[0]
|
|
assert report.ok is False
|
|
|
|
survived = destination._query(
|
|
f"SELECT payload_hash FROM {destination._events} WHERE event_id = 'legacy-1'", ()
|
|
)[0][0]
|
|
assert survived != "diverged", "destination record must not be overwritten"
|
|
|
|
|
|
def test_migrated_events_remain_readable_and_append_only(populated_sqlite, destination):
|
|
path, _ = populated_sqlite
|
|
migrate(path, destination)
|
|
|
|
record = destination.get("legacy-0")
|
|
assert record["tenant"] == "tenant:friendly:binky"
|
|
assert destination.retention_policy.immutable is True
|
|
|
|
from audit_core.interface import BackendUnavailableError
|
|
|
|
with pytest.raises(BackendUnavailableError):
|
|
destination._execute(
|
|
f"DELETE FROM {destination._events} WHERE event_id = %s", ("legacy-0",)
|
|
)
|
|
|
|
|
|
def test_empty_source_migrates_cleanly(tmp_path, destination):
|
|
"""Today's actual case: nothing to move."""
|
|
path = str(tmp_path / "empty.db")
|
|
SQLiteAuditBackend(path)
|
|
|
|
report = migrate(path, destination)
|
|
assert report.events_read == 0
|
|
assert report.events_written == 0
|
|
assert report.ok is True
|
|
|
|
|
|
def test_historical_import_extends_existing_chain_and_preserves_attestation(
|
|
populated_sqlite, destination
|
|
):
|
|
path, _ = populated_sqlite
|
|
event = make_event('native-before-import')
|
|
destination.accept(event, 'native-hash')
|
|
attestation = destination.attest_chain()
|
|
old_link = destination._query(
|
|
f"SELECT chain_hash FROM {destination._events} WHERE event_id = %s",
|
|
(event.event_id,),
|
|
)[0][0]
|
|
|
|
report = migrate(path, destination)
|
|
assert report.ok
|
|
assert destination.verify_chain(attestation).intact
|
|
assert destination.verify_chain(attestation).attestation_match
|
|
assert destination._query(
|
|
f"SELECT chain_hash FROM {destination._events} WHERE event_id = %s",
|
|
(event.event_id,),
|
|
)[0][0] == old_link
|
|
destination.accept(make_event('native-after-import'), 'after-hash')
|
|
assert destination.verify_chain(attestation).intact
|
|
|
|
|
|
def test_replaying_side_tables_does_not_duplicate_evidence(populated_sqlite, destination):
|
|
path, _ = populated_sqlite
|
|
migrate(path, destination, source_id='stable-source')
|
|
replay = migrate(path, destination, source_id='stable-source')
|
|
assert replay.ok
|
|
assert replay.dead_letters_written == replay.findings_written == 0
|
|
assert len(destination.dead_letters()) == 1
|
|
assert destination.secret_findings()[0]['occurrences'] == 1
|
|
|
|
|
|
def test_incremental_finding_import_adds_only_new_occurrences(populated_sqlite, destination):
|
|
from audit_core.redaction import Finding
|
|
|
|
path, source = populated_sqlite
|
|
migrate(path, destination)
|
|
source.count_secret_findings(
|
|
sender='user-engine', source='user-engine', action='membership.added',
|
|
outcome='redacted', findings=[Finding('data.auth_token', True)],
|
|
)
|
|
assert migrate(path, destination).ok
|
|
assert migrate(path, destination).ok
|
|
assert destination.secret_findings()[0]['occurrences'] == 2
|
|
|
|
|
|
def test_import_and_native_accept_share_one_chain_lock(populated_sqlite, destination):
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
|
|
path, _ = populated_sqlite
|
|
with ThreadPoolExecutor(max_workers=3) as executor:
|
|
futures = [executor.submit(migrate, path, destination) for _ in range(2)]
|
|
futures.append(executor.submit(destination.accept, make_event('concurrent'), 'native'))
|
|
results = [future.result(timeout=20) for future in futures]
|
|
assert all(result.ok for result in results[:2])
|
|
assert sum(result.events_written for result in results[:2]) == 3
|
|
assert destination.verify_chain().intact
|
|
assert destination.verify_chain().events == 4
|
|
assert len(destination.dead_letters()) == 1
|
|
assert destination.secret_findings()[0]['occurrences'] == 1
|
|
|
|
|
|
def test_matching_hash_does_not_hide_a_different_acceptance_time(populated_sqlite, destination):
|
|
path, source = populated_sqlite
|
|
migrate(path, destination)
|
|
source.db.execute("UPDATE events SET accepted_at = '2020-01-01T00:00:00+00:00' WHERE event_id = 'legacy-1'")
|
|
report = migrate(path, destination)
|
|
assert not report.ok
|
|
assert any('different historical evidence' in conflict for conflict in report.conflicts)
|
|
assert destination.verify_chain().intact
|
|
|
|
|
|
def test_failed_import_rolls_back_events_and_replay_receipts(
|
|
populated_sqlite, destination, monkeypatch
|
|
):
|
|
import audit_core.migrate_store as importer
|
|
import psycopg
|
|
from audit_core.interface import BackendUnavailableError
|
|
|
|
path, _ = populated_sqlite
|
|
original = importer._copy_findings
|
|
def fail_after_side_tables(*args):
|
|
original(*args)
|
|
raise psycopg.OperationalError('injected import failure')
|
|
monkeypatch.setattr(importer, '_copy_findings', fail_after_side_tables)
|
|
with pytest.raises(BackendUnavailableError):
|
|
migrate(path, destination)
|
|
assert destination.verify_chain().events == 0
|
|
assert destination.dead_letters() == []
|
|
assert destination.secret_findings() == []
|
|
assert destination._query(f'SELECT count(*) FROM "{destination.schema}".import_receipts', ())[0][0] == 0
|
|
monkeypatch.setattr(importer, '_copy_findings', original)
|
|
assert migrate(path, destination).ok
|
|
|
|
|
|
def test_source_relocation_with_stable_id_preserves_side_table_replay(
|
|
populated_sqlite, destination, tmp_path
|
|
):
|
|
path, _ = populated_sqlite
|
|
# Backup captures WAL-backed source data too.
|
|
import sqlite3
|
|
moved = str(tmp_path / 'moved.db')
|
|
with sqlite3.connect(path) as source, sqlite3.connect(moved) as copy:
|
|
source.backup(copy)
|
|
migrate(path, destination, source_id='same-source')
|
|
replay = migrate(moved, destination, source_id='same-source')
|
|
assert replay.ok
|
|
assert replay.events_written == replay.dead_letters_written == replay.findings_written == 0
|