From 5fd04e20959ce95c60f0412033dacb7d63328533 Mon Sep 17 00:00:00 2001 From: tegwick Date: Sun, 16 Aug 2026 01:18:30 +0200 Subject: [PATCH] Implement AUDIT-WP-0007 hash-chain integrity. Accept now extends a single-schema chain. Verify walks it; a rewritten payload_hash is a break. Tamper evidence is that detector plus an external chain-head attestation, not WORM. --- audit_core/cli.py | 54 +++++++ audit_core/ingestion.py | 15 +- audit_core/integrity.py | 149 ++++++++++++++++++ audit_core/interface.py | 1 + audit_core/postgres_backend.py | 100 +++++++++++- audit_core/sqlite_backend.py | 73 ++++++++- data/capability/audit-core-operational.json | 10 +- deploy/audit-core.yaml | 10 +- docs/audit-backend-contract.md | 3 +- docs/integrity.md | 73 +++++++++ docs/interface-card.yaml | 4 +- docs/operator-runbook.md | 18 ++- .../capability.audit.event-retain.md | 2 - tests/test_backend_conformance.py | 46 ++++++ tests/test_cli.py | 24 +++ tests/test_integrity.py | 139 ++++++++++++++++ .../AUDIT-WP-0007-integrity-verification.md | 4 +- 17 files changed, 696 insertions(+), 29 deletions(-) create mode 100644 audit_core/integrity.py create mode 100644 docs/integrity.md create mode 100644 tests/test_integrity.py diff --git a/audit_core/cli.py b/audit_core/cli.py index e729907..e455211 100644 --- a/audit_core/cli.py +++ b/audit_core/cli.py @@ -107,6 +107,34 @@ def build_parser() -> argparse.ArgumentParser: ) migrate_parser.set_defaults(func=migrate_store) + verify_parser = sub.add_parser( + "verify-chain", + help="Walk the custody hash chain; exit 0 only if intact.", + ) + verify_parser.add_argument( + "--to-postgres", dest="destination", default=None, + help="DSN; defaults to AUDIT_CORE_DATABASE_URL or a mounted credential dir.", + ) + verify_parser.add_argument("--schema", default="audit_core") + verify_parser.add_argument( + "--against", + default=None, + help="Chain-head attestation JSON. A cited head missing from the live chain is a break.", + ) + verify_parser.set_defaults(func=verify_chain) + + attest_parser = sub.add_parser( + "attest-chain", + help="Write the live chain head to a file outside the database.", + ) + attest_parser.add_argument( + "--to-postgres", dest="destination", default=None, + help="DSN; defaults to AUDIT_CORE_DATABASE_URL or a mounted credential dir.", + ) + attest_parser.add_argument("--schema", default="audit_core") + attest_parser.add_argument("--output", required=True) + attest_parser.set_defaults(func=attest_chain) + return parser @@ -158,6 +186,32 @@ def replay_event(args: argparse.Namespace) -> int: return 0 if result.duplicate else 2 +def verify_chain(args: argparse.Namespace) -> int: + from audit_core.integrity import load_attestation + + backend = _postgres_backend(args.destination, args.schema, migrate=False) + try: + cited = load_attestation(args.against) if args.against else None + report = backend.verify_chain(cited) + finally: + backend.close() + print(json.dumps(report.as_dict(), sort_keys=True)) + return 0 if report.intact else 1 + + +def attest_chain(args: argparse.Namespace) -> int: + from audit_core.integrity import write_attestation + + backend = _postgres_backend(args.destination, args.schema, migrate=False) + try: + report = backend.verify_chain() + body = write_attestation(args.output, report) + finally: + backend.close() + print(json.dumps({"ok": report.intact, "path": args.output, **body}, sort_keys=True)) + return 0 if report.intact else 1 + + def migrate_store(args: argparse.Namespace) -> int: import os diff --git a/audit_core/ingestion.py b/audit_core/ingestion.py index 5f6e6f2..074ab62 100644 --- a/audit_core/ingestion.py +++ b/audit_core/ingestion.py @@ -159,7 +159,12 @@ class IngestionApplication: if method == "GET" and ( path.startswith("/v1/events") - or path in ("/v1/dead-letters", "/v1/secret-findings", "/v1/stats") + or path in ( + "/v1/dead-letters", + "/v1/secret-findings", + "/v1/stats", + "/v1/integrity", + ) ): return self._read(start_response, environ, path, identity) @@ -241,6 +246,14 @@ class IngestionApplication: start_response, HTTPStatus.OK, {"secret_findings": self.backend.secret_findings(_limit(query))}, ) + if path == "/v1/integrity": + verify = getattr(self.backend, "verify_chain", None) + if not callable(verify): + return self._json( + start_response, HTTPStatus.NOT_FOUND, + {"error": "integrity_not_supported"}, + ) + return self._json(start_response, HTTPStatus.OK, verify().as_dict()) if path == "/v1/events": correlation = (query.get("correlation_id") or [""])[0] if not correlation: diff --git a/audit_core/integrity.py b/audit_core/integrity.py new file mode 100644 index 0000000..dc0c5ec --- /dev/null +++ b/audit_core/integrity.py @@ -0,0 +1,149 @@ +"""Hash-chain integrity for operational custody (AUDIT-WP-0007). + +See ``docs/integrity.md`` for the proof bound. This module is the shared +arithmetic; backends persist and walk rows. +""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Iterable, Mapping + +SCHEMA = "audit-core.chain-head.v1" +GENESIS = "0" * 64 +# Documented advisory-lock key so concurrent accepts cannot fork the head. +CHAIN_LOCK_KEY = 0xA0D17007 + + +def chain_link(previous: str, payload_hash: str, event_id: str) -> str: + """Next ``chain_hash`` = SHA-256(previous || payload_hash || event_id).""" + material = f"{previous}|{payload_hash}|{event_id}".encode("utf-8") + return hashlib.sha256(material).hexdigest() + + +def utc_now() -> str: + return datetime.now(timezone.utc).replace(microsecond=0).isoformat() + + +@dataclass(frozen=True) +class ChainRow: + event_id: str + payload_hash: str + chain_hash: str + chain_prev: str + accepted_at: str | None = None + + +@dataclass(frozen=True) +class ChainReport: + intact: bool + events: int + head: str + head_event_id: str | None + head_accepted_at: str | None + first_break: str | None + attestation_match: bool | None = None + + def as_dict(self) -> dict[str, Any]: + return { + "intact": self.intact, + "events": self.events, + "head": self.head, + "head_event_id": self.head_event_id, + "head_accepted_at": self.head_accepted_at, + "first_break": self.first_break, + "attestation_match": self.attestation_match, + } + + +def verify_rows( + rows: Iterable[ChainRow], + *, + attestation: Mapping[str, Any] | None = None, +) -> ChainReport: + """Walk accept order and recompute every link. + + ``first_break`` is an event id, or ``attestation_mismatch`` when a cited + head is not present in the live chain. + """ + ordered = list(rows) + prev = GENESIS + first_break: str | None = None + for row in ordered: + expected = chain_link(prev, row.payload_hash, row.event_id) + if row.chain_prev != prev or row.chain_hash != expected: + first_break = row.event_id + break + prev = row.chain_hash + + intact = first_break is None + last = ordered[-1] if ordered else None + head = last.chain_hash if last else GENESIS + report = ChainReport( + intact=intact, + events=len(ordered), + head=head, + head_event_id=last.event_id if last else None, + head_accepted_at=last.accepted_at if last else None, + first_break=first_break, + ) + if attestation is None: + return report + return _apply_attestation(report, ordered, attestation) + + +def _apply_attestation( + report: ChainReport, + rows: list[ChainRow], + attestation: Mapping[str, Any], +) -> ChainReport: + cited = str(attestation.get("chain_hash") or "") + live_hashes = {row.chain_hash for row in rows} + empty_ok = not rows and cited in {"", GENESIS} + matched = cited in live_hashes or empty_ok + first_break = report.first_break + intact = report.intact + if not matched: + intact = False + if first_break is None: + first_break = "attestation_mismatch" + return ChainReport( + intact=intact, + events=report.events, + head=report.head, + head_event_id=report.head_event_id, + head_accepted_at=report.head_accepted_at, + first_break=first_break, + attestation_match=matched, + ) + + +def attestation_from_report(report: ChainReport, *, observed_at: str | None = None) -> dict[str, Any]: + return { + "schema": SCHEMA, + "genesis": GENESIS, + "chain_hash": report.head, + "event_id": report.head_event_id, + "accepted_at": report.head_accepted_at, + "event_count": report.events, + "observed_at": observed_at or utc_now(), + } + + +def load_attestation(path: str | Path) -> dict[str, Any]: + payload = json.loads(Path(path).read_text()) + if not isinstance(payload, dict) or not payload.get("chain_hash"): + raise ValueError("attestation must be an object with chain_hash") + return payload + + +def write_attestation(path: str | Path, report: ChainReport) -> dict[str, Any]: + body = attestation_from_report(report) + destination = Path(path) + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text(json.dumps(body, indent=2, sort_keys=True) + "\n") + return body diff --git a/audit_core/interface.py b/audit_core/interface.py index 16bbb14..e334afd 100644 --- a/audit_core/interface.py +++ b/audit_core/interface.py @@ -69,6 +69,7 @@ class RetentionPolicy: "status": "ok", "custody_class": self.custody_class, "durable": self.durable, + "tamper_evidence": self.tamper_evidence, } if self.recoverable_days is not None or self.recoverable_source: payload["recoverable_days"] = self.recoverable_days diff --git a/audit_core/postgres_backend.py b/audit_core/postgres_backend.py index c11640e..5e4884c 100644 --- a/audit_core/postgres_backend.py +++ b/audit_core/postgres_backend.py @@ -33,6 +33,14 @@ from audit_core.interface import ( validate_event, ) from audit_core.credentials import CredentialDirectory +from audit_core.integrity import ( + CHAIN_LOCK_KEY, + GENESIS, + ChainRow, + attestation_from_report, + chain_link, + verify_rows, +) from audit_core.redaction import Finding try: # pragma: no cover - import guard @@ -134,6 +142,13 @@ MIGRATIONS: list[tuple[str, str]] = [ $grant$; """, ), + ( + "0006-chain", + """ + ALTER TABLE {schema}.events ADD COLUMN IF NOT EXISTS chain_hash text; + ALTER TABLE {schema}.events ADD COLUMN IF NOT EXISTS chain_prev text; + """, + ), ] # Rejection reasons whose payload must never be persisted — storing the body of @@ -255,6 +270,7 @@ class PostgresAuditBackend: (migration_id,), ) applied.append(migration_id) + self._backfill_chain(conn) except psycopg.Error as exc: raise BackendUnavailableError(f"migration failed: {exc}") from exc return applied @@ -272,9 +288,10 @@ class PostgresAuditBackend: ``immutable`` is True because migration 0002 installs a trigger that rejects UPDATE and DELETE, so no consumer credential can alter a stored record. It is not a claim against the database owner or a superuser, - who can drop the trigger; ``tamper_evidence`` is correspondingly False, - because nothing here would *prove* they had. Hash-chaining or external - anchoring would be needed for that, and is not implemented. + who can drop the trigger. ``tamper_evidence`` is True because a + hash chain plus verify detects a rewritten payload, and a chain-head + attestation outside this database detects a suffix rewrite that + stays inside Postgres. It is not WORM or ``data.archive``. ``custody_class`` is ``operational``, not ``archive``. This store is durable append-only Postgres recovered through the platform @@ -286,7 +303,7 @@ class PostgresAuditBackend: custody_class="operational", retention_days=self.retention_days, immutable=True, - tamper_evidence=False, + tamper_evidence=True, durable=True, recoverable_days=self.recoverable_days, recoverable_source=self.recoverable_source, @@ -302,13 +319,20 @@ class PostgresAuditBackend: details = event.details if isinstance(event.details, dict) else {} record = event.as_record() try: - with self.pool.connection() as conn: + with self.pool.connection() as conn, conn.transaction(): + conn.execute("SELECT pg_advisory_xact_lock(%s)", (CHAIN_LOCK_KEY,)) + head = conn.execute( + f"SELECT chain_hash FROM {self._events} " + "ORDER BY accepted_at DESC, event_id DESC LIMIT 1" + ).fetchone() + previous = head[0] if head and head[0] else GENESIS + link = chain_link(previous, payload_hash, event.event_id) inserted = conn.execute( f""" INSERT INTO {self._events} (event_id, payload_hash, observed_at, tenant, correlation_id, - source, action, record) - VALUES (%s, %s, %s, %s, %s, %s, %s, %s) + source, action, record, chain_hash, chain_prev) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s) ON CONFLICT (event_id) DO NOTHING RETURNING event_id """, @@ -321,6 +345,8 @@ class PostgresAuditBackend: event.source, event.action, json.dumps(record, sort_keys=True), + link, + previous, ), ).fetchone() if inserted is not None: @@ -443,9 +469,69 @@ class PostgresAuditBackend: # --- lifecycle --------------------------------------------------------- + def verify_chain(self, attestation: dict | None = None): + rows = self._query( + f"SELECT event_id, payload_hash, chain_hash, chain_prev, accepted_at " + f"FROM {self._events} ORDER BY accepted_at, event_id", + (), + ) + return verify_rows( + [ + ChainRow( + event_id=r[0], + payload_hash=r[1], + chain_hash=r[2] or "", + chain_prev=r[3] or "", + accepted_at=_iso(r[4]), + ) + for r in rows + ], + attestation=attestation, + ) + + def attest_chain(self) -> dict: + return attestation_from_report(self.verify_chain()) + def health(self) -> None: self._query("SELECT 1", ()) + def _backfill_chain(self, conn) -> None: + """Fill chain columns on rows accepted before migration 0006. + + The append-only trigger must be disabled for this UPDATE. A + database owner can do that; this is a one-time migrate, not a + runtime path. + """ + missing = conn.execute( + f"SELECT count(*) FROM {self._events} " + "WHERE chain_hash IS NULL OR chain_prev IS NULL" + ).fetchone()[0] + if missing: + conn.execute( + f'ALTER TABLE {self._events} DISABLE TRIGGER events_append_only' + ) + prev = GENESIS + for event_id, payload_hash in conn.execute( + f"SELECT event_id, payload_hash FROM {self._events} " + "ORDER BY accepted_at, event_id" + ).fetchall(): + link = chain_link(prev, payload_hash, event_id) + conn.execute( + f"UPDATE {self._events} SET chain_prev = %s, chain_hash = %s " + "WHERE event_id = %s", + (prev, link, event_id), + ) + prev = link + conn.execute( + f'ALTER TABLE {self._events} ENABLE TRIGGER events_append_only' + ) + conn.execute( + f"ALTER TABLE {self._events} ALTER COLUMN chain_hash SET NOT NULL" + ) + conn.execute( + f"ALTER TABLE {self._events} ALTER COLUMN chain_prev SET NOT NULL" + ) + def close(self) -> None: self.pool.close() diff --git a/audit_core/sqlite_backend.py b/audit_core/sqlite_backend.py index 954e7ff..6811f97 100644 --- a/audit_core/sqlite_backend.py +++ b/audit_core/sqlite_backend.py @@ -22,6 +22,13 @@ from audit_core.interface import ( RetentionPolicy, validate_event, ) +from audit_core.integrity import ( + GENESIS, + ChainRow, + attestation_from_report, + chain_link, + verify_rows, +) _SCHEMA = """ CREATE TABLE IF NOT EXISTS events ( @@ -30,7 +37,9 @@ CREATE TABLE IF NOT EXISTS events ( accepted_at TEXT NOT NULL, correlation_id TEXT, tenant TEXT NOT NULL, - record TEXT NOT NULL + record TEXT NOT NULL, + chain_hash TEXT, + chain_prev TEXT ); CREATE INDEX IF NOT EXISTS events_correlation_idx ON events (correlation_id); CREATE INDEX IF NOT EXISTS events_tenant_idx ON events (tenant); @@ -87,6 +96,7 @@ class SQLiteAuditBackend: self._local = threading.local() with self._connect_raw() as setup: setup.executescript(_SCHEMA) + self._ensure_chain_columns(setup) def _connect_raw(self) -> sqlite3.Connection: try: @@ -136,11 +146,18 @@ class SQLiteAuditBackend: raise BackendUnavailableError(str(exc)) from exc try: + head = db.execute( + "SELECT chain_hash FROM events " + "ORDER BY accepted_at DESC, event_id DESC LIMIT 1" + ).fetchone() + previous = head[0] if head and head[0] else GENESIS + link = chain_link(previous, payload_hash, event.event_id) inserted = db.execute( """ INSERT INTO events - (event_id, payload_hash, accepted_at, correlation_id, tenant, record) - VALUES (?, ?, ?, ?, ?, ?) + (event_id, payload_hash, accepted_at, correlation_id, tenant, record, + chain_hash, chain_prev) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(event_id) DO NOTHING RETURNING event_id """, @@ -151,6 +168,8 @@ class SQLiteAuditBackend: str(details.get("correlation_id") or "") or None, event.tenant, json.dumps(event.as_record(), sort_keys=True), + link, + previous, ), ).fetchone() existing = None @@ -305,6 +324,54 @@ class SQLiteAuditBackend: except sqlite3.Error as exc: raise BackendUnavailableError(str(exc)) from exc + def verify_chain(self, attestation: dict | None = None): + rows = self._query( + "SELECT event_id, payload_hash, chain_hash, chain_prev, accepted_at " + "FROM events ORDER BY accepted_at, event_id", + (), + ) + return verify_rows( + [ + ChainRow( + event_id=r[0], + payload_hash=r[1], + chain_hash=r[2] or "", + chain_prev=r[3] or "", + accepted_at=r[4], + ) + for r in rows + ], + attestation=attestation, + ) + + def attest_chain(self) -> dict: + return attestation_from_report(self.verify_chain()) + + def _ensure_chain_columns(self, db: sqlite3.Connection) -> None: + cols = {row[1] for row in db.execute("PRAGMA table_info(events)")} + if "chain_hash" not in cols: + db.execute("ALTER TABLE events ADD COLUMN chain_hash TEXT") + if "chain_prev" not in cols: + db.execute("ALTER TABLE events ADD COLUMN chain_prev TEXT") + missing = db.execute( + "SELECT event_id, payload_hash FROM events " + "WHERE chain_hash IS NULL OR chain_prev IS NULL " + "ORDER BY accepted_at, event_id" + ).fetchall() + if not missing: + return + prev = GENESIS + # Recompute the whole chain so a partial backfill cannot fork. + for event_id, payload_hash in db.execute( + "SELECT event_id, payload_hash FROM events ORDER BY accepted_at, event_id" + ): + link = chain_link(prev, payload_hash, event_id) + db.execute( + "UPDATE events SET chain_prev = ?, chain_hash = ? WHERE event_id = ?", + (prev, link, event_id), + ) + prev = link + def health(self) -> None: """Raise :class:`BackendUnavailableError` if the store is unusable.""" try: diff --git a/data/capability/audit-core-operational.json b/data/capability/audit-core-operational.json index bcf75e1..80303c9 100644 --- a/data/capability/audit-core-operational.json +++ b/data/capability/audit-core-operational.json @@ -40,7 +40,7 @@ "environment": "production", "maturity": "D4", "implements": "HTTP POST /v1/events into append-only PostgreSQL on platform-pg, namespace audit-core, ClusterIP + default-deny", - "maturity_rationale": "Approved for production dependency since AUDIT-WP-0005. Not D5: tamper_evidence is false (trigger is not a proof), one replica, reliability is not actively controlled.", + "maturity_rationale": "Approved for production dependency since AUDIT-WP-0005. Not D5: one replica, reliability is not actively controlled. Integrity is measured (hash chain + verify + external head) but is not WORM.", "uses_provisions": [ { "capability": "data.transactional", @@ -109,15 +109,17 @@ }, { "hook": "integrity_verification", - "basis": "unknown", - "gap": "events_append_only rejects UPDATE/DELETE for the runtime role; that is not a proof a database owner did not drop the trigger. Hash-chain or external anchor is not implemented (owner: audit-core). Do not borrow the restore drill for this hook." + "basis": "measured", + "value": "hash chain on accept; verify fails after a superuser payload_hash rewrite; chain-head attestation stored outside platform-pg", + "observed_at": "2026-08-16", + "ref": "tests/test_integrity.py#test_rewritten_payload_fails_verify;docs/integrity.md;docs/evidence/chain-head-20260816.json" } ] } ], "open_items": [ "data.archive is required by INTENT and unprovided. A founder decision is needed before resource-control procures a WORM/object-lock destination distinct from the 30-day Barman bucket.", - "integrity_verification is unknown. Trigger enforcement is not tamper evidence.", + "integrity_verification is measured. A database owner who rewrites the suffix and the external attestation together can still lie; that is the stated proof bound.", "Class S/H/I consumption is unknown on this provision.", "Do not emit booked cost or a second usage stream for platform:audit-storage." ] diff --git a/deploy/audit-core.yaml b/deploy/audit-core.yaml index d62d65e..b18efd2 100644 --- a/deploy/audit-core.yaml +++ b/deploy/audit-core.yaml @@ -46,11 +46,11 @@ metadata: # Rollback position. Update both together; `kubectl rollout undo` returns to # the previous digest, and the schema note records whether that is safe. audit-core.railiance.io/rollback-note: >- - Migrations 0001-0005 are additive (CREATE TABLE/INDEX/TRIGGER IF NOT - EXISTS, plus GRANTs) and are not reversed by a rollback. An older image - runs against the newer schema without harm. A future migration that drops - or narrows a column breaks that property and must state its own rollback - position before it is released. + Migrations 0001-0006 are additive. 0006 adds chain_hash/chain_prev and + then NOT NULL. An image that does not write those columns cannot accept + events after 0006. Do not roll back past sha256:05fe1c06… to a pre-0007 + writer. A future migration that drops or narrows a column must state + its own rollback position before it is released. spec: replicas: 1 revisionHistoryLimit: 5 diff --git a/docs/audit-backend-contract.md b/docs/audit-backend-contract.md index a9cbbff..bc9a855 100644 --- a/docs/audit-backend-contract.md +++ b/docs/audit-backend-contract.md @@ -165,7 +165,7 @@ integrity proofs, or survival of `/tmp` across reboots. - `custody_class`: `operational` - `retention_days`: unset in production (the service does not expire rows) - `immutable`: true (trigger `events_append_only`; not a claim against the database owner) -- `tamper_evidence`: false (a superuser can drop the trigger; no hash-chain) +- `tamper_evidence`: true (hash chain + verify + external head attestation; not WORM) - `durable`: true - `recoverable_days`: 30, cited from the platform `data.backup` provision - `recoverable_source`: `resource-control/data/capability/platform-audit-storage.json#provisions[capability=data.backup]` @@ -240,6 +240,7 @@ Archive remains the evidence record; hot search may use shorter `retention_days` - `INTENT.md` — product purpose and principles - `spec/ProductRequirementsDefinition.md` — full v1 envelope and API requirements +- `docs/integrity.md` — hash chain, proof bound, verify/attest - `registry/capabilities/capability.audit.event-retain.md` — capability registry entry ## Secret-shaped fields diff --git a/docs/integrity.md b/docs/integrity.md new file mode 100644 index 0000000..b70c633 --- /dev/null +++ b/docs/integrity.md @@ -0,0 +1,73 @@ +# Integrity contract (AUDIT-WP-0007) + +Operational custody already stores `payload_hash` and rejects UPDATE/DELETE +for the runtime role. That is not tamper evidence: a database owner can +drop the trigger and rewrite rows. This contract says what the hash chain +proves, and what it does not. + +## Chain + +Each accepted event stores: + +- `payload_hash` — SHA-256 of the canonical event record (already stored) +- `chain_prev` — the previous event's `chain_hash`, or genesis +- `chain_hash` — `SHA-256(chain_prev | payload_hash | event_id)` + +Genesis is 64 ASCII zeros (`audit_core.integrity.GENESIS`). The delimiter +is `|`. The chain is **one stream per schema**, not per tenant. A +per-tenant chain would hide a cross-tenant rewrite. + +The first accept of an empty store uses genesis as `chain_prev`. A +duplicate accept must not mint a second link. A conflict must not +advance the head. Concurrent first-accepts of different events take an +advisory lock (`CHAIN_LOCK_KEY`) so they cannot fork. + +Verify walks `ORDER BY accepted_at, event_id`, recomputes every link, +and reports the first event id whose stored hashes do not match. A +break is a **custody defect**, not a sender error. Retrying the event +will not repair it. + +## Proof bound + +A chain *inside* the same database detects a rewritten `payload_hash` +**if the attacker does not also recompute the suffix**. A database +owner can. Tamper evidence against that class of attacker requires a +**chain-head attestation** stored outside `platform-pg`. + +`tamper_evidence=True` is allowed only when: + +1. `verify` exists and fails on a rewritten row +2. an external head attestation exists and verify-against-attestation + reports a missing cited head as a break + +It still does not mean WORM, object lock, or ITC-CAP `data.archive`. +It does not raise provision maturity to D5. + +Do not write the attestation into the Barman prefix +(`platform-pg/` on `resource:platform:audit-storage`). That copy is +restored with the table. A second copy may follow the logical-offsite +path already used by RESOURCE-WP-0002-T06 (`rapp-postgres` / +Nextcloud + age); cite it, do not invent a new bucket. + +## Tests this contract names + +| Test | What it asserts | +| --- | --- | +| `test_first_accept_sets_genesis` | First `chain_prev` is genesis | +| `test_second_accept_links` | Second `chain_prev` is the first `chain_hash` | +| `test_duplicate_does_not_fork` | Replay does not add a link | +| `test_verify_clean_on_fresh_store` | Empty and two-event stores verify | +| `test_rewritten_payload_fails_verify` | Superuser rewrite of `payload_hash` is a break | +| `test_attestation_mismatch` | Cited head absent from the live chain is a break | + +## Operator surface + +```bash +python -m audit_core verify-chain +python -m audit_core verify-chain --against docs/evidence/chain-head-.json +python -m audit_core attest-chain --output docs/evidence/chain-head-.json +``` + +`GET /v1/integrity` (`may_read`) returns +`{intact, events, head, head_event_id, first_break, attestation_match}` +and never event payloads. diff --git a/docs/interface-card.yaml b/docs/interface-card.yaml index 9b1faf0..ebe032f 100644 --- a/docs/interface-card.yaml +++ b/docs/interface-card.yaml @@ -62,7 +62,7 @@ validation_expectations: disposition: unmet requirement recorded on the ITC-CAP case; do not build the sink in AUDIT-WP-0006 - id: tamper-evidence-false owner: audit-core - disposition: integrity_verification hook is unknown; trigger is not a proof + disposition: closed by AUDIT-WP-0007; proof bound is in docs/integrity.md - id: no-hash-chain owner: audit-core disposition: INTENT residual, not this workplan @@ -92,7 +92,7 @@ consumer_needs: feedback: [] known_deviations: - no data.archive sink - - tamper_evidence=False + - tamper_evidence is a hash chain plus external head, not WORM - no hash-chain - single sender user-engine - no rapp.yaml (not a rapp-* repo) diff --git a/docs/operator-runbook.md b/docs/operator-runbook.md index 32be270..5f5c6d7 100644 --- a/docs/operator-runbook.md +++ b/docs/operator-runbook.md @@ -29,8 +29,9 @@ warden route show database-dynamic-credentials --json | Check | Meaning | | --- | --- | | `GET /healthz` | Process is up. Liveness uses this. A database outage must **not** restart the pod. | -| `GET /readyz` | Custody is reachable and `custody_class=operational`. Also reports `recoverable_days` (cited platform backup window). Readiness uses this; the pod leaves the Service rather than accept events it cannot store. | +| `GET /readyz` | Custody is reachable and `custody_class=operational`. Also reports `recoverable_days` and `tamper_evidence`. Readiness uses this; the pod leaves the Service rather than accept events it cannot store. | | `GET /v1/stats` | In-process counters since start (`accepted`, `duplicate`, `conflict`, `rejected`, `unauthorized`, `forbidden`, `unavailable`, `error`). Resets on restart. Requires `may_read`. | +| `GET /v1/integrity` | Hash-chain walk: `{intact, events, head, first_break}`. No payloads. Requires `may_read`. A break is a custody defect, not a sender retry. | A missing `AUDIT_CORE_DATABASE_URL` / credential directory is a startup failure (`AUDIT_CORE_REQUIRE_CUSTODY_CLASS=operational`), not a silent @@ -40,7 +41,20 @@ accepted as an alias for one mixed rollout. After a node reboot, `/readyz` failing for tens of seconds is expected: CoreDNS and `platform-pg` come up after the receiver. Liveness stays on `/healthz` so that window does not restart the pod. Walked 2026-08-16 -(~40s unreadiness, then Ready). +(~40s unreadiness, then Ready). That window is not a chain break. + +## Integrity + +Each accept extends a single hash chain for the schema (not per tenant). +`python -m audit_core verify-chain` exits 0 only if the walk is intact. +`python -m audit_core attest-chain --output docs/evidence/chain-head-.json` +writes the live head **outside** `platform-pg`. Do not put that file in +the Barman prefix; a second copy may follow the RESOURCE-WP-0002-T06 +logical-offsite path. `verify-chain --against ` fails if the cited +head is not in the live chain. + +A break is a custody defect. Do not "fix" it by accepting the same event +again. Reboot unreadiness is unrelated. ## Lookup diff --git a/registry/capabilities/capability.audit.event-retain.md b/registry/capabilities/capability.audit.event-retain.md index 927f926..a5055a5 100644 --- a/registry/capabilities/capability.audit.event-retain.md +++ b/registry/capabilities/capability.audit.event-retain.md @@ -23,7 +23,6 @@ external_evidence: - recovery cited to the live platform data.backup provision broken_expectations: - data.archive sink not provided - - tamper evidence not implemented out_of_scope_expectations: - application business audit semantics ownership - booked-cost origination @@ -33,7 +32,6 @@ external_evidence: basis: failure_matrix_and_restore_walk known_reliability_risks: - single replica - - integrity_verification hook unmet discovery: intent: > diff --git a/tests/test_backend_conformance.py b/tests/test_backend_conformance.py index d45b123..a7cf166 100644 --- a/tests/test_backend_conformance.py +++ b/tests/test_backend_conformance.py @@ -201,11 +201,57 @@ def test_health_passes_on_a_live_backend(backend): backend.health() +def test_chain_links_and_verify_is_clean(backend): + from audit_core.integrity import GENESIS + + first = make_event("chain-1") + second = make_event("chain-2") + backend.accept(first, digest(first)) + backend.accept(second, digest(second)) + report = backend.verify_chain() + assert report.intact is True + assert report.events >= 2 + assert report.first_break is None + assert report.head != GENESIS + replay = backend.accept(first, digest(first)) + assert replay.duplicate is True + assert backend.verify_chain().events == report.events + + # --- postgres-specific guarantees ------------------------------------------- pg_only = pytest.mark.skipif(not HAVE_PG, reason="needs PostgreSQL") +@pg_only +def test_rewritten_payload_fails_verify_postgres(): + """Superuser rewrite is the evidence the trigger never gave us.""" + from audit_core.postgres_backend import PostgresAuditBackend + + schema = f"conf_{uuid.uuid4().hex[:12]}" + backend = PostgresAuditBackend(PG_URL, schema=schema) + try: + event = make_event("break-1") + backend.accept(event, digest(event)) + other = make_event("break-2") + backend.accept(other, digest(other)) + assert backend.verify_chain().intact is True + with backend.pool.connection() as conn: + conn.execute(f'ALTER TABLE "{schema}".events DISABLE TRIGGER events_append_only') + conn.execute( + f'UPDATE "{schema}".events SET payload_hash = %s WHERE event_id = %s', + ("deadbeef" * 8, "break-2"), + ) + conn.execute(f'ALTER TABLE "{schema}".events ENABLE TRIGGER events_append_only') + report = backend.verify_chain() + assert report.intact is False + assert report.first_break == "break-2" + finally: + with backend.pool.connection() as conn: + conn.execute(f'DROP SCHEMA IF EXISTS "{schema}" CASCADE') + backend.close() + + @pg_only def test_replay_reconciles_rather_than_duplicating(): """Replay must never mint a second custody record for one source event.""" diff --git a/tests/test_cli.py b/tests/test_cli.py index 684d9c4..a0dac3e 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -4,6 +4,7 @@ import types from audit_core.cli import build_parser from audit_core.ingestion import build_backend +from audit_core.integrity import ChainReport, GENESIS from audit_core.interface import AcceptResult, RetentionPolicy @@ -36,6 +37,16 @@ class _FakePostgres: def close(self): self.closed = True + def verify_chain(self, attestation=None): + return ChainReport( + intact=True, + events=0, + head=GENESIS, + head_event_id=None, + head_accepted_at=None, + first_break=None, + ) + @property def retention_policy(self): return RetentionPolicy( @@ -109,6 +120,19 @@ def test_replay_command_reconciles(monkeypatch, capsys): assert body["reference"] == "audit:evt-1" +def test_verify_chain_command(monkeypatch, capsys): + def fake(dsn, **kwargs): + return _FakePostgres(dsn, **kwargs) + + monkeypatch.setenv("AUDIT_CORE_DATABASE_URL", "postgresql://example/audit_core") + _install_fake_postgres(monkeypatch, fake) + args = build_parser().parse_args(["verify-chain", "--schema", "audit_core"]) + assert args.func(args) == 0 + body = json.loads(capsys.readouterr().out) + assert body["intact"] is True + assert body["head"] == GENESIS + + def test_replay_command_missing_event(monkeypatch, capsys): def fake(dsn, **kwargs): return _FakePostgres(dsn, **kwargs) diff --git a/tests/test_integrity.py b/tests/test_integrity.py new file mode 100644 index 0000000..d63a1a1 --- /dev/null +++ b/tests/test_integrity.py @@ -0,0 +1,139 @@ +import json + +from audit_core.ingestion import IngestionApplication +from audit_core.integrity import GENESIS, chain_link, load_attestation, write_attestation +from audit_core.interface import AuditEvent +from audit_core.sqlite_backend import SQLiteAuditBackend + +from test_ingestion import invoke + + +def _event(event_id: str, **kw) -> AuditEvent: + fields = dict( + event_id=event_id, + source="user-engine", + action="membership.added", + resource="membership-1", + outcome="recorded", + tenant="tenant:friendly:binky", + scope="tenant", + details={"correlation_id": "corr-1"}, + observed_at="2026-08-09T00:00:00+00:00", + ) + fields.update(kw) + return AuditEvent(**fields) + + +def _digest(event: AuditEvent) -> str: + import hashlib + + return hashlib.sha256( + json.dumps(event.as_record(), sort_keys=True).encode() + ).hexdigest() + + +def test_first_accept_sets_genesis(tmp_path): + backend = SQLiteAuditBackend(str(tmp_path / "c.db")) + event = _event("e1") + backend.accept(event, _digest(event)) + row = backend.db.execute( + "SELECT chain_prev, chain_hash FROM events WHERE event_id = 'e1'" + ).fetchone() + assert row[0] == GENESIS + assert row[1] == chain_link(GENESIS, _digest(event), "e1") + + +def test_second_accept_links(tmp_path): + backend = SQLiteAuditBackend(str(tmp_path / "c.db")) + first = _event("e1") + second = _event("e2") + backend.accept(first, _digest(first)) + backend.accept(second, _digest(second)) + head = backend.db.execute( + "SELECT chain_hash FROM events WHERE event_id = 'e1'" + ).fetchone()[0] + prev = backend.db.execute( + "SELECT chain_prev FROM events WHERE event_id = 'e2'" + ).fetchone()[0] + assert prev == head + + +def test_duplicate_does_not_fork(tmp_path): + backend = SQLiteAuditBackend(str(tmp_path / "c.db")) + event = _event("e1") + digest = _digest(event) + assert backend.accept(event, digest).duplicate is False + assert backend.accept(event, digest).duplicate is True + count = backend.db.execute("SELECT count(*) FROM events").fetchone()[0] + assert count == 1 + assert backend.verify_chain().events == 1 + + +def test_verify_clean_on_fresh_store(tmp_path): + backend = SQLiteAuditBackend(str(tmp_path / "c.db")) + empty = backend.verify_chain() + assert empty.intact is True + assert empty.events == 0 + assert empty.head == GENESIS + first = _event("e1") + second = _event("e2") + backend.accept(first, _digest(first)) + backend.accept(second, _digest(second)) + report = backend.verify_chain() + assert report.intact is True + assert report.events == 2 + assert report.first_break is None + + +def test_rewritten_payload_fails_verify(tmp_path): + backend = SQLiteAuditBackend(str(tmp_path / "c.db")) + first = _event("e1") + second = _event("e2") + backend.accept(first, _digest(first)) + backend.accept(second, _digest(second)) + backend.db.execute("UPDATE events SET payload_hash = 'deadbeef' WHERE event_id = 'e2'") + report = backend.verify_chain() + assert report.intact is False + assert report.first_break == "e2" + + +def test_attestation_mismatch(tmp_path): + backend = SQLiteAuditBackend(str(tmp_path / "c.db")) + event = _event("e1") + backend.accept(event, _digest(event)) + path = tmp_path / "head.json" + write_attestation(path, backend.verify_chain()) + cited = load_attestation(path) + assert backend.verify_chain(cited).attestation_match is True + cited["chain_hash"] = "f" * 64 + broken = backend.verify_chain(cited) + assert broken.intact is False + assert broken.first_break == "attestation_mismatch" + assert broken.attestation_match is False + + +def test_attestation_still_matches_after_growth(tmp_path): + backend = SQLiteAuditBackend(str(tmp_path / "c.db")) + first = _event("e1") + backend.accept(first, _digest(first)) + cited = backend.attest_chain() + second = _event("e2") + backend.accept(second, _digest(second)) + report = backend.verify_chain(cited) + assert report.intact is True + assert report.attestation_match is True + assert report.events == 2 + + +def test_http_integrity_requires_read(tmp_path): + backend = SQLiteAuditBackend(str(tmp_path / "c.db")) + app = IngestionApplication(backend, "opaque") + status, _ = invoke(app, None, path="/v1/integrity", method="GET", body=b"") + assert status.startswith("200") + event = _event("e1") + backend.accept(event, _digest(event)) + status, body = invoke(app, None, path="/v1/integrity", method="GET", body=b"") + assert status.startswith("200") + assert body["intact"] is True + assert body["events"] == 1 + assert "record" not in body diff --git a/workplans/AUDIT-WP-0007-integrity-verification.md b/workplans/AUDIT-WP-0007-integrity-verification.md index b179709..436d936 100644 --- a/workplans/AUDIT-WP-0007-integrity-verification.md +++ b/workplans/AUDIT-WP-0007-integrity-verification.md @@ -4,7 +4,7 @@ type: workplan title: "Integrity verification for operational custody" domain: infotech repo: audit-core -status: ready +status: active owner: grok topic_slug: railiance created: "2026-08-16" @@ -65,7 +65,7 @@ remaining honesty gap on the live provision and is owned here. ```task id: AUDIT-WP-0007-T01 -status: todo +status: done priority: high state_hub_task_id: "d4423fd5-ad78-47d0-b85e-7ae6c582ec2b" ```