diff --git a/SCOPE.md b/SCOPE.md index 94fe26c..60ef68d 100644 --- a/SCOPE.md +++ b/SCOPE.md @@ -31,13 +31,12 @@ audit-core exists to provide the capability described in INTENT.md. - Status: production - Production receiver on railiance01 (`namespace audit-core`), Postgres operational custody on `platform-pg`, sender `user-engine`. -- Recovery is the platform `data.backup` window (30 days). Live `/readyz` - reports `custody_class=operational`, `tamper_evidence=true`, - `recoverable_days=30` (image `sha256:7febc28e…`). -- Hash chain verified on 30 live events - (`docs/evidence/chain-head-20260816.json`). -- ITC-CAP case: `data/capability/audit-core-operational.json` at D4. - `data.archive` is an unmet requirement. +- Recovery is the platform `data.backup` window (30 days, RESOURCE-WP-0002 + live). Live `/readyz` reports `custody_class=operational` and + `recoverable_days=30` (image `sha256:05fe1c06…`, 2026-08-16 cutover). +- ITC-CAP case: `data/capability/audit-core-operational.json`. + `data.archive` is an unmet requirement. `tamper_evidence` is still false. +- Open workplan: `workplans/AUDIT-WP-0007-integrity-verification.md`. ## Getting Oriented diff --git a/audit_core/cli.py b/audit_core/cli.py index e455211..e729907 100644 --- a/audit_core/cli.py +++ b/audit_core/cli.py @@ -107,34 +107,6 @@ def build_parser() -> argparse.ArgumentParser: ) migrate_parser.set_defaults(func=migrate_store) - verify_parser = sub.add_parser( - "verify-chain", - help="Walk the custody hash chain; exit 0 only if intact.", - ) - verify_parser.add_argument( - "--to-postgres", dest="destination", default=None, - help="DSN; defaults to AUDIT_CORE_DATABASE_URL or a mounted credential dir.", - ) - verify_parser.add_argument("--schema", default="audit_core") - verify_parser.add_argument( - "--against", - default=None, - help="Chain-head attestation JSON. A cited head missing from the live chain is a break.", - ) - verify_parser.set_defaults(func=verify_chain) - - attest_parser = sub.add_parser( - "attest-chain", - help="Write the live chain head to a file outside the database.", - ) - attest_parser.add_argument( - "--to-postgres", dest="destination", default=None, - help="DSN; defaults to AUDIT_CORE_DATABASE_URL or a mounted credential dir.", - ) - attest_parser.add_argument("--schema", default="audit_core") - attest_parser.add_argument("--output", required=True) - attest_parser.set_defaults(func=attest_chain) - return parser @@ -186,32 +158,6 @@ def replay_event(args: argparse.Namespace) -> int: return 0 if result.duplicate else 2 -def verify_chain(args: argparse.Namespace) -> int: - from audit_core.integrity import load_attestation - - backend = _postgres_backend(args.destination, args.schema, migrate=False) - try: - cited = load_attestation(args.against) if args.against else None - report = backend.verify_chain(cited) - finally: - backend.close() - print(json.dumps(report.as_dict(), sort_keys=True)) - return 0 if report.intact else 1 - - -def attest_chain(args: argparse.Namespace) -> int: - from audit_core.integrity import write_attestation - - backend = _postgres_backend(args.destination, args.schema, migrate=False) - try: - report = backend.verify_chain() - body = write_attestation(args.output, report) - finally: - backend.close() - print(json.dumps({"ok": report.intact, "path": args.output, **body}, sort_keys=True)) - return 0 if report.intact else 1 - - def migrate_store(args: argparse.Namespace) -> int: import os diff --git a/audit_core/ingestion.py b/audit_core/ingestion.py index 074ab62..5f6e6f2 100644 --- a/audit_core/ingestion.py +++ b/audit_core/ingestion.py @@ -159,12 +159,7 @@ class IngestionApplication: if method == "GET" and ( path.startswith("/v1/events") - or path in ( - "/v1/dead-letters", - "/v1/secret-findings", - "/v1/stats", - "/v1/integrity", - ) + or path in ("/v1/dead-letters", "/v1/secret-findings", "/v1/stats") ): return self._read(start_response, environ, path, identity) @@ -246,14 +241,6 @@ class IngestionApplication: start_response, HTTPStatus.OK, {"secret_findings": self.backend.secret_findings(_limit(query))}, ) - if path == "/v1/integrity": - verify = getattr(self.backend, "verify_chain", None) - if not callable(verify): - return self._json( - start_response, HTTPStatus.NOT_FOUND, - {"error": "integrity_not_supported"}, - ) - return self._json(start_response, HTTPStatus.OK, verify().as_dict()) if path == "/v1/events": correlation = (query.get("correlation_id") or [""])[0] if not correlation: diff --git a/audit_core/integrity.py b/audit_core/integrity.py deleted file mode 100644 index dc0c5ec..0000000 --- a/audit_core/integrity.py +++ /dev/null @@ -1,149 +0,0 @@ -"""Hash-chain integrity for operational custody (AUDIT-WP-0007). - -See ``docs/integrity.md`` for the proof bound. This module is the shared -arithmetic; backends persist and walk rows. -""" - -from __future__ import annotations - -import hashlib -import json -from dataclasses import dataclass -from datetime import datetime, timezone -from pathlib import Path -from typing import Any, Iterable, Mapping - -SCHEMA = "audit-core.chain-head.v1" -GENESIS = "0" * 64 -# Documented advisory-lock key so concurrent accepts cannot fork the head. -CHAIN_LOCK_KEY = 0xA0D17007 - - -def chain_link(previous: str, payload_hash: str, event_id: str) -> str: - """Next ``chain_hash`` = SHA-256(previous || payload_hash || event_id).""" - material = f"{previous}|{payload_hash}|{event_id}".encode("utf-8") - return hashlib.sha256(material).hexdigest() - - -def utc_now() -> str: - return datetime.now(timezone.utc).replace(microsecond=0).isoformat() - - -@dataclass(frozen=True) -class ChainRow: - event_id: str - payload_hash: str - chain_hash: str - chain_prev: str - accepted_at: str | None = None - - -@dataclass(frozen=True) -class ChainReport: - intact: bool - events: int - head: str - head_event_id: str | None - head_accepted_at: str | None - first_break: str | None - attestation_match: bool | None = None - - def as_dict(self) -> dict[str, Any]: - return { - "intact": self.intact, - "events": self.events, - "head": self.head, - "head_event_id": self.head_event_id, - "head_accepted_at": self.head_accepted_at, - "first_break": self.first_break, - "attestation_match": self.attestation_match, - } - - -def verify_rows( - rows: Iterable[ChainRow], - *, - attestation: Mapping[str, Any] | None = None, -) -> ChainReport: - """Walk accept order and recompute every link. - - ``first_break`` is an event id, or ``attestation_mismatch`` when a cited - head is not present in the live chain. - """ - ordered = list(rows) - prev = GENESIS - first_break: str | None = None - for row in ordered: - expected = chain_link(prev, row.payload_hash, row.event_id) - if row.chain_prev != prev or row.chain_hash != expected: - first_break = row.event_id - break - prev = row.chain_hash - - intact = first_break is None - last = ordered[-1] if ordered else None - head = last.chain_hash if last else GENESIS - report = ChainReport( - intact=intact, - events=len(ordered), - head=head, - head_event_id=last.event_id if last else None, - head_accepted_at=last.accepted_at if last else None, - first_break=first_break, - ) - if attestation is None: - return report - return _apply_attestation(report, ordered, attestation) - - -def _apply_attestation( - report: ChainReport, - rows: list[ChainRow], - attestation: Mapping[str, Any], -) -> ChainReport: - cited = str(attestation.get("chain_hash") or "") - live_hashes = {row.chain_hash for row in rows} - empty_ok = not rows and cited in {"", GENESIS} - matched = cited in live_hashes or empty_ok - first_break = report.first_break - intact = report.intact - if not matched: - intact = False - if first_break is None: - first_break = "attestation_mismatch" - return ChainReport( - intact=intact, - events=report.events, - head=report.head, - head_event_id=report.head_event_id, - head_accepted_at=report.head_accepted_at, - first_break=first_break, - attestation_match=matched, - ) - - -def attestation_from_report(report: ChainReport, *, observed_at: str | None = None) -> dict[str, Any]: - return { - "schema": SCHEMA, - "genesis": GENESIS, - "chain_hash": report.head, - "event_id": report.head_event_id, - "accepted_at": report.head_accepted_at, - "event_count": report.events, - "observed_at": observed_at or utc_now(), - } - - -def load_attestation(path: str | Path) -> dict[str, Any]: - payload = json.loads(Path(path).read_text()) - if not isinstance(payload, dict) or not payload.get("chain_hash"): - raise ValueError("attestation must be an object with chain_hash") - return payload - - -def write_attestation(path: str | Path, report: ChainReport) -> dict[str, Any]: - body = attestation_from_report(report) - destination = Path(path) - destination.parent.mkdir(parents=True, exist_ok=True) - destination.write_text(json.dumps(body, indent=2, sort_keys=True) + "\n") - return body diff --git a/audit_core/interface.py b/audit_core/interface.py index e334afd..16bbb14 100644 --- a/audit_core/interface.py +++ b/audit_core/interface.py @@ -69,7 +69,6 @@ class RetentionPolicy: "status": "ok", "custody_class": self.custody_class, "durable": self.durable, - "tamper_evidence": self.tamper_evidence, } if self.recoverable_days is not None or self.recoverable_source: payload["recoverable_days"] = self.recoverable_days diff --git a/audit_core/postgres_backend.py b/audit_core/postgres_backend.py index 5e4884c..c11640e 100644 --- a/audit_core/postgres_backend.py +++ b/audit_core/postgres_backend.py @@ -33,14 +33,6 @@ from audit_core.interface import ( validate_event, ) from audit_core.credentials import CredentialDirectory -from audit_core.integrity import ( - CHAIN_LOCK_KEY, - GENESIS, - ChainRow, - attestation_from_report, - chain_link, - verify_rows, -) from audit_core.redaction import Finding try: # pragma: no cover - import guard @@ -142,13 +134,6 @@ MIGRATIONS: list[tuple[str, str]] = [ $grant$; """, ), - ( - "0006-chain", - """ - ALTER TABLE {schema}.events ADD COLUMN IF NOT EXISTS chain_hash text; - ALTER TABLE {schema}.events ADD COLUMN IF NOT EXISTS chain_prev text; - """, - ), ] # Rejection reasons whose payload must never be persisted — storing the body of @@ -270,7 +255,6 @@ class PostgresAuditBackend: (migration_id,), ) applied.append(migration_id) - self._backfill_chain(conn) except psycopg.Error as exc: raise BackendUnavailableError(f"migration failed: {exc}") from exc return applied @@ -288,10 +272,9 @@ class PostgresAuditBackend: ``immutable`` is True because migration 0002 installs a trigger that rejects UPDATE and DELETE, so no consumer credential can alter a stored record. It is not a claim against the database owner or a superuser, - who can drop the trigger. ``tamper_evidence`` is True because a - hash chain plus verify detects a rewritten payload, and a chain-head - attestation outside this database detects a suffix rewrite that - stays inside Postgres. It is not WORM or ``data.archive``. + who can drop the trigger; ``tamper_evidence`` is correspondingly False, + because nothing here would *prove* they had. Hash-chaining or external + anchoring would be needed for that, and is not implemented. ``custody_class`` is ``operational``, not ``archive``. This store is durable append-only Postgres recovered through the platform @@ -303,7 +286,7 @@ class PostgresAuditBackend: custody_class="operational", retention_days=self.retention_days, immutable=True, - tamper_evidence=True, + tamper_evidence=False, durable=True, recoverable_days=self.recoverable_days, recoverable_source=self.recoverable_source, @@ -319,20 +302,13 @@ class PostgresAuditBackend: details = event.details if isinstance(event.details, dict) else {} record = event.as_record() try: - with self.pool.connection() as conn, conn.transaction(): - conn.execute("SELECT pg_advisory_xact_lock(%s)", (CHAIN_LOCK_KEY,)) - head = conn.execute( - f"SELECT chain_hash FROM {self._events} " - "ORDER BY accepted_at DESC, event_id DESC LIMIT 1" - ).fetchone() - previous = head[0] if head and head[0] else GENESIS - link = chain_link(previous, payload_hash, event.event_id) + with self.pool.connection() as conn: inserted = conn.execute( f""" INSERT INTO {self._events} (event_id, payload_hash, observed_at, tenant, correlation_id, - source, action, record, chain_hash, chain_prev) - VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s) + source, action, record) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s) ON CONFLICT (event_id) DO NOTHING RETURNING event_id """, @@ -345,8 +321,6 @@ class PostgresAuditBackend: event.source, event.action, json.dumps(record, sort_keys=True), - link, - previous, ), ).fetchone() if inserted is not None: @@ -469,69 +443,9 @@ class PostgresAuditBackend: # --- lifecycle --------------------------------------------------------- - def verify_chain(self, attestation: dict | None = None): - rows = self._query( - f"SELECT event_id, payload_hash, chain_hash, chain_prev, accepted_at " - f"FROM {self._events} ORDER BY accepted_at, event_id", - (), - ) - return verify_rows( - [ - ChainRow( - event_id=r[0], - payload_hash=r[1], - chain_hash=r[2] or "", - chain_prev=r[3] or "", - accepted_at=_iso(r[4]), - ) - for r in rows - ], - attestation=attestation, - ) - - def attest_chain(self) -> dict: - return attestation_from_report(self.verify_chain()) - def health(self) -> None: self._query("SELECT 1", ()) - def _backfill_chain(self, conn) -> None: - """Fill chain columns on rows accepted before migration 0006. - - The append-only trigger must be disabled for this UPDATE. A - database owner can do that; this is a one-time migrate, not a - runtime path. - """ - missing = conn.execute( - f"SELECT count(*) FROM {self._events} " - "WHERE chain_hash IS NULL OR chain_prev IS NULL" - ).fetchone()[0] - if missing: - conn.execute( - f'ALTER TABLE {self._events} DISABLE TRIGGER events_append_only' - ) - prev = GENESIS - for event_id, payload_hash in conn.execute( - f"SELECT event_id, payload_hash FROM {self._events} " - "ORDER BY accepted_at, event_id" - ).fetchall(): - link = chain_link(prev, payload_hash, event_id) - conn.execute( - f"UPDATE {self._events} SET chain_prev = %s, chain_hash = %s " - "WHERE event_id = %s", - (prev, link, event_id), - ) - prev = link - conn.execute( - f'ALTER TABLE {self._events} ENABLE TRIGGER events_append_only' - ) - conn.execute( - f"ALTER TABLE {self._events} ALTER COLUMN chain_hash SET NOT NULL" - ) - conn.execute( - f"ALTER TABLE {self._events} ALTER COLUMN chain_prev SET NOT NULL" - ) - def close(self) -> None: self.pool.close() diff --git a/audit_core/sqlite_backend.py b/audit_core/sqlite_backend.py index 6811f97..954e7ff 100644 --- a/audit_core/sqlite_backend.py +++ b/audit_core/sqlite_backend.py @@ -22,13 +22,6 @@ from audit_core.interface import ( RetentionPolicy, validate_event, ) -from audit_core.integrity import ( - GENESIS, - ChainRow, - attestation_from_report, - chain_link, - verify_rows, -) _SCHEMA = """ CREATE TABLE IF NOT EXISTS events ( @@ -37,9 +30,7 @@ CREATE TABLE IF NOT EXISTS events ( accepted_at TEXT NOT NULL, correlation_id TEXT, tenant TEXT NOT NULL, - record TEXT NOT NULL, - chain_hash TEXT, - chain_prev TEXT + record TEXT NOT NULL ); CREATE INDEX IF NOT EXISTS events_correlation_idx ON events (correlation_id); CREATE INDEX IF NOT EXISTS events_tenant_idx ON events (tenant); @@ -96,7 +87,6 @@ class SQLiteAuditBackend: self._local = threading.local() with self._connect_raw() as setup: setup.executescript(_SCHEMA) - self._ensure_chain_columns(setup) def _connect_raw(self) -> sqlite3.Connection: try: @@ -146,18 +136,11 @@ class SQLiteAuditBackend: raise BackendUnavailableError(str(exc)) from exc try: - head = db.execute( - "SELECT chain_hash FROM events " - "ORDER BY accepted_at DESC, event_id DESC LIMIT 1" - ).fetchone() - previous = head[0] if head and head[0] else GENESIS - link = chain_link(previous, payload_hash, event.event_id) inserted = db.execute( """ INSERT INTO events - (event_id, payload_hash, accepted_at, correlation_id, tenant, record, - chain_hash, chain_prev) - VALUES (?, ?, ?, ?, ?, ?, ?, ?) + (event_id, payload_hash, accepted_at, correlation_id, tenant, record) + VALUES (?, ?, ?, ?, ?, ?) ON CONFLICT(event_id) DO NOTHING RETURNING event_id """, @@ -168,8 +151,6 @@ class SQLiteAuditBackend: str(details.get("correlation_id") or "") or None, event.tenant, json.dumps(event.as_record(), sort_keys=True), - link, - previous, ), ).fetchone() existing = None @@ -324,54 +305,6 @@ class SQLiteAuditBackend: except sqlite3.Error as exc: raise BackendUnavailableError(str(exc)) from exc - def verify_chain(self, attestation: dict | None = None): - rows = self._query( - "SELECT event_id, payload_hash, chain_hash, chain_prev, accepted_at " - "FROM events ORDER BY accepted_at, event_id", - (), - ) - return verify_rows( - [ - ChainRow( - event_id=r[0], - payload_hash=r[1], - chain_hash=r[2] or "", - chain_prev=r[3] or "", - accepted_at=r[4], - ) - for r in rows - ], - attestation=attestation, - ) - - def attest_chain(self) -> dict: - return attestation_from_report(self.verify_chain()) - - def _ensure_chain_columns(self, db: sqlite3.Connection) -> None: - cols = {row[1] for row in db.execute("PRAGMA table_info(events)")} - if "chain_hash" not in cols: - db.execute("ALTER TABLE events ADD COLUMN chain_hash TEXT") - if "chain_prev" not in cols: - db.execute("ALTER TABLE events ADD COLUMN chain_prev TEXT") - missing = db.execute( - "SELECT event_id, payload_hash FROM events " - "WHERE chain_hash IS NULL OR chain_prev IS NULL " - "ORDER BY accepted_at, event_id" - ).fetchall() - if not missing: - return - prev = GENESIS - # Recompute the whole chain so a partial backfill cannot fork. - for event_id, payload_hash in db.execute( - "SELECT event_id, payload_hash FROM events ORDER BY accepted_at, event_id" - ): - link = chain_link(prev, payload_hash, event_id) - db.execute( - "UPDATE events SET chain_prev = ?, chain_hash = ? WHERE event_id = ?", - (prev, link, event_id), - ) - prev = link - def health(self) -> None: """Raise :class:`BackendUnavailableError` if the store is unusable.""" try: diff --git a/data/capability/audit-core-operational.json b/data/capability/audit-core-operational.json index 80303c9..bcf75e1 100644 --- a/data/capability/audit-core-operational.json +++ b/data/capability/audit-core-operational.json @@ -40,7 +40,7 @@ "environment": "production", "maturity": "D4", "implements": "HTTP POST /v1/events into append-only PostgreSQL on platform-pg, namespace audit-core, ClusterIP + default-deny", - "maturity_rationale": "Approved for production dependency since AUDIT-WP-0005. Not D5: one replica, reliability is not actively controlled. Integrity is measured (hash chain + verify + external head) but is not WORM.", + "maturity_rationale": "Approved for production dependency since AUDIT-WP-0005. Not D5: tamper_evidence is false (trigger is not a proof), one replica, reliability is not actively controlled.", "uses_provisions": [ { "capability": "data.transactional", @@ -109,17 +109,15 @@ }, { "hook": "integrity_verification", - "basis": "measured", - "value": "hash chain on accept; verify fails after a superuser payload_hash rewrite; chain-head attestation stored outside platform-pg", - "observed_at": "2026-08-16", - "ref": "tests/test_integrity.py#test_rewritten_payload_fails_verify;docs/integrity.md;docs/evidence/chain-head-20260816.json" + "basis": "unknown", + "gap": "events_append_only rejects UPDATE/DELETE for the runtime role; that is not a proof a database owner did not drop the trigger. Hash-chain or external anchor is not implemented (owner: audit-core). Do not borrow the restore drill for this hook." } ] } ], "open_items": [ "data.archive is required by INTENT and unprovided. A founder decision is needed before resource-control procures a WORM/object-lock destination distinct from the 30-day Barman bucket.", - "integrity_verification is measured. A database owner who rewrites the suffix and the external attestation together can still lie; that is the stated proof bound.", + "integrity_verification is unknown. Trigger enforcement is not tamper evidence.", "Class S/H/I consumption is unknown on this provision.", "Do not emit booked cost or a second usage stream for platform:audit-storage." ] diff --git a/deploy/README.md b/deploy/README.md index 2915557..4922b02 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -8,7 +8,7 @@ the server port or export a copy before applying. Apply order is documented in `docs/operator-runbook.md`. Do not apply the Deployment until: -1. The image digest is pinned (currently `sha256:7febc28e…` from commit `5fd04e2`). +1. The image digest is pinned (currently `sha256:05fe1c06…` from commit `40dcadd`). 2. Secrets `audit-core-database`, `audit-core-database-migrate`, and `audit-core-senders` exist. ConfigMap `audit-core-senders-scope` is applied (`deploy/senders-scope.yaml`) before the Deployment mounts it. diff --git a/deploy/audit-core.yaml b/deploy/audit-core.yaml index f71709f..d62d65e 100644 --- a/deploy/audit-core.yaml +++ b/deploy/audit-core.yaml @@ -46,11 +46,11 @@ metadata: # Rollback position. Update both together; `kubectl rollout undo` returns to # the previous digest, and the schema note records whether that is safe. audit-core.railiance.io/rollback-note: >- - Migrations 0001-0006 are additive. 0006 adds chain_hash/chain_prev and - then NOT NULL. An image that does not write those columns cannot accept - events after 0006. Do not roll back past sha256:7febc28e… to a pre-0007 - writer. A future migration that drops or narrows a column must state - its own rollback position before it is released. + Migrations 0001-0005 are additive (CREATE TABLE/INDEX/TRIGGER IF NOT + EXISTS, plus GRANTs) and are not reversed by a rollback. An older image + runs against the newer schema without harm. A future migration that drops + or narrows a column breaks that property and must state its own rollback + position before it is released. spec: replicas: 1 revisionHistoryLimit: 5 @@ -80,7 +80,7 @@ spec: - name: audit-core # REPLACE at release time with the built digest. A mutable tag is not # an immutable image, and `:latest` must never be the only reference. - image: forgejo.coulomb.social/coulomb/audit-core@sha256:7febc28e8a828dbc245144a38e5728e0fbf496b594dd7591170b450a1265fb10 + image: forgejo.coulomb.social/coulomb/audit-core@sha256:05fe1c06f809be29309695c397025da755bfe82afed81a7f9b8771e0b9200c17 imagePullPolicy: IfNotPresent ports: - name: http diff --git a/deploy/migrate-job.yaml b/deploy/migrate-job.yaml index a52e263..dd11e60 100644 --- a/deploy/migrate-job.yaml +++ b/deploy/migrate-job.yaml @@ -33,7 +33,7 @@ spec: type: RuntimeDefault containers: - name: migrate - image: forgejo.coulomb.social/coulomb/audit-core@sha256:7febc28e8a828dbc245144a38e5728e0fbf496b594dd7591170b450a1265fb10 + image: forgejo.coulomb.social/coulomb/audit-core@sha256:05fe1c06f809be29309695c397025da755bfe82afed81a7f9b8771e0b9200c17 imagePullPolicy: IfNotPresent command: ["python", "-m", "audit_core", "migrate"] env: diff --git a/docs/audit-backend-contract.md b/docs/audit-backend-contract.md index bc9a855..a9cbbff 100644 --- a/docs/audit-backend-contract.md +++ b/docs/audit-backend-contract.md @@ -165,7 +165,7 @@ integrity proofs, or survival of `/tmp` across reboots. - `custody_class`: `operational` - `retention_days`: unset in production (the service does not expire rows) - `immutable`: true (trigger `events_append_only`; not a claim against the database owner) -- `tamper_evidence`: true (hash chain + verify + external head attestation; not WORM) +- `tamper_evidence`: false (a superuser can drop the trigger; no hash-chain) - `durable`: true - `recoverable_days`: 30, cited from the platform `data.backup` provision - `recoverable_source`: `resource-control/data/capability/platform-audit-storage.json#provisions[capability=data.backup]` @@ -240,7 +240,6 @@ Archive remains the evidence record; hot search may use shorter `retention_days` - `INTENT.md` — product purpose and principles - `spec/ProductRequirementsDefinition.md` — full v1 envelope and API requirements -- `docs/integrity.md` — hash chain, proof bound, verify/attest - `registry/capabilities/capability.audit.event-retain.md` — capability registry entry ## Secret-shaped fields diff --git a/docs/evidence/chain-head-20260816.json b/docs/evidence/chain-head-20260816.json deleted file mode 100644 index 2da1de6..0000000 --- a/docs/evidence/chain-head-20260816.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "accepted_at": "2026-08-13T15:38:37+00:00", - "chain_hash": "03fd1773c1954e2c39548b0ba297359b9edd739a94ff9b2dcc9529dcd5cf1fcc", - "event_count": 30, - "event_id": "evt_69aad81d75bb49c9b988be7650a5360d", - "genesis": "0000000000000000000000000000000000000000000000000000000000000000", - "observed_at": "2026-08-15T23:23:34+00:00", - "schema": "audit-core.chain-head.v1" -} diff --git a/docs/integrity.md b/docs/integrity.md deleted file mode 100644 index b70c633..0000000 --- a/docs/integrity.md +++ /dev/null @@ -1,73 +0,0 @@ -# Integrity contract (AUDIT-WP-0007) - -Operational custody already stores `payload_hash` and rejects UPDATE/DELETE -for the runtime role. That is not tamper evidence: a database owner can -drop the trigger and rewrite rows. This contract says what the hash chain -proves, and what it does not. - -## Chain - -Each accepted event stores: - -- `payload_hash` — SHA-256 of the canonical event record (already stored) -- `chain_prev` — the previous event's `chain_hash`, or genesis -- `chain_hash` — `SHA-256(chain_prev | payload_hash | event_id)` - -Genesis is 64 ASCII zeros (`audit_core.integrity.GENESIS`). The delimiter -is `|`. The chain is **one stream per schema**, not per tenant. A -per-tenant chain would hide a cross-tenant rewrite. - -The first accept of an empty store uses genesis as `chain_prev`. A -duplicate accept must not mint a second link. A conflict must not -advance the head. Concurrent first-accepts of different events take an -advisory lock (`CHAIN_LOCK_KEY`) so they cannot fork. - -Verify walks `ORDER BY accepted_at, event_id`, recomputes every link, -and reports the first event id whose stored hashes do not match. A -break is a **custody defect**, not a sender error. Retrying the event -will not repair it. - -## Proof bound - -A chain *inside* the same database detects a rewritten `payload_hash` -**if the attacker does not also recompute the suffix**. A database -owner can. Tamper evidence against that class of attacker requires a -**chain-head attestation** stored outside `platform-pg`. - -`tamper_evidence=True` is allowed only when: - -1. `verify` exists and fails on a rewritten row -2. an external head attestation exists and verify-against-attestation - reports a missing cited head as a break - -It still does not mean WORM, object lock, or ITC-CAP `data.archive`. -It does not raise provision maturity to D5. - -Do not write the attestation into the Barman prefix -(`platform-pg/` on `resource:platform:audit-storage`). That copy is -restored with the table. A second copy may follow the logical-offsite -path already used by RESOURCE-WP-0002-T06 (`rapp-postgres` / -Nextcloud + age); cite it, do not invent a new bucket. - -## Tests this contract names - -| Test | What it asserts | -| --- | --- | -| `test_first_accept_sets_genesis` | First `chain_prev` is genesis | -| `test_second_accept_links` | Second `chain_prev` is the first `chain_hash` | -| `test_duplicate_does_not_fork` | Replay does not add a link | -| `test_verify_clean_on_fresh_store` | Empty and two-event stores verify | -| `test_rewritten_payload_fails_verify` | Superuser rewrite of `payload_hash` is a break | -| `test_attestation_mismatch` | Cited head absent from the live chain is a break | - -## Operator surface - -```bash -python -m audit_core verify-chain -python -m audit_core verify-chain --against docs/evidence/chain-head-.json -python -m audit_core attest-chain --output docs/evidence/chain-head-.json -``` - -`GET /v1/integrity` (`may_read`) returns -`{intact, events, head, head_event_id, first_break, attestation_match}` -and never event payloads. diff --git a/docs/interface-card.yaml b/docs/interface-card.yaml index ebe032f..9b1faf0 100644 --- a/docs/interface-card.yaml +++ b/docs/interface-card.yaml @@ -62,7 +62,7 @@ validation_expectations: disposition: unmet requirement recorded on the ITC-CAP case; do not build the sink in AUDIT-WP-0006 - id: tamper-evidence-false owner: audit-core - disposition: closed by AUDIT-WP-0007; proof bound is in docs/integrity.md + disposition: integrity_verification hook is unknown; trigger is not a proof - id: no-hash-chain owner: audit-core disposition: INTENT residual, not this workplan @@ -92,7 +92,7 @@ consumer_needs: feedback: [] known_deviations: - no data.archive sink - - tamper_evidence is a hash chain plus external head, not WORM + - tamper_evidence=False - no hash-chain - single sender user-engine - no rapp.yaml (not a rapp-* repo) diff --git a/docs/operator-runbook.md b/docs/operator-runbook.md index 5f5c6d7..32be270 100644 --- a/docs/operator-runbook.md +++ b/docs/operator-runbook.md @@ -29,9 +29,8 @@ warden route show database-dynamic-credentials --json | Check | Meaning | | --- | --- | | `GET /healthz` | Process is up. Liveness uses this. A database outage must **not** restart the pod. | -| `GET /readyz` | Custody is reachable and `custody_class=operational`. Also reports `recoverable_days` and `tamper_evidence`. Readiness uses this; the pod leaves the Service rather than accept events it cannot store. | +| `GET /readyz` | Custody is reachable and `custody_class=operational`. Also reports `recoverable_days` (cited platform backup window). Readiness uses this; the pod leaves the Service rather than accept events it cannot store. | | `GET /v1/stats` | In-process counters since start (`accepted`, `duplicate`, `conflict`, `rejected`, `unauthorized`, `forbidden`, `unavailable`, `error`). Resets on restart. Requires `may_read`. | -| `GET /v1/integrity` | Hash-chain walk: `{intact, events, head, first_break}`. No payloads. Requires `may_read`. A break is a custody defect, not a sender retry. | A missing `AUDIT_CORE_DATABASE_URL` / credential directory is a startup failure (`AUDIT_CORE_REQUIRE_CUSTODY_CLASS=operational`), not a silent @@ -41,20 +40,7 @@ accepted as an alias for one mixed rollout. After a node reboot, `/readyz` failing for tens of seconds is expected: CoreDNS and `platform-pg` come up after the receiver. Liveness stays on `/healthz` so that window does not restart the pod. Walked 2026-08-16 -(~40s unreadiness, then Ready). That window is not a chain break. - -## Integrity - -Each accept extends a single hash chain for the schema (not per tenant). -`python -m audit_core verify-chain` exits 0 only if the walk is intact. -`python -m audit_core attest-chain --output docs/evidence/chain-head-.json` -writes the live head **outside** `platform-pg`. Do not put that file in -the Barman prefix; a second copy may follow the RESOURCE-WP-0002-T06 -logical-offsite path. `verify-chain --against ` fails if the cited -head is not in the live chain. - -A break is a custody defect. Do not "fix" it by accepting the same event -again. Reboot unreadiness is unrelated. +(~40s unreadiness, then Ready). ## Lookup diff --git a/registry/capabilities/capability.audit.event-retain.md b/registry/capabilities/capability.audit.event-retain.md index a5055a5..927f926 100644 --- a/registry/capabilities/capability.audit.event-retain.md +++ b/registry/capabilities/capability.audit.event-retain.md @@ -23,6 +23,7 @@ external_evidence: - recovery cited to the live platform data.backup provision broken_expectations: - data.archive sink not provided + - tamper evidence not implemented out_of_scope_expectations: - application business audit semantics ownership - booked-cost origination @@ -32,6 +33,7 @@ external_evidence: basis: failure_matrix_and_restore_walk known_reliability_risks: - single replica + - integrity_verification hook unmet discovery: intent: > diff --git a/tests/test_backend_conformance.py b/tests/test_backend_conformance.py index a7cf166..d45b123 100644 --- a/tests/test_backend_conformance.py +++ b/tests/test_backend_conformance.py @@ -201,57 +201,11 @@ def test_health_passes_on_a_live_backend(backend): backend.health() -def test_chain_links_and_verify_is_clean(backend): - from audit_core.integrity import GENESIS - - first = make_event("chain-1") - second = make_event("chain-2") - backend.accept(first, digest(first)) - backend.accept(second, digest(second)) - report = backend.verify_chain() - assert report.intact is True - assert report.events >= 2 - assert report.first_break is None - assert report.head != GENESIS - replay = backend.accept(first, digest(first)) - assert replay.duplicate is True - assert backend.verify_chain().events == report.events - - # --- postgres-specific guarantees ------------------------------------------- pg_only = pytest.mark.skipif(not HAVE_PG, reason="needs PostgreSQL") -@pg_only -def test_rewritten_payload_fails_verify_postgres(): - """Superuser rewrite is the evidence the trigger never gave us.""" - from audit_core.postgres_backend import PostgresAuditBackend - - schema = f"conf_{uuid.uuid4().hex[:12]}" - backend = PostgresAuditBackend(PG_URL, schema=schema) - try: - event = make_event("break-1") - backend.accept(event, digest(event)) - other = make_event("break-2") - backend.accept(other, digest(other)) - assert backend.verify_chain().intact is True - with backend.pool.connection() as conn: - conn.execute(f'ALTER TABLE "{schema}".events DISABLE TRIGGER events_append_only') - conn.execute( - f'UPDATE "{schema}".events SET payload_hash = %s WHERE event_id = %s', - ("deadbeef" * 8, "break-2"), - ) - conn.execute(f'ALTER TABLE "{schema}".events ENABLE TRIGGER events_append_only') - report = backend.verify_chain() - assert report.intact is False - assert report.first_break == "break-2" - finally: - with backend.pool.connection() as conn: - conn.execute(f'DROP SCHEMA IF EXISTS "{schema}" CASCADE') - backend.close() - - @pg_only def test_replay_reconciles_rather_than_duplicating(): """Replay must never mint a second custody record for one source event.""" diff --git a/tests/test_capability_case.py b/tests/test_capability_case.py index d48dc6a..7817620 100644 --- a/tests/test_capability_case.py +++ b/tests/test_capability_case.py @@ -24,8 +24,6 @@ def test_capability_record_exists_and_joins_operations_audit(): unknown = [row for row in audit["consumes"] if row["basis"] == "unknown"] assert unknown assert all(row["quantity"]["value"] is None and row.get("gap") for row in unknown) - hooks = {item["hook"]: item["basis"] for item in audit["evidence"]} - assert hooks["integrity_verification"] == "measured" def test_capability_review_against_live_catalog(): diff --git a/tests/test_cli.py b/tests/test_cli.py index a0dac3e..684d9c4 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -4,7 +4,6 @@ import types from audit_core.cli import build_parser from audit_core.ingestion import build_backend -from audit_core.integrity import ChainReport, GENESIS from audit_core.interface import AcceptResult, RetentionPolicy @@ -37,16 +36,6 @@ class _FakePostgres: def close(self): self.closed = True - def verify_chain(self, attestation=None): - return ChainReport( - intact=True, - events=0, - head=GENESIS, - head_event_id=None, - head_accepted_at=None, - first_break=None, - ) - @property def retention_policy(self): return RetentionPolicy( @@ -120,19 +109,6 @@ def test_replay_command_reconciles(monkeypatch, capsys): assert body["reference"] == "audit:evt-1" -def test_verify_chain_command(monkeypatch, capsys): - def fake(dsn, **kwargs): - return _FakePostgres(dsn, **kwargs) - - monkeypatch.setenv("AUDIT_CORE_DATABASE_URL", "postgresql://example/audit_core") - _install_fake_postgres(monkeypatch, fake) - args = build_parser().parse_args(["verify-chain", "--schema", "audit_core"]) - assert args.func(args) == 0 - body = json.loads(capsys.readouterr().out) - assert body["intact"] is True - assert body["head"] == GENESIS - - def test_replay_command_missing_event(monkeypatch, capsys): def fake(dsn, **kwargs): return _FakePostgres(dsn, **kwargs) diff --git a/tests/test_integrity.py b/tests/test_integrity.py deleted file mode 100644 index d63a1a1..0000000 --- a/tests/test_integrity.py +++ /dev/null @@ -1,139 +0,0 @@ -import json - -from audit_core.ingestion import IngestionApplication -from audit_core.integrity import GENESIS, chain_link, load_attestation, write_attestation -from audit_core.interface import AuditEvent -from audit_core.sqlite_backend import SQLiteAuditBackend - -from test_ingestion import invoke - - -def _event(event_id: str, **kw) -> AuditEvent: - fields = dict( - event_id=event_id, - source="user-engine", - action="membership.added", - resource="membership-1", - outcome="recorded", - tenant="tenant:friendly:binky", - scope="tenant", - details={"correlation_id": "corr-1"}, - observed_at="2026-08-09T00:00:00+00:00", - ) - fields.update(kw) - return AuditEvent(**fields) - - -def _digest(event: AuditEvent) -> str: - import hashlib - - return hashlib.sha256( - json.dumps(event.as_record(), sort_keys=True).encode() - ).hexdigest() - - -def test_first_accept_sets_genesis(tmp_path): - backend = SQLiteAuditBackend(str(tmp_path / "c.db")) - event = _event("e1") - backend.accept(event, _digest(event)) - row = backend.db.execute( - "SELECT chain_prev, chain_hash FROM events WHERE event_id = 'e1'" - ).fetchone() - assert row[0] == GENESIS - assert row[1] == chain_link(GENESIS, _digest(event), "e1") - - -def test_second_accept_links(tmp_path): - backend = SQLiteAuditBackend(str(tmp_path / "c.db")) - first = _event("e1") - second = _event("e2") - backend.accept(first, _digest(first)) - backend.accept(second, _digest(second)) - head = backend.db.execute( - "SELECT chain_hash FROM events WHERE event_id = 'e1'" - ).fetchone()[0] - prev = backend.db.execute( - "SELECT chain_prev FROM events WHERE event_id = 'e2'" - ).fetchone()[0] - assert prev == head - - -def test_duplicate_does_not_fork(tmp_path): - backend = SQLiteAuditBackend(str(tmp_path / "c.db")) - event = _event("e1") - digest = _digest(event) - assert backend.accept(event, digest).duplicate is False - assert backend.accept(event, digest).duplicate is True - count = backend.db.execute("SELECT count(*) FROM events").fetchone()[0] - assert count == 1 - assert backend.verify_chain().events == 1 - - -def test_verify_clean_on_fresh_store(tmp_path): - backend = SQLiteAuditBackend(str(tmp_path / "c.db")) - empty = backend.verify_chain() - assert empty.intact is True - assert empty.events == 0 - assert empty.head == GENESIS - first = _event("e1") - second = _event("e2") - backend.accept(first, _digest(first)) - backend.accept(second, _digest(second)) - report = backend.verify_chain() - assert report.intact is True - assert report.events == 2 - assert report.first_break is None - - -def test_rewritten_payload_fails_verify(tmp_path): - backend = SQLiteAuditBackend(str(tmp_path / "c.db")) - first = _event("e1") - second = _event("e2") - backend.accept(first, _digest(first)) - backend.accept(second, _digest(second)) - backend.db.execute("UPDATE events SET payload_hash = 'deadbeef' WHERE event_id = 'e2'") - report = backend.verify_chain() - assert report.intact is False - assert report.first_break == "e2" - - -def test_attestation_mismatch(tmp_path): - backend = SQLiteAuditBackend(str(tmp_path / "c.db")) - event = _event("e1") - backend.accept(event, _digest(event)) - path = tmp_path / "head.json" - write_attestation(path, backend.verify_chain()) - cited = load_attestation(path) - assert backend.verify_chain(cited).attestation_match is True - cited["chain_hash"] = "f" * 64 - broken = backend.verify_chain(cited) - assert broken.intact is False - assert broken.first_break == "attestation_mismatch" - assert broken.attestation_match is False - - -def test_attestation_still_matches_after_growth(tmp_path): - backend = SQLiteAuditBackend(str(tmp_path / "c.db")) - first = _event("e1") - backend.accept(first, _digest(first)) - cited = backend.attest_chain() - second = _event("e2") - backend.accept(second, _digest(second)) - report = backend.verify_chain(cited) - assert report.intact is True - assert report.attestation_match is True - assert report.events == 2 - - -def test_http_integrity_requires_read(tmp_path): - backend = SQLiteAuditBackend(str(tmp_path / "c.db")) - app = IngestionApplication(backend, "opaque") - status, _ = invoke(app, None, path="/v1/integrity", method="GET", body=b"") - assert status.startswith("200") - event = _event("e1") - backend.accept(event, _digest(event)) - status, body = invoke(app, None, path="/v1/integrity", method="GET", body=b"") - assert status.startswith("200") - assert body["intact"] is True - assert body["events"] == 1 - assert "record" not in body diff --git a/workplans/AUDIT-WP-0007-integrity-verification.md b/workplans/AUDIT-WP-0007-integrity-verification.md index e44155b..b179709 100644 --- a/workplans/AUDIT-WP-0007-integrity-verification.md +++ b/workplans/AUDIT-WP-0007-integrity-verification.md @@ -4,7 +4,7 @@ type: workplan title: "Integrity verification for operational custody" domain: infotech repo: audit-core -status: finished +status: ready owner: grok topic_slug: railiance created: "2026-08-16" @@ -65,7 +65,7 @@ remaining honesty gap on the live provision and is owned here. ```task id: AUDIT-WP-0007-T01 -status: done +status: todo priority: high state_hub_task_id: "d4423fd5-ad78-47d0-b85e-7ae6c582ec2b" ``` @@ -89,14 +89,11 @@ Write a short contract in `docs/audit-backend-contract.md` (or a sibling Done when the contract is written and the tests to be added are named. -Done 2026-08-16: `docs/integrity.md` names the chain, genesis, proof bound, -and the six tests. - ## T02 — Persist the chain on accept ```task id: AUDIT-WP-0007-T02 -status: done +status: todo priority: high state_hub_task_id: "5830bb1a-27b7-4eed-b09f-75ba8cc9f7f1" ``` @@ -115,14 +112,11 @@ suite. Mock file backend stays `tamper_evidence=False`. Done when conformance tests show: first accept sets genesis; second links; duplicate does not fork; verify is clean on a fresh store. -Done 2026-08-16: migration 0006 plus backfill; SQLite and Postgres accept -write the next link under a lock. Conformance + `tests/test_integrity.py`. - ## T03 — Verify surface ```task id: AUDIT-WP-0007-T03 -status: done +status: todo priority: high state_hub_task_id: "fee1f93c-dcac-4fc3-9e41-17bb4436e8d6" ``` @@ -139,14 +133,11 @@ alone never gave us. Done when CLI and HTTP agree, and the broken-row test is red-then-green as a detector, not as a repair. -Done 2026-08-16: `verify-chain` CLI, `GET /v1/integrity`, SQLite rewrite -test and Postgres superuser rewrite test both fail verify. - ## T04 — Attest the chain head outside platform-pg ```task id: AUDIT-WP-0007-T04 -status: done +status: todo priority: medium state_hub_task_id: "b6d03d6a-0605-4982-a9a3-e18c89319ba9" ``` @@ -167,17 +158,11 @@ and report mismatch as a break. Done when one production walk produces an attestation file in `docs/evidence/` and verify-against-attestation is tested. -Done 2026-08-16: live walk 30 events, intact, head -`03fd1773…`. Evidence -`docs/evidence/chain-head-20260816.json`. Verify-against matched. -Do not copy this file into the Barman prefix; a second copy may follow -RESOURCE-WP-0002-T06 logical-offsite. - ## T05 — Declare tamper_evidence only as far as the proof ```task id: AUDIT-WP-0007-T05 -status: done +status: todo priority: medium state_hub_task_id: "9fd70d3c-5a64-4b26-967f-4ef45394d81c" ``` @@ -197,10 +182,6 @@ After T03 and T04: Done when capability-review still `ok` and the hook is no longer `unknown`. -Done 2026-08-16: Postgres `tamper_evidence=True`; live `/readyz` reports -it; capability-review `ok`; `integrity_verification` is `measured`. -Maturity stays D4. `data.archive` stays unmet. - ## Acceptance - A rewritten stored payload makes `verify` fail.