coulomb/audit-core
4
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Implement AUDIT-WP-0007 hash-chain integrity.

Browse source 
Accept now extends a single-schema chain. Verify walks it; a rewritten
payload_hash is a break. Tamper evidence is that detector plus an
external chain-head attestation, not WORM.
This commit is contained in:
tegwick 2026-08-16 01:18:30 +02:00
parent 5faede18fc
commit 5fd04e2095
17 changed files with 696 additions and 29 deletions
Download patch file Download diff file Expand all files Collapse all files

54
audit_core/cli.py
View file

@ -107,6 +107,34 @@ def build_parser() -> argparse.ArgumentParser:
)
migrate_parser.set_defaults(func=migrate_store)
verify_parser = sub.add_parser(
"verify-chain",
help="Walk the custody hash chain; exit 0 only if intact.",
)
verify_parser.add_argument(
"--to-postgres", dest="destination", default=None,
help="DSN; defaults to AUDIT_CORE_DATABASE_URL or a mounted credential dir.",
)
verify_parser.add_argument("--schema", default="audit_core")
verify_parser.add_argument(
"--against",
default=None,
help="Chain-head attestation JSON. A cited head missing from the live chain is a break.",
)
verify_parser.set_defaults(func=verify_chain)
attest_parser = sub.add_parser(
"attest-chain",
help="Write the live chain head to a file outside the database.",
)
attest_parser.add_argument(
"--to-postgres", dest="destination", default=None,
help="DSN; defaults to AUDIT_CORE_DATABASE_URL or a mounted credential dir.",
)
attest_parser.add_argument("--schema", default="audit_core")
attest_parser.add_argument("--output", required=True)
attest_parser.set_defaults(func=attest_chain)
return parser
@ -158,6 +186,32 @@ def replay_event(args: argparse.Namespace) -> int:
return 0 if result.duplicate else 2
def verify_chain(args: argparse.Namespace) -> int:
from audit_core.integrity import load_attestation
backend = _postgres_backend(args.destination, args.schema, migrate=False)
try:
cited = load_attestation(args.against) if args.against else None
report = backend.verify_chain(cited)
finally:
backend.close()
print(json.dumps(report.as_dict(), sort_keys=True))
return 0 if report.intact else 1
def attest_chain(args: argparse.Namespace) -> int:
from audit_core.integrity import write_attestation
backend = _postgres_backend(args.destination, args.schema, migrate=False)
try:
report = backend.verify_chain()
body = write_attestation(args.output, report)
finally:
backend.close()
print(json.dumps({"ok": report.intact, "path": args.output, **body}, sort_keys=True))
return 0 if report.intact else 1
def migrate_store(args: argparse.Namespace) -> int:
import os

15
audit_core/ingestion.py
View file

@ -159,7 +159,12 @@ class IngestionApplication:
if method == "GET" and (
path.startswith("/v1/events")
or path in ("/v1/dead-letters", "/v1/secret-findings", "/v1/stats")
or path in (
"/v1/dead-letters",
"/v1/secret-findings",
"/v1/stats",
"/v1/integrity",
)
):
return self._read(start_response, environ, path, identity)
@ -241,6 +246,14 @@ class IngestionApplication:
start_response, HTTPStatus.OK,
{"secret_findings": self.backend.secret_findings(_limit(query))},
)
if path == "/v1/integrity":
verify = getattr(self.backend, "verify_chain", None)
if not callable(verify):
return self._json(
start_response, HTTPStatus.NOT_FOUND,
{"error": "integrity_not_supported"},
)
return self._json(start_response, HTTPStatus.OK, verify().as_dict())
if path == "/v1/events":
correlation = (query.get("correlation_id") or [""])[0]
if not correlation:

149
audit_core/integrity.py Normal file
View file

@ -0,0 +1,149 @@
"""Hash-chain integrity for operational custody (AUDIT-WP-0007).
See ``docs/integrity.md`` for the proof bound. This module is the shared
arithmetic; backends persist and walk rows.
"""
from __future__ import annotations
import hashlib
import json
from dataclasses import dataclass
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Iterable, Mapping
SCHEMA = "audit-core.chain-head.v1"
GENESIS = "0" * 64
# Documented advisory-lock key so concurrent accepts cannot fork the head.
CHAIN_LOCK_KEY = 0xA0D17007
def chain_link(previous: str, payload_hash: str, event_id: str) -> str:
"""Next ``chain_hash`` = SHA-256(previous || payload_hash || event_id)."""
material = f"{previous}|{payload_hash}|{event_id}".encode("utf-8")
return hashlib.sha256(material).hexdigest()
def utc_now() -> str:
return datetime.now(timezone.utc).replace(microsecond=0).isoformat()
@dataclass(frozen=True)
class ChainRow:
event_id: str
payload_hash: str
chain_hash: str
chain_prev: str
accepted_at: str | None = None
@dataclass(frozen=True)
class ChainReport:
intact: bool
events: int
head: str
head_event_id: str | None
head_accepted_at: str | None
first_break: str | None
attestation_match: bool | None = None
def as_dict(self) -> dict[str, Any]:
return {
"intact": self.intact,
"events": self.events,
"head": self.head,
"head_event_id": self.head_event_id,
"head_accepted_at": self.head_accepted_at,
"first_break": self.first_break,
"attestation_match": self.attestation_match,
}
def verify_rows(
rows: Iterable[ChainRow],
*,
attestation: Mapping[str, Any] | None = None,
) -> ChainReport:
"""Walk accept order and recompute every link.
``first_break`` is an event id, or ``attestation_mismatch`` when a cited
head is not present in the live chain.
"""
ordered = list(rows)
prev = GENESIS
first_break: str | None = None
for row in ordered:
expected = chain_link(prev, row.payload_hash, row.event_id)
if row.chain_prev != prev or row.chain_hash != expected:
first_break = row.event_id
break
prev = row.chain_hash
intact = first_break is None
last = ordered[-1] if ordered else None
head = last.chain_hash if last else GENESIS
report = ChainReport(
intact=intact,
events=len(ordered),
head=head,
head_event_id=last.event_id if last else None,
head_accepted_at=last.accepted_at if last else None,
first_break=first_break,
)
if attestation is None:
return report
return _apply_attestation(report, ordered, attestation)
def _apply_attestation(
report: ChainReport,
rows: list[ChainRow],
attestation: Mapping[str, Any],
) -> ChainReport:
cited = str(attestation.get("chain_hash") or "")
live_hashes = {row.chain_hash for row in rows}
empty_ok = not rows and cited in {"", GENESIS}
matched = cited in live_hashes or empty_ok
first_break = report.first_break
intact = report.intact
if not matched:
intact = False
if first_break is None:
first_break = "attestation_mismatch"
return ChainReport(
intact=intact,
events=report.events,
head=report.head,
head_event_id=report.head_event_id,
head_accepted_at=report.head_accepted_at,
first_break=first_break,
attestation_match=matched,
)
def attestation_from_report(report: ChainReport, *, observed_at: str | None = None) -> dict[str, Any]:
return {
"schema": SCHEMA,
"genesis": GENESIS,
"chain_hash": report.head,
"event_id": report.head_event_id,
"accepted_at": report.head_accepted_at,
"event_count": report.events,
"observed_at": observed_at or utc_now(),
}
def load_attestation(path: str | Path) -> dict[str, Any]:
payload = json.loads(Path(path).read_text())
if not isinstance(payload, dict) or not payload.get("chain_hash"):
raise ValueError("attestation must be an object with chain_hash")
return payload
def write_attestation(path: str | Path, report: ChainReport) -> dict[str, Any]:
body = attestation_from_report(report)
destination = Path(path)
destination.parent.mkdir(parents=True, exist_ok=True)
destination.write_text(json.dumps(body, indent=2, sort_keys=True) + "\n")
return body

1
audit_core/interface.py
View file

@ -69,6 +69,7 @@ class RetentionPolicy:
"status": "ok",
"custody_class": self.custody_class,
"durable": self.durable,
"tamper_evidence": self.tamper_evidence,
}
if self.recoverable_days is not None or self.recoverable_source:
payload["recoverable_days"] = self.recoverable_days

100
audit_core/postgres_backend.py
View file

@ -33,6 +33,14 @@ from audit_core.interface import (
validate_event,
)
from audit_core.credentials import CredentialDirectory
from audit_core.integrity import (
CHAIN_LOCK_KEY,
GENESIS,
ChainRow,
attestation_from_report,
chain_link,
verify_rows,
)
from audit_core.redaction import Finding
try: # pragma: no cover - import guard
@ -134,6 +142,13 @@ MIGRATIONS: list[tuple[str, str]] = [
$grant$;
""",
),
(
"0006-chain",
"""
ALTER TABLE {schema}.events ADD COLUMN IF NOT EXISTS chain_hash text;
ALTER TABLE {schema}.events ADD COLUMN IF NOT EXISTS chain_prev text;
""",
),
]
# Rejection reasons whose payload must never be persisted — storing the body of
@ -255,6 +270,7 @@ class PostgresAuditBackend:
(migration_id,),
)
applied.append(migration_id)
self._backfill_chain(conn)
except psycopg.Error as exc:
raise BackendUnavailableError(f"migration failed: {exc}") from exc
return applied
@ -272,9 +288,10 @@ class PostgresAuditBackend:
``immutable`` is True because migration 0002 installs a trigger that
rejects UPDATE and DELETE, so no consumer credential can alter a stored
record. It is not a claim against the database owner or a superuser,
who can drop the trigger; ``tamper_evidence`` is correspondingly False,
because nothing here would *prove* they had. Hash-chaining or external
anchoring would be needed for that, and is not implemented.
who can drop the trigger. ``tamper_evidence`` is True because a
hash chain plus verify detects a rewritten payload, and a chain-head
attestation outside this database detects a suffix rewrite that
stays inside Postgres. It is not WORM or ``data.archive``.
``custody_class`` is ``operational``, not ``archive``. This store is
durable append-only Postgres recovered through the platform
@ -286,7 +303,7 @@ class PostgresAuditBackend:
custody_class="operational",
retention_days=self.retention_days,
immutable=True,
tamper_evidence=False,
tamper_evidence=True,
durable=True,
recoverable_days=self.recoverable_days,
recoverable_source=self.recoverable_source,
@ -302,13 +319,20 @@ class PostgresAuditBackend:
details = event.details if isinstance(event.details, dict) else {}
record = event.as_record()
try:
with self.pool.connection() as conn:
with self.pool.connection() as conn, conn.transaction():
conn.execute("SELECT pg_advisory_xact_lock(%s)", (CHAIN_LOCK_KEY,))
head = conn.execute(
f"SELECT chain_hash FROM {self._events} "
"ORDER BY accepted_at DESC, event_id DESC LIMIT 1"
).fetchone()
previous = head[0] if head and head[0] else GENESIS
link = chain_link(previous, payload_hash, event.event_id)
inserted = conn.execute(
f"""
INSERT INTO {self._events}
(event_id, payload_hash, observed_at, tenant, correlation_id,
source, action, record)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
source, action, record, chain_hash, chain_prev)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
ON CONFLICT (event_id) DO NOTHING
RETURNING event_id
""",
@ -321,6 +345,8 @@ class PostgresAuditBackend:
event.source,
event.action,
json.dumps(record, sort_keys=True),
link,
previous,
),
).fetchone()
if inserted is not None:
@ -443,9 +469,69 @@ class PostgresAuditBackend:
# --- lifecycle ---------------------------------------------------------
def verify_chain(self, attestation: dict | None = None):
rows = self._query(
f"SELECT event_id, payload_hash, chain_hash, chain_prev, accepted_at "
f"FROM {self._events} ORDER BY accepted_at, event_id",
(),
)
return verify_rows(
[
ChainRow(
event_id=r[0],
payload_hash=r[1],
chain_hash=r[2] or "",
chain_prev=r[3] or "",
accepted_at=_iso(r[4]),
)
for r in rows
],
attestation=attestation,
)
def attest_chain(self) -> dict:
return attestation_from_report(self.verify_chain())
def health(self) -> None:
self._query("SELECT 1", ())
def _backfill_chain(self, conn) -> None:
"""Fill chain columns on rows accepted before migration 0006.
The append-only trigger must be disabled for this UPDATE. A
database owner can do that; this is a one-time migrate, not a
runtime path.
"""
missing = conn.execute(
f"SELECT count(*) FROM {self._events} "
"WHERE chain_hash IS NULL OR chain_prev IS NULL"
).fetchone()[0]
if missing:
conn.execute(
f'ALTER TABLE {self._events} DISABLE TRIGGER events_append_only'
)
prev = GENESIS
for event_id, payload_hash in conn.execute(
f"SELECT event_id, payload_hash FROM {self._events} "
"ORDER BY accepted_at, event_id"
).fetchall():
link = chain_link(prev, payload_hash, event_id)
conn.execute(
f"UPDATE {self._events} SET chain_prev = %s, chain_hash = %s "
"WHERE event_id = %s",
(prev, link, event_id),
)
prev = link
conn.execute(
f'ALTER TABLE {self._events} ENABLE TRIGGER events_append_only'
)
conn.execute(
f"ALTER TABLE {self._events} ALTER COLUMN chain_hash SET NOT NULL"
)
conn.execute(
f"ALTER TABLE {self._events} ALTER COLUMN chain_prev SET NOT NULL"
)
def close(self) -> None:
self.pool.close()

73
audit_core/sqlite_backend.py
View file

@ -22,6 +22,13 @@ from audit_core.interface import (
RetentionPolicy,
validate_event,
)
from audit_core.integrity import (
GENESIS,
ChainRow,
attestation_from_report,
chain_link,
verify_rows,
)
_SCHEMA = """
CREATE TABLE IF NOT EXISTS events (
@ -30,7 +37,9 @@ CREATE TABLE IF NOT EXISTS events (
accepted_at TEXT NOT NULL,
correlation_id TEXT,
tenant TEXT NOT NULL,
record TEXT NOT NULL
record TEXT NOT NULL,
chain_hash TEXT,
chain_prev TEXT
);
CREATE INDEX IF NOT EXISTS events_correlation_idx ON events (correlation_id);
CREATE INDEX IF NOT EXISTS events_tenant_idx ON events (tenant);
@ -87,6 +96,7 @@ class SQLiteAuditBackend:
self._local = threading.local()
with self._connect_raw() as setup:
setup.executescript(_SCHEMA)
self._ensure_chain_columns(setup)
def _connect_raw(self) -> sqlite3.Connection:
try:
@ -136,11 +146,18 @@ class SQLiteAuditBackend:
raise BackendUnavailableError(str(exc)) from exc
try:
head = db.execute(
"SELECT chain_hash FROM events "
"ORDER BY accepted_at DESC, event_id DESC LIMIT 1"
).fetchone()
previous = head[0] if head and head[0] else GENESIS
link = chain_link(previous, payload_hash, event.event_id)
inserted = db.execute(
"""
INSERT INTO events
(event_id, payload_hash, accepted_at, correlation_id, tenant, record)
VALUES (?, ?, ?, ?, ?, ?)
(event_id, payload_hash, accepted_at, correlation_id, tenant, record,
chain_hash, chain_prev)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(event_id) DO NOTHING
RETURNING event_id
""",
@ -151,6 +168,8 @@ class SQLiteAuditBackend:
str(details.get("correlation_id") or "") or None,
event.tenant,
json.dumps(event.as_record(), sort_keys=True),
link,
previous,
),
).fetchone()
existing = None
@ -305,6 +324,54 @@ class SQLiteAuditBackend:
except sqlite3.Error as exc:
raise BackendUnavailableError(str(exc)) from exc
def verify_chain(self, attestation: dict | None = None):
rows = self._query(
"SELECT event_id, payload_hash, chain_hash, chain_prev, accepted_at "
"FROM events ORDER BY accepted_at, event_id",
(),
)
return verify_rows(
[
ChainRow(
event_id=r[0],
payload_hash=r[1],
chain_hash=r[2] or "",
chain_prev=r[3] or "",
accepted_at=r[4],
)
for r in rows
],
attestation=attestation,
)
def attest_chain(self) -> dict:
return attestation_from_report(self.verify_chain())
def _ensure_chain_columns(self, db: sqlite3.Connection) -> None:
cols = {row[1] for row in db.execute("PRAGMA table_info(events)")}
if "chain_hash" not in cols:
db.execute("ALTER TABLE events ADD COLUMN chain_hash TEXT")
if "chain_prev" not in cols:
db.execute("ALTER TABLE events ADD COLUMN chain_prev TEXT")
missing = db.execute(
"SELECT event_id, payload_hash FROM events "
"WHERE chain_hash IS NULL OR chain_prev IS NULL "
"ORDER BY accepted_at, event_id"
).fetchall()
if not missing:
return
prev = GENESIS
# Recompute the whole chain so a partial backfill cannot fork.
for event_id, payload_hash in db.execute(
"SELECT event_id, payload_hash FROM events ORDER BY accepted_at, event_id"
):
link = chain_link(prev, payload_hash, event_id)
db.execute(
"UPDATE events SET chain_prev = ?, chain_hash = ? WHERE event_id = ?",
(prev, link, event_id),
)
prev = link
def health(self) -> None:
"""Raise :class:`BackendUnavailableError` if the store is unusable."""
try: