Bound /readyz so kubelet probes cannot hang the Service
All checks were successful
CI Smoke / host-smoke (push) Successful in 0s
CI Smoke / container-smoke (push) Successful in 2s

/readyz walked the hash chain and opened pooled connections with no
libpq connect_timeout, so a 2s kubelet probe never saw a response and
the pod stayed unready. Informed Decision accept is blocked on that.

Probe health() only, under a 1.5s budget, publish last-known
tamper_evidence, and fail TCP handshake in 1s. Integrity stays on
/v1/integrity.

Assistant: grok
Assistant-Session: 01a0a182-bab7-7f11-b32b-d06f3af52082
This commit is contained in:
tegwick 2026-09-14 22:13:55 +02:00
parent e7e054d8d6
commit b0e6792cf0
9 changed files with 276 additions and 18 deletions

View file

@ -57,6 +57,15 @@ except ImportError as exc: # pragma: no cover
"the postgres backend needs psycopg: pip install 'audit-core[postgres]'"
) from exc
try: # pragma: no cover - older psycopg_pool
from psycopg_pool import PoolTimeout as _PoolTimeout
except ImportError: # pragma: no cover
_PoolTimeout = None
_UNAVAILABLE: tuple[type[BaseException], ...] = (psycopg.Error, TimeoutError, OSError)
if _PoolTimeout is not None:
_UNAVAILABLE = (*_UNAVAILABLE, _PoolTimeout)
DEFAULT_SCHEMA = "audit_core"
# Ordered, append-only. Each entry runs once and is recorded in
@ -181,6 +190,8 @@ class PostgresAuditBackend:
min_size: int = 1,
max_size: int = 8,
statement_timeout_ms: int = 30_000,
connect_timeout_s: int = 1,
health_timeout_s: float = 1.0,
migrate: bool = True,
credential_dir: str | None = None,
attestation_path: str | None = None,
@ -224,8 +235,13 @@ class PostgresAuditBackend:
self._attestation_cache_seconds = float(attestation_cache_seconds)
self._tamper_state: TamperEvidenceState | None = None
self._tamper_state_at = 0.0
self._connect_timeout_s = max(1, int(connect_timeout_s))
self._health_timeout_s = float(health_timeout_s)
base_kwargs = {
"autocommit": True,
# A stalled TCP handshake must fail the probe, not hold it until
# kubelet times out the pod.
"connect_timeout": self._connect_timeout_s,
# A stalled write must surface as unavailable rather than hold a
# request open indefinitely.
"options": f"-c statement_timeout={int(statement_timeout_ms)}",
@ -334,6 +350,25 @@ class PostgresAuditBackend:
recoverable_basis=self.recoverable_basis,
)
def readiness_policy(self) -> RetentionPolicy:
"""``/readyz`` view of custody: class, durability, recovery, last claim.
Does not walk the chain. An unevaluated or expired cache reports
``tamper_evidence=False`` uncomputed is not claimed. Integrity
evaluation stays on ``retention_policy`` / ``/v1/integrity``.
"""
cached = self._tamper_state
return RetentionPolicy(
custody_class="operational",
retention_days=self.retention_days,
immutable=True,
tamper_evidence=bool(cached is not None and cached.claimed),
durable=True,
recoverable_days=self.recoverable_days,
recoverable_source=self.recoverable_source,
recoverable_basis=self.recoverable_basis,
)
def emit(self, event: AuditEvent) -> str:
return self.accept(event, payload_hash=_record_hash(event)).reference
@ -563,9 +598,10 @@ class PostgresAuditBackend:
def tamper_evidence_state(self) -> TamperEvidenceState:
"""Evaluate the tamper-evidence preconditions against live state.
Cached briefly because ``/readyz`` reads it on every probe and the
evaluation walks the chain. The cache only ever delays a *change* of
state; it cannot manufacture one.
Cached briefly because a full walk is too expensive to repeat per
caller. ``/readyz`` uses :meth:`readiness_policy` and never waits
on this. The cache only ever delays a *change* of state; it cannot
manufacture one.
"""
now = time.monotonic()
cached = self._tamper_state
@ -586,7 +622,17 @@ class PostgresAuditBackend:
return state
def health(self) -> None:
self._query("SELECT 1", ())
"""Reachability check for ``/readyz``. Must return inside the probe budget."""
timeout_ms = max(1, int(self._health_timeout_s * 1000))
try:
with self.pool.connection(timeout=self._health_timeout_s) as conn:
with conn.transaction():
conn.execute(f"SET LOCAL statement_timeout = {timeout_ms}")
conn.execute("SELECT 1")
except _UNAVAILABLE as exc:
raise BackendUnavailableError(str(exc)) from exc
except Exception as exc: # pool/libpq can raise outside psycopg.Error
raise BackendUnavailableError(str(exc)) from exc
def _backfill_chain(self, conn) -> None:
"""Fill chain columns on rows accepted before migration 0006.
@ -632,14 +678,14 @@ class PostgresAuditBackend:
try:
with self.pool.connection() as conn:
return conn.execute(sql, params).fetchall()
except psycopg.Error as exc:
except _UNAVAILABLE as exc:
raise BackendUnavailableError(str(exc)) from exc
def _execute(self, sql: str, params: tuple) -> None:
try:
with self.pool.connection() as conn:
conn.execute(sql, params)
except psycopg.Error as exc:
except _UNAVAILABLE as exc:
raise BackendUnavailableError(str(exc)) from exc