Bound /readyz so kubelet probes cannot hang the Service
/readyz walked the hash chain and opened pooled connections with no libpq connect_timeout, so a 2s kubelet probe never saw a response and the pod stayed unready. Informed Decision accept is blocked on that. Probe health() only, under a 1.5s budget, publish last-known tamper_evidence, and fail TCP handshake in 1s. Integrity stays on /v1/integrity. Assistant: grok Assistant-Session: 01a0a182-bab7-7f11-b32b-d06f3af52082
This commit is contained in:
parent
e7e054d8d6
commit
b0e6792cf0
9 changed files with 276 additions and 18 deletions
|
|
@ -57,6 +57,15 @@ except ImportError as exc: # pragma: no cover
|
|||
"the postgres backend needs psycopg: pip install 'audit-core[postgres]'"
|
||||
) from exc
|
||||
|
||||
try: # pragma: no cover - older psycopg_pool
|
||||
from psycopg_pool import PoolTimeout as _PoolTimeout
|
||||
except ImportError: # pragma: no cover
|
||||
_PoolTimeout = None
|
||||
|
||||
_UNAVAILABLE: tuple[type[BaseException], ...] = (psycopg.Error, TimeoutError, OSError)
|
||||
if _PoolTimeout is not None:
|
||||
_UNAVAILABLE = (*_UNAVAILABLE, _PoolTimeout)
|
||||
|
||||
DEFAULT_SCHEMA = "audit_core"
|
||||
|
||||
# Ordered, append-only. Each entry runs once and is recorded in
|
||||
|
|
@ -181,6 +190,8 @@ class PostgresAuditBackend:
|
|||
min_size: int = 1,
|
||||
max_size: int = 8,
|
||||
statement_timeout_ms: int = 30_000,
|
||||
connect_timeout_s: int = 1,
|
||||
health_timeout_s: float = 1.0,
|
||||
migrate: bool = True,
|
||||
credential_dir: str | None = None,
|
||||
attestation_path: str | None = None,
|
||||
|
|
@ -224,8 +235,13 @@ class PostgresAuditBackend:
|
|||
self._attestation_cache_seconds = float(attestation_cache_seconds)
|
||||
self._tamper_state: TamperEvidenceState | None = None
|
||||
self._tamper_state_at = 0.0
|
||||
self._connect_timeout_s = max(1, int(connect_timeout_s))
|
||||
self._health_timeout_s = float(health_timeout_s)
|
||||
base_kwargs = {
|
||||
"autocommit": True,
|
||||
# A stalled TCP handshake must fail the probe, not hold it until
|
||||
# kubelet times out the pod.
|
||||
"connect_timeout": self._connect_timeout_s,
|
||||
# A stalled write must surface as unavailable rather than hold a
|
||||
# request open indefinitely.
|
||||
"options": f"-c statement_timeout={int(statement_timeout_ms)}",
|
||||
|
|
@ -334,6 +350,25 @@ class PostgresAuditBackend:
|
|||
recoverable_basis=self.recoverable_basis,
|
||||
)
|
||||
|
||||
def readiness_policy(self) -> RetentionPolicy:
|
||||
"""``/readyz`` view of custody: class, durability, recovery, last claim.
|
||||
|
||||
Does not walk the chain. An unevaluated or expired cache reports
|
||||
``tamper_evidence=False`` — uncomputed is not claimed. Integrity
|
||||
evaluation stays on ``retention_policy`` / ``/v1/integrity``.
|
||||
"""
|
||||
cached = self._tamper_state
|
||||
return RetentionPolicy(
|
||||
custody_class="operational",
|
||||
retention_days=self.retention_days,
|
||||
immutable=True,
|
||||
tamper_evidence=bool(cached is not None and cached.claimed),
|
||||
durable=True,
|
||||
recoverable_days=self.recoverable_days,
|
||||
recoverable_source=self.recoverable_source,
|
||||
recoverable_basis=self.recoverable_basis,
|
||||
)
|
||||
|
||||
def emit(self, event: AuditEvent) -> str:
|
||||
return self.accept(event, payload_hash=_record_hash(event)).reference
|
||||
|
||||
|
|
@ -563,9 +598,10 @@ class PostgresAuditBackend:
|
|||
def tamper_evidence_state(self) -> TamperEvidenceState:
|
||||
"""Evaluate the tamper-evidence preconditions against live state.
|
||||
|
||||
Cached briefly because ``/readyz`` reads it on every probe and the
|
||||
evaluation walks the chain. The cache only ever delays a *change* of
|
||||
state; it cannot manufacture one.
|
||||
Cached briefly because a full walk is too expensive to repeat per
|
||||
caller. ``/readyz`` uses :meth:`readiness_policy` and never waits
|
||||
on this. The cache only ever delays a *change* of state; it cannot
|
||||
manufacture one.
|
||||
"""
|
||||
now = time.monotonic()
|
||||
cached = self._tamper_state
|
||||
|
|
@ -586,7 +622,17 @@ class PostgresAuditBackend:
|
|||
return state
|
||||
|
||||
def health(self) -> None:
|
||||
self._query("SELECT 1", ())
|
||||
"""Reachability check for ``/readyz``. Must return inside the probe budget."""
|
||||
timeout_ms = max(1, int(self._health_timeout_s * 1000))
|
||||
try:
|
||||
with self.pool.connection(timeout=self._health_timeout_s) as conn:
|
||||
with conn.transaction():
|
||||
conn.execute(f"SET LOCAL statement_timeout = {timeout_ms}")
|
||||
conn.execute("SELECT 1")
|
||||
except _UNAVAILABLE as exc:
|
||||
raise BackendUnavailableError(str(exc)) from exc
|
||||
except Exception as exc: # pool/libpq can raise outside psycopg.Error
|
||||
raise BackendUnavailableError(str(exc)) from exc
|
||||
|
||||
def _backfill_chain(self, conn) -> None:
|
||||
"""Fill chain columns on rows accepted before migration 0006.
|
||||
|
|
@ -632,14 +678,14 @@ class PostgresAuditBackend:
|
|||
try:
|
||||
with self.pool.connection() as conn:
|
||||
return conn.execute(sql, params).fetchall()
|
||||
except psycopg.Error as exc:
|
||||
except _UNAVAILABLE as exc:
|
||||
raise BackendUnavailableError(str(exc)) from exc
|
||||
|
||||
def _execute(self, sql: str, params: tuple) -> None:
|
||||
try:
|
||||
with self.pool.connection() as conn:
|
||||
conn.execute(sql, params)
|
||||
except psycopg.Error as exc:
|
||||
except _UNAVAILABLE as exc:
|
||||
raise BackendUnavailableError(str(exc)) from exc
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue