Bound /readyz so kubelet probes cannot hang the Service
/readyz walked the hash chain and opened pooled connections with no libpq connect_timeout, so a 2s kubelet probe never saw a response and the pod stayed unready. Informed Decision accept is blocked on that. Probe health() only, under a 1.5s budget, publish last-known tamper_evidence, and fail TCP handshake in 1s. Integrity stays on /v1/integrity. Assistant: grok Assistant-Session: 01a0a182-bab7-7f11-b32b-d06f3af52082
This commit is contained in:
parent
e7e054d8d6
commit
b0e6792cf0
9 changed files with 276 additions and 18 deletions
|
|
@ -359,6 +359,32 @@ def test_connects_from_a_brokered_libpq_environment(monkeypatch):
|
|||
backend.close()
|
||||
|
||||
|
||||
def test_postgres_connections_set_a_connect_timeout(monkeypatch):
|
||||
"""A hung TCP handshake must not hold /readyz past the kubelet budget."""
|
||||
try:
|
||||
from audit_core.postgres_backend import PostgresAuditBackend
|
||||
except ImportError:
|
||||
pytest.skip("psycopg is not installed")
|
||||
|
||||
captured: dict = {}
|
||||
|
||||
class FakePool:
|
||||
def __init__(self, dsn, **kwargs):
|
||||
captured["dsn"] = dsn
|
||||
captured.update(kwargs)
|
||||
|
||||
def close(self):
|
||||
return None
|
||||
|
||||
monkeypatch.setattr("audit_core.postgres_backend.ConnectionPool", FakePool)
|
||||
PostgresAuditBackend("postgresql://example/audit", migrate=False)
|
||||
kwargs = captured["kwargs"]
|
||||
if callable(kwargs):
|
||||
kwargs = kwargs()
|
||||
assert int(kwargs["connect_timeout"]) == 1
|
||||
assert "statement_timeout" in kwargs["options"]
|
||||
|
||||
|
||||
def test_missing_connection_information_is_a_clear_error(monkeypatch):
|
||||
for var in ("AUDIT_CORE_DATABASE_URL", "PGHOST", "PGUSER"):
|
||||
monkeypatch.delenv(var, raising=False)
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import io
|
||||
import json
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import pytest
|
||||
|
|
@ -502,6 +503,68 @@ def test_readiness_reports_recovery_fields_for_operational_backend():
|
|||
assert "platform-audit-storage" in body["recoverable_source"]
|
||||
|
||||
|
||||
def test_readyz_does_not_wait_on_chain_verification():
|
||||
"""Kubelet timeoutSeconds is 2; walking the hash chain is not a probe."""
|
||||
|
||||
class _ChainWalk(_BrokenBackend):
|
||||
def health(self):
|
||||
return None
|
||||
|
||||
@property
|
||||
def retention_policy(self):
|
||||
time.sleep(5)
|
||||
raise AssertionError("/readyz must not call retention_policy")
|
||||
|
||||
def readiness_policy(self):
|
||||
return RetentionPolicy(
|
||||
custody_class="operational",
|
||||
retention_days=None,
|
||||
immutable=True,
|
||||
tamper_evidence=False,
|
||||
durable=True,
|
||||
recoverable_days=30,
|
||||
recoverable_source="cited",
|
||||
recoverable_basis="measured",
|
||||
)
|
||||
|
||||
started = time.monotonic()
|
||||
status, body = invoke(
|
||||
IngestionApplication(_ChainWalk(), "opaque"),
|
||||
None, path="/readyz", method="GET", body=b"",
|
||||
)
|
||||
assert time.monotonic() - started < 1.0
|
||||
assert status.startswith("200")
|
||||
assert body["durable"] is True
|
||||
assert body["tamper_evidence"] is False
|
||||
|
||||
|
||||
def test_readyz_treats_a_hung_health_check_as_unavailable():
|
||||
class _HungHealth(_BrokenBackend):
|
||||
def health(self):
|
||||
time.sleep(5)
|
||||
|
||||
app = IngestionApplication(_HungHealth(), "opaque")
|
||||
app._ready_budget = 0.05
|
||||
started = time.monotonic()
|
||||
status, body = invoke(app, None, path="/readyz", method="GET", body=b"")
|
||||
assert time.monotonic() - started < 1.0
|
||||
assert status.startswith("503")
|
||||
assert body["status"] == "unavailable"
|
||||
|
||||
|
||||
def test_readyz_unavailable_backend_is_503():
|
||||
class _Down(_BrokenBackend):
|
||||
def health(self):
|
||||
raise BackendUnavailableError("down")
|
||||
|
||||
status, body = invoke(
|
||||
IngestionApplication(_Down(), "opaque"),
|
||||
None, path="/readyz", method="GET", body=b"",
|
||||
)
|
||||
assert status.startswith("503")
|
||||
assert body["status"] == "unavailable"
|
||||
|
||||
|
||||
def test_counters_track_each_outcome(tmp_path):
|
||||
app, _ = bound_app(tmp_path, may_read=True)
|
||||
invoke(app, event()) # accepted
|
||||
|
|
|
|||
|
|
@ -303,6 +303,32 @@ def test_postgres_policy_treats_an_unreadable_attestation_as_absent(tmp_path):
|
|||
assert backend.tamper_evidence_state().reason == "no_attestation"
|
||||
|
||||
|
||||
def test_postgres_readiness_policy_does_not_walk_the_chain(tmp_path):
|
||||
backend = _bare_postgres_backend(tmp_path)
|
||||
|
||||
def _boom(attestation=None):
|
||||
raise AssertionError("/readyz must not walk the chain")
|
||||
|
||||
backend.verify_chain = _boom
|
||||
policy = backend.readiness_policy()
|
||||
assert policy.tamper_evidence is False
|
||||
assert policy.durable is True
|
||||
assert policy.custody_class == "operational"
|
||||
|
||||
|
||||
def test_postgres_readiness_policy_reports_a_cached_claim(tmp_path):
|
||||
from audit_core.integrity import TamperEvidenceState, utc_now
|
||||
|
||||
backend = _bare_postgres_backend(tmp_path)
|
||||
backend.verify_chain = lambda attestation=None: (_ for _ in ()).throw(
|
||||
AssertionError("cached claim must not walk")
|
||||
)
|
||||
backend._tamper_state = TamperEvidenceState(
|
||||
claimed=True, reason="attested", observed_at=utc_now(),
|
||||
)
|
||||
assert backend.readiness_policy().tamper_evidence is True
|
||||
|
||||
|
||||
def test_postgres_policy_drops_the_claim_when_the_chain_cannot_be_walked(tmp_path):
|
||||
from audit_core.integrity import utc_now
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue