diff --git a/bounded-executor/cli.py b/bounded-executor/cli.py index 09c800c..5d9cae6 100644 --- a/bounded-executor/cli.py +++ b/bounded-executor/cli.py @@ -57,6 +57,8 @@ def build(cfg: dict, logins: list): if cfg["journey"]["k8s_role"] == o["k8s_role"]: raise ValueError("the journey read and the bind write must use separate roles") + OPS.wait_reachable(o["addr"], ca_file=o.get("ca_file")) # new pod: NetworkPolicy may not be applied yet + def login(role: str): client = OPS.OpenBao(o["addr"], k8s_role=role, jwt_file=o["jwt_file"], ca_file=o.get("ca_file")) logins.append(client) @@ -91,6 +93,7 @@ def build(cfg: dict, logins: list): def journey_only(cfg: dict, logins: list) -> dict: o, j = cfg["openbao"], cfg["journey"] + OPS.wait_reachable(o["addr"], ca_file=o.get("ca_file")) # new pod: NetworkPolicy may not be applied yet client = OPS.OpenBao(o["addr"], k8s_role=j["k8s_role"], jwt_file=o["jwt_file"], ca_file=o.get("ca_file")) logins.append(client) receipt = OPS.journey_from_kv(client, j["kv"], j["config"]) diff --git a/bounded-executor/operations.py b/bounded-executor/operations.py index 5130794..2148c47 100644 --- a/bounded-executor/operations.py +++ b/bounded-executor/operations.py @@ -61,6 +61,32 @@ def _opener(ca_file: str | None): # -------------------------------------------------------------------------------------------------- +def wait_reachable(addr: str, *, timeout: float = 30, interval: float = 1, ca_file: str | None = None, + opener=None, clock=time.monotonic, sleep=time.sleep) -> float: + """Wait until OpenBao answers on the network, before the first login. k3s applies NetworkPolicy to a new pod a + moment after it starts, and connections in that window are refused (seen live 2026-10-31: refused at 0 s, + open at 3 s). Polls the unauthenticated sys/health. Any HTTP answer, including 429/473/503, counts as reachable; + only connection errors are retried. Returns the seconds waited; raises Unreachable after `timeout`.""" + open_ = opener or _opener(ca_file).open + start = clock() + while True: + request = urllib.request.Request(addr.rstrip("/") + "/v1/sys/health", method="GET") + try: + with open_(request, timeout=5): + return clock() - start + except urllib.error.HTTPError as error: + error.close() + return clock() - start + except (urllib.error.URLError, OSError): + if clock() - start >= timeout: + raise Unreachable("openbao not reachable") from None + sleep(interval) + + +class Unreachable(Exception): + pass + + class OpenBao: def __init__(self, addr: str, *, token: str | None = None, k8s_role: str | None = None, jwt_file: str | None = None, k8s_mount: str = "kubernetes", ca_file: str | None = None, diff --git a/bounded-executor/tests/test_clients.py b/bounded-executor/tests/test_clients.py index c432cbf..b06a5ca 100644 --- a/bounded-executor/tests/test_clients.py +++ b/bounded-executor/tests/test_clients.py @@ -120,6 +120,9 @@ class CliTests(unittest.TestCase): (root / "cfg.json").write_text(json.dumps(cfg)) admission = CLI.X.REPO / "examples/bounded-execution-admission/bind-rotation-svc-authelia.yaml" import contextlib, io + wait = CLI.OPS.wait_reachable + self.addCleanup(setattr, CLI.OPS, "wait_reachable", wait) + CLI.OPS.wait_reachable = lambda addr, **kw: wait(addr, **{**kw, "timeout": 0.2, "interval": 0.1}) with contextlib.redirect_stdout(io.StringIO()) as out: code = CLI.main(["--admission", str(admission), "--approval-id", "apr-1", "--config", str(root / "cfg.json")]) self.assertEqual(code, 4) diff --git a/bounded-executor/tests/test_openbao_lanes.py b/bounded-executor/tests/test_openbao_lanes.py index 71530ca..db0a471 100644 --- a/bounded-executor/tests/test_openbao_lanes.py +++ b/bounded-executor/tests/test_openbao_lanes.py @@ -162,6 +162,39 @@ class JourneyOnly(Base): self.run_cli(self.cfg(), "--approval-id", "apr-1") +class WaitReachable(unittest.TestCase): + """k3s applies NetworkPolicy to a new pod a moment late: early connections are refused (live, 2026-10-31).""" + + def fake(self, outcomes): + self.now, self.slept, calls = 0.0, [], list(outcomes) + + def open_(request, timeout): + outcome = calls.pop(0) + if isinstance(outcome, BaseException): + raise outcome + return contextlib.nullcontext() + + def sleep(seconds): + self.slept.append(seconds) + self.now += seconds + return dict(opener=open_, clock=lambda: self.now, sleep=sleep) + + def test_refused_then_reachable(self): + refused = OPS.urllib.error.URLError(ConnectionRefusedError()) + waited = OPS.wait_reachable("http://bao:8200", **self.fake([refused, refused, "ok"])) + self.assertEqual((waited, self.slept), (2.0, [1, 1])) + + def test_any_http_answer_is_reachable(self): + sealed = OPS.urllib.error.HTTPError("http://bao/v1/sys/health", 503, "sealed", {}, io.BytesIO(b"")) + self.assertEqual(OPS.wait_reachable("http://bao:8200", **self.fake([sealed])), 0.0) + + def test_gives_up_after_the_timeout(self): + refused = OPS.urllib.error.URLError(ConnectionRefusedError()) + with self.assertRaises(OPS.Unreachable): + OPS.wait_reachable("http://bao:8200", timeout=3, **self.fake([refused] * 10)) + self.assertEqual(self.slept, [1, 1, 1]) + + class Renewal(Base): def test_logs_in_again_before_the_lease_ends_and_revokes_the_old_token(self): self.bao.lease = 1 # renew_at = max(1 - margin, 0.5 s) diff --git a/docs/evidence/2026-10-31-nk56-t10-journey-only/README.md b/docs/evidence/2026-10-31-nk56-t10-journey-only/README.md index 7532a9a..a4277b9 100644 --- a/docs/evidence/2026-10-31-nk56-t10-journey-only/README.md +++ b/docs/evidence/2026-10-31-nk56-t10-journey-only/README.md @@ -22,3 +22,14 @@ token that holds only the journey-read policy: `all_met: true`, `no_values_in_report: true`. This is T10's fail-closed proof. The unattended live run is the rapp's `make journey`. + +## First live attempt and the startup race (2026-10-31) + +- `make journey` (Job `bounded-executor-journey-20261031-192708`) was refused at setup with `URLError`. The OpenBao + login did not connect, so nothing was read or changed. +- `make netcheck` (rapp `5fe1a70`) ran a read-only pod with the executor's labels. At 0 s, OpenBao, KeyCape and + Authelia all refused connections, while DNS was fine. From 3 s on, everything answered, through 42 s. +- Diagnosis: k3s applies NetworkPolicy to a new pod a moment after it starts. The policies themselves are correct. +- Fix: before the first login, the executor polls OpenBao's unauthenticated `sys/health` for up to 30 s + (`operations.wait_reachable`). Only connection errors are retried, and any HTTP answer counts as reachable. + This applies to both CLI modes.