From b4773235bc83cf2a9c4b64e7ae97fc33ba403ccb Mon Sep 17 00:00:00 2001 From: tegwick Date: Fri, 6 Nov 2026 10:33:30 +0100 Subject: [PATCH] window_watch: wait up to 30 s for OpenBao and the API server, and name an unreachable endpoint The first two watch runs after the deploy (2026-11-06 09:20Z, 09:30Z) exited 5 with a bare URLError: k3s programs a new pod's NetworkPolicy a few seconds late (as for the executor, 2026-10-31). The live policies admit the watch pod to both endpoints. NK-WP-0058-T06. Co-Authored-By: Claude Opus 5.5 Assistant: claude-code Assistant-Model: opus Assistant-Process: 943911@bnt-lap001 Assistant-Session: fc322940-1c8e-4bed-ab39-86cec99f23a0 --- bounded-executor/tests/test_window_watch.py | 24 +++++++++++++++++++++ bounded-executor/window_watch.py | 14 +++++++++++- 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/bounded-executor/tests/test_window_watch.py b/bounded-executor/tests/test_window_watch.py index a996779..1434aa4 100644 --- a/bounded-executor/tests/test_window_watch.py +++ b/bounded-executor/tests/test_window_watch.py @@ -106,5 +106,29 @@ class Modes(unittest.TestCase): W.watch(Denied(), FakeKube([]), 'close', NOW) +class Network(unittest.TestCase): + """2026-11-06: the first watch runs failed with a bare URLError before the pod's NetworkPolicy was live.""" + + def test_waits_for_both_endpoints_with_the_api_ca(self): + calls = [] + W.wait_for_network('http://openbao.openbao.svc:8200', wait=lambda addr, **kw: calls.append((addr, kw))) + self.assertEqual([c[0] for c in calls], ['http://openbao.openbao.svc:8200', W.KUBE_API]) + self.assertEqual(calls[1][1]['ca_file'], W.KUBE_CA) + self.assertTrue(all(c[1]['timeout'] == 30 for c in calls)) + + def test_an_unreachable_endpoint_is_named_in_the_line(self): + import contextlib, io, json as js + original = W.wait_for_network + + def unreachable(addr, wait=None): + raise OPS.Unreachable('openbao not reachable') + W.wait_for_network = unreachable + self.addCleanup(setattr, W, 'wait_for_network', original) + with contextlib.redirect_stdout(io.StringIO()) as out: + code = W.main(['--mode', 'flag']) + self.assertEqual(code, 5) + self.assertEqual(js.loads(out.getvalue())['reason'], 'openbao not reachable') + + if __name__ == '__main__': unittest.main() diff --git a/bounded-executor/window_watch.py b/bounded-executor/window_watch.py index 0a643af..a64dc68 100644 --- a/bounded-executor/window_watch.py +++ b/bounded-executor/window_watch.py @@ -34,6 +34,17 @@ WATCH_ROLE = 'net-kingdom-bounded-executor-window-watch' ROLE_PATH = '/v1/auth/kubernetes/role/' + WRITE_ROLE RUN_JOBS = 'app.kubernetes.io/name=bounded-executor,!app.kubernetes.io/component' GRACE = 600 # seconds: the window opens moments before its run Job is created +KUBE_API = 'https://kubernetes.default.svc' +KUBE_CA = '/var/run/secrets/kubernetes.io/serviceaccount/ca.crt' + + +def wait_for_network(openbao_addr: str, wait=None) -> None: + """A new pod's NetworkPolicy is programmed a few seconds after it starts (k3s; 2026-10-31 and the first watch + runs on 2026-11-06 failed with URLError). Wait up to 30 s for each endpoint; Unreachable names the one that + stayed silent. Any HTTP answer counts.""" + wait = wait or OPS.wait_reachable + wait(openbao_addr, timeout=30, path='/v1/sys/health', name='openbao') + wait(KUBE_API, timeout=30, path='/healthz', name='kubernetes-api', ca_file=KUBE_CA) def role_present(bao) -> bool: @@ -82,10 +93,11 @@ def main(argv=None) -> int: a = p.parse_args(argv) bao = None try: + wait_for_network(a.openbao_addr) bao = OPS.OpenBao(a.openbao_addr, k8s_role=WATCH_ROLE, jwt_file=a.jwt_file) result, code = watch(bao, OPS.Kube(), a.mode, dt.datetime.now(dt.timezone.utc)) except Exception as error: # value-free: the clients' own errors carry fixed texts - reason = str(error) if isinstance(error, OPS.OperationError) else type(error).__name__ + reason = str(error) if isinstance(error, (OPS.OperationError, OPS.Unreachable)) else type(error).__name__ result, code = {'status': 'error', 'reason': reason}, 5 finally: if bao is not None: