From dfe5a5ed28777f1719e53da6a07f38ae53775153 Mon Sep 17 00:00:00 2001 From: tegwick Date: Fri, 23 Oct 2026 00:17:42 +0200 Subject: [PATCH] NK-WP-0048-T12: KeyCape restart and post-restart readiness source candidate Co-Authored-By: Claude Opus 5.5 Assistant: claude-code Assistant-Model: opus Assistant-Process: 696317@bnt-lap001 Assistant-Session: bab9e511-0915-45b3-926a-2fab552edff6 --- WORK-RECORDS.md | 2 +- sso-mfa/k8s/keycape/restart-readiness.py | 119 ++++++++++++++++ sso-mfa/k8s/keycape/test_restart_readiness.py | 128 ++++++++++++++++++ workplans/NK-WP-0048-security-case-types.md | 19 ++- 4 files changed, 266 insertions(+), 2 deletions(-) create mode 100644 sso-mfa/k8s/keycape/restart-readiness.py create mode 100644 sso-mfa/k8s/keycape/test_restart_readiness.py diff --git a/WORK-RECORDS.md b/WORK-RECORDS.md index d20d86b..31cffc2 100644 --- a/WORK-RECORDS.md +++ b/WORK-RECORDS.md @@ -216,7 +216,7 @@ | task | NK-WP-0048-T09 | done | — | workplans/NK-WP-0048-security-case-types.md | | task | NK-WP-0048-T10 | done | — | workplans/NK-WP-0048-security-case-types.md | | task | NK-WP-0048-T11 | done | — | workplans/NK-WP-0048-security-case-types.md | -| task | NK-WP-0048-T12 | todo | — | workplans/NK-WP-0048-security-case-types.md | +| task | NK-WP-0048-T12 | wait | — | workplans/NK-WP-0048-security-case-types.md | | task | NK-WP-0048-T13 | done | — | workplans/NK-WP-0048-security-case-types.md | | task | NK-WP-0048-T14 | progress | — | workplans/NK-WP-0048-security-case-types.md | | task | NK-WP-0048-T15 | wait | — | workplans/NK-WP-0048-security-case-types.md | diff --git a/sso-mfa/k8s/keycape/restart-readiness.py b/sso-mfa/k8s/keycape/restart-readiness.py new file mode 100644 index 0000000..5a8f81e --- /dev/null +++ b/sso-mfa/k8s/keycape/restart-readiness.py @@ -0,0 +1,119 @@ +"""Bounded KeyCape restart and post-restart readiness check (NK-WP-0048-T12). + +`keycape-client.py apply` reports `restart_required` because KeyCape loads its +client registry at startup. This module supplies the missing step after a +registration: plan one restart of the single-replica Recreate issuer, then +decide from observed state whether the new instance came up and serves the +expected issuer. It is a source candidate, not wired into a SecurityCaseType or +Warden kernel; the coordinator decides 1.3 placement during the T16 re-freeze. + +No CLI, kubectl or credential handling lives here. read/write/discovery are +injected; pods are inspected for status only, never env or Secret data. A +refused restart (crash loop, failed container) stops sign-in for every client, +so it is reported as `failed` for attended owner rollback, never retried. A +deadline without a decision is `uncertain` and needs owner reconciliation; it +is never converted to success. +""" +import copy + +ANNOTATION = 'keycape.netkingdom/restarted-at' +FAILED_REASONS = {'CrashLoopBackOff', 'Error', 'CreateContainerConfigError', + 'ImagePullBackOff', 'ErrImagePull', 'RunContainerError'} + + +class Refused(Exception): + pass + + +def _issuer_deployment(deployment): + meta = deployment.get('metadata', {}) + if (deployment.get('kind') != 'Deployment' or meta.get('name') != 'keycape' + or meta.get('namespace') != 'sso'): + raise Refused('issuer deployment sso/keycape required') + spec = deployment.get('spec', {}) + if spec.get('replicas') != 1 or spec.get('strategy', {}).get('type') != 'Recreate': + raise Refused('single-replica Recreate issuer required') + return spec + + +def plan_restart(deployment, stamp, expected_resource_version): + """Return the restarted Deployment; the caller writes it with CAS on the observed revision.""" + if not isinstance(stamp, str) or not 0 < len(stamp) <= 64: + raise Refused('bounded restart stamp required') + if deployment.get('metadata', {}).get('resourceVersion') != expected_resource_version: + raise Refused('deployment revision differs from observed') + _issuer_deployment(deployment) + new = copy.deepcopy(deployment) + annotations = new['spec']['template'].setdefault('metadata', {}).setdefault('annotations', {}) + if annotations.get(ANNOTATION) == stamp: + raise Refused('restart stamp already applied') + annotations[ANNOTATION] = stamp + return new + + +def _condition(obj, kind): + return next((c.get('status') for c in obj.get('status', {}).get('conditions', []) + if c.get('type') == kind), None) + + +def _container_failure(pod): + for status in pod.get('status', {}).get('containerStatuses', []): + if status.get('name') != 'keycape': + continue + state = status.get('state', {}) + reason = (state.get('waiting') or {}).get('reason') or (state.get('terminated') or {}).get('reason') + if reason in FAILED_REASONS: + return reason + return None + + +def assess(deployment, pods, discovery, *, stamp, expected_issuer): + """Return (verdict, reasons) with verdict in ready, pending, failed.""" + spec = _issuer_deployment(deployment) + if spec['template'].get('metadata', {}).get('annotations', {}).get(ANNOTATION) != stamp: + return 'failed', ['restart stamp missing from template; deployment changed by another writer'] + reasons = [] + meta, status = deployment['metadata'], deployment.get('status', {}) + if status.get('observedGeneration', -1) < meta.get('generation', 0): + reasons.append('generation not yet observed') + if any(status.get(k) != 1 for k in ('updatedReplicas', 'readyReplicas', 'availableReplicas')): + reasons.append('replica counts not 1/1/1') + live = [p for p in pods if not p.get('metadata', {}).get('deletionTimestamp')] + new = [p for p in live if p.get('metadata', {}).get('annotations', {}).get(ANNOTATION) == stamp] + for pod in new: + failure = _container_failure(pod) + if failure: + return 'failed', [f'restarted keycape container {failure}'] + if len(new) != 1 or len(live) != 1: + reasons.append(f'expected exactly one restarted pod, saw {len(new)} of {len(live)}') + elif new[0].get('status', {}).get('phase') != 'Running' or _condition(new[0], 'Ready') != 'True': + reasons.append('restarted pod not Running/Ready') + if discovery is None: + reasons.append('issuer discovery not yet served') + elif discovery.get('issuer') != expected_issuer: + return 'failed', ['discovery issuer differs from expected'] + elif not all(discovery.get(k) for k in ('authorization_endpoint', 'token_endpoint', 'jwks_uri')): + return 'failed', ['discovery document incomplete'] + return ('pending', reasons) if reasons else ('ready', []) + + +def wait_ready(observe, sleep, *, stamp, expected_issuer, attempts, interval): + """Poll observe()->(deployment, pods, discovery) until a decision or the bound. + + Returns a status-only receipt. observe errors propagate (uncertain to the caller). + """ + if not isinstance(attempts, int) or not 0 < attempts <= 120: + raise Refused('bounded attempts required') + reasons = [] + for attempt in range(1, attempts + 1): + verdict, reasons = assess(*observe(), stamp=stamp, expected_issuer=expected_issuer) + if verdict != 'pending': + break + if attempt < attempts: + sleep(interval) + else: + verdict = 'uncertain' + return {'schema': 'netkingdom.keycape-restart-readiness.v1', 'verdict': verdict, + 'attempts': attempt, 'reasons': reasons, 'stamp': stamp, + 'issuer': expected_issuer, 'values_emitted': False, + 'owner_reconciliation_required': verdict in ('failed', 'uncertain')} diff --git a/sso-mfa/k8s/keycape/test_restart_readiness.py b/sso-mfa/k8s/keycape/test_restart_readiness.py new file mode 100644 index 0000000..216a920 --- /dev/null +++ b/sso-mfa/k8s/keycape/test_restart_readiness.py @@ -0,0 +1,128 @@ +"""Synthetic-only tests for the T12 restart/readiness candidate; no cluster, no values.""" +import importlib.util +from pathlib import Path +import unittest + +ROOT = Path(__file__).parent +spec = importlib.util.spec_from_file_location('restart_readiness', ROOT / 'restart-readiness.py') +r = importlib.util.module_from_spec(spec) +spec.loader.exec_module(r) + +ISSUER = 'https://kc.coulomb.social' +STAMP = '2026-10-23T12:00:00Z' +DISCOVERY = {'issuer': ISSUER, 'authorization_endpoint': ISSUER + '/authorize', + 'token_endpoint': ISSUER + '/token', 'jwks_uri': ISSUER + '/jwks'} + + +def deployment(stamp=None, generation=2, observed=2, counts=1, replicas=1, strategy='Recreate'): + template = {'spec': {'containers': [{'name': 'keycape'}]}} + if stamp: + template['metadata'] = {'annotations': {r.ANNOTATION: stamp}} + return {'kind': 'Deployment', + 'metadata': {'name': 'keycape', 'namespace': 'sso', 'resourceVersion': '7', 'generation': generation}, + 'spec': {'replicas': replicas, 'strategy': {'type': strategy}, 'template': template}, + 'status': {'observedGeneration': observed, 'updatedReplicas': counts, + 'readyReplicas': counts, 'availableReplicas': counts}} + + +def pod(stamp=STAMP, ready='True', phase='Running', waiting=None, deleting=False): + meta = {'annotations': {r.ANNOTATION: stamp}} + if deleting: + meta['deletionTimestamp'] = '2026-10-23T12:00:01Z' + state = {'waiting': {'reason': waiting}} if waiting else {'running': {}} + return {'metadata': meta, 'status': {'phase': phase, 'conditions': [{'type': 'Ready', 'status': ready}], + 'containerStatuses': [{'name': 'keycape', 'state': state}]}} + + +class PlanRestart(unittest.TestCase): + def test_sets_stamp_without_mutating_input(self): + before = deployment() + new = r.plan_restart(before, STAMP, '7') + self.assertEqual(new['spec']['template']['metadata']['annotations'][r.ANNOTATION], STAMP) + self.assertNotIn('metadata', before['spec']['template']) + + def test_refusals(self): + cases = [(deployment(), STAMP, '6'), # stale revision + (deployment(replicas=2), STAMP, '7'), # not single replica + (deployment(strategy='RollingUpdate'), STAMP, '7'), + (deployment(stamp=STAMP), STAMP, '7'), # already applied + (deployment(), '', '7')] + wrong = deployment() + wrong['metadata']['namespace'] = 'default' + cases.append((wrong, STAMP, '7')) + for dep, stamp, rv in cases: + with self.subTest(dep=dep['metadata'], stamp=stamp, rv=rv), self.assertRaises(r.Refused): + r.plan_restart(dep, stamp, rv) + + +class Assess(unittest.TestCase): + def check(self, dep, pods, discovery=DISCOVERY): + return r.assess(dep, pods, discovery, stamp=STAMP, expected_issuer=ISSUER) + + def test_ready(self): + self.assertEqual(self.check(deployment(STAMP), [pod()]), ('ready', [])) + + def test_pending_states(self): + for dep, pods, disc in [(deployment(STAMP, observed=1), [pod()], DISCOVERY), + (deployment(STAMP, counts=0), [pod()], DISCOVERY), + (deployment(STAMP), [pod(ready='False')], DISCOVERY), + (deployment(STAMP), [pod(stamp='old'), pod()], DISCOVERY), + (deployment(STAMP), [], DISCOVERY), + (deployment(STAMP), [pod()], None)]: + with self.subTest(pods=len(pods), disc=disc is not None): + self.assertEqual(self.check(dep, pods, disc)[0], 'pending') + + def test_terminating_old_pod_ignored(self): + self.assertEqual(self.check(deployment(STAMP), [pod(stamp='old', deleting=True), pod()])[0], 'ready') + + def test_failures(self): + incomplete = dict(DISCOVERY, jwks_uri='') + for dep, pods, disc in [(deployment(STAMP), [pod(waiting='CrashLoopBackOff', ready='False')], None), + (deployment(STAMP), [pod(waiting='CreateContainerConfigError')], None), + (deployment(STAMP), [pod()], dict(DISCOVERY, issuer='https://evil')), + (deployment(STAMP), [pod()], incomplete), + (deployment('someone-else'), [pod()], DISCOVERY)]: + with self.subTest(disc=disc): + self.assertEqual(self.check(dep, pods, disc)[0], 'failed') + + +class WaitReady(unittest.TestCase): + def run_wait(self, states, attempts=5): + seq, slept = iter(states), [] + receipt = r.wait_ready(lambda: next(seq), slept.append, stamp=STAMP, + expected_issuer=ISSUER, attempts=attempts, interval=2) + return receipt, slept + + def test_becomes_ready(self): + pending = (deployment(STAMP, counts=0), [pod(ready='False')], None) + receipt, slept = self.run_wait([pending, pending, (deployment(STAMP), [pod()], DISCOVERY)]) + self.assertEqual((receipt['verdict'], receipt['attempts'], slept), ('ready', 3, [2, 2])) + self.assertFalse(receipt['owner_reconciliation_required']) + self.assertFalse(receipt['values_emitted']) + + def test_failure_stops_without_retry(self): + crash = (deployment(STAMP), [pod(waiting='CrashLoopBackOff')], None) + receipt, slept = self.run_wait([crash]) + self.assertEqual((receipt['verdict'], slept), ('failed', [])) + self.assertTrue(receipt['owner_reconciliation_required']) + + def test_deadline_is_uncertain_not_success(self): + pending = (deployment(STAMP, counts=0), [pod()], DISCOVERY) + receipt, slept = self.run_wait([pending] * 3, attempts=3) + self.assertEqual((receipt['verdict'], len(slept)), ('uncertain', 2)) + self.assertTrue(receipt['owner_reconciliation_required']) + + def test_unbounded_attempts_refused(self): + for attempts in (0, 121, 2.0): + with self.subTest(attempts=attempts), self.assertRaises(r.Refused): + self.run_wait([], attempts=attempts) + + def test_observe_error_propagates(self): + def boom(): + raise RuntimeError('readback failed') + with self.assertRaises(RuntimeError): + r.wait_ready(boom, lambda s: None, stamp=STAMP, expected_issuer=ISSUER, attempts=2, interval=1) + + +if __name__ == '__main__': + unittest.main() diff --git a/workplans/NK-WP-0048-security-case-types.md b/workplans/NK-WP-0048-security-case-types.md index 8bfddd2..81a5224 100644 --- a/workplans/NK-WP-0048-security-case-types.md +++ b/workplans/NK-WP-0048-security-case-types.md @@ -628,8 +628,9 @@ publication receipt: Custodian ```task id: NK-WP-0048-T12 -status: todo +status: wait priority: high +blocking_reason: "Restart/readiness source candidate delivered 2026-10-23 (sso-mfa/k8s/keycape/restart-readiness.py, 11 synthetic tests). Awaits the coordinator's 1.3 placement decision (ready_check vs NetKingdom step vs carried limit) at the T16 re-freeze, then owner review and descriptor wiring." state_hub_task_id: "7ce4eee3-cd47-55a9-bfa5-4f1835d4d6fd" ``` @@ -665,6 +666,22 @@ changes the 1.3 type and its descriptor digests, the coordinator decides with the T16 re-freeze (message to the-custodian, 2026-10-23) whether it lands in 1.3 or becomes a carried known limit. Status stays todo. +2026-10-23 source candidate delivered: `sso-mfa/k8s/keycape/restart-readiness.py` +with `test_restart_readiness.py` (11 synthetic tests pass). It is +injection-only. `plan_restart` stamps the pod template of the single-replica +Recreate issuer, with CAS on the observed revision. It refuses a wrong target, a +non-Recreate strategy, more than one replica, a stale revision or a re-applied +stamp. `assess` decides ready/pending/failed from rollout status, exactly one +restarted Ready pod, and issuer discovery (issuer equality plus endpoints). It +reads pods for status only. A crash loop, config error, issuer mismatch or a +foreign template change is `failed` and is never retried. `wait_ready` is +bounded (at most 120 attempts). A deadline is `uncertain`; both failed and +uncertain require owner reconciliation. The module is not wired into the type, +descriptor or kernel; that waits on the coordinator's placement decision. +Separately: `test_adapter_source_closure.py` already fails 7 probes in this +workstation environment, with or without this change, because the probe +subprocess cannot import `warden`. + ## T13 — Prepare the fixed registration adapter source candidate ```task