feat: accept KeyCape approval clients with tested recovery

Assistant: codex
Assistant-Model: gpt-5.6-luna
Assistant-Session: 01a07ff8-19d0-7820-b4d0-1353833cb7fc

sso-mfa/k8s/keycape/approval-clients-rollout.py

@@ -4,6 +4,7 @@ from __future__ import annotations
 import argparse
 import base64
 import copy
+import hashlib
 import importlib.util
 import json
 import os
@@ -26,11 +27,18 @@ spec = importlib.util.spec_from_file_location('pin', Path(__file__).with_name('o
 pin = importlib.util.module_from_spec(spec); spec.loader.exec_module(pin)
 ISSUER = 'https://kc.coulomb.social'
 IMAGE = 'forgejo.coulomb.social/coulomb/key-cape@sha256:7ff54c54e63ee172ae9e6e7fd2da96e427352f712343d74626ee6fe0f6f82611'
+VERIFIER = Path('/home/worsch/.local/share/key-cape/verified-bin/dcebd46/keycape')
+VERIFIER_SHA256 = '4bf93bbe9afe0bf2e21d51c03373a7e2b4be6864586bcc2d472eeb9abc913d92'
 PRIOR_IMAGE = 'forgejo.coulomb.social/coulomb/key-cape:main-153258b'
 IDS = ('secrets-engine-approval', 'approval-engine-operator')
 SECRET_NAMES = ('keycape-secrets-engine-approval-client', 'keycape-approval-engine-operator-client')
 
 
+def verify_artifact():
+    require(VERIFIER.is_file() and not VERIFIER.is_symlink()
+            and hashlib.sha256(VERIFIER.read_bytes()).hexdigest() == VERIFIER_SHA256, 'pinned_verifier_artifact_required')
+
+
 def registrations():
     source = yaml.safe_load((KEYCAPE / 'config/service-clients.example.yaml').read_text())['clients']
     result = [next(c for c in source if c['clientId'] == name) for name in IDS]
@@ -115,7 +123,9 @@ def ready(kube, image, timeout=150):
             podlist = data(command(kube + ['-n', 'sso', 'get', 'pods', '-l', 'app.kubernetes.io/name=keycape', '-o', 'json']))['items']
             if len(podlist) == 1 and not podlist[0]['metadata'].get('deletionTimestamp'):
                 containers = podlist[0].get('status', {}).get('containerStatuses', [])
-                if any(c['name'] == 'keycape' and c['ready'] and c['image'] == image for c in containers):
+                if any(c['name'] == 'keycape' and c['ready'] and
+                       (c.get('imageID', '').removeprefix('docker-pullable://') == image
+                        if '@sha256:' in image else c.get('image') == image) for c in containers):
                     return {'deployment_uid': dep['metadata']['uid'], 'generation': dep['metadata']['generation'],
                             'pod_uid': podlist[0]['metadata']['uid'], 'single_ready_replica': True}
         time.sleep(3)
@@ -141,7 +151,10 @@ def http(path, fields=None, credentials=None):
         with opener.open(request, timeout=20) as response:
             return response.status, json.load(response)
     except urllib.error.HTTPError as error:
-        return error.code, json.load(error)
+        try:
+            return error.code, json.load(error)
+        except (ValueError, TypeError):
+            raise LaneError('http_' + str(error.code) + '_non_json_error') from None
 
 
 def denied(result, status, feature):
@@ -150,53 +163,79 @@ def denied(result, status, feature):
             and body.get('feature') == feature, 'keycape_denial_inconclusive')
 
 
+
+def verified_claims(token, public_key):
+    # Match authclient/client.go: tolerate at most 30s future iat, but no
+    # extension to expiry or not-before. PyJWT's global leeway would extend both.
+    claims = jwt.decode(token, public_key, algorithms=['RS256'], issuer=ISSUER, audience='approval-engine',
+                        options={'verify_iat': False, 'require': ['exp', 'iat', 'sub', 'iss', 'aud']})
+    require(type(claims['iat']) is int and type(claims['exp']) is int
+            and claims['iat'] <= int(time.time()) + 30 and claims['iat'] < claims['exp'], 'issued_at_binding_failed')
+    return claims
+
+
 def acceptance(kube, receipt):
+    receipt['acceptance_phase'] = 'discovery'
     code, discovery = http('/.well-known/openid-configuration')
     require(code == 200 and discovery['issuer'] == ISSUER
             and discovery['token_endpoint'] == ISSUER + '/token'
             and discovery['jwks_uri'].startswith(ISSUER + '/'), 'discovery_binding_mismatch')
+    receipt['acceptance_phase'] = 'jwks'
     code, jwks = http(discovery['jwks_uri'].removeprefix(ISSUER))
     require(code == 200, 'jwks_failed')
     receipt['clients'] = []
     for client, secret_name in zip(registrations(), SECRET_NAMES):
+        receipt['acceptance_client'] = client['clientId']
+        receipt['acceptance_phase'] = 'verifier_secret_read'
         secret = get(kube, 'secret', secret_name)
         credential = base64.b64decode(secret['data']['client-secret'], validate=True).decode()
         scopes = ' '.join(client['allowedScopes'])
+        receipt['acceptance_phase'] = 'token_exchange'
         result, response = http('/token', {'grant_type': 'client_credentials', 'scope': scopes}, (client['clientId'], credential))
         require(result == 200 and response.get('token_type') == 'Bearer' and response.get('expires_in') == 900, 'service_issuance_failed')
+        receipt['acceptance_phase'] = 'signature_and_claims'
         token = response['access_token']; header = jwt.get_unverified_header(token)
         keys = [key for key in jwks['keys'] if key['kid'] == header.get('kid')]
         require(header.get('alg') == 'RS256' and len(keys) == 1, 'signing_key_selection_failed')
-        claims = jwt.decode(token, jwt.PyJWK.from_dict(keys[0]).key, algorithms=['RS256'],
-                            issuer=ISSUER, audience='approval-engine',
-                            options={'require': ['exp', 'iat', 'sub', 'iss', 'aud']})
+        claims = verified_claims(token, jwt.PyJWK.from_dict(keys[0]).key)
         require(claims['sub'] == client['serviceSubject'] and claims['tenant'] == 'tenant:platform'
+                and claims['aud'] == 'approval-engine'
                 and claims['roles'] == client['roles'] and claims['exp'] - claims['iat'] == 900
                 and claims['principal_type'] == 'service'
                 and set(claims['scope'].split()) == set(client['allowedScopes']), 'exact_claims_mismatch')
         excessive = 'approval:approve' if client['clientId'] == IDS[0] else 'approval:consume'
+        receipt['acceptance_phase'] = 'excess_scope_denial'
         denied(http('/token', {'grant_type': 'client_credentials', 'scope': excessive},
                     (client['clientId'], credential)), 400, 'scope')
+        receipt['acceptance_phase'] = 'wrong_secret_denial'
         denied(http('/token', {'grant_type': 'client_credentials', 'scope': scopes},
                     (client['clientId'], secrets.token_urlsafe(48))), 401, 'Authorization')
-        args = kube + ['-n', 'sso', 'exec', 'deployment/keycape', '--', '/keycape', 'verify-client',
+        args = [str(VERIFIER), 'verify-client',
             '-issuer', ISSUER, '-client-id', client['clientId'], '-audience', 'approval-engine', '-scope', scopes,
             '-secret-env', client['secretRef'].removeprefix('env:'), '-expect-subject', client['serviceSubject'],
             '-expect-tenant', 'tenant:platform', '-expect-roles', ','.join(client['roles']), '-deny-scope', excessive]
-        command(args)
+        receipt['acceptance_phase'] = 'pinned_artifact_verifier'
+        verify_artifact()
+        verifier_env = os.environ.copy()
+        verifier_env[client['secretRef'].removeprefix('env:')] = credential
+        command(args, env=verifier_env)
         receipt['clients'].append({'client_id': client['clientId'], 'live_jwks_signature_verified': True,
-            'exact_claims_verified': True, 'lifetime_seconds': 900, 'excess_scope_denied': True,
-            'wrong_secret_denied': True, 'pod_verify_client_passed': True,
+            'exact_claims_verified': True, 'lifetime_seconds': 900, 'maximum_future_iat_seconds': 30, 'expiry_leeway_seconds': 0, 'excess_scope_denied': True,
+            'wrong_secret_denied': True, 'pinned_artifact_verifier_passed': True,
+            'verifier_location': 'attended owner process', 'verifier_sha256': VERIFIER_SHA256,
             'real_predecessor_rotation_tested': False, 'observed_wall_clock_expiry': False})
+    receipt['acceptance_phase'] = 'human_consume_denial'
     query = urllib.parse.urlencode({'client_id': 'openbao-admin', 'response_type': 'code',
         'redirect_uri': 'http://localhost:8250/oidc/callback', 'scope': 'openid approval:consume',
         'code_challenge_method': 'S256', 'code_challenge': 'A' * 43, 'state': secrets.token_urlsafe(32)})
     denied(http('/authorize?' + query), 400, 'scope')
     receipt['human_client_consume_denied'] = True
+    receipt['acceptance_phase'] = 'passed'
 
 
 def rollout(kube, receipt, recovery_path):
     assert_cluster(kube)
+    verify_artifact()
     before = get(kube, 'secret', 'keycape-config')
     deployment = get(kube, 'deployment', 'keycape')
     require(before['metadata']['uid'] == '2e94519d-1550-41c7-9701-2efe47fe1fd3'
@@ -223,7 +262,8 @@ def rollout(kube, receipt, recovery_path):
         receipt.update(status='service_acceptance_passed_pending_fresh_human_login', image=IMAGE,
             config_resource_version=after['metadata']['resourceVersion'], signing_key_unchanged=True,
             unrelated_config_bytes_preserved=True, existing_human_login_after=False)
-    except Exception:
+    except Exception as failure:
+        receipt['failure_class'] = type(failure).__name__
         # Read after uncertain API outcomes too; never assume a timeout means no write.
         now = get(kube, 'secret', 'keycape-config')
         require(now['metadata']['uid'] == before['metadata']['uid'], 'rollback_config_identity_drift')

sso-mfa/k8s/keycape/deployment.yaml

@@ -1,8 +1,8 @@
 # Deployment + Service — KeyCape (namespace: sso)
 #
-# KeyCape is the OIDC orchestration layer. It is stateless: all persistent
-# state lives in Authelia (session), LLDAP (users), and privacyIDEA (MFA tokens).
-# No PVC is required.
+# KeyCape orchestrates OIDC. Pending logins and authorization codes are process-local;
+# use one replica with Recreate during replacement. Persistent identity state remains
+# in Authelia, LLDAP and privacyIDEA. No PVC is required.
 #
 # Configuration is stored entirely in the keycape-config Secret, which holds
 # a complete config.yaml and the RSA private key used to sign OIDC tokens
@@ -33,7 +33,7 @@ spec:
     matchLabels:
       app.kubernetes.io/name: keycape
   strategy:
-    type: RollingUpdate   # stateless — safe to roll
+    type: Recreate        # one issuer instance; process-local login/code state
   template:
     metadata:
       labels:
@@ -50,7 +50,7 @@ spec:
         - name: keycape
           # Image published to the self-hosted Forgejo OCI registry (KEY-WP-0002).
           # KEY-WP-0012: canonical OIDC subject resolution for /userinfo.
-          image: forgejo.coulomb.social/coulomb/key-cape:main-153258b
+          image: forgejo.coulomb.social/coulomb/key-cape@sha256:7ff54c54e63ee172ae9e6e7fd2da96e427352f712343d74626ee6fe0f6f82611
           imagePullPolicy: IfNotPresent
 
           ports:
@@ -67,6 +67,17 @@ spec:
                   name: keycape-rapp-qonto-client
                   key: client-secret
 
+            - name: KEYCAPE_SECRETS_ENGINE_APPROVAL_CLIENT_SECRET
+              valueFrom:
+                secretKeyRef:
+                  name: keycape-secrets-engine-approval-client
+                  key: client-secret
+            - name: KEYCAPE_APPROVAL_ENGINE_OPERATOR_CLIENT_SECRET
+              valueFrom:
+                secretKeyRef:
+                  name: keycape-approval-engine-operator-client
+                  key: client-secret
+
           volumeMounts:
             # keycape-config Secret provides config.yaml and key.pem
             - name: config-secret
@@ -89,7 +100,7 @@ spec:
             failureThreshold: 3
           readinessProbe:
             httpGet:
-              path: /healthz
+              path: /readyz
               port: 8080
             initialDelaySeconds: 0
             periodSeconds: 10

sso-mfa/k8s/keycape/exercise-approval-clients.py

@@ -0,0 +1,92 @@
+import base64, importlib.util, json, os, secrets, socket, subprocess, tempfile, time
+from pathlib import Path
+from unittest.mock import patch
+from cryptography.hazmat.primitives import serialization
+from cryptography.hazmat.primitives.asymmetric import rsa
+import yaml
+import http.server, ssl, threading, urllib.request, urllib.error, datetime, ipaddress
+from cryptography import x509
+from cryptography.x509.oid import NameOID
+import argparse
+parser=argparse.ArgumentParser(description='Exercise the pinned image and verifier over loopback HTTPS using synthetic credentials only.')
+parser.add_argument('--receipt', required=True, type=Path)
+args=parser.parse_args()
+spec=importlib.util.spec_from_file_location('rollout',Path(__file__).with_name('approval-clients-rollout.py'))
+m=importlib.util.module_from_spec(spec); spec.loader.exec_module(m)
+m.verify_artifact()
+def call(args):
+    p=subprocess.run(args,capture_output=True,timeout=45)
+    if p.returncode: raise RuntimeError('contained_docker_command_failed')
+    return p
+sock=socket.socket(); sock.bind(('127.0.0.1',0)); port=sock.getsockname()[1]; sock.close()
+class Proxy(http.server.BaseHTTPRequestHandler):
+    def log_message(self, *args): pass
+    def handle_proxy(self):
+        body = self.rfile.read(int(self.headers.get('Content-Length', 0))) if self.command == 'POST' else None
+        req = urllib.request.Request(f'http://127.0.0.1:{port}'+self.path, data=body,
+              headers={k:self.headers[k] for k in ['Content-Type','Authorization'] if k in self.headers})
+        try:
+            response=urllib.request.urlopen(req,timeout=20)
+        except urllib.error.HTTPError as error: response=error
+        with response:
+            self.send_response(response.code)
+            self.send_header('Content-Type', response.headers.get('Content-Type','application/json'))
+            self.end_headers(); self.wfile.write(response.read())
+    do_GET=handle_proxy
+    do_POST=handle_proxy
+server=http.server.ThreadingHTTPServer(('127.0.0.1',0),Proxy)
+https_port=server.server_address[1]
+name='keycape-approval-exercise-'+secrets.token_hex(5)
+with tempfile.TemporaryDirectory(prefix='keycape-private-exercise-') as temp:
+    root=Path(temp)
+    key=rsa.generate_private_key(public_exponent=65537,key_size=2048)
+    (root/'key.pem').write_bytes(key.private_bytes(serialization.Encoding.PEM,serialization.PrivateFormat.PKCS8,serialization.NoEncryption()))
+    os.chmod(root/'key.pem',0o600)
+    name_attr=x509.Name([x509.NameAttribute(NameOID.COMMON_NAME,'KeyCape local exercise')])
+    cert=(x509.CertificateBuilder().subject_name(name_attr).issuer_name(name_attr).public_key(key.public_key())
+          .serial_number(x509.random_serial_number()).not_valid_before(datetime.datetime.now(datetime.timezone.utc)-datetime.timedelta(minutes=1))
+          .not_valid_after(datetime.datetime.now(datetime.timezone.utc)+datetime.timedelta(hours=1))
+          .add_extension(x509.SubjectAlternativeName([x509.IPAddress(ipaddress.ip_address('127.0.0.1'))]),False)
+          .add_extension(x509.BasicConstraints(ca=True,path_length=None),True))
+    from cryptography.hazmat.primitives import hashes
+    (root/'cert.pem').write_bytes(cert.sign(key,hashes.SHA256()).public_bytes(serialization.Encoding.PEM))
+    context=ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER); context.load_cert_chain(root/'cert.pem',root/'key.pem')
+    server.socket=context.wrap_socket(server.socket,server_side=True)
+    threading.Thread(target=server.serve_forever,daemon=True).start()
+    config=yaml.safe_load((m.KEYCAPE/'config/dev-config.yaml').read_text())
+    config.update(issuer=f'https://127.0.0.1:{https_port}')
+    config['authelia']['issuer']='https://auth.coulomb.social'
+    config['clients'].extend(m.registrations())
+    (root/'config.yaml').write_text(yaml.safe_dump(config,sort_keys=False))
+    os.chmod(root/'config.yaml',0o600)
+    values={c['clientId']:secrets.token_urlsafe(48) for c in m.registrations()}
+    (root/'env').write_text('KEYCAPE_CONFIG=/etc/keycape/config.yaml\n'+''.join(c['secretRef'].removeprefix('env:')+'='+values[c['clientId']]+'\n' for c in m.registrations()))
+    os.chmod(root/'env',0o600)
+    created=False
+    try:
+        call(['docker','run','-d','--rm','--name',name,'--user',str(os.getuid()),'--publish',f'127.0.0.1:{port}:8080',
+              '--env-file',str(root/'env'),'--mount','type=bind,source='+temp+',target=/etc/keycape,readonly',m.IMAGE])
+        created=True
+        with patch.object(m,'ISSUER',config['issuer']), patch.dict(os.environ,{'SSL_CERT_FILE':str(root/'cert.pem')}):
+            for _ in range(30):
+                try:
+                    status,_=m.http('/.well-known/openid-configuration')
+                    if status==200: break
+                except Exception: pass
+                time.sleep(.5)
+            else: raise RuntimeError('scratch_keycape_not_ready')
+            def fake_get(kube,kind,name):
+                index=m.SECRET_NAMES.index(name)
+                return {'data': {'client-secret':base64.b64encode(values[m.IDS[index]].encode()).decode()}}
+            receipt={}
+            with patch.object(m,'get',side_effect=fake_get):
+                m.acceptance([],receipt)
+            receipt.update(target='disposable pinned KeyCape image; synthetic keys and clients only',
+                           pinned_native_cli_verification='passed against loopback HTTPS with synthetic credentials',cleanup_complete=False)
+    finally:
+        if created: call(['docker','stop','--time=5',name])
+        server.shutdown(); server.server_close()
+    receipt['cleanup_complete']=True
+    fd=os.open(args.receipt,os.O_WRONLY|os.O_CREAT|os.O_EXCL,0o600)
+    with os.fdopen(fd,'w') as out: out.write(json.dumps(receipt,indent=2)+'\n')
+print('Pinned-image scratch acceptance passed; synthetic credentials removed.')

sso-mfa/k8s/keycape/test_approval_clients_rollout.py

@@ -83,7 +83,7 @@ class RolloutTests(unittest.TestCase):
                 changed['metadata']['resourceVersion'] = str(int(changed['metadata']['resourceVersion']) + 1)
                 state[kind] = changed
             return copy.deepcopy(changed)
-        with tempfile.TemporaryDirectory() as directory, patch.object(m, 'assert_cluster'), \
+        with tempfile.TemporaryDirectory() as directory, patch.object(m, 'assert_cluster'), patch.object(m, 'verify_artifact'), \
                 patch.object(m, 'get', side_effect=lambda kube, kind, name: copy.deepcopy(state[kind])), \
                 patch.object(m, 'patch_object', side_effect=fake_patch), patch.object(m, 'ready', return_value={}), \
                 patch.object(m, 'acceptance', side_effect=m.LaneError('synthetic_acceptance_failure')):
@@ -95,6 +95,29 @@ class RolloutTests(unittest.TestCase):
             self.assertEqual(state['deployment']['spec'], original['deployment']['spec'])
             self.assertEqual((Path(directory) / 'recovery.json').stat().st_mode & 0o777, 0o600)
 
+    def test_ready_matches_manifest_imageid_not_runtime_config_id(self):
+        dep = deployment(); dep['metadata']['generation'] = 30
+        dep['status'] = {'observedGeneration': 30, 'updatedReplicas': 1, 'readyReplicas': 1, 'availableReplicas': 1, 'replicas': 1}
+        pod = {'metadata': {'uid': 'fixture-pod'}, 'status': {'containerStatuses': [
+            {'name': 'keycape', 'ready': True, 'image': 'sha256:runtime-config-id', 'imageID': m.IMAGE}]}}
+        from types import SimpleNamespace
+        with patch.object(m, 'get', return_value=dep), patch.object(m, 'command', return_value=SimpleNamespace(stdout=json.dumps({'items': [pod]}).encode())):
+            self.assertTrue(m.ready([], m.IMAGE, timeout=1)['single_ready_replica'])
+
+    def test_iat_skew_matches_native_contract_without_extending_expiry(self):
+        from cryptography.hazmat.primitives.asymmetric import rsa
+        key = rsa.generate_private_key(public_exponent=65537, key_size=2048)
+        now = int(m.time.time())
+        claims = {'iss': m.ISSUER, 'aud': 'approval-engine', 'sub': 'synthetic-service', 'iat': now + 2, 'exp': now + 902}
+        encoded = m.jwt.encode(claims, key, algorithm='RS256')
+        self.assertEqual(m.verified_claims(encoded, key.public_key())['iat'], now + 2)
+        future = m.jwt.encode(dict(claims, iat=now+60), key, algorithm='RS256')
+        with self.assertRaisesRegex(m.LaneError, 'issued_at_binding_failed'):
+            m.verified_claims(future, key.public_key())
+        expired = m.jwt.encode(dict(claims, iat=now-900, exp=now-1), key, algorithm='RS256')
+        with self.assertRaises(m.jwt.ExpiredSignatureError):
+            m.verified_claims(expired, key.public_key())
+
     def test_unrelated_refusal_never_passes(self):
         for status, body in [(500, {}), (400, {'error': 'invalid_profile_usage', 'feature': 'client_id'}),
                               (401, {'error': 'invalid_profile_usage', 'feature': 'scope'})]: