#!/usr/bin/env python3 """Contained KeyCape approval-client config/image cutover and acceptance.""" from __future__ import annotations import argparse import base64 import copy import hashlib import importlib.util import json import os from pathlib import Path import secrets import sys import time import urllib.error import urllib.parse import urllib.request import jwt import yaml PLATFORM = Path('/home/worsch/railiance-platform') KEYCAPE = Path('/home/worsch/key-cape') sys.path.insert(0, str(PLATFORM / 'scripts')) from state_hub_preflight_lane import LaneError, assert_cluster, bao, command, data from keycape_approval_custody import require spec = importlib.util.spec_from_file_location('pin', Path(__file__).with_name('openbao-client-config.py')) pin = importlib.util.module_from_spec(spec); spec.loader.exec_module(pin) ISSUER = 'https://kc.coulomb.social' IMAGE = 'forgejo.coulomb.social/coulomb/key-cape@sha256:7ff54c54e63ee172ae9e6e7fd2da96e427352f712343d74626ee6fe0f6f82611' VERIFIER = Path('/home/worsch/.local/share/key-cape/verified-bin/dcebd46/keycape') VERIFIER_SHA256 = '4bf93bbe9afe0bf2e21d51c03373a7e2b4be6864586bcc2d472eeb9abc913d92' PRIOR_IMAGE = 'forgejo.coulomb.social/coulomb/key-cape:main-153258b' IDS = ('secrets-engine-approval', 'approval-engine-operator') SECRET_NAMES = ('keycape-secrets-engine-approval-client', 'keycape-approval-engine-operator-client') def verify_artifact(): require(VERIFIER.is_file() and not VERIFIER.is_symlink() and hashlib.sha256(VERIFIER.read_bytes()).hexdigest() == VERIFIER_SHA256, 'pinned_verifier_artifact_required') def registrations(): source = yaml.safe_load((KEYCAPE / 'config/service-clients.example.yaml').read_text())['clients'] result = [next(c for c in source if c['clientId'] == name) for name in IDS] for i, c in enumerate(result): require(c['audience'] == 'approval-engine' and c['tenant'] == 'tenant:platform' and c['grantTypes'] == ['client_credentials'] and c['clientType'] == 'confidential' and c['tokenLifetime'] == '15m', 'registration_contract_drift') scopes = ['approval:read', 'approval:consume'] if i == 0 else [ 'approval:create', 'approval:read', 'approval:approve', 'approval:revoke', 'approval:supersede', 'approval:observe', 'approval:emit'] require(c['allowedScopes'] == scopes and c['roles'] == (["secrets-engine"] if i == 0 else ['approval-operator']) and c['serviceSubject'] == ('service:secrets-engine' if i == 0 else 'service:approval-engine-operator') and c['secretRef'] == ('env:KEYCAPE_SECRETS_ENGINE_APPROVAL_CLIENT_SECRET' if i == 0 else 'env:KEYCAPE_APPROVAL_ENGINE_OPERATOR_CLIENT_SECRET'), 'registration_authority_drift') return result def replacement(secret, clients): raw, config, _ = pin.issuer_document(secret) require(config['authelia'].get('issuer') == pin.VERIFIED_UPSTREAM_ISSUER, 'verified_issuer_pin_required') require(isinstance(config.get('clients'), list) and config['clients'], 'client_sequence_required') existing = [c['clientId'] for c in config['clients']] require(len(existing) == len(set(existing)) and not set(existing).intersection(IDS), 'existing_client_registration_requires_reconciliation') root = yaml.compose(raw) node = next(v for k, v in root.value if k.value == 'clients') require(isinstance(node, yaml.nodes.SequenceNode) and not node.flow_style, 'block_client_sequence_required') index = node.end_mark.index # Insert at the start of the next top-level line, preserving every old byte. line_start = raw.rfind('\n', 0, index) + 1 if not raw[line_start:index].strip(): index = line_start indent = ' ' * node.start_mark.column fragment = yaml.safe_dump(clients, sort_keys=False, width=120) fragment = ''.join(indent + line if line.strip() else line for line in fragment.splitlines(True)) addition = ('' if index == 0 or raw[index - 1] == '\n' else '\n') + fragment updated = raw[:index] + addition + raw[index:] expected = copy.deepcopy(config); expected['clients'].extend(clients) require(yaml.load(updated, Loader=pin.UniqueLoader) == expected, 'unrelated_configuration_changed') return base64.b64encode(updated.encode()).decode() def candidate_spec(deployment): result = copy.deepcopy(deployment['spec']) require(result.get('replicas', 1) == 1, 'single_replica_required') container = next(c for c in result['template']['spec']['containers'] if c['name'] == 'keycape') require(container['image'] == PRIOR_IMAGE, 'prior_image_drift') patch = yaml.safe_load((KEYCAPE / 'docs/approval-clients-deployment.patch.yaml').read_text())['spec'] approved = patch['template']['spec']['containers'][0] require(approved['image'] == IMAGE and patch['strategy']['type'] == 'Recreate', 'candidate_source_drift') env = container.setdefault('env', []) require(not {x['name'] for x in env}.intersection(x['name'] for x in approved['env']), 'existing_candidate_env') for item, client, secret_name in zip(approved['env'], registrations(), SECRET_NAMES): require(item == {'name': client['secretRef'].removeprefix('env:'), 'valueFrom': { 'secretKeyRef': {'name': secret_name, 'key': 'client-secret'}}}, 'env_delivery_contract_drift') env.extend(approved['env']); container['image'] = IMAGE container['readinessProbe']['httpGet'] = {'path': '/readyz', 'port': 8080} result['strategy'] = {'type': 'Recreate'} return result def patch_object(kube, kind, obj, path, value, dry=False): patch = [{'op': 'test', 'path': '/metadata/uid', 'value': obj['metadata']['uid']}, {'op': 'test', 'path': '/metadata/resourceVersion', 'value': obj['metadata']['resourceVersion']}, {'op': 'replace', 'path': path, 'value': value}] args = kube + ['-n', 'sso', 'patch', kind, obj['metadata']['name'], '--type=json', '--patch-file=/dev/stdin', '-o', 'json'] if dry: args += ['--dry-run=server'] return data(command(args, payload=patch)) def get(kube, kind, name): return data(command(kube + ['-n', 'sso', 'get', kind, name, '-o', 'json'])) def ready(kube, image, timeout=150): deadline = time.monotonic() + timeout while time.monotonic() < deadline: dep = get(kube, 'deployment', 'keycape') status = dep.get('status', {}) if (status.get('observedGeneration') == dep['metadata']['generation'] and status.get('updatedReplicas') == status.get('readyReplicas') == status.get('availableReplicas') == 1 and status.get('replicas') == 1): podlist = data(command(kube + ['-n', 'sso', 'get', 'pods', '-l', 'app.kubernetes.io/name=keycape', '-o', 'json']))['items'] if len(podlist) == 1 and not podlist[0]['metadata'].get('deletionTimestamp'): containers = podlist[0].get('status', {}).get('containerStatuses', []) if any(c['name'] == 'keycape' and c['ready'] and (c.get('imageID', '').removeprefix('docker-pullable://') == image if '@sha256:' in image else c.get('image') == image) for c in containers): return {'deployment_uid': dep['metadata']['uid'], 'generation': dep['metadata']['generation'], 'pod_uid': podlist[0]['metadata']['uid'], 'single_ready_replica': True} time.sleep(3) raise LaneError('keycape_readiness_timeout') class NoRedirect(urllib.request.HTTPRedirectHandler): def redirect_request(self, *args, **kwargs): return None def http(path, fields=None, credentials=None): headers = {'Accept': 'application/json'} if credentials: headers['Authorization'] = 'Basic ' + base64.b64encode(':'.join(credentials).encode()).decode() body = None if fields is not None: body = urllib.parse.urlencode(fields).encode() headers['Content-Type'] = 'application/x-www-form-urlencoded' request = urllib.request.Request(ISSUER + path, data=body, headers=headers) opener = urllib.request.build_opener(NoRedirect()) try: with opener.open(request, timeout=20) as response: return response.status, json.load(response) except urllib.error.HTTPError as error: try: return error.code, json.load(error) except (ValueError, TypeError): raise LaneError('http_' + str(error.code) + '_non_json_error') from None def denied(result, status, feature): code, body = result require(code == status and body.get('error') == 'invalid_profile_usage' and body.get('feature') == feature, 'keycape_denial_inconclusive') def verified_claims(token, public_key): # Match authclient/client.go: tolerate at most 30s future iat, but no # extension to expiry or not-before. PyJWT's global leeway would extend both. claims = jwt.decode(token, public_key, algorithms=['RS256'], issuer=ISSUER, audience='approval-engine', options={'verify_iat': False, 'require': ['exp', 'iat', 'sub', 'iss', 'aud']}) require(type(claims['iat']) is int and type(claims['exp']) is int and claims['iat'] <= int(time.time()) + 30 and claims['iat'] < claims['exp'], 'issued_at_binding_failed') return claims def acceptance(kube, receipt): receipt['acceptance_phase'] = 'discovery' code, discovery = http('/.well-known/openid-configuration') require(code == 200 and discovery['issuer'] == ISSUER and discovery['token_endpoint'] == ISSUER + '/token' and discovery['jwks_uri'].startswith(ISSUER + '/'), 'discovery_binding_mismatch') receipt['acceptance_phase'] = 'jwks' code, jwks = http(discovery['jwks_uri'].removeprefix(ISSUER)) require(code == 200, 'jwks_failed') receipt['clients'] = [] for client, secret_name in zip(registrations(), SECRET_NAMES): receipt['acceptance_client'] = client['clientId'] receipt['acceptance_phase'] = 'verifier_secret_read' secret = get(kube, 'secret', secret_name) credential = base64.b64decode(secret['data']['client-secret'], validate=True).decode() scopes = ' '.join(client['allowedScopes']) receipt['acceptance_phase'] = 'token_exchange' result, response = http('/token', {'grant_type': 'client_credentials', 'scope': scopes}, (client['clientId'], credential)) require(result == 200 and response.get('token_type') == 'Bearer' and response.get('expires_in') == 900, 'service_issuance_failed') receipt['acceptance_phase'] = 'signature_and_claims' token = response['access_token']; header = jwt.get_unverified_header(token) keys = [key for key in jwks['keys'] if key['kid'] == header.get('kid')] require(header.get('alg') == 'RS256' and len(keys) == 1, 'signing_key_selection_failed') claims = verified_claims(token, jwt.PyJWK.from_dict(keys[0]).key) require(claims['sub'] == client['serviceSubject'] and claims['tenant'] == 'tenant:platform' and claims['aud'] == 'approval-engine' and claims['roles'] == client['roles'] and claims['exp'] - claims['iat'] == 900 and claims['principal_type'] == 'service' and set(claims['scope'].split()) == set(client['allowedScopes']), 'exact_claims_mismatch') excessive = 'approval:approve' if client['clientId'] == IDS[0] else 'approval:consume' receipt['acceptance_phase'] = 'excess_scope_denial' denied(http('/token', {'grant_type': 'client_credentials', 'scope': excessive}, (client['clientId'], credential)), 400, 'scope') receipt['acceptance_phase'] = 'wrong_secret_denial' denied(http('/token', {'grant_type': 'client_credentials', 'scope': scopes}, (client['clientId'], secrets.token_urlsafe(48))), 401, 'Authorization') args = [str(VERIFIER), 'verify-client', '-issuer', ISSUER, '-client-id', client['clientId'], '-audience', 'approval-engine', '-scope', scopes, '-secret-env', client['secretRef'].removeprefix('env:'), '-expect-subject', client['serviceSubject'], '-expect-tenant', 'tenant:platform', '-expect-roles', ','.join(client['roles']), '-deny-scope', excessive] receipt['acceptance_phase'] = 'pinned_artifact_verifier' verify_artifact() verifier_env = os.environ.copy() verifier_env[client['secretRef'].removeprefix('env:')] = credential command(args, env=verifier_env) receipt['clients'].append({'client_id': client['clientId'], 'live_jwks_signature_verified': True, 'exact_claims_verified': True, 'lifetime_seconds': 900, 'maximum_future_iat_seconds': 30, 'expiry_leeway_seconds': 0, 'excess_scope_denied': True, 'wrong_secret_denied': True, 'pinned_artifact_verifier_passed': True, 'verifier_location': 'attended owner process', 'verifier_sha256': VERIFIER_SHA256, 'real_predecessor_rotation_tested': False, 'observed_wall_clock_expiry': False}) receipt['acceptance_phase'] = 'human_consume_denial' query = urllib.parse.urlencode({'client_id': 'openbao-admin', 'response_type': 'code', 'redirect_uri': 'http://localhost:8250/oidc/callback', 'scope': 'openid approval:consume', 'code_challenge_method': 'S256', 'code_challenge': 'A' * 43, 'state': secrets.token_urlsafe(32)}) denied(http('/authorize?' + query), 400, 'scope') receipt['human_client_consume_denied'] = True receipt['acceptance_phase'] = 'passed' def rollout(kube, receipt, recovery_path): assert_cluster(kube) verify_artifact() before = get(kube, 'secret', 'keycape-config') deployment = get(kube, 'deployment', 'keycape') require(before['metadata']['uid'] == '2e94519d-1550-41c7-9701-2efe47fe1fd3' and deployment['metadata']['uid'] == '99ddd83c-cb3f-4847-bcf8-35f1aa87627f', 'owner_object_identity_mismatch') config = replacement(before, registrations()); candidate = candidate_spec(deployment) patch_object(kube, 'secret', before, '/data/config.yaml', config, dry=True) patch_object(kube, 'deployment', deployment, '/spec', candidate, dry=True) previous = {'deployment': deployment, 'secret': before} recovery_path.parent.mkdir(mode=0o700, parents=True, exist_ok=True) require(recovery_path.parent.stat().st_mode & 0o077 == 0, 'private_recovery_directory_required') fd = os.open(recovery_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) with os.fdopen(fd, 'w') as out: json.dump(previous, out) receipt['protected_recovery_retained'] = True changed = None; deployed = None try: changed = patch_object(kube, 'secret', before, '/data/config.yaml', config) require(changed['data'] == dict(before['data'], **{'config.yaml': config}), 'secret_readback_mismatch') deployed = patch_object(kube, 'deployment', deployment, '/spec', candidate) receipt.update(ready(kube, IMAGE)) after = get(kube, 'secret', 'keycape-config') require(after['data'] == changed['data'], 'configuration_changed_during_rollout') acceptance(kube, receipt) receipt.update(status='service_acceptance_passed_pending_fresh_human_login', image=IMAGE, config_resource_version=after['metadata']['resourceVersion'], signing_key_unchanged=True, unrelated_config_bytes_preserved=True, existing_human_login_after=False) except Exception as failure: receipt['failure_class'] = type(failure).__name__ # Read after uncertain API outcomes too; never assume a timeout means no write. now = get(kube, 'secret', 'keycape-config') require(now['metadata']['uid'] == before['metadata']['uid'], 'rollback_config_identity_drift') expected_data = dict(before['data'], **{'config.yaml': config}) require(now['data'] in (before['data'], expected_data), 'rollback_config_drift_requires_owner_reconcile') if now['data'] == expected_data: patch_object(kube, 'secret', now, '/data/config.yaml', before['data']['config.yaml']) now = get(kube, 'deployment', 'keycape') require(now['metadata']['uid'] == deployment['metadata']['uid'], 'rollback_deployment_identity_drift') expected_spec = deployed['spec'] if deployed else candidate require(now['spec'] in (deployment['spec'], expected_spec), 'rollback_deployment_drift_requires_owner_reconcile') if now['spec'] == expected_spec: patch_object(kube, 'deployment', now, '/spec', deployment['spec']) ready(kube, PRIOR_IMAGE) receipt['compatible_pair_restored'] = True raise def run(args, receipt): require(Path.home().parent.name == '.warden-attended-login' and not os.environ.get('BAO_TOKEN') and not os.environ.get('VAULT_TOKEN'), 'attended_warden_envelope_required') identity = data(bao(['token', 'lookup', '-format=json']))['data'] require('platform-admin' in identity['policies'] and 'root' not in identity['policies'], 'attended_platform_admin_required') kube = ['kubectl', '--kubeconfig', args.kubeconfig] if args.action == 'rollout': receipt['existing_human_login_before'] = True rollout(kube, receipt, Path(args.recovery)) else: assert_cluster(kube); receipt.update(ready(kube, IMAGE)); acceptance(kube, receipt) receipt.update(status='service_and_existing_human_login_acceptance_passed', existing_human_login_after=True) def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('action', choices=['rollout', 'verify-after-login']) parser.add_argument('--kubeconfig', required=True) parser.add_argument('--recovery') parser.add_argument('--receipt', required=True) args = parser.parse_args() receipt = {'schema': 'netkingdom.keycape-approval-rollout.v1', 'status': 'failed', 'values_emitted': False} fd = os.open(args.receipt, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) try: run(args, receipt) except Exception as exc: receipt['error'] = str(exc) if isinstance(exc, (LaneError, pin.IssuerPinError)) else 'contained_operation_failed' finally: with os.fdopen(fd, 'w') as out: json.dump(receipt, out, indent=2); out.write('\n') return 0 if receipt['status'] != 'failed' else 1 if __name__ == '__main__': raise SystemExit(main())