net-kingdom/sso-mfa/k8s/keycape/approval-clients-rollout.py

#!/usr/bin/env python3
"""Contained KeyCape approval-client config/image cutover and acceptance."""
from __future__ import annotations
import argparse
import base64
import copy
import hashlib
import importlib.util
import json
import os
from pathlib import Path
import secrets
import sys
import time
import urllib.error
import urllib.parse
import urllib.request
import jwt
import yaml

PLATFORM = Path('/home/worsch/railiance-platform')
KEYCAPE = Path('/home/worsch/key-cape')
sys.path.insert(0, str(PLATFORM / 'scripts'))
from state_hub_preflight_lane import LaneError, assert_cluster, bao, command, data
from keycape_approval_custody import require
spec = importlib.util.spec_from_file_location('pin', Path(__file__).with_name('openbao-client-config.py'))
pin = importlib.util.module_from_spec(spec); spec.loader.exec_module(pin)
ISSUER = 'https://kc.coulomb.social'
IMAGE = 'forgejo.coulomb.social/coulomb/key-cape@sha256:7ff54c54e63ee172ae9e6e7fd2da96e427352f712343d74626ee6fe0f6f82611'
VERIFIER = Path('/home/worsch/.local/share/key-cape/verified-bin/dcebd46/keycape')
VERIFIER_SHA256 = '4bf93bbe9afe0bf2e21d51c03373a7e2b4be6864586bcc2d472eeb9abc913d92'
PRIOR_IMAGE = 'forgejo.coulomb.social/coulomb/key-cape:main-153258b'
IDS = ('secrets-engine-approval', 'approval-engine-operator')
SECRET_NAMES = ('keycape-secrets-engine-approval-client', 'keycape-approval-engine-operator-client')


def verify_artifact():
    require(VERIFIER.is_file() and not VERIFIER.is_symlink()
            and hashlib.sha256(VERIFIER.read_bytes()).hexdigest() == VERIFIER_SHA256, 'pinned_verifier_artifact_required')


def registrations():
    source = yaml.safe_load((KEYCAPE / 'config/service-clients.example.yaml').read_text())['clients']
    result = [next(c for c in source if c['clientId'] == name) for name in IDS]
    for i, c in enumerate(result):
        require(c['audience'] == 'approval-engine' and c['tenant'] == 'tenant:platform'
                and c['grantTypes'] == ['client_credentials'] and c['clientType'] == 'confidential'
                and c['tokenLifetime'] == '15m', 'registration_contract_drift')
        scopes = ['approval:read', 'approval:consume'] if i == 0 else [
            'approval:create', 'approval:read', 'approval:approve', 'approval:revoke',
            'approval:supersede', 'approval:observe', 'approval:emit']
        require(c['allowedScopes'] == scopes and c['roles'] == (["secrets-engine"] if i == 0 else ['approval-operator'])
                and c['serviceSubject'] == ('service:secrets-engine' if i == 0 else 'service:approval-engine-operator')
                and c['secretRef'] == ('env:KEYCAPE_SECRETS_ENGINE_APPROVAL_CLIENT_SECRET' if i == 0 else 'env:KEYCAPE_APPROVAL_ENGINE_OPERATOR_CLIENT_SECRET'), 'registration_authority_drift')
    return result


def replacement(secret, clients):
    raw, config, _ = pin.issuer_document(secret)
    require(config['authelia'].get('issuer') == pin.VERIFIED_UPSTREAM_ISSUER, 'verified_issuer_pin_required')
    require(isinstance(config.get('clients'), list) and config['clients'], 'client_sequence_required')
    existing = [c['clientId'] for c in config['clients']]
    require(len(existing) == len(set(existing)) and not set(existing).intersection(IDS), 'existing_client_registration_requires_reconciliation')
    root = yaml.compose(raw)
    node = next(v for k, v in root.value if k.value == 'clients')
    require(isinstance(node, yaml.nodes.SequenceNode) and not node.flow_style, 'block_client_sequence_required')
    index = node.end_mark.index
    # Insert at the start of the next top-level line, preserving every old byte.
    line_start = raw.rfind('\n', 0, index) + 1
    if not raw[line_start:index].strip():
        index = line_start
    indent = ' ' * node.start_mark.column
    fragment = yaml.safe_dump(clients, sort_keys=False, width=120)
    fragment = ''.join(indent + line if line.strip() else line for line in fragment.splitlines(True))
    addition = ('' if index == 0 or raw[index - 1] == '\n' else '\n') + fragment
    updated = raw[:index] + addition + raw[index:]
    expected = copy.deepcopy(config); expected['clients'].extend(clients)
    require(yaml.load(updated, Loader=pin.UniqueLoader) == expected, 'unrelated_configuration_changed')
    return base64.b64encode(updated.encode()).decode()


def candidate_spec(deployment):
    result = copy.deepcopy(deployment['spec'])
    require(result.get('replicas', 1) == 1, 'single_replica_required')
    container = next(c for c in result['template']['spec']['containers'] if c['name'] == 'keycape')
    require(container['image'] == PRIOR_IMAGE, 'prior_image_drift')
    patch = yaml.safe_load((KEYCAPE / 'docs/approval-clients-deployment.patch.yaml').read_text())['spec']
    approved = patch['template']['spec']['containers'][0]
    require(approved['image'] == IMAGE and patch['strategy']['type'] == 'Recreate', 'candidate_source_drift')
    env = container.setdefault('env', [])
    require(not {x['name'] for x in env}.intersection(x['name'] for x in approved['env']), 'existing_candidate_env')
    for item, client, secret_name in zip(approved['env'], registrations(), SECRET_NAMES):
        require(item == {'name': client['secretRef'].removeprefix('env:'), 'valueFrom': {
            'secretKeyRef': {'name': secret_name, 'key': 'client-secret'}}}, 'env_delivery_contract_drift')
    env.extend(approved['env']); container['image'] = IMAGE
    container['readinessProbe']['httpGet'] = {'path': '/readyz', 'port': 8080}
    result['strategy'] = {'type': 'Recreate'}
    return result


def patch_object(kube, kind, obj, path, value, dry=False):
    patch = [{'op': 'test', 'path': '/metadata/uid', 'value': obj['metadata']['uid']},
             {'op': 'test', 'path': '/metadata/resourceVersion', 'value': obj['metadata']['resourceVersion']},
             {'op': 'replace', 'path': path, 'value': value}]
    args = kube + ['-n', 'sso', 'patch', kind, obj['metadata']['name'], '--type=json', '--patch-file=/dev/stdin', '-o', 'json']
    if dry:
        args += ['--dry-run=server']
    return data(command(args, payload=patch))


def get(kube, kind, name):
    return data(command(kube + ['-n', 'sso', 'get', kind, name, '-o', 'json']))


def ready(kube, image, timeout=150):
    deadline = time.monotonic() + timeout
    while time.monotonic() < deadline:
        dep = get(kube, 'deployment', 'keycape')
        status = dep.get('status', {})
        if (status.get('observedGeneration') == dep['metadata']['generation']
                and status.get('updatedReplicas') == status.get('readyReplicas') == status.get('availableReplicas') == 1
                and status.get('replicas') == 1):
            podlist = data(command(kube + ['-n', 'sso', 'get', 'pods', '-l', 'app.kubernetes.io/name=keycape', '-o', 'json']))['items']
            if len(podlist) == 1 and not podlist[0]['metadata'].get('deletionTimestamp'):
                containers = podlist[0].get('status', {}).get('containerStatuses', [])
                if any(c['name'] == 'keycape' and c['ready'] and
                       (c.get('imageID', '').removeprefix('docker-pullable://') == image
                        if '@sha256:' in image else c.get('image') == image) for c in containers):
                    return {'deployment_uid': dep['metadata']['uid'], 'generation': dep['metadata']['generation'],
                            'pod_uid': podlist[0]['metadata']['uid'], 'single_ready_replica': True}
        time.sleep(3)
    raise LaneError('keycape_readiness_timeout')


class NoRedirect(urllib.request.HTTPRedirectHandler):
    def redirect_request(self, *args, **kwargs):
        return None


def http(path, fields=None, credentials=None):
    headers = {'Accept': 'application/json'}
    if credentials:
        headers['Authorization'] = 'Basic ' + base64.b64encode(':'.join(credentials).encode()).decode()
    body = None
    if fields is not None:
        body = urllib.parse.urlencode(fields).encode()
        headers['Content-Type'] = 'application/x-www-form-urlencoded'
    request = urllib.request.Request(ISSUER + path, data=body, headers=headers)
    opener = urllib.request.build_opener(NoRedirect())
    try:
        with opener.open(request, timeout=20) as response:
            return response.status, json.load(response)
    except urllib.error.HTTPError as error:
        try:
            return error.code, json.load(error)
        except (ValueError, TypeError):
            raise LaneError('http_' + str(error.code) + '_non_json_error') from None


def denied(result, status, feature):
    code, body = result
    require(code == status and body.get('error') == 'invalid_profile_usage'
            and body.get('feature') == feature, 'keycape_denial_inconclusive')



def verified_claims(token, public_key):
    # Match authclient/client.go: tolerate at most 30s future iat, but no
    # extension to expiry or not-before. PyJWT's global leeway would extend both.
    claims = jwt.decode(token, public_key, algorithms=['RS256'], issuer=ISSUER, audience='approval-engine',
                        options={'verify_iat': False, 'require': ['exp', 'iat', 'sub', 'iss', 'aud']})
    require(type(claims['iat']) is int and type(claims['exp']) is int
            and claims['iat'] <= int(time.time()) + 30 and claims['iat'] < claims['exp'], 'issued_at_binding_failed')
    return claims


def acceptance(kube, receipt):
    receipt['acceptance_phase'] = 'discovery'
    code, discovery = http('/.well-known/openid-configuration')
    require(code == 200 and discovery['issuer'] == ISSUER
            and discovery['token_endpoint'] == ISSUER + '/token'
            and discovery['jwks_uri'].startswith(ISSUER + '/'), 'discovery_binding_mismatch')
    receipt['acceptance_phase'] = 'jwks'
    code, jwks = http(discovery['jwks_uri'].removeprefix(ISSUER))
    require(code == 200, 'jwks_failed')
    receipt['clients'] = []
    for client, secret_name in zip(registrations(), SECRET_NAMES):
        receipt['acceptance_client'] = client['clientId']
        receipt['acceptance_phase'] = 'verifier_secret_read'
        secret = get(kube, 'secret', secret_name)
        credential = base64.b64decode(secret['data']['client-secret'], validate=True).decode()
        scopes = ' '.join(client['allowedScopes'])
        receipt['acceptance_phase'] = 'token_exchange'
        result, response = http('/token', {'grant_type': 'client_credentials', 'scope': scopes}, (client['clientId'], credential))
        require(result == 200 and response.get('token_type') == 'Bearer' and response.get('expires_in') == 900, 'service_issuance_failed')
        receipt['acceptance_phase'] = 'signature_and_claims'
        token = response['access_token']; header = jwt.get_unverified_header(token)
        keys = [key for key in jwks['keys'] if key['kid'] == header.get('kid')]
        require(header.get('alg') == 'RS256' and len(keys) == 1, 'signing_key_selection_failed')
        claims = verified_claims(token, jwt.PyJWK.from_dict(keys[0]).key)
        require(claims['sub'] == client['serviceSubject'] and claims['tenant'] == 'tenant:platform'
                and claims['aud'] == 'approval-engine'
                and claims['roles'] == client['roles'] and claims['exp'] - claims['iat'] == 900
                and claims['principal_type'] == 'service'
                and set(claims['scope'].split()) == set(client['allowedScopes']), 'exact_claims_mismatch')
        excessive = 'approval:approve' if client['clientId'] == IDS[0] else 'approval:consume'
        receipt['acceptance_phase'] = 'excess_scope_denial'
        denied(http('/token', {'grant_type': 'client_credentials', 'scope': excessive},
                    (client['clientId'], credential)), 400, 'scope')
        receipt['acceptance_phase'] = 'wrong_secret_denial'
        denied(http('/token', {'grant_type': 'client_credentials', 'scope': scopes},
                    (client['clientId'], secrets.token_urlsafe(48))), 401, 'Authorization')
        args = [str(VERIFIER), 'verify-client',
            '-issuer', ISSUER, '-client-id', client['clientId'], '-audience', 'approval-engine', '-scope', scopes,
            '-secret-env', client['secretRef'].removeprefix('env:'), '-expect-subject', client['serviceSubject'],
            '-expect-tenant', 'tenant:platform', '-expect-roles', ','.join(client['roles']), '-deny-scope', excessive]
        receipt['acceptance_phase'] = 'pinned_artifact_verifier'
        verify_artifact()
        verifier_env = os.environ.copy()
        verifier_env[client['secretRef'].removeprefix('env:')] = credential
        command(args, env=verifier_env)
        receipt['clients'].append({'client_id': client['clientId'], 'live_jwks_signature_verified': True,
            'exact_claims_verified': True, 'lifetime_seconds': 900, 'maximum_future_iat_seconds': 30, 'expiry_leeway_seconds': 0, 'excess_scope_denied': True,
            'wrong_secret_denied': True, 'pinned_artifact_verifier_passed': True,
            'verifier_location': 'attended owner process', 'verifier_sha256': VERIFIER_SHA256,
            'real_predecessor_rotation_tested': False, 'observed_wall_clock_expiry': False})
    receipt['acceptance_phase'] = 'human_consume_denial'
    query = urllib.parse.urlencode({'client_id': 'openbao-admin', 'response_type': 'code',
        'redirect_uri': 'http://localhost:8250/oidc/callback', 'scope': 'openid approval:consume',
        'code_challenge_method': 'S256', 'code_challenge': 'A' * 43, 'state': secrets.token_urlsafe(32)})
    denied(http('/authorize?' + query), 400, 'scope')
    receipt['human_client_consume_denied'] = True
    receipt['acceptance_phase'] = 'passed'


def rollout(kube, receipt, recovery_path):
    assert_cluster(kube)
    verify_artifact()
    before = get(kube, 'secret', 'keycape-config')
    deployment = get(kube, 'deployment', 'keycape')
    require(before['metadata']['uid'] == '2e94519d-1550-41c7-9701-2efe47fe1fd3'
            and deployment['metadata']['uid'] == '99ddd83c-cb3f-4847-bcf8-35f1aa87627f', 'owner_object_identity_mismatch')
    config = replacement(before, registrations()); candidate = candidate_spec(deployment)
    patch_object(kube, 'secret', before, '/data/config.yaml', config, dry=True)
    patch_object(kube, 'deployment', deployment, '/spec', candidate, dry=True)
    previous = {'deployment': deployment, 'secret': before}
    recovery_path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
    require(recovery_path.parent.stat().st_mode & 0o077 == 0, 'private_recovery_directory_required')
    fd = os.open(recovery_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
    with os.fdopen(fd, 'w') as out:
        json.dump(previous, out)
    receipt['protected_recovery_retained'] = True
    changed = None; deployed = None
    try:
        changed = patch_object(kube, 'secret', before, '/data/config.yaml', config)
        require(changed['data'] == dict(before['data'], **{'config.yaml': config}), 'secret_readback_mismatch')
        deployed = patch_object(kube, 'deployment', deployment, '/spec', candidate)
        receipt.update(ready(kube, IMAGE))
        after = get(kube, 'secret', 'keycape-config')
        require(after['data'] == changed['data'], 'configuration_changed_during_rollout')
        acceptance(kube, receipt)
        receipt.update(status='service_acceptance_passed_pending_fresh_human_login', image=IMAGE,
            config_resource_version=after['metadata']['resourceVersion'], signing_key_unchanged=True,
            unrelated_config_bytes_preserved=True, existing_human_login_after=False)
    except Exception as failure:
        receipt['failure_class'] = type(failure).__name__
        # Read after uncertain API outcomes too; never assume a timeout means no write.
        now = get(kube, 'secret', 'keycape-config')
        require(now['metadata']['uid'] == before['metadata']['uid'], 'rollback_config_identity_drift')
        expected_data = dict(before['data'], **{'config.yaml': config})
        require(now['data'] in (before['data'], expected_data), 'rollback_config_drift_requires_owner_reconcile')
        if now['data'] == expected_data:
            patch_object(kube, 'secret', now, '/data/config.yaml', before['data']['config.yaml'])
        now = get(kube, 'deployment', 'keycape')
        require(now['metadata']['uid'] == deployment['metadata']['uid'], 'rollback_deployment_identity_drift')
        expected_spec = deployed['spec'] if deployed else candidate
        require(now['spec'] in (deployment['spec'], expected_spec), 'rollback_deployment_drift_requires_owner_reconcile')
        if now['spec'] == expected_spec:
            patch_object(kube, 'deployment', now, '/spec', deployment['spec'])
            ready(kube, PRIOR_IMAGE)
        receipt['compatible_pair_restored'] = True
        raise


def run(args, receipt):
    require(Path.home().parent.name == '.warden-attended-login' and not os.environ.get('BAO_TOKEN')
            and not os.environ.get('VAULT_TOKEN'), 'attended_warden_envelope_required')
    identity = data(bao(['token', 'lookup', '-format=json']))['data']
    require('platform-admin' in identity['policies'] and 'root' not in identity['policies'], 'attended_platform_admin_required')
    kube = ['kubectl', '--kubeconfig', args.kubeconfig]
    if args.action == 'rollout':
        receipt['existing_human_login_before'] = True
        rollout(kube, receipt, Path(args.recovery))
    else:
        assert_cluster(kube); receipt.update(ready(kube, IMAGE)); acceptance(kube, receipt)
        receipt.update(status='service_and_existing_human_login_acceptance_passed', existing_human_login_after=True)


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('action', choices=['rollout', 'verify-after-login'])
    parser.add_argument('--kubeconfig', required=True)
    parser.add_argument('--recovery')
    parser.add_argument('--receipt', required=True)
    args = parser.parse_args()
    receipt = {'schema': 'netkingdom.keycape-approval-rollout.v1', 'status': 'failed', 'values_emitted': False}
    fd = os.open(args.receipt, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
    try:
        run(args, receipt)
    except Exception as exc:
        receipt['error'] = str(exc) if isinstance(exc, (LaneError, pin.IssuerPinError)) else 'contained_operation_failed'
    finally:
        with os.fdopen(fd, 'w') as out:
            json.dump(receipt, out, indent=2); out.write('\n')
    return 0 if receipt['status'] != 'failed' else 1

if __name__ == '__main__':
    raise SystemExit(main())