#!/usr/bin/env python3 """Attended exact-lane ESO auth repair; no credential values in output/evidence.""" import argparse import base64 import json import os from pathlib import Path import time import yaml from state_hub_preflight_lane import ROOT, LaneError, assert_cluster, bao, command, data, capabilities, revoke def require(condition, label): if not condition: raise LaneError(label) def role_payload(lane): return {'bound_service_account_names': [lane['service_account']], 'bound_service_account_namespaces': [lane['namespace']], 'token_policies': [lane['policy']], 'audience': 'openbao', 'token_ttl': '15m', 'token_max_ttl': '15m', 'token_explicit_max_ttl': '15m', 'token_no_default_policy': True} def check_role(actual, lane): for key, value in role_payload(lane).items(): require(actual.get(key) == (900 if value == '15m' else value), 'role_drift') require(not actual.get('token_period') and not actual.get('bound_service_account_namespace_selector'), 'role_drift') def invalid_token_lookup(result): error = result.stderr.lower() return (result.returncode != 0 and (b'400' in error or b'403' in error) and (b'bad token' in error or b'invalid token' in error) and b'permission denied' not in error) def compare_custody(kube, lane): es = data(command(kube + ['-n', lane['namespace'], 'get', 'externalsecret', lane['name'], '-o', 'json'])) require(es['spec']['secretStoreRef'] == {'kind': 'ClusterSecretStore', 'name': lane['store']}, 'external_secret_store_drift') target = es['spec']['target'].get('name', lane['name']) deployed = data(command(kube + ['-n', lane['namespace'], 'get', 'secret', target, '-o', 'json'])) native = data(bao(['read', '-format=json', lane['kv_path']]))['data'] for entry in es['spec']['data']: ref = entry['remoteRef'] require('platform/data/' + ref['key'] == lane['kv_path'], 'remote_path_drift') require(str(native['data'][ref['property']]).encode() == base64.b64decode(deployed['data'][entry['secretKey']]), 'custody_disagrees_stop_before_resync') return native['metadata']['version'] def verify_login(kube, lane, row): def login(sa, audience, negative=False, namespace=None): jwt = command(kube + ['-n', namespace or lane['namespace'], 'create', 'token', sa, '--audience=' + audience, '--duration=10m']).stdout.decode().strip() result = bao(['write', '-format=json', 'auth/kubernetes/login', '-'], payload={'role': lane['role'], 'jwt': jwt}, allow_failure=negative) if negative: if result.returncode == 0: revoke(data(result)['auth']['client_token']) raise LaneError('negative_login_succeeded') require(b'403' in result.stderr or b'400' in result.stderr, 'negative_login_inconclusive') return None return data(result)['auth'] auth = login(lane['service_account'], 'openbao') token = auth['client_token'] try: require(auth['token_policies'] == [lane['policy']] and 0 < auth['lease_duration'] <= 900, 'effective_policy_or_ttl_drift') paths = [lane['kv_path'], lane['kv_path'].replace('/data/', '/metadata/'), lane['kv_path'] + '-unrelated', 'auth/token/create', 'auth/token/lookup-self', 'auth/token/revoke-self'] caps = capabilities(token, paths) require(caps[paths[0]] == ['read'], 'read_or_write_scope_drift') require(all(caps[p] == ['deny'] for p in paths[1:4]), 'negative_scope_failed') require(caps[paths[4]] == ['read'] and caps[paths[5]] == ['update'], 'token_self_scope_failed') bao(['read', '-format=json', lane['kv_path']], token=token) bao(['token', 'lookup', '-format=json'], token=token) finally: revoke(token) login('default', 'openbao', negative=True) login(lane['service_account'], 'wrong-audience', negative=True) command(kube + ['-n', 'default', 'create', 'serviceaccount', lane['service_account']]) try: login(lane['service_account'], 'openbao', negative=True, namespace='default') finally: command(kube + ['-n', 'default', 'delete', 'serviceaccount', lane['service_account']]) row.update(exact_read=True, secret_write_and_sibling_denied=True, wrong_sa_denied=True, wrong_namespace_denied=True, wrong_audience_denied=True, bounded_ttl=True) def run(args, receipt): kube = ['kubectl', '--kubeconfig', args.kubeconfig] assert_cluster(kube) ident = data(bao(['token', 'lookup', '-format=json']))['data'] require('platform-admin' in ident['policies'] and 'root' not in ident['policies'], 'attended_platform_admin_required') lanes = json.loads((ROOT / 'openbao/eso-auth-recovery/lanes.json').read_text()) # Refuse all mutations if any lane would replace divergent deployed key material. for lane in lanes: compare_custody(kube, lane) if args.action == 'apply': command(kube + ['apply', '-f', str(ROOT / 'openbao/eso-auth-recovery/serviceaccounts.yaml')]) for lane in lanes: row = {'name': lane['name'], 'namespace': lane['namespace'], 'status': 'pending'} receipt['lanes'].append(row) source = ROOT / 'argocd/platform-addons/openbao-secretstore' / (lane['store'] + '.clustersecretstore.yaml') desired = yaml.safe_load(source.read_text()) live = data(command(kube + ['get', 'clustersecretstore', lane['store'], '-o', 'json'])) expected_old = {'tokenSecretRef': {'name': lane['store'] + '-eso-token', 'namespace': 'external-secrets', 'key': 'token'}} a, b = live['spec']['provider']['vault'], desired['spec']['provider']['vault'] require({k:v for k,v in a.items() if k != 'auth'} == {k:v for k,v in b.items() if k != 'auth'} and live['spec']['conditions'] == desired['spec']['conditions'], 'store_scope_drift') require(a['auth'] in [expected_old, b['auth']], 'store_auth_drift') policy = (ROOT / 'openbao/policies' / (lane['policy'] + '.hcl')).read_text() if args.action == 'apply': old = bao(['read', '-format=json', 'sys/policies/acl/' + lane['policy']], allow_failure=True) if old.returncode == 0: require(data(old)['data']['policy'] == policy, 'policy_drift') else: require(b'404' in old.stderr or b'No value found' in old.stderr, 'policy_absence_unproven') old_role = bao(['read', '-format=json', 'auth/kubernetes/role/' + lane['role']], allow_failure=True) if old_role.returncode == 0: check_role(data(old_role)['data'], lane) else: require(b'404' in old_role.stderr or b'No value found' in old_role.stderr, 'role_absence_unproven') bao(['write', 'sys/policies/acl/' + lane['policy'], '-'], payload={'policy': policy}) bao(['write', 'auth/kubernetes/role/' + lane['role'], '-'], payload=role_payload(lane)) check_role(data(bao(['read', '-format=json', 'auth/kubernetes/role/' + lane['role']]))['data'], lane) verify_login(kube, lane, row) row['kv_version'] = compare_custody(kube, lane) if args.action == 'apply': # JSON Patch tests the observed spec atomically; replace only authentication. patch = [{'op':'test','path':'/spec','value':live['spec']}, {'op':'replace','path':'/spec/provider/vault/auth','value':b['auth']}] command(kube + ['patch', 'clustersecretstore', lane['store'], '--type=json', '-p', json.dumps(patch)]) before = data(command(kube + ['-n', lane['namespace'], 'get', 'externalsecret', lane['name'], '-o', 'json'])).get('status', {}).get('refreshTime') command(kube + ['annotate', 'clustersecretstore', lane['store'], 'force-sync=' + str(time.time_ns()), '--overwrite']) command(kube + ['-n', lane['namespace'], 'annotate', 'externalsecret', lane['name'], 'force-sync=' + str(time.time_ns()), '--overwrite']) command(kube + ['wait', '--for=condition=Ready', 'clustersecretstore/' + lane['store'], '--timeout=45s']) command(kube + ['-n', lane['namespace'], 'wait', '--for=condition=Ready', 'externalsecret/' + lane['name'], '--timeout=45s']) for attempt in range(20): status = data(command(kube + ['-n', lane['namespace'], 'get', 'externalsecret', lane['name'], '-o', 'json'])).get('status', {}) if status.get('refreshTime') != before and any(c['type'] == 'Ready' and c['status'] == 'True' for c in status.get('conditions', [])): row['refresh_time'] = status['refreshTime'] break time.sleep(2) else: raise LaneError('fresh_reconcile_not_observed') compare_custody(kube, lane) row.update(status='passed', unchanged_workload_values=True, store_ready=True, external_secret_ready=True) if args.action == 'cleanup': # Retired delivery tokens only; never delete workload Secrets or KV data. stores = data(command(kube + ['get', 'clustersecretstores,secretstores', '-A', '-o', 'json']))['items'] for lane, row in zip(lanes, receipt['lanes']): name = lane['store'] + '-eso-token' for store in stores: ref = store['spec'].get('provider', {}).get('vault', {}).get('auth', {}).get('tokenSecretRef', {}) require(ref.get('name') != name, 'retired_token_still_referenced') secret = data(command(kube + ['-n', 'external-secrets', 'get', 'secret', name, '-o', 'json'])) old_token = base64.b64decode(secret['data']['token']).decode() lookup = bao(['write', '-format=json', 'auth/token/lookup', '-'], payload={'token': old_token}, allow_failure=True) if lookup.returncode == 0: policies = set(data(lookup)['data']['policies']) require(policies <= {lane['policy'].removesuffix('-eso'), 'default'}, 'retired_token_scope_requires_review') revoke(old_token) row['retired_token_revoked'] = True else: require(invalid_token_lookup(lookup), 'retired_token_invalidity_unproven') row['retired_token_already_invalid'] = True check = bao(['token', 'lookup', '-format=json'], token=old_token, allow_failure=True) require(check.returncode != 0 and b'403' in check.stderr, 'retired_token_denial_unproven') options = {'apiVersion':'v1','kind':'DeleteOptions','preconditions':{'uid':secret['metadata']['uid'],'resourceVersion':secret['metadata']['resourceVersion']}} command(kube + ['delete', '--raw', '/api/v1/namespaces/external-secrets/secrets/' + name, '-f', '-'], payload=options) row['retired_token_secret_removed'] = True receipt['status'] = 'passed' def main(): p = argparse.ArgumentParser(description=__doc__) p.add_argument('action', choices=['apply', 'verify', 'cleanup']) p.add_argument('--kubeconfig', required=True) p.add_argument('--receipt', required=True) p.add_argument('--confirm', required=True) args = p.parse_args() receipt = {'schema':'platform.eso-kubernetes-recovery.v1','status':'failed','lanes':[]} fd = os.open(args.receipt, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600) try: require(args.confirm == 'APPLY RPF-WP-0037', 'confirmation_mismatch') run(args, receipt) except Exception as exc: receipt['error'] = str(exc) if isinstance(exc, LaneError) else 'internal_error' finally: with os.fdopen(fd, 'w') as out: json.dump(receipt, out, indent=2) out.write('\n') return 0 if receipt['status'] == 'passed' else 1 if __name__ == '__main__': raise SystemExit(main())