3759850cc4
Assistant: codex Assistant-Model: gpt-6-astra Assistant-Session: 01a06ecb-456a-71c2-b41e-0755d336e883
195 lines
12 KiB
Python
195 lines
12 KiB
Python
#!/usr/bin/env python3
|
|
"""Attended exact-lane ESO auth repair; no credential values in output/evidence."""
|
|
import argparse
|
|
import base64
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import time
|
|
|
|
import yaml
|
|
from state_hub_preflight_lane import ROOT, LaneError, assert_cluster, bao, command, data, capabilities, revoke
|
|
|
|
|
|
def require(condition, label):
|
|
if not condition:
|
|
raise LaneError(label)
|
|
|
|
|
|
def role_payload(lane):
|
|
return {'bound_service_account_names': [lane['service_account']],
|
|
'bound_service_account_namespaces': [lane['namespace']],
|
|
'token_policies': [lane['policy']], 'audience': 'openbao',
|
|
'token_ttl': '15m', 'token_max_ttl': '15m',
|
|
'token_explicit_max_ttl': '15m', 'token_no_default_policy': True}
|
|
|
|
|
|
def check_role(actual, lane):
|
|
for key, value in role_payload(lane).items():
|
|
require(actual.get(key) == (900 if value == '15m' else value), 'role_drift')
|
|
require(not actual.get('token_period') and not actual.get('bound_service_account_namespace_selector'), 'role_drift')
|
|
|
|
|
|
def invalid_token_lookup(result):
|
|
error = result.stderr.lower()
|
|
return (result.returncode != 0 and (b'400' in error or b'403' in error)
|
|
and (b'bad token' in error or b'invalid token' in error)
|
|
and b'permission denied' not in error)
|
|
|
|
|
|
def compare_custody(kube, lane):
|
|
es = data(command(kube + ['-n', lane['namespace'], 'get', 'externalsecret', lane['name'], '-o', 'json']))
|
|
require(es['spec']['secretStoreRef'] == {'kind': 'ClusterSecretStore', 'name': lane['store']}, 'external_secret_store_drift')
|
|
target = es['spec']['target'].get('name', lane['name'])
|
|
deployed = data(command(kube + ['-n', lane['namespace'], 'get', 'secret', target, '-o', 'json']))
|
|
native = data(bao(['read', '-format=json', lane['kv_path']]))['data']
|
|
for entry in es['spec']['data']:
|
|
ref = entry['remoteRef']
|
|
require('platform/data/' + ref['key'] == lane['kv_path'], 'remote_path_drift')
|
|
require(str(native['data'][ref['property']]).encode() == base64.b64decode(deployed['data'][entry['secretKey']]), 'custody_disagrees_stop_before_resync')
|
|
return native['metadata']['version']
|
|
|
|
|
|
def verify_login(kube, lane, row):
|
|
def login(sa, audience, negative=False, namespace=None):
|
|
jwt = command(kube + ['-n', namespace or lane['namespace'], 'create', 'token', sa, '--audience=' + audience, '--duration=10m']).stdout.decode().strip()
|
|
result = bao(['write', '-format=json', 'auth/kubernetes/login', '-'], payload={'role': lane['role'], 'jwt': jwt}, allow_failure=negative)
|
|
if negative:
|
|
if result.returncode == 0:
|
|
revoke(data(result)['auth']['client_token'])
|
|
raise LaneError('negative_login_succeeded')
|
|
require(b'403' in result.stderr or b'400' in result.stderr, 'negative_login_inconclusive')
|
|
return None
|
|
return data(result)['auth']
|
|
auth = login(lane['service_account'], 'openbao')
|
|
token = auth['client_token']
|
|
try:
|
|
require(auth['token_policies'] == [lane['policy']] and 0 < auth['lease_duration'] <= 900, 'effective_policy_or_ttl_drift')
|
|
paths = [lane['kv_path'], lane['kv_path'].replace('/data/', '/metadata/'),
|
|
lane['kv_path'] + '-unrelated', 'auth/token/create',
|
|
'auth/token/lookup-self', 'auth/token/revoke-self']
|
|
caps = capabilities(token, paths)
|
|
require(caps[paths[0]] == ['read'], 'read_or_write_scope_drift')
|
|
require(all(caps[p] == ['deny'] for p in paths[1:4]), 'negative_scope_failed')
|
|
require(caps[paths[4]] == ['read'] and caps[paths[5]] == ['update'], 'token_self_scope_failed')
|
|
bao(['read', '-format=json', lane['kv_path']], token=token)
|
|
bao(['token', 'lookup', '-format=json'], token=token)
|
|
finally:
|
|
revoke(token)
|
|
login('default', 'openbao', negative=True)
|
|
login(lane['service_account'], 'wrong-audience', negative=True)
|
|
command(kube + ['-n', 'default', 'create', 'serviceaccount', lane['service_account']])
|
|
try:
|
|
login(lane['service_account'], 'openbao', negative=True, namespace='default')
|
|
finally:
|
|
command(kube + ['-n', 'default', 'delete', 'serviceaccount', lane['service_account']])
|
|
row.update(exact_read=True, secret_write_and_sibling_denied=True, wrong_sa_denied=True, wrong_namespace_denied=True, wrong_audience_denied=True, bounded_ttl=True)
|
|
|
|
|
|
def run(args, receipt):
|
|
kube = ['kubectl', '--kubeconfig', args.kubeconfig]
|
|
assert_cluster(kube)
|
|
ident = data(bao(['token', 'lookup', '-format=json']))['data']
|
|
require('platform-admin' in ident['policies'] and 'root' not in ident['policies'], 'attended_platform_admin_required')
|
|
lanes = json.loads((ROOT / 'openbao/eso-auth-recovery/lanes.json').read_text())
|
|
# Refuse all mutations if any lane would replace divergent deployed key material.
|
|
for lane in lanes:
|
|
compare_custody(kube, lane)
|
|
if args.action == 'apply':
|
|
command(kube + ['apply', '-f', str(ROOT / 'openbao/eso-auth-recovery/serviceaccounts.yaml')])
|
|
for lane in lanes:
|
|
row = {'name': lane['name'], 'namespace': lane['namespace'], 'status': 'pending'}
|
|
receipt['lanes'].append(row)
|
|
source = ROOT / 'argocd/platform-addons/openbao-secretstore' / (lane['store'] + '.clustersecretstore.yaml')
|
|
desired = yaml.safe_load(source.read_text())
|
|
live = data(command(kube + ['get', 'clustersecretstore', lane['store'], '-o', 'json']))
|
|
expected_old = {'tokenSecretRef': {'name': lane['store'] + '-eso-token', 'namespace': 'external-secrets', 'key': 'token'}}
|
|
a, b = live['spec']['provider']['vault'], desired['spec']['provider']['vault']
|
|
require({k:v for k,v in a.items() if k != 'auth'} == {k:v for k,v in b.items() if k != 'auth'} and live['spec']['conditions'] == desired['spec']['conditions'], 'store_scope_drift')
|
|
require(a['auth'] in [expected_old, b['auth']], 'store_auth_drift')
|
|
policy = (ROOT / 'openbao/policies' / (lane['policy'] + '.hcl')).read_text()
|
|
if args.action == 'apply':
|
|
old = bao(['read', '-format=json', 'sys/policies/acl/' + lane['policy']], allow_failure=True)
|
|
if old.returncode == 0:
|
|
require(data(old)['data']['policy'] == policy, 'policy_drift')
|
|
else:
|
|
require(b'404' in old.stderr or b'No value found' in old.stderr, 'policy_absence_unproven')
|
|
old_role = bao(['read', '-format=json', 'auth/kubernetes/role/' + lane['role']], allow_failure=True)
|
|
if old_role.returncode == 0:
|
|
check_role(data(old_role)['data'], lane)
|
|
else:
|
|
require(b'404' in old_role.stderr or b'No value found' in old_role.stderr, 'role_absence_unproven')
|
|
bao(['write', 'sys/policies/acl/' + lane['policy'], '-'], payload={'policy': policy})
|
|
bao(['write', 'auth/kubernetes/role/' + lane['role'], '-'], payload=role_payload(lane))
|
|
check_role(data(bao(['read', '-format=json', 'auth/kubernetes/role/' + lane['role']]))['data'], lane)
|
|
verify_login(kube, lane, row)
|
|
row['kv_version'] = compare_custody(kube, lane)
|
|
if args.action == 'apply':
|
|
# JSON Patch tests the observed spec atomically; replace only authentication.
|
|
patch = [{'op':'test','path':'/spec','value':live['spec']}, {'op':'replace','path':'/spec/provider/vault/auth','value':b['auth']}]
|
|
command(kube + ['patch', 'clustersecretstore', lane['store'], '--type=json', '-p', json.dumps(patch)])
|
|
before = data(command(kube + ['-n', lane['namespace'], 'get', 'externalsecret', lane['name'], '-o', 'json'])).get('status', {}).get('refreshTime')
|
|
command(kube + ['annotate', 'clustersecretstore', lane['store'], 'force-sync=' + str(time.time_ns()), '--overwrite'])
|
|
command(kube + ['-n', lane['namespace'], 'annotate', 'externalsecret', lane['name'], 'force-sync=' + str(time.time_ns()), '--overwrite'])
|
|
command(kube + ['wait', '--for=condition=Ready', 'clustersecretstore/' + lane['store'], '--timeout=45s'])
|
|
command(kube + ['-n', lane['namespace'], 'wait', '--for=condition=Ready', 'externalsecret/' + lane['name'], '--timeout=45s'])
|
|
for attempt in range(20):
|
|
status = data(command(kube + ['-n', lane['namespace'], 'get', 'externalsecret', lane['name'], '-o', 'json'])).get('status', {})
|
|
if status.get('refreshTime') != before and any(c['type'] == 'Ready' and c['status'] == 'True' for c in status.get('conditions', [])):
|
|
row['refresh_time'] = status['refreshTime']
|
|
break
|
|
time.sleep(2)
|
|
else:
|
|
raise LaneError('fresh_reconcile_not_observed')
|
|
compare_custody(kube, lane)
|
|
row.update(status='passed', unchanged_workload_values=True, store_ready=True, external_secret_ready=True)
|
|
if args.action == 'cleanup':
|
|
# Retired delivery tokens only; never delete workload Secrets or KV data.
|
|
stores = data(command(kube + ['get', 'clustersecretstores,secretstores', '-A', '-o', 'json']))['items']
|
|
for lane, row in zip(lanes, receipt['lanes']):
|
|
name = lane['store'] + '-eso-token'
|
|
for store in stores:
|
|
ref = store['spec'].get('provider', {}).get('vault', {}).get('auth', {}).get('tokenSecretRef', {})
|
|
require(ref.get('name') != name, 'retired_token_still_referenced')
|
|
secret = data(command(kube + ['-n', 'external-secrets', 'get', 'secret', name, '-o', 'json']))
|
|
old_token = base64.b64decode(secret['data']['token']).decode()
|
|
lookup = bao(['write', '-format=json', 'auth/token/lookup', '-'], payload={'token': old_token}, allow_failure=True)
|
|
if lookup.returncode == 0:
|
|
policies = set(data(lookup)['data']['policies'])
|
|
require(policies <= {lane['policy'].removesuffix('-eso'), 'default'}, 'retired_token_scope_requires_review')
|
|
revoke(old_token)
|
|
row['retired_token_revoked'] = True
|
|
else:
|
|
require(invalid_token_lookup(lookup), 'retired_token_invalidity_unproven')
|
|
row['retired_token_already_invalid'] = True
|
|
check = bao(['token', 'lookup', '-format=json'], token=old_token, allow_failure=True)
|
|
require(check.returncode != 0 and b'403' in check.stderr, 'retired_token_denial_unproven')
|
|
options = {'apiVersion':'v1','kind':'DeleteOptions','preconditions':{'uid':secret['metadata']['uid'],'resourceVersion':secret['metadata']['resourceVersion']}}
|
|
command(kube + ['delete', '--raw', '/api/v1/namespaces/external-secrets/secrets/' + name, '-f', '-'], payload=options)
|
|
row['retired_token_secret_removed'] = True
|
|
receipt['status'] = 'passed'
|
|
|
|
|
|
def main():
|
|
p = argparse.ArgumentParser(description=__doc__)
|
|
p.add_argument('action', choices=['apply', 'verify', 'cleanup'])
|
|
p.add_argument('--kubeconfig', required=True)
|
|
p.add_argument('--receipt', required=True)
|
|
p.add_argument('--confirm', required=True)
|
|
args = p.parse_args()
|
|
receipt = {'schema':'platform.eso-kubernetes-recovery.v1','status':'failed','lanes':[]}
|
|
fd = os.open(args.receipt, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600)
|
|
try:
|
|
require(args.confirm == 'APPLY RPF-WP-0037', 'confirmation_mismatch')
|
|
run(args, receipt)
|
|
except Exception as exc:
|
|
receipt['error'] = str(exc) if isinstance(exc, LaneError) else 'internal_error'
|
|
finally:
|
|
with os.fdopen(fd, 'w') as out:
|
|
json.dump(receipt, out, indent=2)
|
|
out.write('\n')
|
|
return 0 if receipt['status'] == 'passed' else 1
|
|
|
|
|
|
if __name__ == '__main__':
|
|
raise SystemExit(main())
|