From bd2e6e86b8502164470bf912fe364d0aa245ec93 Mon Sep 17 00:00:00 2001 From: codex Date: Sat, 5 Sep 2026 18:38:19 +0200 Subject: [PATCH] Prepare bounded Kubernetes authentication recovery for three ESO lanes Assistant: codex Assistant-Model: gpt-6-astra Assistant-Session: 01a06ecb-456a-71c2-b41e-0755d336e883 --- .../openbao-forgejo.clustersecretstore.yaml | 22 +-- .../openbao-reuse.clustersecretstore.yaml | 25 ++- ...bao-target-revenue.clustersecretstore.yaml | 23 ++- openbao/eso-auth-recovery/lanes.json | 29 ++++ .../eso-auth-recovery/serviceaccounts.yaml | 20 +++ .../workload-kv-read-forgejo-mailer-eso.hcl | 11 ++ ...load-kv-read-reuse-surface-runtime-eso.hcl | 11 ++ ...oad-kv-read-target-revenue-runtime-eso.hcl | 11 ++ scripts/repair_eso_kubernetes_auth.py | 155 ++++++++++++++++++ tests/test_eso_auth_recovery.py | 44 +++++ .../RPF-WP-0037-eso-static-token-recovery.md | 58 +++++++ 11 files changed, 372 insertions(+), 37 deletions(-) create mode 100644 openbao/eso-auth-recovery/lanes.json create mode 100644 openbao/eso-auth-recovery/serviceaccounts.yaml create mode 100644 openbao/policies/workload-kv-read-forgejo-mailer-eso.hcl create mode 100644 openbao/policies/workload-kv-read-reuse-surface-runtime-eso.hcl create mode 100644 openbao/policies/workload-kv-read-target-revenue-runtime-eso.hcl create mode 100644 scripts/repair_eso_kubernetes_auth.py create mode 100644 tests/test_eso_auth_recovery.py create mode 100644 workplans/RPF-WP-0037-eso-static-token-recovery.md diff --git a/argocd/platform-addons/openbao-secretstore/openbao-forgejo.clustersecretstore.yaml b/argocd/platform-addons/openbao-secretstore/openbao-forgejo.clustersecretstore.yaml index bccd870..7f27aef 100644 --- a/argocd/platform-addons/openbao-secretstore/openbao-forgejo.clustersecretstore.yaml +++ b/argocd/platform-addons/openbao-secretstore/openbao-forgejo.clustersecretstore.yaml @@ -1,8 +1,4 @@ -# Forgejo reads SMTP credentials from the private railiance01 OpenBao service. -# -# Prereq: Secret external-secrets/openbao-forgejo-eso-token (key: token) with a -# policy-limited OpenBao token that can read platform/workloads/forgejo/forgejo-mailer. -# Bootstrap: railiance-apps make forgejo-openbao-eso-token-apply +# RPF-WP-0037: exact namespace/SA/audience Kubernetes auth; no static token. apiVersion: external-secrets.io/v1beta1 kind: ClusterSecretStore metadata: @@ -17,10 +13,14 @@ spec: path: platform version: v2 auth: - tokenSecretRef: - name: openbao-forgejo-eso-token - namespace: external-secrets - key: token + kubernetes: + mountPath: kubernetes + role: forgejo-mailer-eso + serviceAccountRef: + name: forgejo-mailer-eso + namespace: forgejo + audiences: + - openbao conditions: - - namespaces: - - forgejo + - namespaces: + - forgejo diff --git a/argocd/platform-addons/openbao-secretstore/openbao-reuse.clustersecretstore.yaml b/argocd/platform-addons/openbao-secretstore/openbao-reuse.clustersecretstore.yaml index b821ab4..57ff19e 100644 --- a/argocd/platform-addons/openbao-secretstore/openbao-reuse.clustersecretstore.yaml +++ b/argocd/platform-addons/openbao-secretstore/openbao-reuse.clustersecretstore.yaml @@ -1,11 +1,4 @@ -# reuse-surface uses the private OpenBao service on railiance01. Public -# bao.coulomb.social remains the operator/API address and is not a workload -# runtime dependency. -# -# Prereq: Secret external-secrets/openbao-reuse-eso-token (key: token) with a -# policy-limited OpenBao token that can read -# platform/workloads/reuse/reuse-surface/runtime-secrets. -# Bootstrap: railiance-apps make reuse-openbao-eso-token-apply +# RPF-WP-0037: exact namespace/SA/audience Kubernetes auth; no static token. apiVersion: external-secrets.io/v1beta1 kind: ClusterSecretStore metadata: @@ -20,10 +13,14 @@ spec: path: platform version: v2 auth: - tokenSecretRef: - name: openbao-reuse-eso-token - namespace: external-secrets - key: token + kubernetes: + mountPath: kubernetes + role: reuse-surface-runtime-eso + serviceAccountRef: + name: reuse-surface-eso + namespace: reuse + audiences: + - openbao conditions: - - namespaces: - - reuse + - namespaces: + - reuse diff --git a/argocd/platform-addons/openbao-secretstore/openbao-target-revenue.clustersecretstore.yaml b/argocd/platform-addons/openbao-secretstore/openbao-target-revenue.clustersecretstore.yaml index c4fba08..337de4c 100644 --- a/argocd/platform-addons/openbao-secretstore/openbao-target-revenue.clustersecretstore.yaml +++ b/argocd/platform-addons/openbao-secretstore/openbao-target-revenue.clustersecretstore.yaml @@ -1,9 +1,4 @@ -# target-revenue Trust Service runtime secrets on railiance01 (WP-0011). -# Pattern matches openbao-reuse / openbao-activity-core: token auth to the -# in-cluster OpenBao service, store limited to the target-revenue namespace. -# -# Prereq: Secret external-secrets/openbao-target-revenue-eso-token (key: token) -# Bootstrap: target-revenue scripts/openbao-eso-token-apply.sh +# RPF-WP-0037: exact namespace/SA/audience Kubernetes auth; no static token. apiVersion: external-secrets.io/v1beta1 kind: ClusterSecretStore metadata: @@ -19,10 +14,14 @@ spec: path: platform version: v2 auth: - tokenSecretRef: - name: openbao-target-revenue-eso-token - namespace: external-secrets - key: token + kubernetes: + mountPath: kubernetes + role: target-revenue-runtime-eso + serviceAccountRef: + name: target-revenue-eso + namespace: target-revenue + audiences: + - openbao conditions: - - namespaces: - - target-revenue + - namespaces: + - target-revenue diff --git a/openbao/eso-auth-recovery/lanes.json b/openbao/eso-auth-recovery/lanes.json new file mode 100644 index 0000000..594c130 --- /dev/null +++ b/openbao/eso-auth-recovery/lanes.json @@ -0,0 +1,29 @@ +[ + { + "namespace": "forgejo", + "name": "forgejo-mailer", + "store": "openbao-forgejo", + "service_account": "forgejo-mailer-eso", + "kv_path": "platform/data/workloads/forgejo/forgejo-mailer", + "policy": "workload-kv-read-forgejo-mailer-eso", + "role": "forgejo-mailer-eso" + }, + { + "namespace": "reuse", + "name": "reuse-surface-runtime", + "store": "openbao-reuse", + "service_account": "reuse-surface-eso", + "kv_path": "platform/data/workloads/reuse/reuse-surface/runtime-secrets", + "policy": "workload-kv-read-reuse-surface-runtime-eso", + "role": "reuse-surface-runtime-eso" + }, + { + "namespace": "target-revenue", + "name": "target-revenue-runtime", + "store": "openbao-target-revenue", + "service_account": "target-revenue-eso", + "kv_path": "platform/data/workloads/target-revenue/runtime-secrets", + "policy": "workload-kv-read-target-revenue-runtime-eso", + "role": "target-revenue-runtime-eso" + } +] diff --git a/openbao/eso-auth-recovery/serviceaccounts.yaml b/openbao/eso-auth-recovery/serviceaccounts.yaml new file mode 100644 index 0000000..18b1a51 --- /dev/null +++ b/openbao/eso-auth-recovery/serviceaccounts.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: forgejo-mailer-eso + namespace: forgejo +automountServiceAccountToken: false +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: reuse-surface-eso + namespace: reuse +automountServiceAccountToken: false +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: target-revenue-eso + namespace: target-revenue +automountServiceAccountToken: false diff --git a/openbao/policies/workload-kv-read-forgejo-mailer-eso.hcl b/openbao/policies/workload-kv-read-forgejo-mailer-eso.hcl new file mode 100644 index 0000000..477b70a --- /dev/null +++ b/openbao/policies/workload-kv-read-forgejo-mailer-eso.hcl @@ -0,0 +1,11 @@ +path "platform/data/workloads/forgejo/forgejo-mailer" { + capabilities = ["read"] +} + +path "auth/token/lookup-self" { + capabilities = ["read"] +} + +path "auth/token/revoke-self" { + capabilities = ["update"] +} diff --git a/openbao/policies/workload-kv-read-reuse-surface-runtime-eso.hcl b/openbao/policies/workload-kv-read-reuse-surface-runtime-eso.hcl new file mode 100644 index 0000000..93739c6 --- /dev/null +++ b/openbao/policies/workload-kv-read-reuse-surface-runtime-eso.hcl @@ -0,0 +1,11 @@ +path "platform/data/workloads/reuse/reuse-surface/runtime-secrets" { + capabilities = ["read"] +} + +path "auth/token/lookup-self" { + capabilities = ["read"] +} + +path "auth/token/revoke-self" { + capabilities = ["update"] +} diff --git a/openbao/policies/workload-kv-read-target-revenue-runtime-eso.hcl b/openbao/policies/workload-kv-read-target-revenue-runtime-eso.hcl new file mode 100644 index 0000000..44b9523 --- /dev/null +++ b/openbao/policies/workload-kv-read-target-revenue-runtime-eso.hcl @@ -0,0 +1,11 @@ +path "platform/data/workloads/target-revenue/runtime-secrets" { + capabilities = ["read"] +} + +path "auth/token/lookup-self" { + capabilities = ["read"] +} + +path "auth/token/revoke-self" { + capabilities = ["update"] +} diff --git a/scripts/repair_eso_kubernetes_auth.py b/scripts/repair_eso_kubernetes_auth.py new file mode 100644 index 0000000..1d30888 --- /dev/null +++ b/scripts/repair_eso_kubernetes_auth.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python3 +"""Attended exact-lane ESO auth repair; no credential values in output/evidence.""" +import argparse +import base64 +import json +import os +from pathlib import Path +import time + +import yaml +from state_hub_preflight_lane import ROOT, LaneError, assert_cluster, bao, command, data, capabilities, revoke + + +def require(condition, label): + if not condition: + raise LaneError(label) + + +def role_payload(lane): + return {'bound_service_account_names': [lane['service_account']], + 'bound_service_account_namespaces': [lane['namespace']], + 'token_policies': [lane['policy']], 'audience': 'openbao', + 'token_ttl': '15m', 'token_max_ttl': '15m', + 'token_explicit_max_ttl': '15m', 'token_no_default_policy': True} + + +def check_role(actual, lane): + for key, value in role_payload(lane).items(): + require(actual.get(key) == (900 if value == '15m' else value), 'role_drift') + require(not actual.get('token_period') and not actual.get('bound_service_account_namespace_selector'), 'role_drift') + + +def compare_custody(kube, lane): + es = data(command(kube + ['-n', lane['namespace'], 'get', 'externalsecret', lane['name'], '-o', 'json'])) + require(es['spec']['secretStoreRef'] == {'kind': 'ClusterSecretStore', 'name': lane['store']}, 'external_secret_store_drift') + target = es['spec']['target'].get('name', lane['name']) + deployed = data(command(kube + ['-n', lane['namespace'], 'get', 'secret', target, '-o', 'json'])) + native = data(bao(['read', '-format=json', lane['kv_path']]))['data'] + for entry in es['spec']['data']: + ref = entry['remoteRef'] + require('platform/data/' + ref['key'] == lane['kv_path'], 'remote_path_drift') + require(str(native['data'][ref['property']]).encode() == base64.b64decode(deployed['data'][entry['secretKey']]), 'custody_disagrees_stop_before_resync') + return native['metadata']['version'] + + +def verify_login(kube, lane, row): + def login(sa, audience, negative=False, namespace=None): + jwt = command(kube + ['-n', namespace or lane['namespace'], 'create', 'token', sa, '--audience=' + audience, '--duration=10m']).stdout.decode().strip() + result = bao(['write', '-format=json', 'auth/kubernetes/login', '-'], payload={'role': lane['role'], 'jwt': jwt}, allow_failure=negative) + if negative: + if result.returncode == 0: + revoke(data(result)['auth']['client_token']) + raise LaneError('negative_login_succeeded') + require(b'403' in result.stderr or b'400' in result.stderr, 'negative_login_inconclusive') + return None + return data(result)['auth'] + auth = login(lane['service_account'], 'openbao') + token = auth['client_token'] + try: + require(auth['token_policies'] == [lane['policy']] and 0 < auth['lease_duration'] <= 900, 'effective_policy_or_ttl_drift') + paths = [lane['kv_path'], lane['kv_path'].replace('/data/', '/metadata/'), + lane['kv_path'] + '-unrelated', 'auth/token/create', + 'auth/token/lookup-self', 'auth/token/revoke-self'] + caps = capabilities(token, paths) + require(caps[paths[0]] == ['read'], 'read_or_write_scope_drift') + require(all(caps[p] == ['deny'] for p in paths[1:4]), 'negative_scope_failed') + require(caps[paths[4]] == ['read'] and caps[paths[5]] == ['update'], 'token_self_scope_failed') + bao(['read', '-format=json', lane['kv_path']], token=token) + bao(['token', 'lookup', '-format=json'], token=token) + finally: + revoke(token) + login('default', 'openbao', negative=True) + login(lane['service_account'], 'wrong-audience', negative=True) + command(kube + ['-n', 'default', 'create', 'serviceaccount', lane['service_account']]) + try: + login(lane['service_account'], 'openbao', negative=True, namespace='default') + finally: + command(kube + ['-n', 'default', 'delete', 'serviceaccount', lane['service_account']]) + row.update(exact_read=True, secret_write_and_sibling_denied=True, wrong_sa_denied=True, wrong_namespace_denied=True, wrong_audience_denied=True, bounded_ttl=True) + + +def run(args, receipt): + kube = ['kubectl', '--kubeconfig', args.kubeconfig] + assert_cluster(kube) + ident = data(bao(['token', 'lookup', '-format=json']))['data'] + require('platform-admin' in ident['policies'] and 'root' not in ident['policies'], 'attended_platform_admin_required') + lanes = json.loads((ROOT / 'openbao/eso-auth-recovery/lanes.json').read_text()) + # Refuse all mutations if any lane would replace divergent deployed key material. + for lane in lanes: + compare_custody(kube, lane) + if args.action == 'apply': + command(kube + ['apply', '-f', str(ROOT / 'openbao/eso-auth-recovery/serviceaccounts.yaml')]) + for lane in lanes: + row = {'name': lane['name'], 'namespace': lane['namespace'], 'status': 'pending'} + receipt['lanes'].append(row) + source = ROOT / 'argocd/platform-addons/openbao-secretstore' / (lane['store'] + '.clustersecretstore.yaml') + desired = yaml.safe_load(source.read_text()) + live = data(command(kube + ['get', 'clustersecretstore', lane['store'], '-o', 'json'])) + expected_old = {'tokenSecretRef': {'name': lane['store'] + '-eso-token', 'namespace': 'external-secrets', 'key': 'token'}} + a, b = live['spec']['provider']['vault'], desired['spec']['provider']['vault'] + require({k:v for k,v in a.items() if k != 'auth'} == {k:v for k,v in b.items() if k != 'auth'} and live['spec']['conditions'] == desired['spec']['conditions'], 'store_scope_drift') + require(a['auth'] in [expected_old, b['auth']], 'store_auth_drift') + policy = (ROOT / 'openbao/policies' / (lane['policy'] + '.hcl')).read_text() + if args.action == 'apply': + old = bao(['read', '-format=json', 'sys/policies/acl/' + lane['policy']], allow_failure=True) + if old.returncode == 0: + require(data(old)['data']['policy'] == policy, 'policy_drift') + else: + require(b'404' in old.stderr or b'No value found' in old.stderr, 'policy_absence_unproven') + old_role = bao(['read', '-format=json', 'auth/kubernetes/role/' + lane['role']], allow_failure=True) + if old_role.returncode == 0: + check_role(data(old_role)['data'], lane) + else: + require(b'404' in old_role.stderr or b'No value found' in old_role.stderr, 'role_absence_unproven') + bao(['write', 'sys/policies/acl/' + lane['policy'], '-'], payload={'policy': policy}) + bao(['write', 'auth/kubernetes/role/' + lane['role'], '-'], payload=role_payload(lane)) + check_role(data(bao(['read', '-format=json', 'auth/kubernetes/role/' + lane['role']]))['data'], lane) + verify_login(kube, lane, row) + row['kv_version'] = compare_custody(kube, lane) + if args.action == 'apply': + # JSON Patch tests the observed spec atomically; replace only authentication. + patch = [{'op':'test','path':'/spec','value':live['spec']}, {'op':'replace','path':'/spec/provider/vault/auth','value':b['auth']}] + command(kube + ['patch', 'clustersecretstore', lane['store'], '--type=json', '-p', json.dumps(patch)]) + command(kube + ['annotate', 'clustersecretstore', lane['store'], 'force-sync=' + str(time.time_ns()), '--overwrite']) + command(kube + ['-n', lane['namespace'], 'annotate', 'externalsecret', lane['name'], 'force-sync=' + str(time.time_ns()), '--overwrite']) + command(kube + ['wait', '--for=condition=Ready', 'clustersecretstore/' + lane['store'], '--timeout=45s']) + command(kube + ['-n', lane['namespace'], 'wait', '--for=condition=Ready', 'externalsecret/' + lane['name'], '--timeout=45s']) + compare_custody(kube, lane) + row.update(status='passed', unchanged_workload_values=True, store_ready=True, external_secret_ready=True) + receipt['status'] = 'passed' + + +def main(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument('action', choices=['apply', 'verify']) + p.add_argument('--kubeconfig', required=True) + p.add_argument('--receipt', required=True) + p.add_argument('--confirm', required=True) + args = p.parse_args() + receipt = {'schema':'platform.eso-kubernetes-recovery.v1','status':'failed','lanes':[]} + fd = os.open(args.receipt, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600) + try: + require(args.confirm == 'APPLY RPF-WP-0037', 'confirmation_mismatch') + run(args, receipt) + except Exception as exc: + receipt['error'] = str(exc) if isinstance(exc, LaneError) else 'internal_error' + finally: + with os.fdopen(fd, 'w') as out: + json.dump(receipt, out, indent=2) + out.write('\n') + return 0 if receipt['status'] == 'passed' else 1 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/tests/test_eso_auth_recovery.py b/tests/test_eso_auth_recovery.py new file mode 100644 index 0000000..50ae08f --- /dev/null +++ b/tests/test_eso_auth_recovery.py @@ -0,0 +1,44 @@ +import importlib.util +import json +from pathlib import Path +import sys +from types import SimpleNamespace +import unittest +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'scripts')) +import repair_eso_kubernetes_auth as repair + + +class EsoRecoveryTests(unittest.TestCase): + def setUp(self): + self.lane = json.loads((ROOT / 'openbao/eso-auth-recovery/lanes.json').read_text())[0] + + def test_role_rejects_wildcard_namespace_added_policy_and_unbounded_ttl(self): + native = {k:900 if v == '15m' else v for k,v in repair.role_payload(self.lane).items()} + repair.check_role(native, self.lane) + for key,value in [('bound_service_account_namespaces',['*']),('token_policies',['root']),('token_explicit_max_ttl',0),('audience','other')]: + with self.assertRaises(repair.LaneError): + repair.check_role(dict(native, **{key:value}), self.lane) + + def test_custody_disagreement_stops_before_serviceaccount_or_auth_write(self): + identity = SimpleNamespace(stdout=json.dumps({'data':{'policies':['platform-admin']}}).encode()) + with patch.object(repair,'assert_cluster'), patch.object(repair,'bao',return_value=identity) as bao, patch.object(repair,'command') as kube, patch.object(repair,'compare_custody',side_effect=repair.LaneError('custody_disagrees')): + with self.assertRaises(repair.LaneError): + repair.run(SimpleNamespace(action='apply',kubeconfig='/fixture'), {'lanes':[]}) + kube.assert_not_called() + self.assertEqual(bao.call_count,1) + + def test_no_credential_values_or_parent_access_in_generated_policies(self): + for lane in json.loads((ROOT / 'openbao/eso-auth-recovery/lanes.json').read_text()): + policy=(ROOT / 'openbao/policies' / (lane['policy']+'.hcl')).read_text() + self.assertNotIn('*',policy) + self.assertNotIn('/metadata/',policy) + self.assertNotIn('"list"',policy) + self.assertEqual(policy.count('path "'),3) + self.assertIn('path "'+lane['kv_path']+'"',policy) + + +if __name__=='__main__': + unittest.main() diff --git a/workplans/RPF-WP-0037-eso-static-token-recovery.md b/workplans/RPF-WP-0037-eso-static-token-recovery.md new file mode 100644 index 0000000..253ed1a --- /dev/null +++ b/workplans/RPF-WP-0037-eso-static-token-recovery.md @@ -0,0 +1,58 @@ +--- +id: RPF-WP-0037 +type: workplan +title: "Recover three active ESO lanes from static-token authentication failure" +domain: financials +repo: railiance-platform +status: active +owner: codex +created: "2026-09-05" +updated: "2026-09-05" +--- + +User authorized continuing with the identified delivery failures. Live metadata +confirms active Forgejo, reuse-surface and target-revenue consumers. All three +ClusterSecretStores fail OpenBao token lookup-self with 403. This plan repairs +delivery authentication; it does not rotate application credentials or restart +consumers. RPF-WP-0036-T04 retains the separate telemetry acceptance gate. + +## Diagnose authentication and compare custody without exposing values + +```task +id: RPF-WP-0037-T01 +status: progress +priority: high +``` + +Use attended platform-admin and pinned cluster access. Capture token validity, +policy coordinates and boolean equality between native KV data and the existing +consumer Secret. No values, tokens, hashes or fingerprints in evidence. Stop +before any resync if custody disagrees with the deployed Secret. + +## Replace static authentication with exact Kubernetes bindings + +```task +id: RPF-WP-0037-T02 +status: todo +priority: high +``` + +Stage dedicated namespace/SA/audience-bound roles with 15-minute maximum TTL, +no default policy and exact KV data read plus token self-lookup/revocation. +Keep namespace restrictions and existing delivery paths. Refuse live drift. +Validate positive read and wrong-identity/sibling/write denial before changing +each store. Persist source before apply. Do not overwrite workload key material. + +## Verify sustained delivery and record closure + +```task +id: RPF-WP-0037-T03 +status: todo +priority: high +``` + +Require Ready stores and ExternalSecrets, a second successful forced reconcile, +unchanged deployed key material and healthy consumers. Record safe receipts and +retire obsolete bootstrap instructions. Remove obsolete static token references +from stores; retain old token Secret objects until their invalidity is proven. +Commit and synchronize file-backed work and report any remaining gates.