Prepare bounded Kubernetes authentication recovery for three ESO lanes

Assistant: codex
Assistant-Model: gpt-6-astra
Assistant-Session: 01a06ecb-456a-71c2-b41e-0755d336e883

argocd/platform-addons/openbao-secretstore/openbao-forgejo.clustersecretstore.yaml

@@ -1,8 +1,4 @@
-# Forgejo reads SMTP credentials from the private railiance01 OpenBao service.
-#
-# Prereq: Secret external-secrets/openbao-forgejo-eso-token (key: token) with a
-# policy-limited OpenBao token that can read platform/workloads/forgejo/forgejo-mailer.
-# Bootstrap: railiance-apps make forgejo-openbao-eso-token-apply
+# RPF-WP-0037: exact namespace/SA/audience Kubernetes auth; no static token.
 apiVersion: external-secrets.io/v1beta1
 kind: ClusterSecretStore
 metadata:
@@ -17,10 +13,14 @@ spec:
       path: platform
       version: v2
       auth:
-        tokenSecretRef:
-          name: openbao-forgejo-eso-token
-          namespace: external-secrets
-          key: token
+        kubernetes:
+          mountPath: kubernetes
+          role: forgejo-mailer-eso
+          serviceAccountRef:
+            name: forgejo-mailer-eso
+            namespace: forgejo
+            audiences:
+            - openbao
   conditions:
-    - namespaces:
-        - forgejo
+  - namespaces:
+    - forgejo

argocd/platform-addons/openbao-secretstore/openbao-reuse.clustersecretstore.yaml

@@ -1,11 +1,4 @@
-# reuse-surface uses the private OpenBao service on railiance01. Public
-# bao.coulomb.social remains the operator/API address and is not a workload
-# runtime dependency.
-#
-# Prereq: Secret external-secrets/openbao-reuse-eso-token (key: token) with a
-# policy-limited OpenBao token that can read
-# platform/workloads/reuse/reuse-surface/runtime-secrets.
-# Bootstrap: railiance-apps make reuse-openbao-eso-token-apply
+# RPF-WP-0037: exact namespace/SA/audience Kubernetes auth; no static token.
 apiVersion: external-secrets.io/v1beta1
 kind: ClusterSecretStore
 metadata:
@@ -20,10 +13,14 @@ spec:
       path: platform
       version: v2
       auth:
-        tokenSecretRef:
-          name: openbao-reuse-eso-token
-          namespace: external-secrets
-          key: token
+        kubernetes:
+          mountPath: kubernetes
+          role: reuse-surface-runtime-eso
+          serviceAccountRef:
+            name: reuse-surface-eso
+            namespace: reuse
+            audiences:
+            - openbao
   conditions:
-    - namespaces:
-        - reuse
+  - namespaces:
+    - reuse

argocd/platform-addons/openbao-secretstore/openbao-target-revenue.clustersecretstore.yaml

@@ -1,9 +1,4 @@
-# target-revenue Trust Service runtime secrets on railiance01 (WP-0011).
-# Pattern matches openbao-reuse / openbao-activity-core: token auth to the
-# in-cluster OpenBao service, store limited to the target-revenue namespace.
-#
-# Prereq: Secret external-secrets/openbao-target-revenue-eso-token (key: token)
-# Bootstrap: target-revenue scripts/openbao-eso-token-apply.sh
+# RPF-WP-0037: exact namespace/SA/audience Kubernetes auth; no static token.
 apiVersion: external-secrets.io/v1beta1
 kind: ClusterSecretStore
 metadata:
@@ -19,10 +14,14 @@ spec:
       path: platform
       version: v2
       auth:
-        tokenSecretRef:
-          name: openbao-target-revenue-eso-token
-          namespace: external-secrets
-          key: token
+        kubernetes:
+          mountPath: kubernetes
+          role: target-revenue-runtime-eso
+          serviceAccountRef:
+            name: target-revenue-eso
+            namespace: target-revenue
+            audiences:
+            - openbao
   conditions:
-    - namespaces:
-        - target-revenue
+  - namespaces:
+    - target-revenue

openbao/eso-auth-recovery/lanes.json

@@ -0,0 +1,29 @@
+[
+  {
+    "namespace": "forgejo",
+    "name": "forgejo-mailer",
+    "store": "openbao-forgejo",
+    "service_account": "forgejo-mailer-eso",
+    "kv_path": "platform/data/workloads/forgejo/forgejo-mailer",
+    "policy": "workload-kv-read-forgejo-mailer-eso",
+    "role": "forgejo-mailer-eso"
+  },
+  {
+    "namespace": "reuse",
+    "name": "reuse-surface-runtime",
+    "store": "openbao-reuse",
+    "service_account": "reuse-surface-eso",
+    "kv_path": "platform/data/workloads/reuse/reuse-surface/runtime-secrets",
+    "policy": "workload-kv-read-reuse-surface-runtime-eso",
+    "role": "reuse-surface-runtime-eso"
+  },
+  {
+    "namespace": "target-revenue",
+    "name": "target-revenue-runtime",
+    "store": "openbao-target-revenue",
+    "service_account": "target-revenue-eso",
+    "kv_path": "platform/data/workloads/target-revenue/runtime-secrets",
+    "policy": "workload-kv-read-target-revenue-runtime-eso",
+    "role": "target-revenue-runtime-eso"
+  }
+]

openbao/eso-auth-recovery/serviceaccounts.yaml

@@ -0,0 +1,20 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: forgejo-mailer-eso
+  namespace: forgejo
+automountServiceAccountToken: false
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: reuse-surface-eso
+  namespace: reuse
+automountServiceAccountToken: false
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: target-revenue-eso
+  namespace: target-revenue
+automountServiceAccountToken: false

openbao/policies/workload-kv-read-forgejo-mailer-eso.hcl

@@ -0,0 +1,11 @@
+path "platform/data/workloads/forgejo/forgejo-mailer" {
+  capabilities = ["read"]
+}
+
+path "auth/token/lookup-self" {
+  capabilities = ["read"]
+}
+
+path "auth/token/revoke-self" {
+  capabilities = ["update"]
+}

openbao/policies/workload-kv-read-reuse-surface-runtime-eso.hcl

@@ -0,0 +1,11 @@
+path "platform/data/workloads/reuse/reuse-surface/runtime-secrets" {
+  capabilities = ["read"]
+}
+
+path "auth/token/lookup-self" {
+  capabilities = ["read"]
+}
+
+path "auth/token/revoke-self" {
+  capabilities = ["update"]
+}

openbao/policies/workload-kv-read-target-revenue-runtime-eso.hcl

@@ -0,0 +1,11 @@
+path "platform/data/workloads/target-revenue/runtime-secrets" {
+  capabilities = ["read"]
+}
+
+path "auth/token/lookup-self" {
+  capabilities = ["read"]
+}
+
+path "auth/token/revoke-self" {
+  capabilities = ["update"]
+}

scripts/repair_eso_kubernetes_auth.py

@@ -0,0 +1,155 @@
+#!/usr/bin/env python3
+"""Attended exact-lane ESO auth repair; no credential values in output/evidence."""
+import argparse
+import base64
+import json
+import os
+from pathlib import Path
+import time
+
+import yaml
+from state_hub_preflight_lane import ROOT, LaneError, assert_cluster, bao, command, data, capabilities, revoke
+
+
+def require(condition, label):
+    if not condition:
+        raise LaneError(label)
+
+
+def role_payload(lane):
+    return {'bound_service_account_names': [lane['service_account']],
+            'bound_service_account_namespaces': [lane['namespace']],
+            'token_policies': [lane['policy']], 'audience': 'openbao',
+            'token_ttl': '15m', 'token_max_ttl': '15m',
+            'token_explicit_max_ttl': '15m', 'token_no_default_policy': True}
+
+
+def check_role(actual, lane):
+    for key, value in role_payload(lane).items():
+        require(actual.get(key) == (900 if value == '15m' else value), 'role_drift')
+    require(not actual.get('token_period') and not actual.get('bound_service_account_namespace_selector'), 'role_drift')
+
+
+def compare_custody(kube, lane):
+    es = data(command(kube + ['-n', lane['namespace'], 'get', 'externalsecret', lane['name'], '-o', 'json']))
+    require(es['spec']['secretStoreRef'] == {'kind': 'ClusterSecretStore', 'name': lane['store']}, 'external_secret_store_drift')
+    target = es['spec']['target'].get('name', lane['name'])
+    deployed = data(command(kube + ['-n', lane['namespace'], 'get', 'secret', target, '-o', 'json']))
+    native = data(bao(['read', '-format=json', lane['kv_path']]))['data']
+    for entry in es['spec']['data']:
+        ref = entry['remoteRef']
+        require('platform/data/' + ref['key'] == lane['kv_path'], 'remote_path_drift')
+        require(str(native['data'][ref['property']]).encode() == base64.b64decode(deployed['data'][entry['secretKey']]), 'custody_disagrees_stop_before_resync')
+    return native['metadata']['version']
+
+
+def verify_login(kube, lane, row):
+    def login(sa, audience, negative=False, namespace=None):
+        jwt = command(kube + ['-n', namespace or lane['namespace'], 'create', 'token', sa, '--audience=' + audience, '--duration=10m']).stdout.decode().strip()
+        result = bao(['write', '-format=json', 'auth/kubernetes/login', '-'], payload={'role': lane['role'], 'jwt': jwt}, allow_failure=negative)
+        if negative:
+            if result.returncode == 0:
+                revoke(data(result)['auth']['client_token'])
+                raise LaneError('negative_login_succeeded')
+            require(b'403' in result.stderr or b'400' in result.stderr, 'negative_login_inconclusive')
+            return None
+        return data(result)['auth']
+    auth = login(lane['service_account'], 'openbao')
+    token = auth['client_token']
+    try:
+        require(auth['token_policies'] == [lane['policy']] and 0 < auth['lease_duration'] <= 900, 'effective_policy_or_ttl_drift')
+        paths = [lane['kv_path'], lane['kv_path'].replace('/data/', '/metadata/'),
+                 lane['kv_path'] + '-unrelated', 'auth/token/create',
+                 'auth/token/lookup-self', 'auth/token/revoke-self']
+        caps = capabilities(token, paths)
+        require(caps[paths[0]] == ['read'], 'read_or_write_scope_drift')
+        require(all(caps[p] == ['deny'] for p in paths[1:4]), 'negative_scope_failed')
+        require(caps[paths[4]] == ['read'] and caps[paths[5]] == ['update'], 'token_self_scope_failed')
+        bao(['read', '-format=json', lane['kv_path']], token=token)
+        bao(['token', 'lookup', '-format=json'], token=token)
+    finally:
+        revoke(token)
+    login('default', 'openbao', negative=True)
+    login(lane['service_account'], 'wrong-audience', negative=True)
+    command(kube + ['-n', 'default', 'create', 'serviceaccount', lane['service_account']])
+    try:
+        login(lane['service_account'], 'openbao', negative=True, namespace='default')
+    finally:
+        command(kube + ['-n', 'default', 'delete', 'serviceaccount', lane['service_account']])
+    row.update(exact_read=True, secret_write_and_sibling_denied=True, wrong_sa_denied=True, wrong_namespace_denied=True, wrong_audience_denied=True, bounded_ttl=True)
+
+
+def run(args, receipt):
+    kube = ['kubectl', '--kubeconfig', args.kubeconfig]
+    assert_cluster(kube)
+    ident = data(bao(['token', 'lookup', '-format=json']))['data']
+    require('platform-admin' in ident['policies'] and 'root' not in ident['policies'], 'attended_platform_admin_required')
+    lanes = json.loads((ROOT / 'openbao/eso-auth-recovery/lanes.json').read_text())
+    # Refuse all mutations if any lane would replace divergent deployed key material.
+    for lane in lanes:
+        compare_custody(kube, lane)
+    if args.action == 'apply':
+        command(kube + ['apply', '-f', str(ROOT / 'openbao/eso-auth-recovery/serviceaccounts.yaml')])
+    for lane in lanes:
+        row = {'name': lane['name'], 'namespace': lane['namespace'], 'status': 'pending'}
+        receipt['lanes'].append(row)
+        source = ROOT / 'argocd/platform-addons/openbao-secretstore' / (lane['store'] + '.clustersecretstore.yaml')
+        desired = yaml.safe_load(source.read_text())
+        live = data(command(kube + ['get', 'clustersecretstore', lane['store'], '-o', 'json']))
+        expected_old = {'tokenSecretRef': {'name': lane['store'] + '-eso-token', 'namespace': 'external-secrets', 'key': 'token'}}
+        a, b = live['spec']['provider']['vault'], desired['spec']['provider']['vault']
+        require({k:v for k,v in a.items() if k != 'auth'} == {k:v for k,v in b.items() if k != 'auth'} and live['spec']['conditions'] == desired['spec']['conditions'], 'store_scope_drift')
+        require(a['auth'] in [expected_old, b['auth']], 'store_auth_drift')
+        policy = (ROOT / 'openbao/policies' / (lane['policy'] + '.hcl')).read_text()
+        if args.action == 'apply':
+            old = bao(['read', '-format=json', 'sys/policies/acl/' + lane['policy']], allow_failure=True)
+            if old.returncode == 0:
+                require(data(old)['data']['policy'] == policy, 'policy_drift')
+            else:
+                require(b'404' in old.stderr or b'No value found' in old.stderr, 'policy_absence_unproven')
+            old_role = bao(['read', '-format=json', 'auth/kubernetes/role/' + lane['role']], allow_failure=True)
+            if old_role.returncode == 0:
+                check_role(data(old_role)['data'], lane)
+            else:
+                require(b'404' in old_role.stderr or b'No value found' in old_role.stderr, 'role_absence_unproven')
+            bao(['write', 'sys/policies/acl/' + lane['policy'], '-'], payload={'policy': policy})
+            bao(['write', 'auth/kubernetes/role/' + lane['role'], '-'], payload=role_payload(lane))
+        check_role(data(bao(['read', '-format=json', 'auth/kubernetes/role/' + lane['role']]))['data'], lane)
+        verify_login(kube, lane, row)
+        row['kv_version'] = compare_custody(kube, lane)
+        if args.action == 'apply':
+            # JSON Patch tests the observed spec atomically; replace only authentication.
+            patch = [{'op':'test','path':'/spec','value':live['spec']}, {'op':'replace','path':'/spec/provider/vault/auth','value':b['auth']}]
+            command(kube + ['patch', 'clustersecretstore', lane['store'], '--type=json', '-p', json.dumps(patch)])
+        command(kube + ['annotate', 'clustersecretstore', lane['store'], 'force-sync=' + str(time.time_ns()), '--overwrite'])
+        command(kube + ['-n', lane['namespace'], 'annotate', 'externalsecret', lane['name'], 'force-sync=' + str(time.time_ns()), '--overwrite'])
+        command(kube + ['wait', '--for=condition=Ready', 'clustersecretstore/' + lane['store'], '--timeout=45s'])
+        command(kube + ['-n', lane['namespace'], 'wait', '--for=condition=Ready', 'externalsecret/' + lane['name'], '--timeout=45s'])
+        compare_custody(kube, lane)
+        row.update(status='passed', unchanged_workload_values=True, store_ready=True, external_secret_ready=True)
+    receipt['status'] = 'passed'
+
+
+def main():
+    p = argparse.ArgumentParser(description=__doc__)
+    p.add_argument('action', choices=['apply', 'verify'])
+    p.add_argument('--kubeconfig', required=True)
+    p.add_argument('--receipt', required=True)
+    p.add_argument('--confirm', required=True)
+    args = p.parse_args()
+    receipt = {'schema':'platform.eso-kubernetes-recovery.v1','status':'failed','lanes':[]}
+    fd = os.open(args.receipt, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600)
+    try:
+        require(args.confirm == 'APPLY RPF-WP-0037', 'confirmation_mismatch')
+        run(args, receipt)
+    except Exception as exc:
+        receipt['error'] = str(exc) if isinstance(exc, LaneError) else 'internal_error'
+    finally:
+        with os.fdopen(fd, 'w') as out:
+            json.dump(receipt, out, indent=2)
+            out.write('\n')
+    return 0 if receipt['status'] == 'passed' else 1
+
+
+if __name__ == '__main__':
+    raise SystemExit(main())

tests/test_eso_auth_recovery.py

@@ -0,0 +1,44 @@
+import importlib.util
+import json
+from pathlib import Path
+import sys
+from types import SimpleNamespace
+import unittest
+from unittest.mock import patch
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT / 'scripts'))
+import repair_eso_kubernetes_auth as repair
+
+
+class EsoRecoveryTests(unittest.TestCase):
+    def setUp(self):
+        self.lane = json.loads((ROOT / 'openbao/eso-auth-recovery/lanes.json').read_text())[0]
+
+    def test_role_rejects_wildcard_namespace_added_policy_and_unbounded_ttl(self):
+        native = {k:900 if v == '15m' else v for k,v in repair.role_payload(self.lane).items()}
+        repair.check_role(native, self.lane)
+        for key,value in [('bound_service_account_namespaces',['*']),('token_policies',['root']),('token_explicit_max_ttl',0),('audience','other')]:
+            with self.assertRaises(repair.LaneError):
+                repair.check_role(dict(native, **{key:value}), self.lane)
+
+    def test_custody_disagreement_stops_before_serviceaccount_or_auth_write(self):
+        identity = SimpleNamespace(stdout=json.dumps({'data':{'policies':['platform-admin']}}).encode())
+        with patch.object(repair,'assert_cluster'), patch.object(repair,'bao',return_value=identity) as bao, patch.object(repair,'command') as kube, patch.object(repair,'compare_custody',side_effect=repair.LaneError('custody_disagrees')):
+            with self.assertRaises(repair.LaneError):
+                repair.run(SimpleNamespace(action='apply',kubeconfig='/fixture'), {'lanes':[]})
+            kube.assert_not_called()
+            self.assertEqual(bao.call_count,1)
+
+    def test_no_credential_values_or_parent_access_in_generated_policies(self):
+        for lane in json.loads((ROOT / 'openbao/eso-auth-recovery/lanes.json').read_text()):
+            policy=(ROOT / 'openbao/policies' / (lane['policy']+'.hcl')).read_text()
+            self.assertNotIn('*',policy)
+            self.assertNotIn('/metadata/',policy)
+            self.assertNotIn('"list"',policy)
+            self.assertEqual(policy.count('path "'),3)
+            self.assertIn('path "'+lane['kv_path']+'"',policy)
+
+
+if __name__=='__main__':
+    unittest.main()

workplans/RPF-WP-0037-eso-static-token-recovery.md

@@ -0,0 +1,58 @@
+---
+id: RPF-WP-0037
+type: workplan
+title: "Recover three active ESO lanes from static-token authentication failure"
+domain: financials
+repo: railiance-platform
+status: active
+owner: codex
+created: "2026-09-05"
+updated: "2026-09-05"
+---
+
+User authorized continuing with the identified delivery failures. Live metadata
+confirms active Forgejo, reuse-surface and target-revenue consumers. All three
+ClusterSecretStores fail OpenBao token lookup-self with 403. This plan repairs
+delivery authentication; it does not rotate application credentials or restart
+consumers. RPF-WP-0036-T04 retains the separate telemetry acceptance gate.
+
+## Diagnose authentication and compare custody without exposing values
+
+```task
+id: RPF-WP-0037-T01
+status: progress
+priority: high
+```
+
+Use attended platform-admin and pinned cluster access. Capture token validity,
+policy coordinates and boolean equality between native KV data and the existing
+consumer Secret. No values, tokens, hashes or fingerprints in evidence. Stop
+before any resync if custody disagrees with the deployed Secret.
+
+## Replace static authentication with exact Kubernetes bindings
+
+```task
+id: RPF-WP-0037-T02
+status: todo
+priority: high
+```
+
+Stage dedicated namespace/SA/audience-bound roles with 15-minute maximum TTL,
+no default policy and exact KV data read plus token self-lookup/revocation.
+Keep namespace restrictions and existing delivery paths. Refuse live drift.
+Validate positive read and wrong-identity/sibling/write denial before changing
+each store. Persist source before apply. Do not overwrite workload key material.
+
+## Verify sustained delivery and record closure
+
+```task
+id: RPF-WP-0037-T03
+status: todo
+priority: high
+```
+
+Require Ready stores and ExternalSecrets, a second successful forced reconcile,
+unchanged deployed key material and healthy consumers. Record safe receipts and
+retire obsolete bootstrap instructions. Remove obsolete static token references
+from stores; retain old token Secret objects until their invalidity is proven.
+Commit and synchronize file-backed work and report any remaining gates.