Close three-lane ESO recovery with live verification and cleanup evidence

Assistant: codex
Assistant-Model: gpt-6-astra
Assistant-Session: 01a06ecb-456a-71c2-b41e-0755d336e883

WORK-RECORDS.md

@@ -14,6 +14,7 @@
 | workplan | RPF-WP-0029 | blocked | — | workplans/RPF-WP-0029-backup-credential-default-removal.md |
 | workplan | RPF-WP-0035 | blocked | — | workplans/RPF-WP-0035-credential-lane-implementation.md |
 | workplan | RPF-WP-0036 | blocked | — | workplans/RPF-WP-0036-platform-service-assurance.md |
+| workplan | RPF-WP-0037 | finished | — | workplans/RPF-WP-0037-eso-static-token-recovery.md |
 | task | RPF-WP-0015-T01 | done | — | workplans/RPF-WP-0015-audit-core-custody-and-recovery-coordination.md |
 | task | RPF-WP-0015-T02 | wait | — | workplans/RPF-WP-0015-audit-core-custody-and-recovery-coordination.md |
 | task | RPF-WP-0015-T03 | wait | — | workplans/RPF-WP-0015-audit-core-custody-and-recovery-coordination.md |
@@ -40,3 +41,6 @@
 | task | RPF-WP-0036-T05 | done | — | workplans/RPF-WP-0036-platform-service-assurance.md |
 | task | RPF-WP-0036-T06 | wait | — | workplans/RPF-WP-0036-platform-service-assurance.md |
 | task | RPF-WP-0036-T07 | done | — | workplans/RPF-WP-0036-platform-service-assurance.md |
+| task | RPF-WP-0037-T01 | done | — | workplans/RPF-WP-0037-eso-static-token-recovery.md |
+| task | RPF-WP-0037-T02 | done | — | workplans/RPF-WP-0037-eso-static-token-recovery.md |
+| task | RPF-WP-0037-T03 | done | — | workplans/RPF-WP-0037-eso-static-token-recovery.md |

credential-change-requests/CCR-2026-0005-reuse-surface-runtime-secrets-lane.yaml

@@ -5,7 +5,7 @@ request_type: workload-kv-read
 title: reuse-surface runtime secrets lane
 status: active
 created: '2026-07-07'
-updated: '2026-07-07'
+updated: '2026-09-05'
 requester:
   agent: grok
   reason: Promote reuse-surface hub runtime secrets from bootstrap Kubernetes Secret
@@ -38,6 +38,14 @@ review:
     comment: 'Approved in chat acting as platform-operator and reuse-surface-owner:
       promote reuse-surface runtime secrets to OpenBao workload KV with Railiance01
       interim ESO delivery per CCR-2026-0005.'
+  - at: '2026-09-05'
+    reviewer: codex under user-authorized delivery recovery
+    decision: delivery_auth_migration_verified
+    comment: RPF-WP-0037 replaces rejected static ESO token with exact namespace/SA/audience
+      binding and data-only/self-token policy. Workload values unchanged; positive,
+      wrong-SA/namespace/audience and write/sibling negatives passed. Existing caller-facing
+      route retained. Historical July interim/bootstrap notes below are superseded
+      for ESO delivery.
 target:
   domain: financials
   tenant: reuse
@@ -51,21 +59,27 @@ openbao:
   fields:
   - REUSE_SURFACE_TOKEN
   - REUSE_SURFACE_FORGEJO_WEBHOOK_SECRET
-  policy_name: workload-kv-read-reuse-surface-runtime
+  policy_name: workload-kv-read-reuse-surface-runtime-eso
-  policy_file: openbao/policies/workload-kv-read-reuse-surface-runtime.hcl
+  policy_file: openbao/policies/workload-kv-read-reuse-surface-runtime-eso.hcl
   auth:
     method: kubernetes
     mount: kubernetes
-    role: external-secrets-reuse-surface
+    role: reuse-surface-runtime-eso
     bound_claims:
       service_account_names:
-      - external-secrets
+      - reuse-surface-eso
       service_account_namespaces:
-      - external-secrets
+      - reuse
     bound_claims_confirmed: true
     policies:
-    - workload-kv-read-reuse-surface-runtime
+    - workload-kv-read-reuse-surface-runtime-eso
     ttl: 15m
+    audience: openbao
+    token_max_ttl: 15m
+    token_explicit_max_ttl: 15m
+    token_no_default_policy: true
+  metadata_read: false
+  token_self_lifecycle: true
 access_frontdoor:
   type: ops-warden
   catalog_id: reuse-surface-hub-write-token
@@ -76,10 +90,10 @@ access_frontdoor:
   activation: verified-positive-and-negative-access-frontdoor-active-2026-07-07
 delivery:
   surface: external-secrets
-  target: ExternalSecret reuse/reuse-surface-runtime -> Secret reuse-surface-env in
+  target: ExternalSecret reuse/reuse-surface-runtime -> Secret reuse-surface-env;
-    the reuse namespace on Railiance01; interim ClusterSecretStore openbao-reuse (token
+    ClusterSecretStore openbao-reuse uses private railiance01 OpenBao and exact reuse/reuse-surface-eso
-    auth to coulombcore OpenBao, namespace-limited to reuse) until railiance01-local
+    Kubernetes identity. Caller-facing OIDC policy workload-kv-read-reuse-surface-runtime
-    OpenBao Wave 7 bootstrap
+    remains separately retained.
 risk:
   classification: high
   notes:
@@ -149,6 +163,12 @@ verification:
     details:
     - ops-warden catalog reuse-surface-hub-write-token migrated to bao kv get handoff;
       resolvable=true; playbook and workload-kv-access-lanes updated (RAILIANCE-WP-0011-T03)
+  - at: '2026-09-05'
+    actor: codex
+    kind: kubernetes_delivery_recovery
+    result: passed
+    details:
+    - RPF-WP-0037; docs/evidence/RPF-WP-0037-eso-auth-recovery-2026-09-05.json
 lifecycle:
   deactivate: Disable ops-warden catalog entry, remove ExternalSecret, and detach
     auth role policy; delete materialized reuse-surface-env only after confirming

docs/evidence/RPF-WP-0037-eso-auth-recovery-2026-09-05.json

@@ -0,0 +1,188 @@
+{
+  "schema": "platform.eso-auth-recovery-evidence.v1",
+  "date": "2026-09-05",
+  "cluster_uid": "a553c742-0115-43d4-99a4-a5ca56fe0786",
+  "diagnosis": {
+    "status": "diagnosed",
+    "lanes": [
+      {
+        "namespace": "forgejo",
+        "name": "forgejo-mailer",
+        "static_token_valid": false,
+        "static_token_denied": true,
+        "fields_match": {
+          "MAILER_PASSWD": true
+        },
+        "versions": {
+          "platform/data/workloads/forgejo/forgejo-mailer": 2
+        }
+      },
+      {
+        "namespace": "reuse",
+        "name": "reuse-surface-runtime",
+        "static_token_valid": false,
+        "static_token_denied": true,
+        "fields_match": {
+          "REUSE_SURFACE_TOKEN": true,
+          "REUSE_SURFACE_FORGEJO_WEBHOOK_SECRET": true
+        },
+        "versions": {
+          "platform/data/workloads/reuse/reuse-surface/runtime-secrets": 1
+        }
+      },
+      {
+        "namespace": "target-revenue",
+        "name": "target-revenue-runtime",
+        "static_token_valid": false,
+        "static_token_denied": true,
+        "fields_match": {
+          "TRF_SIGNING_KEY_HEX": true,
+          "TRF_CONTROL_PLANE_SECRET_KEY": true,
+          "TRF_DATABASE_URL": true,
+          "TRF_MIGRATE_DATABASE_URL": true
+        },
+        "versions": {
+          "platform/data/workloads/target-revenue/runtime-secrets": 1
+        }
+      }
+    ]
+  },
+  "apply": {
+    "schema": "platform.eso-kubernetes-recovery.v1",
+    "status": "passed",
+    "lanes": [
+      {
+        "name": "forgejo-mailer",
+        "namespace": "forgejo",
+        "status": "passed",
+        "exact_read": true,
+        "secret_write_and_sibling_denied": true,
+        "wrong_sa_denied": true,
+        "wrong_namespace_denied": true,
+        "wrong_audience_denied": true,
+        "bounded_ttl": true,
+        "kv_version": 2,
+        "unchanged_workload_values": true,
+        "store_ready": true,
+        "external_secret_ready": true
+      },
+      {
+        "name": "reuse-surface-runtime",
+        "namespace": "reuse",
+        "status": "passed",
+        "exact_read": true,
+        "secret_write_and_sibling_denied": true,
+        "wrong_sa_denied": true,
+        "wrong_namespace_denied": true,
+        "wrong_audience_denied": true,
+        "bounded_ttl": true,
+        "kv_version": 1,
+        "unchanged_workload_values": true,
+        "store_ready": true,
+        "external_secret_ready": true
+      },
+      {
+        "name": "target-revenue-runtime",
+        "namespace": "target-revenue",
+        "status": "passed",
+        "exact_read": true,
+        "secret_write_and_sibling_denied": true,
+        "wrong_sa_denied": true,
+        "wrong_namespace_denied": true,
+        "wrong_audience_denied": true,
+        "bounded_ttl": true,
+        "kv_version": 1,
+        "unchanged_workload_values": true,
+        "store_ready": true,
+        "external_secret_ready": true
+      }
+    ]
+  },
+  "final_verification_and_cleanup": {
+    "schema": "platform.eso-kubernetes-recovery.v1",
+    "status": "passed",
+    "lanes": [
+      {
+        "name": "forgejo-mailer",
+        "namespace": "forgejo",
+        "status": "passed",
+        "exact_read": true,
+        "secret_write_and_sibling_denied": true,
+        "wrong_sa_denied": true,
+        "wrong_namespace_denied": true,
+        "wrong_audience_denied": true,
+        "bounded_ttl": true,
+        "kv_version": 2,
+        "refresh_time": "2026-09-05T16:51:56Z",
+        "unchanged_workload_values": true,
+        "store_ready": true,
+        "external_secret_ready": true,
+        "retired_token_already_invalid": true,
+        "retired_token_secret_removed": true
+      },
+      {
+        "name": "reuse-surface-runtime",
+        "namespace": "reuse",
+        "status": "passed",
+        "exact_read": true,
+        "secret_write_and_sibling_denied": true,
+        "wrong_sa_denied": true,
+        "wrong_namespace_denied": true,
+        "wrong_audience_denied": true,
+        "bounded_ttl": true,
+        "kv_version": 1,
+        "refresh_time": "2026-09-05T16:52:03Z",
+        "unchanged_workload_values": true,
+        "store_ready": true,
+        "external_secret_ready": true,
+        "retired_token_already_invalid": true,
+        "retired_token_secret_removed": true
+      },
+      {
+        "name": "target-revenue-runtime",
+        "namespace": "target-revenue",
+        "status": "passed",
+        "exact_read": true,
+        "secret_write_and_sibling_denied": true,
+        "wrong_sa_denied": true,
+        "wrong_namespace_denied": true,
+        "wrong_audience_denied": true,
+        "bounded_ttl": true,
+        "kv_version": 1,
+        "refresh_time": "2026-09-05T16:52:11Z",
+        "unchanged_workload_values": true,
+        "store_ready": true,
+        "external_secret_ready": true,
+        "retired_token_already_invalid": true,
+        "retired_token_secret_removed": true
+      }
+    ]
+  },
+  "fleet_external_secrets": {
+    "total": 27,
+    "ready": 27
+  },
+  "consumers": [
+    {
+      "namespace": "forgejo",
+      "deployment": "forgejo-gitea",
+      "ready_replicas": 1,
+      "generation": 11
+    },
+    {
+      "namespace": "reuse",
+      "deployment": "reuse-surface",
+      "ready_replicas": 1,
+      "generation": 6
+    },
+    {
+      "namespace": "target-revenue",
+      "deployment": "target-revenue",
+      "ready_replicas": 1,
+      "generation": 5
+    }
+  ],
+  "application_credentials_rotated": false,
+  "consumer_rollouts_requested": false,
+  "unit_tests_passed": 187
+}

history/2026-09-05-eso-auth-recovery.md

@@ -0,0 +1,35 @@
+# Three failing secret deliveries recovered
+
+User authorized the next operational follow-up after signing activation.
+RPF-WP-0037 records the bounded recovery; RPF-WP-0036 retains telemetry ownership.
+
+Forgejo mailer, reuse-surface and target-revenue were active consumers. All three
+stores used static tokens rejected during self-validation. Attended native KV
+reads and captured Kubernetes reads proved every workload field identical before
+resync. No application values, tokens, hashes or fingerprints were recorded.
+
+Dedicated namespace/SA/audience Kubernetes bindings replaced static token auth.
+New policies grant only exact data read and self-token lookup/revocation;
+15-minute TTL/max/explicit max and no default policy are enforced. Source was
+committed and pushed before apply. Existing namespace restrictions were retained
+and JSON Patch tested live store scope before replacing authentication.
+
+Each lane passed native read, wrong-SA/namespace/audience, sibling/write and TTL
+checks. Fresh reconciliation timestamps advanced and workload value equality
+remained true. All three consumers remained ready; no rollout was requested.
+All 27 ExternalSecrets report Ready/SecretSynced at final verification.
+
+Cleanup initially stopped because native lookup returned 403 with `bad token`,
+while the helper expected 400. Fixed classification distinguishes explicit bad
+token from permission denial; regression tests cover both. Native lookup proved
+all three predecessors invalid. After checking zero store references, their
+obsolete Kubernetes Secrets were deleted with UID/resourceVersion preconditions.
+
+Updated deployment manifests, runbook, bootstrap documentation and CCR-2026-0005
+delivery binding. Caller-facing policies and catalog route remain separately
+retained. 187 unit tests and credential validation passed; the signing test
+fixture now clears active-frontdoor state when constructing an approved draft.
+
+Evidence: `docs/evidence/RPF-WP-0037-eso-auth-recovery-2026-09-05.json`.
+No residual delivery repair remains. Backup predecessor invalidation/recovery
+and OpenBao operator-only access retain their separate WP-0029/WP-0025 gates.

workplans/RPF-WP-0036-platform-service-assurance.md

@@ -221,3 +221,11 @@ The output explicitly says unmonitored and unsupported guarantees.
 Evidence: `docs/evidence/RPF-WP-0036-assurance-2026-09-05.json`.
 The plan is blocked on these explicit live/owner gates, not finished merely
 because its repository implementation and tests pass.
+
+
+2026-09-05 ESO recovery follow-up: RPF-WP-0037 resolved all three delivery
+failures using exact Kubernetes authentication. Custody values were unchanged,
+negative checks and repeated refresh passed, obsolete invalid delivery tokens
+were removed, and all 27 ExternalSecrets now report Ready. The earlier dated
+assurance snapshot is retained as historical evidence. T04 still waits on the
+accepted Q2 receiver and controlled failure/absence transport proof.

workplans/RPF-WP-0037-eso-static-token-recovery.md

@@ -4,7 +4,7 @@ type: workplan
 title: "Recover three active ESO lanes from static-token authentication failure"
 domain: financials
 repo: railiance-platform
-status: active
+status: finished
 owner: codex
 created: "2026-09-05"
 updated: "2026-09-05"
@@ -21,7 +21,7 @@ consumers. RPF-WP-0036-T04 retains the separate telemetry acceptance gate.
 
 ```task
 id: RPF-WP-0037-T01
-status: progress
+status: done
 priority: high
 state_hub_task_id: "e0e7ff05-01bf-58db-a18e-4c5e48a697fc"
 ```
@@ -35,7 +35,7 @@ before any resync if custody disagrees with the deployed Secret.
 
 ```task
 id: RPF-WP-0037-T02
-status: todo
+status: done
 priority: high
 state_hub_task_id: "01dd799d-f967-57d6-9cf0-9b01971e5bf1"
 ```
@@ -50,7 +50,7 @@ each store. Persist source before apply. Do not overwrite workload key material.
 
 ```task
 id: RPF-WP-0037-T03
-status: todo
+status: done
 priority: high
 state_hub_task_id: "ee47707c-289b-5bea-a2ad-fdf40fed7034"
 ```
@@ -60,3 +60,21 @@ unchanged deployed key material and healthy consumers. Record safe receipts and
 retire obsolete bootstrap instructions. Remove obsolete static token references
 from stores; retain old token Secret objects until their invalidity is proven.
 Commit and synchronize file-backed work and report any remaining gates.
+
+
+## Completion — 2026-09-05
+
+All three lanes migrated and passed fresh reconciliation, exact reads,
+wrong-SA/namespace/audience rejection, bounded TTL and sibling/write denial.
+OpenBao workload values matched deployed Secrets before and after migration;
+no application credential rotation or consumer rollout was requested. Native
+admin lookup proved old tokens invalid; after proving zero store references,
+all three obsolete token Secrets were removed with deletion preconditions.
+All 27 cluster ExternalSecrets are Ready and all three consumers have one ready
+replica. 187 tests and credential request validation passed. Current source,
+CCR-2026-0005 delivery binding and bootstrap guidance are updated.
+
+Evidence: `docs/evidence/RPF-WP-0037-eso-auth-recovery-2026-09-05.json`.
+Runbook: `docs/eso-auth-recovery.md`.
+RPF-WP-0036-T04 retains the independent Q2 signal-delivery acceptance gate;
+backup credential recovery and operator-only cutover remain in WP-0029/0025.