Prepare OpenBao authority migration

2026-08-03 18:50:13 +02:00 · 2026-08-03 18:50:13 +02:00 · ceed081c8d
commit ceed081c8d
parent c4124b6a7f
4 changed files with 211 additions and 9 deletions
--- a/WORK-RECORDS.md
+++ b/WORK-RECORDS.md
@ -12,7 +12,7 @@
 | workplan | RAILIANCE-WP-0017 | finished | — | workplans/RAILIANCE-WP-0017-rail-rapp-reef-repo-separation.md |
 | workplan | RAILIANCE-WP-0018 | finished | — | workplans/RAILIANCE-WP-0018-first-wave-repo-family-materialization.md |
 | workplan | RAILIANCE-WP-0019 | finished | — | workplans/RAILIANCE-WP-0019-knative-qonto-runtime-on-reef-railiance.md |
-| workplan | RAILIANCE-WP-0020 | ready | — | workplans/RAILIANCE-WP-0020-openbao-migration-to-reef-railiance.md |
+| workplan | RAILIANCE-WP-0020 | active | — | workplans/RAILIANCE-WP-0020-openbao-migration-to-reef-railiance.md |
 | task | ADHOC-2026-07-30-T01 | done | — | workplans/ADHOC-2026-07-30.md |
 | task | RAILIANCE-WP-0017-T01 | done | — | workplans/RAILIANCE-WP-0017-rail-rapp-reef-repo-separation.md |
 | task | RAILIANCE-WP-0017-T02 | done | — | workplans/RAILIANCE-WP-0017-rail-rapp-reef-repo-separation.md |
@ -37,9 +37,9 @@
 | task | RAILIANCE-WP-0019-T08 | done | — | workplans/RAILIANCE-WP-0019-knative-qonto-runtime-on-reef-railiance.md |
 | task | RAILIANCE-WP-0019-T09 | done | — | workplans/RAILIANCE-WP-0019-knative-qonto-runtime-on-reef-railiance.md |
 | task | RAILIANCE-WP-0019-T10 | done | — | workplans/RAILIANCE-WP-0019-knative-qonto-runtime-on-reef-railiance.md |
-| task | RAILIANCE-WP-0020-T01 | todo | — | workplans/RAILIANCE-WP-0020-openbao-migration-to-reef-railiance.md |
-| task | RAILIANCE-WP-0020-T02 | todo | — | workplans/RAILIANCE-WP-0020-openbao-migration-to-reef-railiance.md |
-| task | RAILIANCE-WP-0020-T03 | todo | — | workplans/RAILIANCE-WP-0020-openbao-migration-to-reef-railiance.md |
+| task | RAILIANCE-WP-0020-T01 | done | — | workplans/RAILIANCE-WP-0020-openbao-migration-to-reef-railiance.md |
+| task | RAILIANCE-WP-0020-T02 | done | — | workplans/RAILIANCE-WP-0020-openbao-migration-to-reef-railiance.md |
+| task | RAILIANCE-WP-0020-T03 | wait | — | workplans/RAILIANCE-WP-0020-openbao-migration-to-reef-railiance.md |
 | task | RAILIANCE-WP-0020-T04 | wait | — | workplans/RAILIANCE-WP-0020-openbao-migration-to-reef-railiance.md |
 | task | RAILIANCE-WP-0020-T05 | wait | — | workplans/RAILIANCE-WP-0020-openbao-migration-to-reef-railiance.md |
 | task | RAILIANCE-WP-0020-T06 | wait | — | workplans/RAILIANCE-WP-0020-openbao-migration-to-reef-railiance.md |
--- a/docs/evidence/openbao-migration-inventory-2026-08-03.json
+++ b/docs/evidence/openbao-migration-inventory-2026-08-03.json
@ -0,0 +1,81 @@
+{
+  "captured_at": "2026-08-03",
+  "consumer_contracts": {
+    "cluster_secret_stores": [
+      "openbao-activity-core",
+      "openbao-forgejo",
+      "openbao-rapp-qonto",
+      "openbao-reuse",
+      "openbao-sso-user-engine-runtime",
+      "openbao-user-engine-runtime"
+    ],
+    "cluster_secret_store_count": 6,
+    "external_secret_count": 9,
+    "public_address": "https://bao.coulomb.social",
+    "rollback_owner": "railiance-platform",
+    "target_namespaces": {
+      "activity-core": 3,
+      "forgejo": 1,
+      "rapp-qonto": 1,
+      "reuse": 1,
+      "sso": 1,
+      "user-engine": 2
+    },
+    "transitional_auth": "AppRole and token-based stores; migrate to namespace-restricted Kubernetes auth after private restore"
+  },
+  "source": {
+    "address": "https://bao.coulomb.social",
+    "alias_count": 10,
+    "approles": [
+      "agent-harness-binky-mail",
+      "external-secrets-rapp-qonto",
+      "external-secrets-sso-user-engine",
+      "external-secrets-user-engine",
+      "rein-openweights",
+      "se-prod-whynot-design-npm-publish",
+      "warden-sign"
+    ],
+    "audit_devices": ["file/"],
+    "auth_methods": {
+      "approle/": "approle",
+      "keycape/": "oidc",
+      "kubernetes/": "kubernetes",
+      "netkingdom/": "oidc",
+      "token/": "token"
+    },
+    "cluster_id": "fd28df5d-98ec-57dd-42ec-9b3e4f4e53bf",
+    "entity_count": 10,
+    "initialized": true,
+    "kubernetes_roles": [
+      "external-secrets-activity-core",
+      "external-secrets-issue-core",
+      "external-secrets-reuse-surface"
+    ],
+    "mounts": {
+      "cubbyhole/": "cubbyhole",
+      "identity/": "identity",
+      "platform/": "kv",
+      "reins/": "kv",
+      "secret/": "kv",
+      "ssh/": "ssh",
+      "sys/": "system",
+      "tenants/": "kv"
+    },
+    "policy_count": 26,
+    "raft_peers": [{"leader": true, "node_id": "openbao-0"}],
+    "sealed": false,
+    "seal": {"shares": 3, "threshold": 2, "type": "shamir"},
+    "ssh_roles": ["adm-role", "agt-role", "atm-role"],
+    "storage": {"audit_pvc": "2Gi", "data_pvc": "5Gi", "type": "raft"},
+    "version": "2.5.4"
+  },
+  "target": {
+    "address": "http://openbao.openbao.svc:8200",
+    "initialized": false,
+    "public_ingress": false,
+    "sealed": true,
+    "storage": {"audit_pvc": "2Gi", "data_pvc": "5Gi", "type": "raft"},
+    "version_after_alignment": "2.5.4"
+  },
+  "secret_values_observed": false
+}
--- a/docs/openbao-coulombcore-to-railiance01-runbook.md
+++ b/docs/openbao-coulombcore-to-railiance01-runbook.md
@ -0,0 +1,102 @@
+# OpenBao CoulombCore to railiance01 migration runbook
+
+This runbook moves the authoritative OpenBao Raft state without changing the
+public API name. OpenBao remains a stateful `rail-kubernetes` workload. It must
+not run on Knative or scale to zero.
+
+## Ratified design
+
+- Restore a source OpenBao 2.5.4 Raft snapshot into a target OpenBao 2.5.4
+  single-node Raft release. Never initialize a second lasting authority.
+- Preserve `bao.coulomb.social` and its TLS identity. Consumers move first to
+  the private `openbao.openbao.svc:8200` endpoint in bounded waves; public DNS
+  changes only after private probes pass.
+- Preserve the source Shamir barrier for this migration. Two of the three
+  existing unseal shares are required through an attended, non-logged
+  ceremony. A later auto-unseal migration is separate work and also requires
+  the existing threshold once.
+- Encrypt snapshots before offsite transfer with the established Railiance
+  backup recipient. Never store plaintext snapshots, tokens, or shares in Git,
+  State Hub, shell history, or chat.
+- The source remains unsealed and authoritative until the target passes two
+  restart/unseal cycles and all private consumer waves pass. During cutover it
+  is write-frozen, then scaled to zero while its PVCs, audit PVC, manifests,
+  TLS material, and encrypted snapshot remain retained for at least 30 days.
+
+## Proven preparation
+
+On 2026-08-03 an authoritative snapshot was encrypted and uploaded through the
+approved offsite backup lane. Its non-secret fingerprints are:
+
+- plaintext size: 106837 bytes
+- plaintext SHA-256: `4933da9837f00054fc6017cbe7921e83248456af3b4b003945beed08eda9f22b`
+- encrypted size: 107053 bytes
+- encrypted SHA-256: `662538c3e94486b51de316d2d308ae8c4671825841823616370d7f550165fd30`
+- recipient: `age1zvryunvjhvpkmasskauga2heeg0ztnte9ymgppvjge36ekumk50syr3tsz`
+
+A second fresh snapshot restored successfully into an isolated OpenBao 2.5.4
+pod on railiance01. OpenBao correctly resealed immediately after restore under
+the source Shamir barrier. Full semantic and restart validation therefore
+requires the source 2-of-3 unseal quorum; the temporary instance was not made
+serving. The empty target Helm release has been aligned from 2.5.3 to 2.5.4.
+
+## Preconditions and abort points
+
+1. Confirm the encrypted offsite artifact is retrievable and its encrypted
+   fingerprint matches. Abort if it does not.
+2. Confirm two custodians can supply source shares without disclosure to the
+   agent, logs, or terminal history. Abort if quorum is unavailable.
+3. Freeze administrative writes and record the final Raft index and snapshot
+   fingerprint. Abort on writes after the final snapshot.
+4. Restore only to the empty target PVC, unseal with the source quorum, and
+   probe through a private endpoint. Abort before consumer changes if counts,
+   exact-path capability, sibling denial, audit output, OIDC metadata, or SSH
+   role metadata differ.
+5. Restart and unseal twice. Abort before consumer changes if either cycle
+   fails.
+6. Move consumers one namespace at a time. Reverse the current wave on any
+   ExternalSecret, rotation, rollout, or negative-access failure.
+7. Change DNS only after every railiance01 consumer has no CoulombCore runtime
+   dependency. Restore the prior A record on any public-path regression.
+
+## Consumer waves
+
+The six ClusterSecretStores cover nine ExternalSecrets. Move low-blast-radius
+namespaces first, then application and identity control planes:
+
+1. `reuse`
+2. `activity-core`
+3. `rapp-qonto`
+4. `forgejo`
+5. `user-engine`
+6. `sso`
+
+Initially reuse the restored AppRoles against the private endpoint. For each
+wave, construct namespace- and service-account-restricted Kubernetes auth,
+prove the exact path and sibling denial, switch the store, force refresh,
+observe workload rollout, and retain the AppRole until reversal is proven.
+The restored Kubernetes auth configuration must be rewritten for the
+railiance01 API and reviewer identity before use.
+
+## Rollback, retention, RTO, and RPO
+
+- Rollback owner: `railiance-platform`; DNS owner: the Coulomb DNS operator;
+  workload owners validate their namespace waves.
+- Before DNS cutover, reverse a store endpoint/auth change and refresh its
+  ExternalSecrets. The source remains authoritative.
+- After DNS cutover, restore the CoulombCore workloads and previous DNS A
+  record, then reverse private store endpoints. Do not restore a newer target
+  snapshot over the retained source without a new reviewed recovery plan.
+- Target RTO is 60 minutes from an abort decision. RPO is the bounded final
+  write-freeze window; the intended RPO is zero acknowledged writes.
+- Keep the stopped source state and final encrypted snapshot for at least 30
+  days. PVC, Raft, Secret, audit, or TLS deletion requires fresh explicit
+  approval after a successful railiance01 disaster-recovery drill.
+
+## Completion evidence
+
+Completion requires machine-readable source/target inventories, two successful
+restart/unseal cycles, capability tests with no secret values recorded, all
+nine ExternalSecrets Ready on local identity, public TLS/OIDC/operator probes,
+backup retrieval proof, absence of source traffic throughout the observation
+window, and State Hub/file consistency.
--- a/workplans/RAILIANCE-WP-0020-openbao-migration-to-reef-railiance.md
+++ b/workplans/RAILIANCE-WP-0020-openbao-migration-to-reef-railiance.md
@ -4,11 +4,11 @@ type: workplan
 title: "Migrate authoritative OpenBao from CoulombCore to reef-railiance"
 domain: financials
 repo: railiance-master
-status: ready
+status: active
 owner: codex
 topic_slug: railiance
 created: "2026-07-30"
-updated: "2026-07-30"
+updated: "2026-08-03"
 depends_on:
  - NK-WP-0022
 state_hub_workstream_id: "0616a297-18c5-4c4e-a4fc-69135b3f9a15"
@ -42,7 +42,7 @@ run on scale-to-zero Knative.

 ```task
 id: RAILIANCE-WP-0020-T01
-status: todo
+status: done
 priority: high
 state_hub_task_id: "0d40e879-ae4e-45de-aa31-81bf110924dc"
 ```
@ -56,11 +56,17 @@ all consumers that bypass ESO or depend on the public address.
 Done when source, empty target, and every consumer have a machine-readable
 dependency map plus explicit rollback owners.

+Completed 2026-08-03. The non-secret source, target, and consumer dependency
+map is recorded in
+`docs/evidence/openbao-migration-inventory-2026-08-03.json`. It includes six
+ClusterSecretStores, nine ExternalSecrets, state/auth counts, storage and
+version topology, and rollback ownership without secret values.
+
 ## T02 - Ratify migration, seal, and custody design

 ```task
 id: RAILIANCE-WP-0020-T02
-status: todo
+status: done
 priority: high
 state_hub_task_id: "c266dce7-7d89-454a-a299-48fc169efea1"
 ```
@ -75,11 +81,16 @@ its executive approval gate.
 Done when the procedure has preconditions, abort points, rollback commands,
 RTO/RPO, and no secret material in Git or State Hub.

+Completed 2026-08-03. The ratified procedure, custody boundary, consumer
+waves, abort points, rollback, retention, RTO, and RPO are documented in
+`docs/openbao-coulombcore-to-railiance01-runbook.md`. The empty target release
+was aligned to OpenBao 2.5.4, matching the source.
+
 ## T03 - Prove backup and isolated restore before cutover

 ```task
 id: RAILIANCE-WP-0020-T03
-status: todo
+status: wait
 priority: high
 state_hub_task_id: "88671108-cf8c-4de6-b1ce-06e99b1c3d70"
 ```
@ -93,6 +104,14 @@ restart/unseal behavior.
 Done when restore evidence is machine-readable and rollback does not depend on
 the live source remaining healthy.

+2026-08-03: Created a fresh authoritative snapshot, encrypted it before
+offsite upload, removed plaintext staging, and recorded only fingerprints.
+A second fresh snapshot restored successfully into an isolated OpenBao 2.5.4
+pod on railiance01. The restored instance correctly resealed under the source
+Shamir barrier. The remaining semantic and repeated restart/unseal proof needs
+two of the three existing source shares through the attended custody ceremony;
+no supported automated custody route exposes those shares.
+
 ## T04 - Restore authoritative state on railiance01

 ```task