diff --git a/WORK-RECORDS.md b/WORK-RECORDS.md index d89f62a..224836b 100644 --- a/WORK-RECORDS.md +++ b/WORK-RECORDS.md @@ -12,7 +12,7 @@ | workplan | RAILIANCE-WP-0017 | finished | — | workplans/RAILIANCE-WP-0017-rail-rapp-reef-repo-separation.md | | workplan | RAILIANCE-WP-0018 | finished | — | workplans/RAILIANCE-WP-0018-first-wave-repo-family-materialization.md | | workplan | RAILIANCE-WP-0019 | finished | — | workplans/RAILIANCE-WP-0019-knative-qonto-runtime-on-reef-railiance.md | -| workplan | RAILIANCE-WP-0020 | ready | — | workplans/RAILIANCE-WP-0020-openbao-migration-to-reef-railiance.md | +| workplan | RAILIANCE-WP-0020 | active | — | workplans/RAILIANCE-WP-0020-openbao-migration-to-reef-railiance.md | | task | ADHOC-2026-07-30-T01 | done | — | workplans/ADHOC-2026-07-30.md | | task | RAILIANCE-WP-0017-T01 | done | — | workplans/RAILIANCE-WP-0017-rail-rapp-reef-repo-separation.md | | task | RAILIANCE-WP-0017-T02 | done | — | workplans/RAILIANCE-WP-0017-rail-rapp-reef-repo-separation.md | @@ -37,9 +37,9 @@ | task | RAILIANCE-WP-0019-T08 | done | — | workplans/RAILIANCE-WP-0019-knative-qonto-runtime-on-reef-railiance.md | | task | RAILIANCE-WP-0019-T09 | done | — | workplans/RAILIANCE-WP-0019-knative-qonto-runtime-on-reef-railiance.md | | task | RAILIANCE-WP-0019-T10 | done | — | workplans/RAILIANCE-WP-0019-knative-qonto-runtime-on-reef-railiance.md | -| task | RAILIANCE-WP-0020-T01 | todo | — | workplans/RAILIANCE-WP-0020-openbao-migration-to-reef-railiance.md | -| task | RAILIANCE-WP-0020-T02 | todo | — | workplans/RAILIANCE-WP-0020-openbao-migration-to-reef-railiance.md | -| task | RAILIANCE-WP-0020-T03 | todo | — | workplans/RAILIANCE-WP-0020-openbao-migration-to-reef-railiance.md | +| task | RAILIANCE-WP-0020-T01 | done | — | workplans/RAILIANCE-WP-0020-openbao-migration-to-reef-railiance.md | +| task | RAILIANCE-WP-0020-T02 | done | — | workplans/RAILIANCE-WP-0020-openbao-migration-to-reef-railiance.md | +| task | RAILIANCE-WP-0020-T03 | wait | — | workplans/RAILIANCE-WP-0020-openbao-migration-to-reef-railiance.md | | task | RAILIANCE-WP-0020-T04 | wait | — | workplans/RAILIANCE-WP-0020-openbao-migration-to-reef-railiance.md | | task | RAILIANCE-WP-0020-T05 | wait | — | workplans/RAILIANCE-WP-0020-openbao-migration-to-reef-railiance.md | | task | RAILIANCE-WP-0020-T06 | wait | — | workplans/RAILIANCE-WP-0020-openbao-migration-to-reef-railiance.md | diff --git a/docs/evidence/openbao-migration-inventory-2026-08-03.json b/docs/evidence/openbao-migration-inventory-2026-08-03.json new file mode 100644 index 0000000..332b9a8 --- /dev/null +++ b/docs/evidence/openbao-migration-inventory-2026-08-03.json @@ -0,0 +1,81 @@ +{ + "captured_at": "2026-08-03", + "consumer_contracts": { + "cluster_secret_stores": [ + "openbao-activity-core", + "openbao-forgejo", + "openbao-rapp-qonto", + "openbao-reuse", + "openbao-sso-user-engine-runtime", + "openbao-user-engine-runtime" + ], + "cluster_secret_store_count": 6, + "external_secret_count": 9, + "public_address": "https://bao.coulomb.social", + "rollback_owner": "railiance-platform", + "target_namespaces": { + "activity-core": 3, + "forgejo": 1, + "rapp-qonto": 1, + "reuse": 1, + "sso": 1, + "user-engine": 2 + }, + "transitional_auth": "AppRole and token-based stores; migrate to namespace-restricted Kubernetes auth after private restore" + }, + "source": { + "address": "https://bao.coulomb.social", + "alias_count": 10, + "approles": [ + "agent-harness-binky-mail", + "external-secrets-rapp-qonto", + "external-secrets-sso-user-engine", + "external-secrets-user-engine", + "rein-openweights", + "se-prod-whynot-design-npm-publish", + "warden-sign" + ], + "audit_devices": ["file/"], + "auth_methods": { + "approle/": "approle", + "keycape/": "oidc", + "kubernetes/": "kubernetes", + "netkingdom/": "oidc", + "token/": "token" + }, + "cluster_id": "fd28df5d-98ec-57dd-42ec-9b3e4f4e53bf", + "entity_count": 10, + "initialized": true, + "kubernetes_roles": [ + "external-secrets-activity-core", + "external-secrets-issue-core", + "external-secrets-reuse-surface" + ], + "mounts": { + "cubbyhole/": "cubbyhole", + "identity/": "identity", + "platform/": "kv", + "reins/": "kv", + "secret/": "kv", + "ssh/": "ssh", + "sys/": "system", + "tenants/": "kv" + }, + "policy_count": 26, + "raft_peers": [{"leader": true, "node_id": "openbao-0"}], + "sealed": false, + "seal": {"shares": 3, "threshold": 2, "type": "shamir"}, + "ssh_roles": ["adm-role", "agt-role", "atm-role"], + "storage": {"audit_pvc": "2Gi", "data_pvc": "5Gi", "type": "raft"}, + "version": "2.5.4" + }, + "target": { + "address": "http://openbao.openbao.svc:8200", + "initialized": false, + "public_ingress": false, + "sealed": true, + "storage": {"audit_pvc": "2Gi", "data_pvc": "5Gi", "type": "raft"}, + "version_after_alignment": "2.5.4" + }, + "secret_values_observed": false +} diff --git a/docs/openbao-coulombcore-to-railiance01-runbook.md b/docs/openbao-coulombcore-to-railiance01-runbook.md new file mode 100644 index 0000000..c5c97b6 --- /dev/null +++ b/docs/openbao-coulombcore-to-railiance01-runbook.md @@ -0,0 +1,102 @@ +# OpenBao CoulombCore to railiance01 migration runbook + +This runbook moves the authoritative OpenBao Raft state without changing the +public API name. OpenBao remains a stateful `rail-kubernetes` workload. It must +not run on Knative or scale to zero. + +## Ratified design + +- Restore a source OpenBao 2.5.4 Raft snapshot into a target OpenBao 2.5.4 + single-node Raft release. Never initialize a second lasting authority. +- Preserve `bao.coulomb.social` and its TLS identity. Consumers move first to + the private `openbao.openbao.svc:8200` endpoint in bounded waves; public DNS + changes only after private probes pass. +- Preserve the source Shamir barrier for this migration. Two of the three + existing unseal shares are required through an attended, non-logged + ceremony. A later auto-unseal migration is separate work and also requires + the existing threshold once. +- Encrypt snapshots before offsite transfer with the established Railiance + backup recipient. Never store plaintext snapshots, tokens, or shares in Git, + State Hub, shell history, or chat. +- The source remains unsealed and authoritative until the target passes two + restart/unseal cycles and all private consumer waves pass. During cutover it + is write-frozen, then scaled to zero while its PVCs, audit PVC, manifests, + TLS material, and encrypted snapshot remain retained for at least 30 days. + +## Proven preparation + +On 2026-08-03 an authoritative snapshot was encrypted and uploaded through the +approved offsite backup lane. Its non-secret fingerprints are: + +- plaintext size: 106837 bytes +- plaintext SHA-256: `4933da9837f00054fc6017cbe7921e83248456af3b4b003945beed08eda9f22b` +- encrypted size: 107053 bytes +- encrypted SHA-256: `662538c3e94486b51de316d2d308ae8c4671825841823616370d7f550165fd30` +- recipient: `age1zvryunvjhvpkmasskauga2heeg0ztnte9ymgppvjge36ekumk50syr3tsz` + +A second fresh snapshot restored successfully into an isolated OpenBao 2.5.4 +pod on railiance01. OpenBao correctly resealed immediately after restore under +the source Shamir barrier. Full semantic and restart validation therefore +requires the source 2-of-3 unseal quorum; the temporary instance was not made +serving. The empty target Helm release has been aligned from 2.5.3 to 2.5.4. + +## Preconditions and abort points + +1. Confirm the encrypted offsite artifact is retrievable and its encrypted + fingerprint matches. Abort if it does not. +2. Confirm two custodians can supply source shares without disclosure to the + agent, logs, or terminal history. Abort if quorum is unavailable. +3. Freeze administrative writes and record the final Raft index and snapshot + fingerprint. Abort on writes after the final snapshot. +4. Restore only to the empty target PVC, unseal with the source quorum, and + probe through a private endpoint. Abort before consumer changes if counts, + exact-path capability, sibling denial, audit output, OIDC metadata, or SSH + role metadata differ. +5. Restart and unseal twice. Abort before consumer changes if either cycle + fails. +6. Move consumers one namespace at a time. Reverse the current wave on any + ExternalSecret, rotation, rollout, or negative-access failure. +7. Change DNS only after every railiance01 consumer has no CoulombCore runtime + dependency. Restore the prior A record on any public-path regression. + +## Consumer waves + +The six ClusterSecretStores cover nine ExternalSecrets. Move low-blast-radius +namespaces first, then application and identity control planes: + +1. `reuse` +2. `activity-core` +3. `rapp-qonto` +4. `forgejo` +5. `user-engine` +6. `sso` + +Initially reuse the restored AppRoles against the private endpoint. For each +wave, construct namespace- and service-account-restricted Kubernetes auth, +prove the exact path and sibling denial, switch the store, force refresh, +observe workload rollout, and retain the AppRole until reversal is proven. +The restored Kubernetes auth configuration must be rewritten for the +railiance01 API and reviewer identity before use. + +## Rollback, retention, RTO, and RPO + +- Rollback owner: `railiance-platform`; DNS owner: the Coulomb DNS operator; + workload owners validate their namespace waves. +- Before DNS cutover, reverse a store endpoint/auth change and refresh its + ExternalSecrets. The source remains authoritative. +- After DNS cutover, restore the CoulombCore workloads and previous DNS A + record, then reverse private store endpoints. Do not restore a newer target + snapshot over the retained source without a new reviewed recovery plan. +- Target RTO is 60 minutes from an abort decision. RPO is the bounded final + write-freeze window; the intended RPO is zero acknowledged writes. +- Keep the stopped source state and final encrypted snapshot for at least 30 + days. PVC, Raft, Secret, audit, or TLS deletion requires fresh explicit + approval after a successful railiance01 disaster-recovery drill. + +## Completion evidence + +Completion requires machine-readable source/target inventories, two successful +restart/unseal cycles, capability tests with no secret values recorded, all +nine ExternalSecrets Ready on local identity, public TLS/OIDC/operator probes, +backup retrieval proof, absence of source traffic throughout the observation +window, and State Hub/file consistency. diff --git a/workplans/RAILIANCE-WP-0020-openbao-migration-to-reef-railiance.md b/workplans/RAILIANCE-WP-0020-openbao-migration-to-reef-railiance.md index 34bbfed..cc1b593 100644 --- a/workplans/RAILIANCE-WP-0020-openbao-migration-to-reef-railiance.md +++ b/workplans/RAILIANCE-WP-0020-openbao-migration-to-reef-railiance.md @@ -4,11 +4,11 @@ type: workplan title: "Migrate authoritative OpenBao from CoulombCore to reef-railiance" domain: financials repo: railiance-master -status: ready +status: active owner: codex topic_slug: railiance created: "2026-07-30" -updated: "2026-07-30" +updated: "2026-08-03" depends_on: - NK-WP-0022 state_hub_workstream_id: "0616a297-18c5-4c4e-a4fc-69135b3f9a15" @@ -42,7 +42,7 @@ run on scale-to-zero Knative. ```task id: RAILIANCE-WP-0020-T01 -status: todo +status: done priority: high state_hub_task_id: "0d40e879-ae4e-45de-aa31-81bf110924dc" ``` @@ -56,11 +56,17 @@ all consumers that bypass ESO or depend on the public address. Done when source, empty target, and every consumer have a machine-readable dependency map plus explicit rollback owners. +Completed 2026-08-03. The non-secret source, target, and consumer dependency +map is recorded in +`docs/evidence/openbao-migration-inventory-2026-08-03.json`. It includes six +ClusterSecretStores, nine ExternalSecrets, state/auth counts, storage and +version topology, and rollback ownership without secret values. + ## T02 - Ratify migration, seal, and custody design ```task id: RAILIANCE-WP-0020-T02 -status: todo +status: done priority: high state_hub_task_id: "c266dce7-7d89-454a-a299-48fc169efea1" ``` @@ -75,11 +81,16 @@ its executive approval gate. Done when the procedure has preconditions, abort points, rollback commands, RTO/RPO, and no secret material in Git or State Hub. +Completed 2026-08-03. The ratified procedure, custody boundary, consumer +waves, abort points, rollback, retention, RTO, and RPO are documented in +`docs/openbao-coulombcore-to-railiance01-runbook.md`. The empty target release +was aligned to OpenBao 2.5.4, matching the source. + ## T03 - Prove backup and isolated restore before cutover ```task id: RAILIANCE-WP-0020-T03 -status: todo +status: wait priority: high state_hub_task_id: "88671108-cf8c-4de6-b1ce-06e99b1c3d70" ``` @@ -93,6 +104,14 @@ restart/unseal behavior. Done when restore evidence is machine-readable and rollback does not depend on the live source remaining healthy. +2026-08-03: Created a fresh authoritative snapshot, encrypted it before +offsite upload, removed plaintext staging, and recorded only fingerprints. +A second fresh snapshot restored successfully into an isolated OpenBao 2.5.4 +pod on railiance01. The restored instance correctly resealed under the source +Shamir barrier. The remaining semantic and repeated restart/unseal proof needs +two of the three existing source shares through the attended custody ceremony; +no supported automated custody route exposes those shares. + ## T04 - Restore authoritative state on railiance01 ```task