Prepare OpenBao authority migration
This commit is contained in:
parent
c4124b6a7f
commit
ceed081c8d
4 changed files with 211 additions and 9 deletions
|
|
@ -12,7 +12,7 @@
|
||||||
| workplan | RAILIANCE-WP-0017 | finished | — | workplans/RAILIANCE-WP-0017-rail-rapp-reef-repo-separation.md |
|
| workplan | RAILIANCE-WP-0017 | finished | — | workplans/RAILIANCE-WP-0017-rail-rapp-reef-repo-separation.md |
|
||||||
| workplan | RAILIANCE-WP-0018 | finished | — | workplans/RAILIANCE-WP-0018-first-wave-repo-family-materialization.md |
|
| workplan | RAILIANCE-WP-0018 | finished | — | workplans/RAILIANCE-WP-0018-first-wave-repo-family-materialization.md |
|
||||||
| workplan | RAILIANCE-WP-0019 | finished | — | workplans/RAILIANCE-WP-0019-knative-qonto-runtime-on-reef-railiance.md |
|
| workplan | RAILIANCE-WP-0019 | finished | — | workplans/RAILIANCE-WP-0019-knative-qonto-runtime-on-reef-railiance.md |
|
||||||
| workplan | RAILIANCE-WP-0020 | ready | — | workplans/RAILIANCE-WP-0020-openbao-migration-to-reef-railiance.md |
|
| workplan | RAILIANCE-WP-0020 | active | — | workplans/RAILIANCE-WP-0020-openbao-migration-to-reef-railiance.md |
|
||||||
| task | ADHOC-2026-07-30-T01 | done | — | workplans/ADHOC-2026-07-30.md |
|
| task | ADHOC-2026-07-30-T01 | done | — | workplans/ADHOC-2026-07-30.md |
|
||||||
| task | RAILIANCE-WP-0017-T01 | done | — | workplans/RAILIANCE-WP-0017-rail-rapp-reef-repo-separation.md |
|
| task | RAILIANCE-WP-0017-T01 | done | — | workplans/RAILIANCE-WP-0017-rail-rapp-reef-repo-separation.md |
|
||||||
| task | RAILIANCE-WP-0017-T02 | done | — | workplans/RAILIANCE-WP-0017-rail-rapp-reef-repo-separation.md |
|
| task | RAILIANCE-WP-0017-T02 | done | — | workplans/RAILIANCE-WP-0017-rail-rapp-reef-repo-separation.md |
|
||||||
|
|
@ -37,9 +37,9 @@
|
||||||
| task | RAILIANCE-WP-0019-T08 | done | — | workplans/RAILIANCE-WP-0019-knative-qonto-runtime-on-reef-railiance.md |
|
| task | RAILIANCE-WP-0019-T08 | done | — | workplans/RAILIANCE-WP-0019-knative-qonto-runtime-on-reef-railiance.md |
|
||||||
| task | RAILIANCE-WP-0019-T09 | done | — | workplans/RAILIANCE-WP-0019-knative-qonto-runtime-on-reef-railiance.md |
|
| task | RAILIANCE-WP-0019-T09 | done | — | workplans/RAILIANCE-WP-0019-knative-qonto-runtime-on-reef-railiance.md |
|
||||||
| task | RAILIANCE-WP-0019-T10 | done | — | workplans/RAILIANCE-WP-0019-knative-qonto-runtime-on-reef-railiance.md |
|
| task | RAILIANCE-WP-0019-T10 | done | — | workplans/RAILIANCE-WP-0019-knative-qonto-runtime-on-reef-railiance.md |
|
||||||
| task | RAILIANCE-WP-0020-T01 | todo | — | workplans/RAILIANCE-WP-0020-openbao-migration-to-reef-railiance.md |
|
| task | RAILIANCE-WP-0020-T01 | done | — | workplans/RAILIANCE-WP-0020-openbao-migration-to-reef-railiance.md |
|
||||||
| task | RAILIANCE-WP-0020-T02 | todo | — | workplans/RAILIANCE-WP-0020-openbao-migration-to-reef-railiance.md |
|
| task | RAILIANCE-WP-0020-T02 | done | — | workplans/RAILIANCE-WP-0020-openbao-migration-to-reef-railiance.md |
|
||||||
| task | RAILIANCE-WP-0020-T03 | todo | — | workplans/RAILIANCE-WP-0020-openbao-migration-to-reef-railiance.md |
|
| task | RAILIANCE-WP-0020-T03 | wait | — | workplans/RAILIANCE-WP-0020-openbao-migration-to-reef-railiance.md |
|
||||||
| task | RAILIANCE-WP-0020-T04 | wait | — | workplans/RAILIANCE-WP-0020-openbao-migration-to-reef-railiance.md |
|
| task | RAILIANCE-WP-0020-T04 | wait | — | workplans/RAILIANCE-WP-0020-openbao-migration-to-reef-railiance.md |
|
||||||
| task | RAILIANCE-WP-0020-T05 | wait | — | workplans/RAILIANCE-WP-0020-openbao-migration-to-reef-railiance.md |
|
| task | RAILIANCE-WP-0020-T05 | wait | — | workplans/RAILIANCE-WP-0020-openbao-migration-to-reef-railiance.md |
|
||||||
| task | RAILIANCE-WP-0020-T06 | wait | — | workplans/RAILIANCE-WP-0020-openbao-migration-to-reef-railiance.md |
|
| task | RAILIANCE-WP-0020-T06 | wait | — | workplans/RAILIANCE-WP-0020-openbao-migration-to-reef-railiance.md |
|
||||||
|
|
|
||||||
81
docs/evidence/openbao-migration-inventory-2026-08-03.json
Normal file
81
docs/evidence/openbao-migration-inventory-2026-08-03.json
Normal file
|
|
@ -0,0 +1,81 @@
|
||||||
|
{
|
||||||
|
"captured_at": "2026-08-03",
|
||||||
|
"consumer_contracts": {
|
||||||
|
"cluster_secret_stores": [
|
||||||
|
"openbao-activity-core",
|
||||||
|
"openbao-forgejo",
|
||||||
|
"openbao-rapp-qonto",
|
||||||
|
"openbao-reuse",
|
||||||
|
"openbao-sso-user-engine-runtime",
|
||||||
|
"openbao-user-engine-runtime"
|
||||||
|
],
|
||||||
|
"cluster_secret_store_count": 6,
|
||||||
|
"external_secret_count": 9,
|
||||||
|
"public_address": "https://bao.coulomb.social",
|
||||||
|
"rollback_owner": "railiance-platform",
|
||||||
|
"target_namespaces": {
|
||||||
|
"activity-core": 3,
|
||||||
|
"forgejo": 1,
|
||||||
|
"rapp-qonto": 1,
|
||||||
|
"reuse": 1,
|
||||||
|
"sso": 1,
|
||||||
|
"user-engine": 2
|
||||||
|
},
|
||||||
|
"transitional_auth": "AppRole and token-based stores; migrate to namespace-restricted Kubernetes auth after private restore"
|
||||||
|
},
|
||||||
|
"source": {
|
||||||
|
"address": "https://bao.coulomb.social",
|
||||||
|
"alias_count": 10,
|
||||||
|
"approles": [
|
||||||
|
"agent-harness-binky-mail",
|
||||||
|
"external-secrets-rapp-qonto",
|
||||||
|
"external-secrets-sso-user-engine",
|
||||||
|
"external-secrets-user-engine",
|
||||||
|
"rein-openweights",
|
||||||
|
"se-prod-whynot-design-npm-publish",
|
||||||
|
"warden-sign"
|
||||||
|
],
|
||||||
|
"audit_devices": ["file/"],
|
||||||
|
"auth_methods": {
|
||||||
|
"approle/": "approle",
|
||||||
|
"keycape/": "oidc",
|
||||||
|
"kubernetes/": "kubernetes",
|
||||||
|
"netkingdom/": "oidc",
|
||||||
|
"token/": "token"
|
||||||
|
},
|
||||||
|
"cluster_id": "fd28df5d-98ec-57dd-42ec-9b3e4f4e53bf",
|
||||||
|
"entity_count": 10,
|
||||||
|
"initialized": true,
|
||||||
|
"kubernetes_roles": [
|
||||||
|
"external-secrets-activity-core",
|
||||||
|
"external-secrets-issue-core",
|
||||||
|
"external-secrets-reuse-surface"
|
||||||
|
],
|
||||||
|
"mounts": {
|
||||||
|
"cubbyhole/": "cubbyhole",
|
||||||
|
"identity/": "identity",
|
||||||
|
"platform/": "kv",
|
||||||
|
"reins/": "kv",
|
||||||
|
"secret/": "kv",
|
||||||
|
"ssh/": "ssh",
|
||||||
|
"sys/": "system",
|
||||||
|
"tenants/": "kv"
|
||||||
|
},
|
||||||
|
"policy_count": 26,
|
||||||
|
"raft_peers": [{"leader": true, "node_id": "openbao-0"}],
|
||||||
|
"sealed": false,
|
||||||
|
"seal": {"shares": 3, "threshold": 2, "type": "shamir"},
|
||||||
|
"ssh_roles": ["adm-role", "agt-role", "atm-role"],
|
||||||
|
"storage": {"audit_pvc": "2Gi", "data_pvc": "5Gi", "type": "raft"},
|
||||||
|
"version": "2.5.4"
|
||||||
|
},
|
||||||
|
"target": {
|
||||||
|
"address": "http://openbao.openbao.svc:8200",
|
||||||
|
"initialized": false,
|
||||||
|
"public_ingress": false,
|
||||||
|
"sealed": true,
|
||||||
|
"storage": {"audit_pvc": "2Gi", "data_pvc": "5Gi", "type": "raft"},
|
||||||
|
"version_after_alignment": "2.5.4"
|
||||||
|
},
|
||||||
|
"secret_values_observed": false
|
||||||
|
}
|
||||||
102
docs/openbao-coulombcore-to-railiance01-runbook.md
Normal file
102
docs/openbao-coulombcore-to-railiance01-runbook.md
Normal file
|
|
@ -0,0 +1,102 @@
|
||||||
|
# OpenBao CoulombCore to railiance01 migration runbook
|
||||||
|
|
||||||
|
This runbook moves the authoritative OpenBao Raft state without changing the
|
||||||
|
public API name. OpenBao remains a stateful `rail-kubernetes` workload. It must
|
||||||
|
not run on Knative or scale to zero.
|
||||||
|
|
||||||
|
## Ratified design
|
||||||
|
|
||||||
|
- Restore a source OpenBao 2.5.4 Raft snapshot into a target OpenBao 2.5.4
|
||||||
|
single-node Raft release. Never initialize a second lasting authority.
|
||||||
|
- Preserve `bao.coulomb.social` and its TLS identity. Consumers move first to
|
||||||
|
the private `openbao.openbao.svc:8200` endpoint in bounded waves; public DNS
|
||||||
|
changes only after private probes pass.
|
||||||
|
- Preserve the source Shamir barrier for this migration. Two of the three
|
||||||
|
existing unseal shares are required through an attended, non-logged
|
||||||
|
ceremony. A later auto-unseal migration is separate work and also requires
|
||||||
|
the existing threshold once.
|
||||||
|
- Encrypt snapshots before offsite transfer with the established Railiance
|
||||||
|
backup recipient. Never store plaintext snapshots, tokens, or shares in Git,
|
||||||
|
State Hub, shell history, or chat.
|
||||||
|
- The source remains unsealed and authoritative until the target passes two
|
||||||
|
restart/unseal cycles and all private consumer waves pass. During cutover it
|
||||||
|
is write-frozen, then scaled to zero while its PVCs, audit PVC, manifests,
|
||||||
|
TLS material, and encrypted snapshot remain retained for at least 30 days.
|
||||||
|
|
||||||
|
## Proven preparation
|
||||||
|
|
||||||
|
On 2026-08-03 an authoritative snapshot was encrypted and uploaded through the
|
||||||
|
approved offsite backup lane. Its non-secret fingerprints are:
|
||||||
|
|
||||||
|
- plaintext size: 106837 bytes
|
||||||
|
- plaintext SHA-256: `4933da9837f00054fc6017cbe7921e83248456af3b4b003945beed08eda9f22b`
|
||||||
|
- encrypted size: 107053 bytes
|
||||||
|
- encrypted SHA-256: `662538c3e94486b51de316d2d308ae8c4671825841823616370d7f550165fd30`
|
||||||
|
- recipient: `age1zvryunvjhvpkmasskauga2heeg0ztnte9ymgppvjge36ekumk50syr3tsz`
|
||||||
|
|
||||||
|
A second fresh snapshot restored successfully into an isolated OpenBao 2.5.4
|
||||||
|
pod on railiance01. OpenBao correctly resealed immediately after restore under
|
||||||
|
the source Shamir barrier. Full semantic and restart validation therefore
|
||||||
|
requires the source 2-of-3 unseal quorum; the temporary instance was not made
|
||||||
|
serving. The empty target Helm release has been aligned from 2.5.3 to 2.5.4.
|
||||||
|
|
||||||
|
## Preconditions and abort points
|
||||||
|
|
||||||
|
1. Confirm the encrypted offsite artifact is retrievable and its encrypted
|
||||||
|
fingerprint matches. Abort if it does not.
|
||||||
|
2. Confirm two custodians can supply source shares without disclosure to the
|
||||||
|
agent, logs, or terminal history. Abort if quorum is unavailable.
|
||||||
|
3. Freeze administrative writes and record the final Raft index and snapshot
|
||||||
|
fingerprint. Abort on writes after the final snapshot.
|
||||||
|
4. Restore only to the empty target PVC, unseal with the source quorum, and
|
||||||
|
probe through a private endpoint. Abort before consumer changes if counts,
|
||||||
|
exact-path capability, sibling denial, audit output, OIDC metadata, or SSH
|
||||||
|
role metadata differ.
|
||||||
|
5. Restart and unseal twice. Abort before consumer changes if either cycle
|
||||||
|
fails.
|
||||||
|
6. Move consumers one namespace at a time. Reverse the current wave on any
|
||||||
|
ExternalSecret, rotation, rollout, or negative-access failure.
|
||||||
|
7. Change DNS only after every railiance01 consumer has no CoulombCore runtime
|
||||||
|
dependency. Restore the prior A record on any public-path regression.
|
||||||
|
|
||||||
|
## Consumer waves
|
||||||
|
|
||||||
|
The six ClusterSecretStores cover nine ExternalSecrets. Move low-blast-radius
|
||||||
|
namespaces first, then application and identity control planes:
|
||||||
|
|
||||||
|
1. `reuse`
|
||||||
|
2. `activity-core`
|
||||||
|
3. `rapp-qonto`
|
||||||
|
4. `forgejo`
|
||||||
|
5. `user-engine`
|
||||||
|
6. `sso`
|
||||||
|
|
||||||
|
Initially reuse the restored AppRoles against the private endpoint. For each
|
||||||
|
wave, construct namespace- and service-account-restricted Kubernetes auth,
|
||||||
|
prove the exact path and sibling denial, switch the store, force refresh,
|
||||||
|
observe workload rollout, and retain the AppRole until reversal is proven.
|
||||||
|
The restored Kubernetes auth configuration must be rewritten for the
|
||||||
|
railiance01 API and reviewer identity before use.
|
||||||
|
|
||||||
|
## Rollback, retention, RTO, and RPO
|
||||||
|
|
||||||
|
- Rollback owner: `railiance-platform`; DNS owner: the Coulomb DNS operator;
|
||||||
|
workload owners validate their namespace waves.
|
||||||
|
- Before DNS cutover, reverse a store endpoint/auth change and refresh its
|
||||||
|
ExternalSecrets. The source remains authoritative.
|
||||||
|
- After DNS cutover, restore the CoulombCore workloads and previous DNS A
|
||||||
|
record, then reverse private store endpoints. Do not restore a newer target
|
||||||
|
snapshot over the retained source without a new reviewed recovery plan.
|
||||||
|
- Target RTO is 60 minutes from an abort decision. RPO is the bounded final
|
||||||
|
write-freeze window; the intended RPO is zero acknowledged writes.
|
||||||
|
- Keep the stopped source state and final encrypted snapshot for at least 30
|
||||||
|
days. PVC, Raft, Secret, audit, or TLS deletion requires fresh explicit
|
||||||
|
approval after a successful railiance01 disaster-recovery drill.
|
||||||
|
|
||||||
|
## Completion evidence
|
||||||
|
|
||||||
|
Completion requires machine-readable source/target inventories, two successful
|
||||||
|
restart/unseal cycles, capability tests with no secret values recorded, all
|
||||||
|
nine ExternalSecrets Ready on local identity, public TLS/OIDC/operator probes,
|
||||||
|
backup retrieval proof, absence of source traffic throughout the observation
|
||||||
|
window, and State Hub/file consistency.
|
||||||
|
|
@ -4,11 +4,11 @@ type: workplan
|
||||||
title: "Migrate authoritative OpenBao from CoulombCore to reef-railiance"
|
title: "Migrate authoritative OpenBao from CoulombCore to reef-railiance"
|
||||||
domain: financials
|
domain: financials
|
||||||
repo: railiance-master
|
repo: railiance-master
|
||||||
status: ready
|
status: active
|
||||||
owner: codex
|
owner: codex
|
||||||
topic_slug: railiance
|
topic_slug: railiance
|
||||||
created: "2026-07-30"
|
created: "2026-07-30"
|
||||||
updated: "2026-07-30"
|
updated: "2026-08-03"
|
||||||
depends_on:
|
depends_on:
|
||||||
- NK-WP-0022
|
- NK-WP-0022
|
||||||
state_hub_workstream_id: "0616a297-18c5-4c4e-a4fc-69135b3f9a15"
|
state_hub_workstream_id: "0616a297-18c5-4c4e-a4fc-69135b3f9a15"
|
||||||
|
|
@ -42,7 +42,7 @@ run on scale-to-zero Knative.
|
||||||
|
|
||||||
```task
|
```task
|
||||||
id: RAILIANCE-WP-0020-T01
|
id: RAILIANCE-WP-0020-T01
|
||||||
status: todo
|
status: done
|
||||||
priority: high
|
priority: high
|
||||||
state_hub_task_id: "0d40e879-ae4e-45de-aa31-81bf110924dc"
|
state_hub_task_id: "0d40e879-ae4e-45de-aa31-81bf110924dc"
|
||||||
```
|
```
|
||||||
|
|
@ -56,11 +56,17 @@ all consumers that bypass ESO or depend on the public address.
|
||||||
Done when source, empty target, and every consumer have a machine-readable
|
Done when source, empty target, and every consumer have a machine-readable
|
||||||
dependency map plus explicit rollback owners.
|
dependency map plus explicit rollback owners.
|
||||||
|
|
||||||
|
Completed 2026-08-03. The non-secret source, target, and consumer dependency
|
||||||
|
map is recorded in
|
||||||
|
`docs/evidence/openbao-migration-inventory-2026-08-03.json`. It includes six
|
||||||
|
ClusterSecretStores, nine ExternalSecrets, state/auth counts, storage and
|
||||||
|
version topology, and rollback ownership without secret values.
|
||||||
|
|
||||||
## T02 - Ratify migration, seal, and custody design
|
## T02 - Ratify migration, seal, and custody design
|
||||||
|
|
||||||
```task
|
```task
|
||||||
id: RAILIANCE-WP-0020-T02
|
id: RAILIANCE-WP-0020-T02
|
||||||
status: todo
|
status: done
|
||||||
priority: high
|
priority: high
|
||||||
state_hub_task_id: "c266dce7-7d89-454a-a299-48fc169efea1"
|
state_hub_task_id: "c266dce7-7d89-454a-a299-48fc169efea1"
|
||||||
```
|
```
|
||||||
|
|
@ -75,11 +81,16 @@ its executive approval gate.
|
||||||
Done when the procedure has preconditions, abort points, rollback commands,
|
Done when the procedure has preconditions, abort points, rollback commands,
|
||||||
RTO/RPO, and no secret material in Git or State Hub.
|
RTO/RPO, and no secret material in Git or State Hub.
|
||||||
|
|
||||||
|
Completed 2026-08-03. The ratified procedure, custody boundary, consumer
|
||||||
|
waves, abort points, rollback, retention, RTO, and RPO are documented in
|
||||||
|
`docs/openbao-coulombcore-to-railiance01-runbook.md`. The empty target release
|
||||||
|
was aligned to OpenBao 2.5.4, matching the source.
|
||||||
|
|
||||||
## T03 - Prove backup and isolated restore before cutover
|
## T03 - Prove backup and isolated restore before cutover
|
||||||
|
|
||||||
```task
|
```task
|
||||||
id: RAILIANCE-WP-0020-T03
|
id: RAILIANCE-WP-0020-T03
|
||||||
status: todo
|
status: wait
|
||||||
priority: high
|
priority: high
|
||||||
state_hub_task_id: "88671108-cf8c-4de6-b1ce-06e99b1c3d70"
|
state_hub_task_id: "88671108-cf8c-4de6-b1ce-06e99b1c3d70"
|
||||||
```
|
```
|
||||||
|
|
@ -93,6 +104,14 @@ restart/unseal behavior.
|
||||||
Done when restore evidence is machine-readable and rollback does not depend on
|
Done when restore evidence is machine-readable and rollback does not depend on
|
||||||
the live source remaining healthy.
|
the live source remaining healthy.
|
||||||
|
|
||||||
|
2026-08-03: Created a fresh authoritative snapshot, encrypted it before
|
||||||
|
offsite upload, removed plaintext staging, and recorded only fingerprints.
|
||||||
|
A second fresh snapshot restored successfully into an isolated OpenBao 2.5.4
|
||||||
|
pod on railiance01. The restored instance correctly resealed under the source
|
||||||
|
Shamir barrier. The remaining semantic and repeated restart/unseal proof needs
|
||||||
|
two of the three existing source shares through the attended custody ceremony;
|
||||||
|
no supported automated custody route exposes those shares.
|
||||||
|
|
||||||
## T04 - Restore authoritative state on railiance01
|
## T04 - Restore authoritative state on railiance01
|
||||||
|
|
||||||
```task
|
```task
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue