diff --git a/WORK-RECORDS.md b/WORK-RECORDS.md index c90fa24..eb451a5 100644 --- a/WORK-RECORDS.md +++ b/WORK-RECORDS.md @@ -19,6 +19,7 @@ | workplan | RAILIANCE-WP-0015 | finished | — | workplans/RAILIANCE-WP-0015-platform-rapp-consistency.md | | workplan | RAILIANCE-WP-0016 | finished | — | workplans/RAILIANCE-WP-0016-apps-pg-resource-evidence.md | | workplan | RAILIANCE-WP-0016 | active | — | workplans/RAILIANCE-WP-0016-architecture-cleanup-backlog.md | +| workplan | RAILIANCE-WP-0017 | proposed | — | workplans/RAILIANCE-WP-0017-consumption-mode-enforcement.md | | task | RAILIANCE-WP-0005-T01 | done | — | workplans/RAILIANCE-WP-0005-credential-request-and-lease-broker.md | | task | RAILIANCE-WP-0005-T02 | done | — | workplans/RAILIANCE-WP-0005-credential-request-and-lease-broker.md | | task | RAILIANCE-WP-0005-T03 | done | — | workplans/RAILIANCE-WP-0005-credential-request-and-lease-broker.md | @@ -78,3 +79,4 @@ | task | RAILIANCE-WP-0016-T03 | done | — | workplans/RAILIANCE-WP-0016-architecture-cleanup-backlog.md | | task | RAILIANCE-WP-0016-T04 | done | — | workplans/RAILIANCE-WP-0016-architecture-cleanup-backlog.md | | task | RAILIANCE-WP-0016-T05 | todo | — | workplans/RAILIANCE-WP-0016-architecture-cleanup-backlog.md | +| task | RAILIANCE-WP-0017-T01 | todo | — | workplans/RAILIANCE-WP-0017-consumption-mode-enforcement.md | diff --git a/argocd/platform-addons/openbao-secretstore/openbao-backup-object-storage.clustersecretstore.yaml b/argocd/platform-addons/openbao-secretstore/openbao-backup-object-storage.clustersecretstore.yaml new file mode 100644 index 0000000..a582e58 --- /dev/null +++ b/argocd/platform-addons/openbao-secretstore/openbao-backup-object-storage.clustersecretstore.yaml @@ -0,0 +1,35 @@ +# CCR-2026-0012. Deploy on railiance01 (first consumer lives in +# databases), not the CoulombCore ArgoCD kustomization. +# +# Prereq: Secret external-secrets/openbao-backup-object-storage-approle +# (role-id, secret-id) for AppRole external-secrets-backup-object-storage. +# Same-cluster Kubernetes auth remains the follow-up. +# Add further namespace conditions when another backup consumer is bound. +apiVersion: external-secrets.io/v1 +kind: ClusterSecretStore +metadata: + name: openbao-backup-object-storage + labels: + app.kubernetes.io/part-of: railiance-gitops + railiance-platform/component: external-secrets + app.kubernetes.io/name: backup-object-storage +spec: + provider: + vault: + server: http://openbao.openbao.svc:8200 + path: platform + version: v2 + auth: + appRole: + path: approle + roleRef: + name: openbao-backup-object-storage-approle + namespace: external-secrets + key: role-id + secretRef: + name: openbao-backup-object-storage-approle + namespace: external-secrets + key: secret-id + conditions: + - namespaces: + - databases diff --git a/credential-change-requests/CCR-2026-0011-scaleway-object-storage-bootstrap.yaml b/credential-change-requests/CCR-2026-0011-scaleway-object-storage-bootstrap.yaml new file mode 100644 index 0000000..67b74cd --- /dev/null +++ b/credential-change-requests/CCR-2026-0011-scaleway-object-storage-bootstrap.yaml @@ -0,0 +1,68 @@ +id: CCR-2026-0011 +kind: credential-change-request +schema_version: 1 +request_type: workload-kv-read +title: Scaleway bootstrap API key for reef-storage / WP-0002 bucket create +status: apply_pending +created: '2026-08-14' +updated: '2026-08-14' +requester: + agent: grok + reason: >- + RESOURCE-WP-0002 T03 is human-approved to create a private Scaleway + Multi-AZ bucket in nl-ams. This session has no Scaleway CLI profile. + Founder will provision a bootstrap org/project API key into OpenBao; + the agent reads it only to create the bucket and then writes the + scoped bucket key to a separate path. Values must not appear in Git + or chat. +review: + required: true + required_approvers: + - platform-operator + comments: + - at: '2026-08-14T00:00:00+00:00' + reviewer: bernd.worsch + decision: approved + comment: >- + Approved in chat (Grok session): put Scaleway bootstrap key in + OpenBao first, then create the bucket. +target: + domain: financials + tenant: railiance + workload: scaleway-bootstrap + environment: production + purpose: Create reef-storage object-store members; not the Barman runtime key +openbao: + mount: platform + kv_path: platform/workloads/railiance/scaleway/bootstrap + fields: + - ACCESS_KEY + - SECRET_KEY + - DEFAULT_ORGANIZATION_ID + - DEFAULT_PROJECT_ID + policy_name: operator-kv-scaleway-bootstrap +access_frontdoor: + type: operator-local + catalog_id: scaleway-bootstrap + selector: scaleway bootstrap api + command: bao kv put platform/workloads/railiance/scaleway/bootstrap + resolvable: false + readiness: waiting-on-ui-replace-of-xxx-placeholders +delivery: + surface: operator-workstation + target: reef-storage/tools/create-platform-audit-bucket.sh (reads, never prints) +risk: + classification: high + notes: + - Bootstrap key can create projects/buckets; rotate or delete after scoped key exists. + - Scoped backup key is a different path: platform/workloads/railiance/backup/object-storage + - Values must not appear in Git, State Hub, or chat. +verification: + positive: + - Field names present on the KV path; values not printed. + negative: + - default-policy token denied on the data path. +lifecycle: + deactivate: Delete bootstrap key at Scaleway after the scoped bucket key works. + rotate: Put a new bootstrap key; do not reuse the scoped backup key. + compromised: Revoke both Scaleway keys; rotate bucket policy. diff --git a/credential-change-requests/CCR-2026-0012-backup-object-storage.yaml b/credential-change-requests/CCR-2026-0012-backup-object-storage.yaml new file mode 100644 index 0000000..6b95741 --- /dev/null +++ b/credential-change-requests/CCR-2026-0012-backup-object-storage.yaml @@ -0,0 +1,123 @@ +id: CCR-2026-0012 +kind: credential-change-request +schema_version: 1 +request_type: workload-kv-read +title: Scaleway scoped backup object-storage key +status: applied +created: '2026-08-14' +updated: '2026-08-14' +requester: + agent: grok + reason: >- + RESOURCE-WP-0002 T04 needs a renewable, revocable, project-scoped + Scaleway backup key in OpenBao. Custody is the resource-control + procuring identity, not a Barman-specific application. First + consumer is rapp-postgres (Secret platform-pg-backup-s3). The + bootstrap key (CCR-2026-0011) must not be the runtime key. +review: + required: true + required_approvers: + - platform-operator + comments: + - at: '2026-08-14T17:30:00+00:00' + reviewer: bernd.worsch + decision: approved + comment: >- + Approved in chat (Grok session): CCR-2026-0012 and “the backup + key is in bao.” Founder used ACCESS_KEY/SECRET_KEY field names. +target: + domain: financials + tenant: railiance + workload: backup + rapp: rapp-postgres + environment: production + purpose: >- + Object-store credentials for Railiance backup destinations. + Secret keys ACCESS_KEY and SECRET_KEY (projected to Kubernetes as + ACCESS_KEY_ID / ACCESS_SECRET_KEY). Org and project ids are + non-secret companions. APPLICATION_ID can be added later for a + bucket policy. Endpoint, bucket, prefix, and region stay on + reef-storage. +openbao: + mount: platform + kv_path: platform/workloads/railiance/backup/object-storage + fields: + - ACCESS_KEY + - SECRET_KEY + - DEFAULT_ORGANIZATION_ID + - DEFAULT_PROJECT_ID + policy_name: workload-kv-read-backup-object-storage + policy_file: openbao/policies/workload-kv-read-backup-object-storage.hcl + auth: + method: kubernetes + mount: kubernetes + role: external-secrets-backup-object-storage + bound_claims: + service_account_names: + - external-secrets + service_account_namespaces: + - external-secrets + bound_claims_confirmed: false + policies: + - workload-kv-read-backup-object-storage + ttl: 15m +access_frontdoor: + type: ops-warden + catalog_id: backup-object-storage + selector: resource-control scoped backup access + command: warden access backup-object-storage --fetch ACCESS_KEY + resolvable: false + readiness: applied-pending-verify +delivery: + surface: external-secrets + target: >- + ClusterSecretStore openbao-backup-object-storage (first namespace + condition: databases) → ExternalSecret databases/platform-pg-backup-s3 + → Secret platform-pg-backup-s3 (ACCESS_KEY_ID / ACCESS_SECRET_KEY + projected from ACCESS_KEY / SECRET_KEY). Drafts live + under railiance-platform/argocd/platform-addons/openbao-secretstore/ + and rapp-postgres/helm/platform-pg-backup-s3.externalsecret.yaml. + Do not apply until this CCR is approved and the KV values are real. +risk: + classification: high + notes: + - Scaleway policies attach to the IAM application, not to one API key. + Application resource-control must hold only backup-scoped keys. + - The runtime key can write and delete backup objects. Compromise can + destroy recovery points or fill the bucket. + - The bootstrap key (CCR-2026-0011) must stay off this application and + be revoked after this key works. + - Do not enable continuous WAL archiving until the empty-archive + preflight passes (RESOURCE-WP-0002 T05). + - Values must not appear in Git, State Hub, logs, or chat. +verification: + positive: + - Field names present on the KV path; values not printed. + - An approved databases-namespace ExternalSecret can project ACCESS_KEY + and SECRET_KEY to Secret platform-pg-backup-s3 as ACCESS_KEY_ID and + ACCESS_SECRET_KEY. + - The scoped key can list/put/delete backup prefixes on the procured + bucket. + negative: + - A token without this policy cannot read the KV path. + - A namespace outside the ClusterSecretStore condition cannot use the store. + - A second, unused key (or revoked key) cannot access the bucket. + - The scoped key cannot create compute resources. + activation_conditions: + - Platform-operator approves this CCR. + - Founder creates or reuses IAM application resource-control, binds + policy Scoped backup access, mints the API key of that name, then + puts ACCESS_KEY and SECRET_KEY in OpenBao. Done 2026-08-14. + - ESO store and ExternalSecret applied only after values are real. +lifecycle: + deactivate: Disable the catalog entry, delete consumer ExternalSecrets, + revoke the Scoped backup access API key. Keep application + resource-control if other backup keys remain. + rotate: Overlap-first. Put a new key in OpenBao, wait for ESO refresh, + revoke the old Scaleway key. + compromised: Revoke the Scoped backup access key and the bootstrap key + at Scaleway, rotate this path, review bucket contents, open incident + follow-up. +state_hub: + workplan_id: RESOURCE-WP-0002 + task_id: RESOURCE-WP-0002-T04 diff --git a/docs/evidence/CCR-2026-0012-backup-key-verify-2026-08-14.md b/docs/evidence/CCR-2026-0012-backup-key-verify-2026-08-14.md new file mode 100644 index 0000000..c51a459 --- /dev/null +++ b/docs/evidence/CCR-2026-0012-backup-key-verify-2026-08-14.md @@ -0,0 +1,47 @@ +# CCR-2026-0012 — scoped backup key verification + +Date: 2026-08-14 +Status: **applied — Secret `databases/platform-pg-backup-s3` synced** +CCR: approved in chat 2026-08-14 +Path: `platform/workloads/railiance/backup/object-storage` (KV v2 version 1) + +No secret values in this file. + +## What is in OpenBao + +| Field | Present | +| --- | --- | +| `ACCESS_KEY` | yes (len 20, `SCW…`) | +| `SECRET_KEY` | yes | +| `DEFAULT_ORGANIZATION_ID` | yes | +| `DEFAULT_PROJECT_ID` | yes | +| `APPLICATION_ID` | yes as of version 2 (UUID, not printed here) | + +Distinct from the CCR-2026-0011 bootstrap key (`keys_differ=yes`). + +## Positive + +- `scw object bucket list` / `get railiance-platform-pg-backup` in `nl-ams` succeeded. +- boto3 put/get/delete of a probe object under `platform-pg/` succeeded; probe deleted. +- Sibling prefix write is now denied (`AccessDenied`) after the + `resource-control-scoped-backup` bucket policy + (`BackupListPrefix`, `BackupObjectsUnderPrefix`) applied 2026-08-14. + +## Negative + +- Bogus secret against the same access key: `SignatureDoesNotMatch`. +- `scw iam application list` / `user list` / `api-key get`: insufficient permissions. +- `scw billing budget list`: not allowed. +- `scw k8s cluster list`: insufficient permissions. + +`instance` / `vpc` / `rdb` / `registry` / `lb` list returned empty success. That is consistent with a project that has no those resources; it is not a proven compute-write deny. Do not create a paid resource to prove it. + +## Cluster delivery (same day) + +- Policy `workload-kv-read-backup-object-storage` written. +- AppRole `external-secrets-backup-object-storage` (15m/30m, 8 uses). +- Secret `external-secrets/openbao-backup-object-storage-approle` applied; local delivery shredded. +- ClusterSecretStore `openbao-backup-object-storage` Ready/Valid, condition `databases` only. +- ExternalSecret `databases/platform-pg-backup-s3` SecretSynced. Keys `ACCESS_KEY_ID` (len 20) and `ACCESS_SECRET_KEY` (len 36). Values not read. +- Prefix `platform-pg/` object count: 0. +- `platform-pg` still has no `barmanObjectStore`. WAL is T05. diff --git a/docs/put-backup-object-storage.md b/docs/put-backup-object-storage.md new file mode 100644 index 0000000..ceff57b --- /dev/null +++ b/docs/put-backup-object-storage.md @@ -0,0 +1,89 @@ +# Put the scoped backup object-store key (founder, local only) + +Do this on a trusted terminal. **Do not paste ACCESS_KEY_ID or +ACCESS_SECRET_KEY into chat, Git, or State Hub.** + +CCR: `credential-change-requests/CCR-2026-0012-backup-object-storage.yaml` +Path: `platform/workloads/railiance/backup/object-storage` + +This is **not** the bootstrap key (`…/scaleway/bootstrap`). That key +created the bucket. This key is the general **backup** runtime identity +for application `resource-control`. First consumer is `rapp-postgres` +(Secret `platform-pg-backup-s3`). + +## 1. Use IAM application `resource-control` + +In [console.scaleway.com](https://console.scaleway.com) → IAM → Applications: + +1. Create application `resource-control` if it does not exist. Do **not** + create a Barman- or postgres-specific application. +2. Create an API key **on that application** (not on your user): + - Description / name: `Scoped backup access` + - **Preferred Project for Object Storage:** the project that owns + bucket `railiance-platform-pg-backup` +3. Copy the access key and secret key into a local scratch file + (`chmod 600`). The secret is shown once. + +Put **only** backup-scoped keys on this application. Scaleway attaches +policies to the application, not to one key. The bootstrap key must stay +off `resource-control`. + +## 2. Bind policy `Scoped backup access` + +IAM → Policies → create (or reuse) `Scoped backup access`: + +- Principal: application `resource-control` +- Scope: **that one project**, not the whole Organization +- Permission sets, if the console lists them: + - `ObjectStorageBucketsRead` + - `ObjectStorageObjectsRead` + - `ObjectStorageObjectsWrite` + - `ObjectStorageObjectsDelete` +- If those sets are not listed, `ObjectStorageFullAccess` **on this + project only** is acceptable because the project should contain only + backup buckets. + +Do **not** attach `IAM*` or compute permission sets. + +## 3. Put the values in OpenBao + +On this host, with a token that can write the `platform` mount: + +```bash +bao kv put platform/workloads/railiance/backup/object-storage \ + ACCESS_KEY='SCWxxxxxxxx' \ + SECRET_KEY='xxxxxxxx' \ + DEFAULT_ORGANIZATION_ID='xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx' \ + DEFAULT_PROJECT_ID='xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx' +``` + +Same four field names as the bootstrap path. External Secrets projects +`ACCESS_KEY` / `SECRET_KEY` to Kubernetes `ACCESS_KEY_ID` / +`ACCESS_SECRET_KEY`. + +Optional field for the bucket policy (not a secret): + +```bash +bao kv patch platform/workloads/railiance/backup/object-storage \ + APPLICATION_ID='xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx' +``` + +Copy the UUID from IAM → Applications → `resource-control`. Then tell +the agent **“APPLICATION_ID is in bao.”** They will run +`reef-storage/tools/apply-backup-bucket-policy.sh`. + +## 4. Confirm without printing values + +```bash +bao kv metadata get platform/workloads/railiance/backup/object-storage +``` + +You should see a current version greater than any placeholder version. +Then tell the agent: **“the backup key is in bao.”** + +Do not enable WAL archiving yourself. T04 still has to prove a negative +key cannot access the bucket and to vend the first consumer Secret. +T05 enables continuous archiving. + +After the scoped key works, delete or lock down the bootstrap key from +CCR-2026-0011. diff --git a/docs/rapp-credential-lane-binding.md b/docs/rapp-credential-lane-binding.md index 2962312..c604f63 100644 --- a/docs/rapp-credential-lane-binding.md +++ b/docs/rapp-credential-lane-binding.md @@ -47,7 +47,8 @@ the list does not say who provisions the thing that satisfies a name. `openbao-database-secrets-engine` and `s3-backup-target` are capabilities. The database engine is configured by S3 from statements the package owns; the backup bucket is a `resource-control` resource and the -provider credential is S3 custody (`Secret platform-pg-backup-s3`). +provider credential is S3 custody (`secret:railiance-platform/backup`, +first projection Secret `platform-pg-backup-s3`). Fail-closed until that handoff lands is correct, not a hole in the package. @@ -180,7 +181,7 @@ shape unless it *is* a secrets engine. | --- | --- | --- | --- | | `rapp-openbao` | none | none — package is the store | correct | | `rapp-postgres` | `consumer_contract` + `openbao-database-secrets-engine` | `rapp-postgres/audit-core-runtime`, `rapp-postgres/audit-core-migration` | bound | -| `rapp-postgres` | `s3-backup-target` | no lane yet; Secret `platform-pg-backup-s3` is S3 custody after RESOURCE-WP-0002 | fail-closed on purpose | +| `rapp-postgres` | `s3-backup-target` / `secret_references: platform/workloads/railiance/backup/object-storage` | `CCR-2026-0012` | pointer set; CCR `proposed`; first consumer of the general backup key; Secret not vended | | `rapp-qonto` | `secret_references: tenants/binky/qonto-api` | `CCR-2026-0009` | pointer set; CCR itself is still `proposed` | Live CCRs without `target.rapp` are un-rapped workloads. They stay on diff --git a/openbao/policies/workload-kv-read-backup-object-storage.hcl b/openbao/policies/workload-kv-read-backup-object-storage.hcl new file mode 100644 index 0000000..e9aa431 --- /dev/null +++ b/openbao/policies/workload-kv-read-backup-object-storage.hcl @@ -0,0 +1,11 @@ +# Least-privilege read of the resource-control backup object-store key. +# ESO (or the later Kubernetes auth role) is the only in-cluster reader. +# Values never belong in Git. + +path "platform/data/workloads/railiance/backup/object-storage" { + capabilities = ["read"] +} + +path "platform/metadata/workloads/railiance/backup/object-storage" { + capabilities = ["read"] +} diff --git a/tools/cmd/cnpg-option-a-backup b/tools/cmd/cnpg-option-a-backup index 744ad77..490d9db 100755 --- a/tools/cmd/cnpg-option-a-backup +++ b/tools/cmd/cnpg-option-a-backup @@ -50,6 +50,7 @@ DEFAULT_TARGETS=( "core-net-kingdom-pg|KUBECONFIG_CORE|databases|net-kingdom-pg|interhub,privacyidea_db" "core-state-hub-db|KUBECONFIG_CORE|databases|state-hub-db|state_hub" "r01-forgejo-db|KUBECONFIG_R01|databases|forgejo-db|forgejo" + "r01-platform-pg|KUBECONFIG_R01|databases|platform-pg|audit_core,platform_meta,isolation_probe" "r01-net-kingdom-pg|KUBECONFIG_R01|databases|net-kingdom-pg|interhub,privacyidea_db" "r01-state-hub-db|KUBECONFIG_R01|databases|state-hub-db|state_hub" "r01-user-engine-pg|KUBECONFIG_R01|user-engine|user-engine-pg|user_engine" diff --git a/workplans/RAILIANCE-WP-0012-rapp-openbao-extraction-boundary.md b/workplans/RAILIANCE-WP-0012-rapp-openbao-extraction-boundary.md index 8dacbfe..76c86dc 100644 --- a/workplans/RAILIANCE-WP-0012-rapp-openbao-extraction-boundary.md +++ b/workplans/RAILIANCE-WP-0012-rapp-openbao-extraction-boundary.md @@ -127,4 +127,4 @@ copies without losing source-of-truth clarity. ## Notes This workplan is repo-local. The framework-level sequencing stays in -`railiance-master/workplans/RAILIANCE-WP-0017-rail-rapp-reef-repo-separation.md`. +`railiance-master/workplans/RMASTER-WP-0017-rail-rapp-reef-repo-separation.md`. diff --git a/workplans/RAILIANCE-WP-0015-platform-rapp-consistency.md b/workplans/RAILIANCE-WP-0015-platform-rapp-consistency.md index 1eb0861..f8cd4fe 100644 --- a/workplans/RAILIANCE-WP-0015-platform-rapp-consistency.md +++ b/workplans/RAILIANCE-WP-0015-platform-rapp-consistency.md @@ -37,7 +37,7 @@ conforms, and to supply the platform-service reference shape. Surveyed 2026-08-11 against the live cluster and all six family repos. The concepts are in better shape than their enforcement. The four families are -cleanly defined and `RAILIANCE-WP-0018`/`WP-0019` in `railiance-master` are both +cleanly defined and `RMASTER-WP-0018`/`RMASTER-WP-0019` in `railiance-master` are both `finished`. Nothing, however, validates a declaration file, so the model has been drifting quietly since the first wave. diff --git a/workplans/RAILIANCE-WP-0017-consumption-mode-enforcement.md b/workplans/RAILIANCE-WP-0017-consumption-mode-enforcement.md new file mode 100644 index 0000000..90a408d --- /dev/null +++ b/workplans/RAILIANCE-WP-0017-consumption-mode-enforcement.md @@ -0,0 +1,55 @@ +--- +id: RAILIANCE-WP-0017 +type: workplan +title: "Enforce resource-control consumption mode" +domain: financials +repo: railiance-platform +status: proposed +owner: codex +topic_slug: railiance +created: "2026-08-14" +updated: "2026-08-14" +related: + - RESOURCE-WP-0005 +origin: residual +origin_ref: RESOURCE-WP-0005 +state_hub_workstream_id: "f33be601-51b5-4bcf-82bf-31f8dba59b9f" +--- + +# RAILIANCE-WP-0017 — enforce consumption mode + +## Goal + +Consume the `open` / `restricted` signal published by +`resource-control` and enforce it on new orders and elastic usage. +`resource-control` does not throttle the cluster. + +Origin: `RESOURCE-WP-0005-T07`. Terms: +`resource-control/docs/TermsOfResourceProcurement.md` § 11. +Runbook: `resource-control/docs/resource-procurement-facility.md`. + +## Boundaries + +- Restricted monthly new-consumption is €50 transfer price minus that + month's interest. At the default €1 000 overdue balance that is + €45.83 transfer price / €38.19 delivered cost. +- Already-committed provider contracts are flagged to human financial + authority, not cancelled (OQ-2). +- Safety paths (backup, identity, ingress needed to avoid data loss) + may continue; overage is an exception line, not a hidden charge. + +## T01 — Subscribe to the consumption-mode signal + +```task +id: RAILIANCE-WP-0017-T01 +status: todo +priority: high +state_hub_task_id: "1c872918-0872-4b2a-8596-89b106239e54" +``` + +Define how platform procurement and admission read the latest +per-entity consumption mode and remaining transfer-price allowance for +the current period. + +Done when a restricted entity cannot place a new order that would +exceed the published allowance, and an `open` entity is unchanged.