From ce0be1efad401276f6ff1e9d396afcad192d87ad Mon Sep 17 00:00:00 2001 From: tegwick Date: Fri, 14 Aug 2026 15:53:30 +0200 Subject: [PATCH 01/11] bootstrap: planned object-storage reef, no rail Declare reef-storage as substrate_kind object-storage. Hold non-secret attribute paths for Scaleway S3. Do not host a compute rail. --- .repo-classification.yaml | 23 ++++++++++ AGENTS.md | 17 +++++++ INTENT.md | 31 +++++++++++++ README.md | 11 ++++- SCOPE.md | 29 ++++++++++++ declarations/reef.yaml | 20 +++++++++ schemas/object-store-attributes.schema.json | 22 ++++++++++ .../object-stores/platform-audit-storage.yaml | 15 +++++++ workplans/REEF-STORAGE-WP-0001-bootstrap.md | 44 +++++++++++++++++++ 9 files changed, 211 insertions(+), 1 deletion(-) create mode 100644 .repo-classification.yaml create mode 100644 AGENTS.md create mode 100644 INTENT.md create mode 100644 SCOPE.md create mode 100644 declarations/reef.yaml create mode 100644 schemas/object-store-attributes.schema.json create mode 100644 substrate/object-stores/platform-audit-storage.yaml create mode 100644 workplans/REEF-STORAGE-WP-0001-bootstrap.md diff --git a/.repo-classification.yaml b/.repo-classification.yaml new file mode 100644 index 0000000..d5a5941 --- /dev/null +++ b/.repo-classification.yaml @@ -0,0 +1,23 @@ +repo_classification: + standard: Repo Classification Standard + version: "1.0" + classified_at: "2026-08-14" + classified_by: agent + category: project + domain: financials + secondary_domains: + - infotech + capability_tags: + - platform + - operations + - storage + - governance + business_stake: + - technology + - operations + business_mechanics: + - coordination + - operation + repo_family: reef + ownership_repo: railiance-platform + substrate_kind: object-storage diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..d6283bb --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,17 @@ +# reef-storage — Agent Instructions + +## Repo Identity + +**Purpose:** Provider-delegated S3-compatible storage substrate. Owns +non-secret operating attributes. Does not host a rail. + +**Domain:** financials +**Repo slug:** reef-storage +**Workplan prefix:** `REEF-STORAGE-WP-` + +## Standing rules + +- No rail. Scaleway (or a successor provider) operates S3. +- No secrets in this repo. Cite `secret:railiance-platform/`. +- Attribute files are the source for `reef:storage/…#key` references. +- Do not fold this substrate into `reef-railiance`. diff --git a/INTENT.md b/INTENT.md new file mode 100644 index 0000000..d6ac90e --- /dev/null +++ b/INTENT.md @@ -0,0 +1,31 @@ +# INTENT + +## Why This Repo Exists + +`reef-storage` is the substrate boundary for **provider-delegated +S3-compatible object storage**. The first instance is Scaleway, used for +postgres backup and other large, slow-tolerant volumes. + +We consume the S3 capability. We do **not** host a rail and we do **not** +operate how Scaleway provides or runs object storage. This reef names the +boundary, holds the committed non-secret attributes, and is the place +operators look for “where is our object store?” + +It is a different reef from `reef-railiance`. Backup is not bolted onto +the home compute substrate. + +## What This Repo Must Protect + +- the split: procure/cost in `resource-control`, operate-by-reference here, + secrets in `railiance-platform`, consume in `rapp-*` +- no rail invented to satisfy a compute-centric schema +- no secret material in git +- independent failure domain from Host Europe / `reef-railiance` + +## What This Repo Is Not + +- a Scaleway account manager or billing ledger +- a Barman/CNPG operator (`rapp-postgres`) +- a secret store +- a generic S3 client library +- an extension of `reef-railiance` diff --git a/README.md b/README.md index 60e7492..9a73d78 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,12 @@ # reef-storage -Low cost data secure storage. Runtime independent S3 compatible storage facility for backup and large volume storage that can tolerate rather slow access. \ No newline at end of file +Low-cost, data-secure, runtime-independent **S3-compatible** storage +substrate for backup and large volumes that can tolerate slow access. + +Scaleway provides the S3 capability. This reef does **not** host a rail +and does not operate the object store. It names the boundary and holds +non-secret attributes. + +- Declaration: `declarations/reef.yaml` +- Attributes: `substrate/object-stores/` +- Join: `reef:storage/#` from `resource-control` \ No newline at end of file diff --git a/SCOPE.md b/SCOPE.md new file mode 100644 index 0000000..90a9642 --- /dev/null +++ b/SCOPE.md @@ -0,0 +1,29 @@ +# SCOPE + +## One-liner + +Provider-delegated S3-compatible storage reef: identity and non-secret +attributes, not a rail, not a secret store. + +## In Scope + +- Reef declaration (`substrate_kind: object-storage`, no rail) +- Committed non-secret object-store attributes (endpoint, region, bucket, + prefix, versioning/lifecycle flags, provider project ref) +- Runbooks for “how we use the delegated store,” not how Scaleway runs it +- Binding notes for consumers (`rapp-postgres` first) + +## Out of Scope + +- Purchase approval, forecasts, settlement (`resource-control`) +- Access keys and OpenBao delivery (`railiance-platform`) +- WAL archive, backup schedule, restore (`rapp-postgres`) +- Hosting Kubernetes or any rail +- Implementing S3 + +## Getting Oriented + +- Declaration: `declarations/reef.yaml` +- Attribute contract: `schemas/object-store-attributes.schema.json` +- Attribute files: `substrate/object-stores/` +- Join convention: `resource-control/docs/operational-reference-convention.md` diff --git a/declarations/reef.yaml b/declarations/reef.yaml new file mode 100644 index 0000000..2668875 --- /dev/null +++ b/declarations/reef.yaml @@ -0,0 +1,20 @@ +kind: substrate-reef +reef_id: reef-storage +repo: reef-storage +ownership_repo: railiance-platform +substrate_kind: object-storage +lifecycle_state: planned +criticality: high +current_members: [] +source_documents: + - repo: railiance-master + path: docs/reef-substrate-model.md + - repo: railiance-master + path: workplans/RMASTER-WP-0022-establish-reef-storage.md + - repo: resource-control + path: docs/operational-reference-convention.md +compatibility_notes: + - "S3-compatible capability is delegated to Scaleway. This reef does not host a rail and does not operate the object store internals." + - "Non-secret attributes (endpoint, bucket, region, prefix, lifecycle) live under substrate/object-stores/." + - "Credentials live in railiance-platform OpenBao. resource-control cites reef: and secret: references only." + - "This substrate is independent of reef-railiance. Backup is not provided on the home compute reef." diff --git a/schemas/object-store-attributes.schema.json b/schemas/object-store-attributes.schema.json new file mode 100644 index 0000000..165b957 --- /dev/null +++ b/schemas/object-store-attributes.schema.json @@ -0,0 +1,22 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://railiance.dev/reef-storage/object-store-attributes.schema.json", + "title": "Non-secret object-store attributes", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "resource_id", "provider", "status"], + "properties": { + "schema_version": {"const": "0.1"}, + "resource_id": {"type": "string", "pattern": "^resource:"}, + "provider": {"type": "string", "minLength": 1}, + "status": {"enum": ["planned", "active", "retired"]}, + "endpoint": {"type": ["string", "null"]}, + "region": {"type": ["string", "null"]}, + "bucket": {"type": ["string", "null"]}, + "prefix": {"type": ["string", "null"]}, + "versioning": {"type": ["boolean", "null"]}, + "lifecycle": {"type": ["string", "null"]}, + "provider_project_ref": {"type": ["string", "null"]}, + "notes": {"type": "array", "items": {"type": "string"}} + } +} diff --git a/substrate/object-stores/platform-audit-storage.yaml b/substrate/object-stores/platform-audit-storage.yaml new file mode 100644 index 0000000..7e6dffb --- /dev/null +++ b/substrate/object-stores/platform-audit-storage.yaml @@ -0,0 +1,15 @@ +# Planned. Fill after RESOURCE-WP-0002 purchase. No secrets. +schema_version: "0.1" +resource_id: resource:platform:audit-storage +provider: Scaleway +status: planned +endpoint: null +region: null +bucket: null +prefix: null +versioning: null +lifecycle: null +provider_project_ref: null +notes: + - Destination for rapp-postgres WAL and base backups once procured. + - resource-control cites reef:storage/substrate/object-stores/platform-audit-storage.yaml# diff --git a/workplans/REEF-STORAGE-WP-0001-bootstrap.md b/workplans/REEF-STORAGE-WP-0001-bootstrap.md new file mode 100644 index 0000000..5d0eb31 --- /dev/null +++ b/workplans/REEF-STORAGE-WP-0001-bootstrap.md @@ -0,0 +1,44 @@ +--- +id: REEF-STORAGE-WP-0001 +type: workplan +title: "Bootstrap reef-storage" +domain: financials +repo: reef-storage +status: active +owner: grok +topic_slug: railiance +created: "2026-08-14" +updated: "2026-08-14" +related: + - RMASTER-WP-0022 + - RESOURCE-WP-0002 +--- + +# REEF-STORAGE-WP-0001 — bootstrap + +## Goal + +Stand up the planned storage reef: declaration without a rail, attribute +file layout, and the contract for non-secret object-store facts. + +## T01 — Declare the reef + +```task +id: REEF-STORAGE-WP-0001-T01 +status: done +priority: high +``` + +`declarations/reef.yaml`: `substrate_kind: object-storage`, no +`primary_rail`, no `hosted_rails`, empty `current_members`, planned. + +## T02 — Attribute contract + +```task +id: REEF-STORAGE-WP-0001-T02 +status: done +priority: high +``` + +`schemas/object-store-attributes.schema.json` and a planned +`platform-audit-storage.yaml` with null operating fields until purchase. From fdde22d33d44ddf6061a16206f36f47a4e63a493 Mon Sep 17 00:00:00 2001 From: tegwick Date: Fri, 14 Aug 2026 16:18:16 +0200 Subject: [PATCH 02/11] seed: planned Scaleway backup store for WP-0002 Identity, topology, consumer bindings, and the planned attribute file (region nl-ams, endpoint/bucket still null). Post-purchase fill is REEF-STORAGE-WP-0002. --- bindings/consumers.yaml | 13 +++++++ docs/post-purchase-seed.md | 25 ++++++++++++++ schemas/object-store-attributes.schema.json | 2 ++ substrate/identity.yaml | 14 ++++++++ .../object-stores/platform-audit-storage.yaml | 13 +++++-- substrate/topology.yaml | 21 ++++++++++++ ...EEF-STORAGE-WP-0002-fill-after-purchase.md | 34 +++++++++++++++++++ 7 files changed, 120 insertions(+), 2 deletions(-) create mode 100644 bindings/consumers.yaml create mode 100644 docs/post-purchase-seed.md create mode 100644 substrate/identity.yaml create mode 100644 substrate/topology.yaml create mode 100644 workplans/REEF-STORAGE-WP-0002-fill-after-purchase.md diff --git a/bindings/consumers.yaml b/bindings/consumers.yaml new file mode 100644 index 0000000..e734c2c --- /dev/null +++ b/bindings/consumers.yaml @@ -0,0 +1,13 @@ +reef_id: reef-storage +consumers_potential: + - consumer_id: rapp-postgres + resource_id: resource:platform:audit-storage + purpose: WAL archive and physical base backups for platform-pg + readiness_state: declared + evidence: + - repo: resource-control + path: workplans/RESOURCE-WP-0002-procure-postgres-backup-storage.md +consumers_actual: [] +notes: + - Actual consumer is recorded only after WAL flows (RESOURCE-WP-0002-T05). + - rapp-postgres consumes a reviewed destination; it does not procure the bucket. diff --git a/docs/post-purchase-seed.md b/docs/post-purchase-seed.md new file mode 100644 index 0000000..3091987 --- /dev/null +++ b/docs/post-purchase-seed.md @@ -0,0 +1,25 @@ +# Fill reef-storage after Scaleway purchase + +Run only after human financial authority approves +`resource-control/docs/evidence/RESOURCE-WP-0002-primary-selection-2026-08-14.md`. + +1. Create a Scaleway project on the Railiance (or GmbH) paying account. +2. Create a **private** bucket in `nl-ams`, Standard Multi-AZ. +3. Enable versioning. Set lifecycle to the 30-day recovery window. +4. Create a key that can only list/get/put/delete on that bucket/prefix. +5. Turn on a project cost alert. +6. Write the non-secret facts into + `substrate/object-stores/platform-audit-storage.yaml`: + `endpoint`, `bucket`, `prefix`, `provider_project_ref`, `versioning`, + `lifecycle`. Status → `active`. +7. Append the bucket/project to `declarations/reef.yaml` `current_members` + and `substrate/topology.yaml` `members`. Reef `lifecycle_state` may stay + `planned` until the first restore (T05), or become `active` when the + bucket exists — prefer `active` only after T05. +8. Hand `secret:railiance-platform/platform-pg-backup-s3` to OpenBao (T04). + Do not commit the key. +9. Tell resource-control: set inventory `decision.status: approved`, + `status: ordered`, `ordered_on`, `provider.provider_resource_id`. + +Do not paste endpoint or keys into chat as the source of truth. The YAML +file is the source for `reef:storage/…` references. diff --git a/schemas/object-store-attributes.schema.json b/schemas/object-store-attributes.schema.json index 165b957..728a4d0 100644 --- a/schemas/object-store-attributes.schema.json +++ b/schemas/object-store-attributes.schema.json @@ -9,6 +9,7 @@ "schema_version": {"const": "0.1"}, "resource_id": {"type": "string", "pattern": "^resource:"}, "provider": {"type": "string", "minLength": 1}, + "product": {"type": ["string", "null"]}, "status": {"enum": ["planned", "active", "retired"]}, "endpoint": {"type": ["string", "null"]}, "region": {"type": ["string", "null"]}, @@ -17,6 +18,7 @@ "versioning": {"type": ["boolean", "null"]}, "lifecycle": {"type": ["string", "null"]}, "provider_project_ref": {"type": ["string", "null"]}, + "intended": {"type": ["object", "null"]}, "notes": {"type": "array", "items": {"type": "string"}} } } diff --git a/substrate/identity.yaml b/substrate/identity.yaml new file mode 100644 index 0000000..50f67be --- /dev/null +++ b/substrate/identity.yaml @@ -0,0 +1,14 @@ +reef_id: reef-storage +display_name: Railiance delegated object storage +substrate_kind: object-storage +ownership_repo: railiance-platform +lifecycle_state: planned +criticality: high +primary_rail: null +current_members: [] +current_role: Provider-delegated S3-compatible store. Scaleway operates the capability; this reef names the boundary and non-secret attributes. +source_of_truth: + repo: reef-storage + paths: + - declarations/reef.yaml + - substrate/object-stores/platform-audit-storage.yaml diff --git a/substrate/object-stores/platform-audit-storage.yaml b/substrate/object-stores/platform-audit-storage.yaml index 7e6dffb..eb7de56 100644 --- a/substrate/object-stores/platform-audit-storage.yaml +++ b/substrate/object-stores/platform-audit-storage.yaml @@ -1,15 +1,24 @@ -# Planned. Fill after RESOURCE-WP-0002 purchase. No secrets. +# Planned primary for RESOURCE-WP-0002. Fill endpoint/bucket/prefix after purchase. +# No secrets in this file. schema_version: "0.1" resource_id: resource:platform:audit-storage provider: Scaleway +product: standard-multi-az status: planned endpoint: null -region: null +region: nl-ams bucket: null prefix: null versioning: null lifecycle: null provider_project_ref: null +intended: + public_access: disabled + versioning: true + lifecycle: 30-day recovery window matching data/demand/platform-audit-storage.json + encryption_at_rest: provider-managed + identity: bucket-and-prefix scoped key only notes: - Destination for rapp-postgres WAL and base backups once procured. - resource-control cites reef:storage/substrate/object-stores/platform-audit-storage.yaml# + - "Decision record: resource-control/docs/evidence/RESOURCE-WP-0002-primary-selection-2026-08-14.md" diff --git a/substrate/topology.yaml b/substrate/topology.yaml new file mode 100644 index 0000000..0f3d8c3 --- /dev/null +++ b/substrate/topology.yaml @@ -0,0 +1,21 @@ +reef_id: reef-storage +membership_policy: Add a member when a purchased object-store (bucket/project) is live. Members are provider resources, not hosts. +members: [] +intended_members: + - member_id: platform-audit-storage + role: primary-backup-object-store + provider: Scaleway + product: standard-multi-az + region: nl-ams + resource_id: resource:platform:audit-storage + status: planned + evidence: + - repo: resource-control + path: docs/evidence/RESOURCE-WP-0002-primary-selection-2026-08-14.md +access_surfaces: + - type: s3-https + note: Endpoint and bucket are filled after purchase in substrate/object-stores/platform-audit-storage.yaml. No rail. +unresolved_questions: + - Scaleway project/organization id on the paying Railiance (or GmbH) account + - Exact bucket name and prefix after create + - Whether Host Europe Backup Storage or Nextcloud is the T06 second copy diff --git a/workplans/REEF-STORAGE-WP-0002-fill-after-purchase.md b/workplans/REEF-STORAGE-WP-0002-fill-after-purchase.md new file mode 100644 index 0000000..56acbd6 --- /dev/null +++ b/workplans/REEF-STORAGE-WP-0002-fill-after-purchase.md @@ -0,0 +1,34 @@ +--- +id: REEF-STORAGE-WP-0002 +type: workplan +title: "Fill Scaleway attributes after purchase" +domain: financials +repo: reef-storage +status: wait +owner: grok +topic_slug: railiance +created: "2026-08-14" +updated: "2026-08-14" +related: + - RESOURCE-WP-0002 +--- + +# REEF-STORAGE-WP-0002 — fill after purchase + +Wait for human approval of +`resource-control/docs/evidence/RESOURCE-WP-0002-primary-selection-2026-08-14.md` +and creation of the Scaleway project/bucket. + +Then follow `docs/post-purchase-seed.md`. + +## T01 — Write live non-secret attributes + +```task +id: REEF-STORAGE-WP-0002-T01 +status: wait +priority: high +``` + +Fill `substrate/object-stores/platform-audit-storage.yaml` endpoint, +bucket, prefix, project ref, versioning, lifecycle. Promote topology +members. No secrets. From 07d75fabdd0dc855e653f5922afa21acf13e9c0f Mon Sep 17 00:00:00 2001 From: tegwick Date: Fri, 14 Aug 2026 16:58:44 +0200 Subject: [PATCH 03/11] feat: Scaleway bucket create script and founder put instructions Read bootstrap creds from OpenBao only. Write non-secret attributes after create. Never print keys. --- docs/put-scaleway-bootstrap.md | 37 +++++++++ tools/create-platform-audit-bucket.sh | 103 ++++++++++++++++++++++++++ 2 files changed, 140 insertions(+) create mode 100644 docs/put-scaleway-bootstrap.md create mode 100755 tools/create-platform-audit-bucket.sh diff --git a/docs/put-scaleway-bootstrap.md b/docs/put-scaleway-bootstrap.md new file mode 100644 index 0000000..1175920 --- /dev/null +++ b/docs/put-scaleway-bootstrap.md @@ -0,0 +1,37 @@ +# Put the Scaleway bootstrap key (founder, local only) + +Do this on a trusted terminal. **Do not paste ACCESS_KEY or SECRET_KEY +into chat, Git, or State Hub.** + +CCR: `railiance-platform/credential-change-requests/CCR-2026-0011-scaleway-object-storage-bootstrap.yaml` +Path: `platform/workloads/railiance/scaleway/bootstrap` + +1. In [console.scaleway.com](https://console.scaleway.com) create or reuse + the Railiance (or GmbH) project. IAM → API keys: create a key that can + create Object Storage buckets in that project. Copy org id and project id + from the project dashboard (those two are not as sensitive as the secret + key, but still keep them out of git). +2. On this host, with a token that can write the `platform` mount: + +```bash +bao kv put platform/workloads/railiance/scaleway/bootstrap \ + ACCESS_KEY='SCWxxxxxxxx' \ + SECRET_KEY='xxxxxxxx' \ + DEFAULT_ORGANIZATION_ID='xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx' \ + DEFAULT_PROJECT_ID='xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx' +``` + +3. Confirm **without printing values**: + +```bash +bao kv metadata get platform/workloads/railiance/scaleway/bootstrap +``` + +You should see a current version. Then tell the agent “bootstrap is in +OpenBao.” They will run `tools/create-platform-audit-bucket.sh`, which +creates the private Multi-AZ bucket, 30-day lifecycle, and writes only +endpoint/bucket/region into `substrate/object-stores/platform-audit-storage.yaml`. + +The Barman runtime key is a **different** path +(`platform/workloads/railiance/backup/platform-pg-backup-s3`) and is T04. +After the scoped key works, delete or lock down this bootstrap key. diff --git a/tools/create-platform-audit-bucket.sh b/tools/create-platform-audit-bucket.sh new file mode 100755 index 0000000..50ca36f --- /dev/null +++ b/tools/create-platform-audit-bucket.sh @@ -0,0 +1,103 @@ +#!/usr/bin/env bash +# Create the planned Scaleway backup bucket. Reads bootstrap creds from +# OpenBao. Never prints secret values. Writes only non-secret attributes +# into substrate/object-stores/platform-audit-storage.yaml. +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +ATTR="$ROOT/substrate/object-stores/platform-audit-storage.yaml" +BAO_ADDR="${BAO_ADDR:-${VAULT_ADDR:-https://bao.coulomb.social}}" +BOOTSTRAP_PATH="platform/workloads/railiance/scaleway/bootstrap" +SCOPED_PATH="platform/workloads/railiance/backup/platform-pg-backup-s3" +REGION="nl-ams" +BUCKET="${BUCKET:-railiance-platform-pg-backup}" +PREFIX="${PREFIX:-platform-pg/}" +ENDPOINT="https://s3.nl-ams.scw.cloud" + +need() { command -v "$1" >/dev/null || { echo "missing $1" >&2; exit 2; }; } +need python3 +need curl + +TOKEN="${OPENBAO_TOKEN:-${VAULT_TOKEN:-}}" +if [[ -z "$TOKEN" && -f "$HOME/.vault-token" ]]; then + TOKEN="$(cat "$HOME/.vault-token")" +fi +[[ -n "$TOKEN" ]] || { echo "no OpenBao token" >&2; exit 2; } + +read_kv() { + local path="$1" + curl -fsS -H "X-Vault-Token: $TOKEN" \ + "$BAO_ADDR/v1/platform/data/${path#platform/}" \ + | python3 -c 'import json,sys; d=json.load(sys.stdin); print(json.dumps(d["data"]["data"]))' +} + +if ! BOOTSTRAP_JSON="$(read_kv workloads/railiance/scaleway/bootstrap 2>/dev/null)"; then + echo "Bootstrap secret missing at $BOOTSTRAP_PATH" >&2 + echo "Founder: put ACCESS_KEY SECRET_KEY DEFAULT_ORGANIZATION_ID DEFAULT_PROJECT_ID locally (not in chat)." >&2 + echo "See docs/put-scaleway-bootstrap.md" >&2 + exit 3 +fi + +eval "$(BOOTSTRAP_JSON="$BOOTSTRAP_JSON" python3 - <<'PY' +import json, os, sys +data = json.loads(os.environ["BOOTSTRAP_JSON"]) +missing = [k for k in ("ACCESS_KEY", "SECRET_KEY", "DEFAULT_ORGANIZATION_ID", "DEFAULT_PROJECT_ID") if not (data.get(k) or data.get(k.lower()))] +if missing: + sys.stderr.write("bootstrap fields missing: " + ",".join(missing) + "\n") + sys.exit(4) +def g(k): + return data.get(k) or data.get(k.lower()) +print("export SCW_ACCESS_KEY=" + json.dumps(g("ACCESS_KEY"))) +print("export SCW_SECRET_KEY=" + json.dumps(g("SECRET_KEY"))) +print("export SCW_DEFAULT_ORGANIZATION_ID=" + json.dumps(g("DEFAULT_ORGANIZATION_ID"))) +print("export SCW_DEFAULT_PROJECT_ID=" + json.dumps(g("DEFAULT_PROJECT_ID"))) +PY +)" + +export SCW_DEFAULT_REGION="$REGION" +need scw +need aws + +echo "creating private bucket $BUCKET in $REGION (versioning on)" +scw object bucket create "$BUCKET" region="$REGION" acl=private enable-versioning=true + +echo "applying 30-day lifecycle (current + noncurrent versions)" +aws --endpoint-url "$ENDPOINT" s3api put-bucket-lifecycle-configuration \ + --bucket "$BUCKET" \ + --lifecycle-configuration '{ + "Rules": [ + { + "ID": "retain-30-days", + "Status": "Enabled", + "Filter": {"Prefix": ""}, + "Expiration": {"Days": 30}, + "NoncurrentVersionExpiration": {"NoncurrentDays": 30} + } + ] + }' + +echo "writing non-secret attributes (no keys)" +python3 - < Date: Fri, 14 Aug 2026 17:36:17 +0200 Subject: [PATCH 04/11] feat: founder ingest of local scaleway.auto.tfvars Parse the four Terraform fields and bao kv put via @file. Never print values. Operator runs this; agents do not. --- tools/ingest-scaleway-tfvars.sh | 54 +++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100755 tools/ingest-scaleway-tfvars.sh diff --git a/tools/ingest-scaleway-tfvars.sh b/tools/ingest-scaleway-tfvars.sh new file mode 100755 index 0000000..1723330 --- /dev/null +++ b/tools/ingest-scaleway-tfvars.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# Founder-only. Reads a local scaleway.auto.tfvars (or equivalent) and +# writes the four fields to OpenBao. Never prints values. Never commit +# the tfvars file. +set -euo pipefail + +TFVARS="${1:-}" +[[ -n "$TFVARS" && -f "$TFVARS" ]] || { + echo "usage: $0 /path/to/scaleway.auto.tfvars" >&2 + exit 2 +} +command -v bao >/dev/null || { echo "missing bao" >&2; exit 2; } + +mapfile -t EXPORTS < <(python3 - "$TFVARS" <<'PY' +import re, json, sys +text = open(sys.argv[1], encoding="utf-8").read() +# HCL-ish: key = "value" (ignore comments and the main.tf provider block) +found = {} +for name, dest in ( + ("access_key", "ACCESS_KEY"), + ("secret_key", "SECRET_KEY"), + ("organization_id", "DEFAULT_ORGANIZATION_ID"), + ("project_id", "DEFAULT_PROJECT_ID"), +): + m = re.search(rf'(?m)^\s*{name}\s*=\s*"([^"]*)"\s*$', text) + if not m or not m.group(1) or m.group(1) in {"xxx", "redacted"}: + sys.stderr.write(f"missing or placeholder field: {name}\n") + sys.exit(4) + found[dest] = m.group(1) +for dest, val in found.items(): + print(f"{dest}={json.dumps(val)}") +PY +) + +# Write via env to bao without putting values on argv. +# bao kv put supports @file; we use a mode-0600 temp dir. +umask 077 +tmp="$(mktemp -d)" +trap 'rm -rf "$tmp"' EXIT +for line in "${EXPORTS[@]}"; do + field="${line%%=*}" + python3 -c 'import json,os,sys; open(sys.argv[2],"w").write(json.loads(sys.argv[1]))' \ + "${line#*=}" "$tmp/$field" +done + +bao kv put platform/workloads/railiance/scaleway/bootstrap \ + ACCESS_KEY=@"$tmp/ACCESS_KEY" \ + SECRET_KEY=@"$tmp/SECRET_KEY" \ + DEFAULT_ORGANIZATION_ID=@"$tmp/DEFAULT_ORGANIZATION_ID" \ + DEFAULT_PROJECT_ID=@"$tmp/DEFAULT_PROJECT_ID" + +echo "wrote four fields to platform/workloads/railiance/scaleway/bootstrap" +bao kv metadata get platform/workloads/railiance/scaleway/bootstrap +echo "shred or keep your tfvars; do not commit it" From d6cc36939262541b8c53e9ced73ed16b03efc99b Mon Sep 17 00:00:00 2001 From: tegwick Date: Fri, 14 Aug 2026 17:40:09 +0200 Subject: [PATCH 05/11] fix: refuse xxx placeholders before Scaleway bucket create --- tools/create-platform-audit-bucket.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tools/create-platform-audit-bucket.sh b/tools/create-platform-audit-bucket.sh index 50ca36f..7fdb44a 100755 --- a/tools/create-platform-audit-bucket.sh +++ b/tools/create-platform-audit-bucket.sh @@ -41,10 +41,15 @@ fi eval "$(BOOTSTRAP_JSON="$BOOTSTRAP_JSON" python3 - <<'PY' import json, os, sys data = json.loads(os.environ["BOOTSTRAP_JSON"]) -missing = [k for k in ("ACCESS_KEY", "SECRET_KEY", "DEFAULT_ORGANIZATION_ID", "DEFAULT_PROJECT_ID") if not (data.get(k) or data.get(k.lower()))] +needed = ("ACCESS_KEY", "SECRET_KEY", "DEFAULT_ORGANIZATION_ID", "DEFAULT_PROJECT_ID") +missing = [k for k in needed if not (data.get(k) or data.get(k.lower()))] if missing: sys.stderr.write("bootstrap fields missing: " + ",".join(missing) + "\n") sys.exit(4) +placeholders = [k for k in needed if (data.get(k) or data.get(k.lower()) or "") in {"xxx", "redacted", "changeme"}] +if placeholders: + sys.stderr.write("bootstrap still has example placeholders; replace them in the OpenBao UI first\n") + sys.exit(5) def g(k): return data.get(k) or data.get(k.lower()) print("export SCW_ACCESS_KEY=" + json.dumps(g("ACCESS_KEY"))) From c545d08f399549de6527126a763e136b807f0450 Mon Sep 17 00:00:00 2001 From: tegwick Date: Fri, 14 Aug 2026 17:48:04 +0200 Subject: [PATCH 06/11] fix: apply lifecycle with boto3; require scw on PATH aws CLI is not installed here. Create still needs Object Storage permission on the bootstrap key. --- tools/create-platform-audit-bucket.sh | 59 +++++++++++++++++++-------- 1 file changed, 43 insertions(+), 16 deletions(-) diff --git a/tools/create-platform-audit-bucket.sh b/tools/create-platform-audit-bucket.sh index 7fdb44a..4ec8f35 100755 --- a/tools/create-platform-audit-bucket.sh +++ b/tools/create-platform-audit-bucket.sh @@ -15,8 +15,10 @@ PREFIX="${PREFIX:-platform-pg/}" ENDPOINT="https://s3.nl-ams.scw.cloud" need() { command -v "$1" >/dev/null || { echo "missing $1" >&2; exit 2; }; } +export PATH="${HOME}/.local/bin:${PATH}" need python3 need curl +need scw TOKEN="${OPENBAO_TOKEN:-${VAULT_TOKEN:-}}" if [[ -z "$TOKEN" && -f "$HOME/.vault-token" ]]; then @@ -60,26 +62,51 @@ PY )" export SCW_DEFAULT_REGION="$REGION" -need scw -need aws echo "creating private bucket $BUCKET in $REGION (versioning on)" -scw object bucket create "$BUCKET" region="$REGION" acl=private enable-versioning=true +scw object bucket create "$BUCKET" region="$REGION" acl=private enable-versioning=true -o json echo "applying 30-day lifecycle (current + noncurrent versions)" -aws --endpoint-url "$ENDPOINT" s3api put-bucket-lifecycle-configuration \ - --bucket "$BUCKET" \ - --lifecycle-configuration '{ - "Rules": [ - { - "ID": "retain-30-days", - "Status": "Enabled", - "Filter": {"Prefix": ""}, - "Expiration": {"Days": 30}, - "NoncurrentVersionExpiration": {"NoncurrentDays": 30} - } - ] - }' +VENV="${TMPDIR:-/tmp}/reef-storage-boto3" +if [[ ! -x "$VENV/bin/python" ]]; then + python3 -m venv "$VENV" + "$VENV/bin/pip" -q install boto3 +fi +BUCKET="$BUCKET" AWS_ACCESS_KEY_ID="$SCW_ACCESS_KEY" AWS_SECRET_ACCESS_KEY="$SCW_SECRET_KEY" \ + "$VENV/bin/python" - <<'PY' +import os +import boto3 +client = boto3.client( + "s3", + region_name="nl-ams", + endpoint_url="https://s3.nl-ams.scw.cloud", + aws_access_key_id=os.environ["AWS_ACCESS_KEY_ID"], + aws_secret_access_key=os.environ["AWS_SECRET_ACCESS_KEY"], +) +client.put_bucket_lifecycle_configuration( + Bucket=os.environ["BUCKET"], + LifecycleConfiguration={ + "Rules": [ + { + "ID": "retain-30-days", + "Status": "Enabled", + "Filter": {"Prefix": ""}, + "Expiration": {"Days": 30}, + "NoncurrentVersionExpiration": {"NoncurrentDays": 30}, + } + ] + }, +) +print("lifecycle applied") +PY + +echo "creating a 20 EUR monthly budget alert if none exists" +scw billing budget create consumption-limit=20 enabled=true -o json | python3 -c ' +import json,sys +b=json.load(sys.stdin) +print("budget_id", b.get("id") or b.get("budget",{}).get("id")) +print("limit", b.get("consumption_limit") or b.get("budget",{}).get("consumption_limit")) +' || echo "budget create skipped (permissions or API shape)" echo "writing non-secret attributes (no keys)" python3 - < Date: Fri, 14 Aug 2026 18:28:27 +0200 Subject: [PATCH 07/11] feat: record live Scaleway backup bucket attributes Bucket railiance-platform-pg-backup is private in nl-ams with versioning and a 30-day lifecycle. Fill reef attributes and topology. Create script is idempotent. Billing budget still needs a console click. --- declarations/reef.yaml | 3 ++- substrate/identity.yaml | 3 ++- .../object-stores/platform-audit-storage.yaml | 19 ++++++++++--------- substrate/topology.yaml | 13 +++++++------ tools/create-platform-audit-bucket.sh | 8 ++++++-- ...EEF-STORAGE-WP-0002-fill-after-purchase.md | 16 +++++++++------- 6 files changed, 36 insertions(+), 26 deletions(-) diff --git a/declarations/reef.yaml b/declarations/reef.yaml index 2668875..5827590 100644 --- a/declarations/reef.yaml +++ b/declarations/reef.yaml @@ -5,7 +5,8 @@ ownership_repo: railiance-platform substrate_kind: object-storage lifecycle_state: planned criticality: high -current_members: [] +current_members: + - railiance-platform-pg-backup source_documents: - repo: railiance-master path: docs/reef-substrate-model.md diff --git a/substrate/identity.yaml b/substrate/identity.yaml index 50f67be..a9571e5 100644 --- a/substrate/identity.yaml +++ b/substrate/identity.yaml @@ -5,7 +5,8 @@ ownership_repo: railiance-platform lifecycle_state: planned criticality: high primary_rail: null -current_members: [] +current_members: + - railiance-platform-pg-backup current_role: Provider-delegated S3-compatible store. Scaleway operates the capability; this reef names the boundary and non-secret attributes. source_of_truth: repo: reef-storage diff --git a/substrate/object-stores/platform-audit-storage.yaml b/substrate/object-stores/platform-audit-storage.yaml index eb7de56..fff617d 100644 --- a/substrate/object-stores/platform-audit-storage.yaml +++ b/substrate/object-stores/platform-audit-storage.yaml @@ -1,17 +1,16 @@ -# Planned primary for RESOURCE-WP-0002. Fill endpoint/bucket/prefix after purchase. -# No secrets in this file. +# Live attributes for RESOURCE-WP-0002. No secrets in this file. schema_version: "0.1" resource_id: resource:platform:audit-storage provider: Scaleway product: standard-multi-az -status: planned -endpoint: null +status: active +endpoint: https://s3.nl-ams.scw.cloud region: nl-ams -bucket: null -prefix: null -versioning: null -lifecycle: null -provider_project_ref: null +bucket: railiance-platform-pg-backup +prefix: platform-pg/ +versioning: true +lifecycle: 30-day current and noncurrent expiration +provider_project_ref: e1a0dd0e-04b8-4ea9-8b30-d53f8c35c688 intended: public_access: disabled versioning: true @@ -22,3 +21,5 @@ notes: - Destination for rapp-postgres WAL and base backups once procured. - resource-control cites reef:storage/substrate/object-stores/platform-audit-storage.yaml# - "Decision record: resource-control/docs/evidence/RESOURCE-WP-0002-primary-selection-2026-08-14.md" + - Created 2026-08-14. Versioning enabled. ACL owner-only FULL_CONTROL. + - Cost alert not set via API. Bootstrap key lacks write billing_budgets. Founder must create a 20 EUR monthly budget in the Scaleway console. diff --git a/substrate/topology.yaml b/substrate/topology.yaml index 0f3d8c3..4def036 100644 --- a/substrate/topology.yaml +++ b/substrate/topology.yaml @@ -1,21 +1,22 @@ reef_id: reef-storage membership_policy: Add a member when a purchased object-store (bucket/project) is live. Members are provider resources, not hosts. -members: [] -intended_members: +members: - member_id: platform-audit-storage role: primary-backup-object-store provider: Scaleway product: standard-multi-az region: nl-ams + bucket: railiance-platform-pg-backup resource_id: resource:platform:audit-storage - status: planned + status: live evidence: - repo: resource-control path: docs/evidence/RESOURCE-WP-0002-primary-selection-2026-08-14.md + - path: substrate/object-stores/platform-audit-storage.yaml +intended_members: [] access_surfaces: - type: s3-https - note: Endpoint and bucket are filled after purchase in substrate/object-stores/platform-audit-storage.yaml. No rail. + note: Endpoint and bucket live in substrate/object-stores/platform-audit-storage.yaml. No rail. unresolved_questions: - - Scaleway project/organization id on the paying Railiance (or GmbH) account - - Exact bucket name and prefix after create - Whether Host Europe Backup Storage or Nextcloud is the T06 second copy + - Founder must create a 20 EUR monthly Scaleway budget. This API key cannot write billing_budgets. diff --git a/tools/create-platform-audit-bucket.sh b/tools/create-platform-audit-bucket.sh index 4ec8f35..b7bef01 100755 --- a/tools/create-platform-audit-bucket.sh +++ b/tools/create-platform-audit-bucket.sh @@ -63,8 +63,12 @@ PY export SCW_DEFAULT_REGION="$REGION" -echo "creating private bucket $BUCKET in $REGION (versioning on)" -scw object bucket create "$BUCKET" region="$REGION" acl=private enable-versioning=true -o json +if scw object bucket get "$BUCKET" region="$REGION" -o json >/dev/null 2>&1; then + echo "bucket $BUCKET already exists in $REGION; skipping create" +else + echo "creating private bucket $BUCKET in $REGION (versioning on)" + scw object bucket create "$BUCKET" region="$REGION" acl=private enable-versioning=true -o json +fi echo "applying 30-day lifecycle (current + noncurrent versions)" VENV="${TMPDIR:-/tmp}/reef-storage-boto3" diff --git a/workplans/REEF-STORAGE-WP-0002-fill-after-purchase.md b/workplans/REEF-STORAGE-WP-0002-fill-after-purchase.md index 56acbd6..fcc3c37 100644 --- a/workplans/REEF-STORAGE-WP-0002-fill-after-purchase.md +++ b/workplans/REEF-STORAGE-WP-0002-fill-after-purchase.md @@ -4,7 +4,7 @@ type: workplan title: "Fill Scaleway attributes after purchase" domain: financials repo: reef-storage -status: wait +status: active owner: grok topic_slug: railiance created: "2026-08-14" @@ -15,20 +15,22 @@ related: # REEF-STORAGE-WP-0002 — fill after purchase -Wait for human approval of -`resource-control/docs/evidence/RESOURCE-WP-0002-primary-selection-2026-08-14.md` -and creation of the Scaleway project/bucket. - -Then follow `docs/post-purchase-seed.md`. +Purchase happened 2026-08-14. Follow `docs/post-purchase-seed.md`. ## T01 — Write live non-secret attributes ```task id: REEF-STORAGE-WP-0002-T01 -status: wait +status: done priority: high ``` Fill `substrate/object-stores/platform-audit-storage.yaml` endpoint, bucket, prefix, project ref, versioning, lifecycle. Promote topology members. No secrets. + +Done 2026-08-14: bucket `railiance-platform-pg-backup` in `nl-ams` is +live (versioning on, 30-day current and noncurrent expiration, owner-only +ACL). Attributes, topology member, and declaration `current_members` +written. Reef `lifecycle_state` stays `planned` until RESOURCE-WP-0002-T05. +Cost alert is not set: bootstrap key lacks `write billing_budgets`. From b14d8268fedaf48a9ece8399c9c6df1fc2703b8a Mon Sep 17 00:00:00 2001 From: tegwick Date: Fri, 14 Aug 2026 18:53:56 +0200 Subject: [PATCH 08/11] note: record email billing alert as the cost backstop --- substrate/object-stores/platform-audit-storage.yaml | 2 +- substrate/topology.yaml | 1 - workplans/REEF-STORAGE-WP-0002-fill-after-purchase.md | 3 ++- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/substrate/object-stores/platform-audit-storage.yaml b/substrate/object-stores/platform-audit-storage.yaml index fff617d..50ef4fb 100644 --- a/substrate/object-stores/platform-audit-storage.yaml +++ b/substrate/object-stores/platform-audit-storage.yaml @@ -22,4 +22,4 @@ notes: - resource-control cites reef:storage/substrate/object-stores/platform-audit-storage.yaml# - "Decision record: resource-control/docs/evidence/RESOURCE-WP-0002-primary-selection-2026-08-14.md" - Created 2026-08-14. Versioning enabled. ACL owner-only FULL_CONTROL. - - Cost alert not set via API. Bootstrap key lacks write billing_budgets. Founder must create a 20 EUR monthly budget in the Scaleway console. + - Cost backstop is a founder email billing alert. Scaleway has no euro hard-stop. diff --git a/substrate/topology.yaml b/substrate/topology.yaml index 4def036..7b6d936 100644 --- a/substrate/topology.yaml +++ b/substrate/topology.yaml @@ -19,4 +19,3 @@ access_surfaces: note: Endpoint and bucket live in substrate/object-stores/platform-audit-storage.yaml. No rail. unresolved_questions: - Whether Host Europe Backup Storage or Nextcloud is the T06 second copy - - Founder must create a 20 EUR monthly Scaleway budget. This API key cannot write billing_budgets. diff --git a/workplans/REEF-STORAGE-WP-0002-fill-after-purchase.md b/workplans/REEF-STORAGE-WP-0002-fill-after-purchase.md index fcc3c37..4320070 100644 --- a/workplans/REEF-STORAGE-WP-0002-fill-after-purchase.md +++ b/workplans/REEF-STORAGE-WP-0002-fill-after-purchase.md @@ -33,4 +33,5 @@ Done 2026-08-14: bucket `railiance-platform-pg-backup` in `nl-ams` is live (versioning on, 30-day current and noncurrent expiration, owner-only ACL). Attributes, topology member, and declaration `current_members` written. Reef `lifecycle_state` stays `planned` until RESOURCE-WP-0002-T05. -Cost alert is not set: bootstrap key lacks `write billing_budgets`. +Cost backstop is a founder email billing alert. Scaleway has no +euro hard-stop; that gap is accepted. From 7965a7295f64f9e6332500cb98c23af679d0efca Mon Sep 17 00:00:00 2001 From: tegwick Date: Fri, 14 Aug 2026 19:19:56 +0200 Subject: [PATCH 09/11] docs: point T04 at the general backup object-storage path --- docs/post-purchase-seed.md | 2 +- docs/put-scaleway-bootstrap.md | 4 ++-- tools/create-platform-audit-bucket.sh | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/post-purchase-seed.md b/docs/post-purchase-seed.md index 3091987..759058d 100644 --- a/docs/post-purchase-seed.md +++ b/docs/post-purchase-seed.md @@ -16,7 +16,7 @@ Run only after human financial authority approves and `substrate/topology.yaml` `members`. Reef `lifecycle_state` may stay `planned` until the first restore (T05), or become `active` when the bucket exists — prefer `active` only after T05. -8. Hand `secret:railiance-platform/platform-pg-backup-s3` to OpenBao (T04). +8. Hand `secret:railiance-platform/backup` to OpenBao (T04). Do not commit the key. 9. Tell resource-control: set inventory `decision.status: approved`, `status: ordered`, `ordered_on`, `provider.provider_resource_id`. diff --git a/docs/put-scaleway-bootstrap.md b/docs/put-scaleway-bootstrap.md index 1175920..f7badd1 100644 --- a/docs/put-scaleway-bootstrap.md +++ b/docs/put-scaleway-bootstrap.md @@ -32,6 +32,6 @@ OpenBao.” They will run `tools/create-platform-audit-bucket.sh`, which creates the private Multi-AZ bucket, 30-day lifecycle, and writes only endpoint/bucket/region into `substrate/object-stores/platform-audit-storage.yaml`. -The Barman runtime key is a **different** path -(`platform/workloads/railiance/backup/platform-pg-backup-s3`) and is T04. +The scoped backup key is a **different** path +(`platform/workloads/railiance/backup/object-storage`) and is T04. After the scoped key works, delete or lock down this bootstrap key. diff --git a/tools/create-platform-audit-bucket.sh b/tools/create-platform-audit-bucket.sh index b7bef01..eb491ca 100755 --- a/tools/create-platform-audit-bucket.sh +++ b/tools/create-platform-audit-bucket.sh @@ -8,7 +8,7 @@ ROOT="$(cd "$(dirname "$0")/.." && pwd)" ATTR="$ROOT/substrate/object-stores/platform-audit-storage.yaml" BAO_ADDR="${BAO_ADDR:-${VAULT_ADDR:-https://bao.coulomb.social}}" BOOTSTRAP_PATH="platform/workloads/railiance/scaleway/bootstrap" -SCOPED_PATH="platform/workloads/railiance/backup/platform-pg-backup-s3" +SCOPED_PATH="platform/workloads/railiance/backup/object-storage" REGION="nl-ams" BUCKET="${BUCKET:-railiance-platform-pg-backup}" PREFIX="${PREFIX:-platform-pg/}" @@ -135,5 +135,5 @@ print(f"endpoint={ENDPOINT} bucket={BUCKET} region={REGION}") PY echo "Cost alert: set a project budget in the Scaleway console if scw billing is unavailable." -echo "Scoped Barman key is T04: $SCOPED_PATH (not written here)." +echo "Scoped backup key is T04: $SCOPED_PATH (not written here)." echo "done. Commit the YAML; do not commit any key." From 9ee3a265bbd0d9308cd5b098e7eaa858a41da7e7 Mon Sep 17 00:00:00 2001 From: tegwick Date: Fri, 14 Aug 2026 19:52:06 +0200 Subject: [PATCH 10/11] feat: apply prefix-scoped backup bucket policy when APPLICATION_ID is in bao --- tools/apply-backup-bucket-policy.sh | 165 ++++++++++++++++++++++++++++ 1 file changed, 165 insertions(+) create mode 100755 tools/apply-backup-bucket-policy.sh diff --git a/tools/apply-backup-bucket-policy.sh b/tools/apply-backup-bucket-policy.sh new file mode 100755 index 0000000..58c7337 --- /dev/null +++ b/tools/apply-backup-bucket-policy.sh @@ -0,0 +1,165 @@ +#!/usr/bin/env bash +# Apply a prefix-scoped bucket policy for the resource-control backup key. +# Reads APPLICATION_ID from OpenBao. Never prints secret values. +# Uses the bootstrap key to PutBucketPolicy (org owner can always edit policies). +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +BAO_ADDR="${BAO_ADDR:-${VAULT_ADDR:-https://bao.coulomb.social}}" +REGION="nl-ams" +BUCKET="${BUCKET:-railiance-platform-pg-backup}" +PREFIX="${PREFIX:-platform-pg/}" + +need() { command -v "$1" >/dev/null || { echo "missing $1" >&2; exit 2; }; } +export PATH="${HOME}/.local/bin:${PATH}" +need python3 +need curl + +TOKEN="${OPENBAO_TOKEN:-${VAULT_TOKEN:-}}" +if [[ -z "$TOKEN" && -f "$HOME/.vault-token" ]]; then + TOKEN="$(cat "$HOME/.vault-token")" +fi +[[ -n "$TOKEN" ]] || { echo "no OpenBao token" >&2; exit 2; } + +read_kv() { + local path="$1" + curl -fsS -H "X-Vault-Token: $TOKEN" \ + "$BAO_ADDR/v1/platform/data/${path#platform/}" \ + | python3 -c 'import json,sys; d=json.load(sys.stdin); print(json.dumps(d["data"]["data"]))' +} + +BACKUP_JSON="$(read_kv workloads/railiance/backup/object-storage)" +BOOTSTRAP_JSON="$(read_kv workloads/railiance/scaleway/bootstrap)" + +EXPORTS="$( +BACKUP_JSON="$BACKUP_JSON" BOOTSTRAP_JSON="$BOOTSTRAP_JSON" python3 - <<'PY' +import json, os, sys, re +backup = json.loads(os.environ["BACKUP_JSON"]) +boot = json.loads(os.environ["BOOTSTRAP_JSON"]) +app = (backup.get("APPLICATION_ID") or "").strip() +if not app or app in {"xxx", "redacted", "changeme"}: + sys.stderr.write( + "APPLICATION_ID missing on platform/workloads/railiance/backup/object-storage\n" + "Founder: add that field in the OpenBao UI (IAM application resource-control UUID).\n" + "It is not a secret. Then re-run this script.\n" + ) + sys.exit(6) +if not re.fullmatch(r"[0-9a-fA-F-]{36}", app): + sys.stderr.write("APPLICATION_ID is not a 36-char UUID\n") + sys.exit(7) +def g(data, *ks): + for k in ks: + if data.get(k): + return data[k] + return "" +print("export APPLICATION_ID=" + json.dumps(app)) +print("export SCW_ACCESS_KEY=" + json.dumps(g(boot, "ACCESS_KEY"))) +print("export SCW_SECRET_KEY=" + json.dumps(g(boot, "SECRET_KEY"))) +print("export BACKUP_ACCESS_KEY=" + json.dumps(g(backup, "ACCESS_KEY", "ACCESS_KEY_ID"))) +print("export BACKUP_SECRET_KEY=" + json.dumps(g(backup, "SECRET_KEY", "ACCESS_SECRET_KEY"))) +PY +)" +eval "$EXPORTS" + +export SCW_DEFAULT_REGION="$REGION" +export AWS_ACCESS_KEY_ID="$SCW_ACCESS_KEY" +export AWS_SECRET_ACCESS_KEY="$SCW_SECRET_KEY" + +VENV="${TMPDIR:-/tmp}/reef-storage-boto3" +if [[ ! -x "$VENV/bin/python" ]]; then + python3 -m venv "$VENV" + "$VENV/bin/pip" -q install boto3 +fi + +echo "putting prefix-scoped bucket policy on $BUCKET (application id not printed)" +BUCKET="$BUCKET" PREFIX="$PREFIX" APPLICATION_ID="$APPLICATION_ID" \ + "$VENV/bin/python" - <<'PY' +import json, os +import boto3 +bucket = os.environ["BUCKET"] +prefix = os.environ["PREFIX"] +app = os.environ["APPLICATION_ID"] +policy = { + "Version": "2023-04-17", + "Id": "resource-control-scoped-backup", + "Statement": [ + { + "Sid": "BackupListPrefix", + "Effect": "Allow", + "Principal": {"SCW": f"application_id:{app}"}, + "Action": [ + "s3:ListBucket", + "s3:ListBucketMultipartUploads", + "s3:ListBucketVersions", + "s3:GetBucketLocation", + "s3:GetBucketVersioning", + "s3:GetLifecycleConfiguration", + ], + "Resource": [bucket], + "Condition": {"StringLike": {"s3:prefix": [prefix, prefix + "*"]}}, + }, + { + "Sid": "BackupObjectsUnderPrefix", + "Effect": "Allow", + "Principal": {"SCW": f"application_id:{app}"}, + "Action": [ + "s3:GetObject", + "s3:GetObjectVersion", + "s3:PutObject", + "s3:DeleteObject", + "s3:DeleteObjectVersion", + "s3:AbortMultipartUpload", + "s3:ListMultipartUploadParts", + ], + "Resource": [f"{bucket}/{prefix}*"], + }, + ], +} +client = boto3.client( + "s3", + region_name="nl-ams", + endpoint_url="https://s3.nl-ams.scw.cloud", + aws_access_key_id=os.environ["AWS_ACCESS_KEY_ID"], + aws_secret_access_key=os.environ["AWS_SECRET_ACCESS_KEY"], +) +client.put_bucket_policy(Bucket=bucket, Policy=json.dumps(policy)) +print("bucket_policy=put") +got = client.get_bucket_policy(Bucket=bucket) +body = json.loads(got["Policy"]) +print("statements", [s.get("Sid") for s in body.get("Statement", [])]) +PY + +echo "verifying backup key: prefix ok, sibling denied" +BUCKET="$BUCKET" PREFIX="$PREFIX" \ +AWS_ACCESS_KEY_ID="$BACKUP_ACCESS_KEY" AWS_SECRET_ACCESS_KEY="$BACKUP_SECRET_KEY" \ + "$VENV/bin/python" - <<'PY' +import os +from botocore.exceptions import ClientError +import boto3 +c = boto3.client( + "s3", + region_name="nl-ams", + endpoint_url="https://s3.nl-ams.scw.cloud", + aws_access_key_id=os.environ["AWS_ACCESS_KEY_ID"], + aws_secret_access_key=os.environ["AWS_SECRET_ACCESS_KEY"], +) +bucket = os.environ["BUCKET"] +prefix = os.environ["PREFIX"] +ok = prefix + ".policy-probe" +sib = "other-consumer/.policy-probe" +try: + c.put_object(Bucket=bucket, Key=ok, Body=b"ok") + c.delete_object(Bucket=bucket, Key=ok) + print("prefix_write=ok") +except ClientError as e: + print("prefix_write=fail", e.response.get("Error", {}).get("Code")) +try: + c.put_object(Bucket=bucket, Key=sib, Body=b"no") + print("sibling_write=UNEXPECTED_OK") + c.delete_object(Bucket=bucket, Key=sib) +except ClientError as e: + print("sibling_write=denied", e.response.get("Error", {}).get("Code")) +PY + +echo "done" +unset SCW_SECRET_KEY SCW_ACCESS_KEY BACKUP_SECRET_KEY BACKUP_ACCESS_KEY AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY APPLICATION_ID From 6b8dc23b2c6adc3c5880639c159008456d06aa86 Mon Sep 17 00:00:00 2001 From: tegwick Date: Fri, 14 Aug 2026 20:24:30 +0200 Subject: [PATCH 11/11] note: rapp-postgres is an actual backup consumer after T05 --- bindings/consumers.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/bindings/consumers.yaml b/bindings/consumers.yaml index e734c2c..bb525b7 100644 --- a/bindings/consumers.yaml +++ b/bindings/consumers.yaml @@ -7,7 +7,13 @@ consumers_potential: evidence: - repo: resource-control path: workplans/RESOURCE-WP-0002-procure-postgres-backup-storage.md -consumers_actual: [] +consumers_actual: + - consumer_id: rapp-postgres + resource_id: resource:platform:audit-storage + readiness_state: wal-archiving + evidence: + - repo: resource-control + path: docs/evidence/RESOURCE-WP-0002-T05-restore-2026-08-14.md notes: - Actual consumer is recorded only after WAL flows (RESOURCE-WP-0002-T05). - rapp-postgres consumes a reviewed destination; it does not procure the bucket.