WP-0013/0014: warden credential routing, CoulombCore kubeconfig, apps-pg backup dry-run

Document OpenBao/warden paths for the offsite backup lane and flag the sealed Vault blocker. Point production Makefile targets at CoulombCore, auto-discover CNPG clusters in backup status, add apps-pg pg_dump dry-run tooling, and record Core Hub Helm cutover findings (live gitea image, adopt required).
2026-07-10 15:46:02 +02:00 · 2026-07-10 15:46:02 +02:00 · a689270f18
commit a689270f18
parent cd7e7035ce
7 changed files with 229 additions and 41 deletions
--- a/57
+++ b/57
@ -29,7 +29,9 @@ REUSE_VALUES    ?= helm/reuse-surface-values.yaml
 REUSE_URL       ?= https://reuse.coulomb.social
 REUSE_CERTIFICATE ?= reuse-surface-tls

+COULOMBCORE_KUBECONFIG ?= $(HOME)/.kube/config
 RAILIANCE01_KUBECONFIG ?= $(HOME)/.kube/config-hosteurope
+PRODUCTION_KUBECONFIG ?= $(COULOMBCORE_KUBECONFIG)
 INTER_HUB_KUBECONFIG ?= $(RAILIANCE01_KUBECONFIG)
 REUSE_KUBECONFIG ?= $(RAILIANCE01_KUBECONFIG)
 FORGEJO_KUBECONFIG ?= $(RAILIANCE01_KUBECONFIG)
@ -64,7 +66,8 @@ CORE_HUB_STAGING_VALUES ?= helm/core-hub-staging-values.yaml
 CORE_HUB_IMAGE_REPOSITORY ?= forgejo.coulomb.social/coulomb/core-hub
 CORE_HUB_IMAGE_TAG ?=
 CORE_HUB_BASE_URL  ?= https://hub.coulomb.social
-CORE_HUB_KUBECONFIG ?= $(RAILIANCE01_KUBECONFIG)
+CORE_HUB_KUBECONFIG ?= $(PRODUCTION_KUBECONFIG)
+VERGABE_KUBECONFIG ?= $(PRODUCTION_KUBECONFIG)
 CORE_HUB_IMAGE_REF = $(CORE_HUB_IMAGE_REPOSITORY):$(CORE_HUB_IMAGE_TAG)
 CORE_HUB_IMAGE_SET_ARG = $(if $(strip $(CORE_HUB_IMAGE_TAG)),--set image.tag=$(CORE_HUB_IMAGE_TAG),)

@ -99,6 +102,20 @@ check-railiance01-kubeconfig: ## Verify Railiance01 production kubeconfig exists
 		exit 1; \
 	}

+check-coulombcore-kubeconfig: ## Verify CoulombCore kubeconfig and k3s API tunnel
+	@test -r "$(COULOMBCORE_KUBECONFIG)" || { \
+		echo "Missing CoulombCore kubeconfig: $(COULOMBCORE_KUBECONFIG)" >&2; \
+		echo "Ensure ops-bridge tunnel is up: bridge up k3s-api-coulombcore" >&2; \
+		exit 1; \
+	}
+	@KUBECONFIG="$(COULOMBCORE_KUBECONFIG)" kubectl get --raw=/readyz >/dev/null 2>&1 || { \
+		echo "CoulombCore API unreachable via $(COULOMBCORE_KUBECONFIG)." >&2; \
+		echo "Run: bridge up k3s-api-coulombcore" >&2; \
+		exit 1; \
+	}
+
+check-production-kubeconfig: check-coulombcore-kubeconfig ## Verify production app cluster API (CoulombCore)
+
 check-inter-hub-image-tag: ## Require an explicit inter-hub image tag for production deploys
 	@test -n "$(INTER_HUB_IMAGE_TAG)" || { \
 		echo "Set INTER_HUB_IMAGE_TAG=<sha> for inter-hub production deploys." >&2; \
@ -119,8 +136,12 @@ check-core-hub-image-tag: ## Require an explicit core-hub image tag for producti
 check-core-hub-image: check-core-hub-image-tag ## Verify the core-hub OCI image tag exists before deploy
 	tools/check-oci-image.sh "$(CORE_HUB_IMAGE_REF)"

-cnpg-backup-status: check-railiance01-kubeconfig ## Report CNPG backup spec and ScheduledBackup coverage
-	KUBECONFIG="$(RAILIANCE01_KUBECONFIG)" tools/cnpg-backup-status.sh
+cnpg-backup-status: check-production-kubeconfig ## Report CNPG backup spec and ScheduledBackup coverage
+	KUBECONFIG="$(PRODUCTION_KUBECONFIG)" tools/cnpg-backup-status.sh
+
+apps-pg-backup-dry-run: check-production-kubeconfig ## Logical pg_dump + age encrypt for vergabe_db (no upload)
+	chmod +x tools/apps-pg-backup-dry-run.sh
+	KUBECONFIG="$(PRODUCTION_KUBECONFIG)" PLATFORM_REPO="$(PLATFORM_REPO)" tools/apps-pg-backup-dry-run.sh

 ##@ Vergabe Teilnahme

@ -129,16 +150,16 @@ vergabe-dry-run: ## helm template render (no apply) for inspection
 		--namespace $(VERGABE_NAMESPACE) \
 		-f $(VERGABE_VALUES)

-vergabe-deploy: ## Deploy / upgrade vergabe-teilnahme Helm release
-	helm upgrade --install $(VERGABE_RELEASE) $(VERGABE_CHART) \
+vergabe-deploy: check-production-kubeconfig ## Deploy / upgrade vergabe-teilnahme Helm release
+	KUBECONFIG="$(VERGABE_KUBECONFIG)" helm upgrade --install $(VERGABE_RELEASE) $(VERGABE_CHART) \
 		--namespace $(VERGABE_NAMESPACE) --create-namespace \
 		-f $(VERGABE_VALUES) --wait --timeout 3m

 vergabe-ingress-deploy: ## Apply the vergabe-teilnahme ingress (whywhynot.de)
 	kubectl apply -f $(VERGABE_INGRESS)

-vergabe-status: ## Show vergabe-teilnahme pod / svc / ingress / cert state
-	kubectl get pods,svc,ingress,certificate -n $(VERGABE_NAMESPACE) -l app.kubernetes.io/instance=$(VERGABE_RELEASE) --ignore-not-found
+vergabe-status: check-production-kubeconfig ## Show vergabe-teilnahme pod / svc / ingress / cert state
+	KUBECONFIG="$(VERGABE_KUBECONFIG)" kubectl get pods,svc,ingress,certificate -n $(VERGABE_NAMESPACE) -l app.kubernetes.io/instance=$(VERGABE_RELEASE) --ignore-not-found

 vergabe-migrate: ## Run Django migrations against the live deployment
 	kubectl exec -n $(VERGABE_NAMESPACE) deploy/$(VERGABE_RELEASE) -- python manage.py migrate --noinput
@ -206,22 +227,22 @@ core-hub-render-baseline: ## Render checked-in core-hub values for chart validat
 		--namespace $(CORE_HUB_NAMESPACE) \
 		-f $(CORE_HUB_VALUES)

-core-hub-dry-run: check-railiance01-kubeconfig check-core-hub-image-tag ## helm template render with an explicit production image tag
+core-hub-dry-run: check-production-kubeconfig check-core-hub-image-tag ## helm template render with an explicit production image tag
 	KUBECONFIG="$(CORE_HUB_KUBECONFIG)" helm template $(CORE_HUB_RELEASE) $(CORE_HUB_CHART) \
 		--namespace $(CORE_HUB_NAMESPACE) \
 		-f $(CORE_HUB_VALUES) $(CORE_HUB_IMAGE_SET_ARG)

-core-hub-server-dry-run: check-railiance01-kubeconfig check-core-hub-image ## Helm server dry-run core-hub upgrade on Railiance01
+core-hub-server-dry-run: check-production-kubeconfig check-core-hub-image ## Helm server dry-run core-hub upgrade on CoulombCore
 	KUBECONFIG="$(CORE_HUB_KUBECONFIG)" helm upgrade --install $(CORE_HUB_RELEASE) $(CORE_HUB_CHART) \
 		--namespace $(CORE_HUB_NAMESPACE) --create-namespace \
 		-f $(CORE_HUB_VALUES) $(CORE_HUB_IMAGE_SET_ARG) --dry-run=server --timeout 5m

-core-hub-deploy: check-railiance01-kubeconfig check-core-hub-image ## Deploy / upgrade core-hub Helm release on Railiance01
+core-hub-deploy: check-production-kubeconfig check-core-hub-image ## Deploy / upgrade core-hub Helm release on CoulombCore
 	KUBECONFIG="$(CORE_HUB_KUBECONFIG)" helm upgrade --install $(CORE_HUB_RELEASE) $(CORE_HUB_CHART) \
 		--namespace $(CORE_HUB_NAMESPACE) --create-namespace \
 		-f $(CORE_HUB_VALUES) $(CORE_HUB_IMAGE_SET_ARG) --wait --timeout 5m

-core-hub-migrate: check-railiance01-kubeconfig check-core-hub-image ## Run Alembic migrations via a one-off Job
+core-hub-migrate: check-production-kubeconfig check-core-hub-image ## Run Alembic migrations via a one-off Job
 	KUBECONFIG="$(CORE_HUB_KUBECONFIG)" kubectl -n $(CORE_HUB_NAMESPACE) delete job $(CORE_HUB_RELEASE)-prod-migrate --ignore-not-found
 	KUBECONFIG="$(CORE_HUB_KUBECONFIG)" helm template $(CORE_HUB_RELEASE) $(CORE_HUB_CHART) \
 		--namespace $(CORE_HUB_NAMESPACE) \
@ -230,11 +251,11 @@ core-hub-migrate: check-railiance01-kubeconfig check-core-hub-image ## Run Alemb
 	KUBECONFIG="$(CORE_HUB_KUBECONFIG)" kubectl -n $(CORE_HUB_NAMESPACE) wait --for=condition=complete \
 		job/$(CORE_HUB_RELEASE)-prod-migrate --timeout=180s

-core-hub-status: check-railiance01-kubeconfig ## Show core-hub pod / svc / ingress / cert state on Railiance01
+core-hub-status: check-production-kubeconfig ## Show core-hub pod / svc / ingress / cert state on CoulombCore
 	KUBECONFIG="$(CORE_HUB_KUBECONFIG)" kubectl get pods,svc,ingress,certificate -n $(CORE_HUB_NAMESPACE) \
 		-l app.kubernetes.io/name=core-hub --ignore-not-found

-core-hub-release-info: check-railiance01-kubeconfig ## Show core-hub Helm history, values, deployment, and pods
+core-hub-release-info: check-production-kubeconfig ## Show core-hub Helm history, values, deployment, and pods
 	KUBECONFIG="$(CORE_HUB_KUBECONFIG)" helm history $(CORE_HUB_RELEASE) -n $(CORE_HUB_NAMESPACE)
 	KUBECONFIG="$(CORE_HUB_KUBECONFIG)" helm get values $(CORE_HUB_RELEASE) -n $(CORE_HUB_NAMESPACE)
 	KUBECONFIG="$(CORE_HUB_KUBECONFIG)" kubectl describe deploy/$(CORE_HUB_RELEASE)-api -n $(CORE_HUB_NAMESPACE)
@ -243,21 +264,21 @@ core-hub-release-info: check-railiance01-kubeconfig ## Show core-hub Helm histor
 core-hub-smoke: ## Verify public Core Hub health and v2 API surface after rollout
 	CORE_HUB_BASE_URL="$(CORE_HUB_BASE_URL)" tools/core-hub-smoke.sh

-core-hub-logs: check-railiance01-kubeconfig ## Tail core-hub app logs from Railiance01
+core-hub-logs: check-production-kubeconfig ## Tail core-hub app logs from CoulombCore
 	KUBECONFIG="$(CORE_HUB_KUBECONFIG)" kubectl logs -n $(CORE_HUB_NAMESPACE) \
 		-l app.kubernetes.io/name=core-hub,app.kubernetes.io/component=api -f --tail=50

-core-hub-staging-dry-run: check-railiance01-kubeconfig check-core-hub-image-tag ## helm template render for core-hub staging
+core-hub-staging-dry-run: check-production-kubeconfig check-core-hub-image-tag ## helm template render for core-hub staging
 	KUBECONFIG="$(CORE_HUB_KUBECONFIG)" helm template $(CORE_HUB_RELEASE) $(CORE_HUB_CHART) \
 		--namespace $(CORE_HUB_STAGING_NAMESPACE) \
 		-f $(CORE_HUB_STAGING_VALUES) $(CORE_HUB_IMAGE_SET_ARG)

-core-hub-staging-deploy: check-railiance01-kubeconfig check-core-hub-image ## Deploy / upgrade core-hub staging release
+core-hub-staging-deploy: check-production-kubeconfig check-core-hub-image ## Deploy / upgrade core-hub staging release
 	KUBECONFIG="$(CORE_HUB_KUBECONFIG)" helm upgrade --install $(CORE_HUB_RELEASE) $(CORE_HUB_CHART) \
 		--namespace $(CORE_HUB_STAGING_NAMESPACE) --create-namespace \
 		-f $(CORE_HUB_STAGING_VALUES) $(CORE_HUB_IMAGE_SET_ARG) --wait --timeout 5m

-core-hub-staging-migrate: check-railiance01-kubeconfig check-core-hub-image ## Run Alembic migrations for core-hub staging
+core-hub-staging-migrate: check-production-kubeconfig check-core-hub-image ## Run Alembic migrations for core-hub staging
 	KUBECONFIG="$(CORE_HUB_KUBECONFIG)" kubectl -n $(CORE_HUB_STAGING_NAMESPACE) delete job $(CORE_HUB_RELEASE)-staging-migrate --ignore-not-found
 	KUBECONFIG="$(CORE_HUB_KUBECONFIG)" helm template $(CORE_HUB_RELEASE) $(CORE_HUB_CHART) \
 		--namespace $(CORE_HUB_STAGING_NAMESPACE) \
@ -266,7 +287,7 @@ core-hub-staging-migrate: check-railiance01-kubeconfig check-core-hub-image ## R
 	KUBECONFIG="$(CORE_HUB_KUBECONFIG)" kubectl -n $(CORE_HUB_STAGING_NAMESPACE) wait --for=condition=complete \
 		job/$(CORE_HUB_RELEASE)-staging-migrate --timeout=180s

-core-hub-staging-status: check-railiance01-kubeconfig ## Show core-hub staging pod / svc state
+core-hub-staging-status: check-production-kubeconfig ## Show core-hub staging pod / svc state
 	KUBECONFIG="$(CORE_HUB_KUBECONFIG)" kubectl get pods,svc -n $(CORE_HUB_STAGING_NAMESPACE) \
 		-l app.kubernetes.io/name=core-hub --ignore-not-found

--- a/docs/core-hub-on-railiance01.md
+++ b/docs/core-hub-on-railiance01.md
@ -1,13 +1,18 @@
-# Core Hub on railiance01
+# Core Hub on CoulombCore (production)

-`hub.coulomb.social` serves Core Hub in production. This repo owns the S5 Helm
-release path; Core Hub owns the application image, migrations, and API behavior.
+`hub.coulomb.social` serves Core Hub in production on **CoulombCore**
+(`92.205.130.254`). This repo owns the S5 Helm release path; Core Hub owns the
+application image, migrations, and API behavior.
+
+Railiance01 (`92.205.62.239`) hosts Forgejo and overlapping platform services but
+not the live Core Hub or `vergabe-teilnahme` releases as of 2026-07-10.

 ## Hosts and release surface

 | Server | IP | Role |
 |---|---|---|
-| Railiance01 | `92.205.62.239` | Production k3s; deploy S5 apps here |
+| CoulombCore | `92.205.130.254` | Production k3s for Core Hub and S5 apps |
+| Railiance01 | `92.205.62.239` | Forgejo and platform overlap |

 | Item | Production | Staging |
 |---|---|---|
@ -34,8 +39,10 @@ is the only schema path.

 ## Deploy

-Use the Railiance01 kubeconfig. The Makefile defaults to
-`~/.kube/config-hosteurope` and fails fast when it is missing.
+Use the CoulombCore kubeconfig. The Makefile defaults to `~/.kube/config`
+(tunnel `bridge up k3s-api-coulombcore`) and fails fast when the API is
+unreachable. See `docs/credential-routing-railiance-apps.md` for cluster and
+credential boundaries.

 ```bash
 make core-hub-render-baseline
--- a/docs/credential-routing-railiance-apps.md
+++ b/docs/credential-routing-railiance-apps.md
@ -0,0 +1,74 @@
+# Credential routing for railiance-apps operators
+
+Use `warden route find "<need>" --json` before requesting secrets. ops-warden
+issues SSH certificates only; other credentials route to OpenBao, key-cape, or
+platform tooling.
+
+## Production cluster access
+
+| Host | IP | Kubeconfig | Tunnel |
+| --- | --- | --- | --- |
+| CoulombCore (production apps) | `92.205.130.254` | `~/.kube/config` via `k3s-api-coulombcore` | `bridge up k3s-api-coulombcore` |
+| Railiance01 (Forgejo S5) | `92.205.62.239` | `~/.kube/config-hosteurope` | `bridge up k3s-api-railiance01` if configured |
+
+Core Hub, `vergabe-teilnahme`, and `apps-pg` currently run on **CoulombCore**.
+Railiance01 hosts Forgejo and a overlapping subset of platform databases.
+
+## Backup lane (Phase 1 — logical pg_dump + age + Nextcloud)
+
+| Item | Value |
+| --- | --- |
+| Owner | `railiance-platform` |
+| OpenBao path | `platform/workloads/railiance/backup/offsite-lane` |
+| Fields | `NC_WEBDAV_TOKEN`, `NC_WEBDAV_URL`, `AGE_PRIVATE_KEY` |
+| OIDC role | `railiance-backup-workload-kv-read` on auth mount `netkingdom` |
+| Age key (local) | `~/.config/age/railiance-backup.key` |
+| warden advisory | `warden access "nextcloud backup webdav token railiance"` |
+| Platform tool | `make -C ~/railiance-platform forgejo-backup` / `forgejo-backup-dry-run` |
+| S5 tool | `make apps-pg-backup-dry-run` (encrypt only; upload needs OpenBao) |
+
+Login and fetch (operator attended — do not log values):
+
+```bash
+warden access "openbao login oidc netkingdom backup" --fetch --no-policy
+# or after browser OIDC:
+bao login -method=oidc -path=netkingdom role=railiance-backup-workload-kv-read
+export RAILIANCE_BACKUP_NC_TOKEN=$(
+  bao kv get -field=NC_WEBDAV_TOKEN platform/workloads/railiance/backup/offsite-lane
+)
+```
+
+**Blocker observed 2026-07-10:** OpenBao at `https://bao.coulomb.social` reports
+`Vault is sealed`. `warden access issue-core-ingestion-api-key --fetch --no-policy`
+reaches the broker but fails with the same sealed error. Unseal requires operator
+shamir keys before any KV fetch succeeds.
+
+## CNPG barman ObjectStore (Phase 2 — deferred)
+
+Barman `ObjectStore` + `ScheduledBackup` templates live in
+`manifests/cnpg-backup-readiness.yaml`. Apply only after platform provisions
+object-store credentials at a confirmed path. Coordinate via `railiance-platform`;
+do not invent S3 secrets in this repo.
+
+## Core Hub runtime secrets
+
+Core Hub production uses the in-cluster Secret `core-hub-prod-env` on CoulombCore
+(keys: `CORE_HUB_DATABASE_URL`, `CORE_HUB_API_TOKEN`). Custody is operator/OpenBao;
+Helm cutover reuses the existing Secret — no secret fetch is required for deploy
+dry-runs when the Secret already exists.
+
+warden advisory for new secret material:
+
+```bash
+warden access "core hub database url api token" 
+```
+
+## SSH certificates
+
+```bash
+warden inventory list
+warden sign <actor> --pubkey <path>
+```
+
+Bridge tunnels (`agt-claude-coulombcore`, `agt-claude-railiance01`) normally
+provide reachability; refresh certs when `warden status` reports EXPIRED.
--- a/tools/apps-pg-backup-dry-run.sh
+++ b/tools/apps-pg-backup-dry-run.sh
@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+# Phase 1 logical backup for apps-pg consumer databases (Option A lane).
+# Encrypts locally with age; Nextcloud upload requires OpenBao offsite-lane creds.
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+PLATFORM_REPO="${PLATFORM_REPO:-$HOME/railiance-platform}"
+COMMON_SH="${PLATFORM_REPO}/lib/railiance-backup-common.sh"
+
+APPS_PG_NAMESPACE="${APPS_PG_NAMESPACE:-databases}"
+APPS_PG_CLUSTER="${APPS_PG_CLUSTER:-apps-pg}"
+APPS_PG_POD="${APPS_PG_POD:-}"
+APPS_PG_DATABASE="${APPS_PG_DATABASE:-vergabe_db}"
+BACKUP_DIR="${BACKUP_DIR:-$HOME/.cache/railiance/backups/apps-pg}"
+DRY_RUN="${RAILIANCE_BACKUP_DRY_RUN:-1}"
+
+if [[ ! -f "$COMMON_SH" ]]; then
+  echo "ERROR: missing ${COMMON_SH}" >&2
+  exit 1
+fi
+# shellcheck source=/dev/null
+source "$COMMON_SH"
+
+railiance_backup_require_tools
+mkdir -p "$BACKUP_DIR"
+
+if [[ -z "$APPS_PG_POD" ]]; then
+  APPS_PG_POD="$(
+    kubectl get pods -n "$APPS_PG_NAMESPACE" \
+      -l "cnpg.io/cluster=${APPS_PG_CLUSTER},role=primary" \
+      -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true
+  )"
+fi
+if [[ -z "$APPS_PG_POD" ]]; then
+  echo "ERROR: primary pod for ${APPS_PG_CLUSTER} not found in ${APPS_PG_NAMESPACE}" >&2
+  exit 1
+fi
+
+TS="$(date -u +%Y%m%dT%H%M%SZ)"
+DB_PLAIN="${BACKUP_DIR}/apps-pg-${APPS_PG_DATABASE}-${TS}.dump"
+DB_AGE="${DB_PLAIN}.age"
+
+echo "apps-pg backup dry-run"
+echo "  cluster: ${APPS_PG_CLUSTER}"
+echo "  pod:     ${APPS_PG_POD}"
+echo "  database:${APPS_PG_DATABASE}"
+echo
+
+kubectl exec -n "$APPS_PG_NAMESPACE" "$APPS_PG_POD" -c postgres -- \
+  pg_dump -U postgres -d "$APPS_PG_DATABASE" -Fc --no-owner --no-acl > "$DB_PLAIN"
+echo "ok: pg_dump $(du -h "$DB_PLAIN" | awk '{print $1}')"
+
+railiance_backup_encrypt "$DB_PLAIN" "$DB_AGE"
+echo "ok: age encrypted $(basename "$DB_AGE")"
+
+if [[ "$DRY_RUN" == "1" ]]; then
+  echo "ok: dry-run — skipping Nextcloud upload (OpenBao offsite-lane required)"
+  echo "     Decrypt with ~/.config/age/railiance-backup.key"
+  exit 0
+fi
+
+railiance_backup_require_openbao_lane
+RAILIANCE_BACKUP_NC_PREFIX="${RAILIANCE_BACKUP_NC_PREFIX:-apps-pg}"
+railiance_backup_nc_upload "$DB_AGE" "apps-pg-${APPS_PG_DATABASE}-${TS}.dump.age"
+railiance_backup_record_success "$BACKUP_DIR" "$TS"
+echo "ok: uploaded apps-pg-${APPS_PG_DATABASE}-${TS}.dump.age"
--- a/tools/cnpg-backup-status.sh
+++ b/tools/cnpg-backup-status.sh
@ -2,12 +2,6 @@
 set -euo pipefail

 namespace="${CNPG_NAMESPACE:-databases}"
-clusters=(
-  apps-pg
-  forgejo-db
-  net-kingdom-pg
-  state-hub-db
-)

 if ! kubectl get ns "$namespace" >/dev/null 2>&1; then
  echo "ERROR: namespace $namespace is not reachable from the current kubeconfig" >&2
@ -15,9 +9,23 @@ if ! kubectl get ns "$namespace" >/dev/null 2>&1; then
 fi

 echo "CNPG backup posture in namespace ${namespace}"
+context="$(kubectl config current-context 2>/dev/null || echo unknown)"
+server="$(kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}' 2>/dev/null || echo unknown)"
+echo "kube-context: ${context}"
+echo "api-server:   ${server}"
 echo

+mapfile -t clusters < <(
+  kubectl get cluster -n "$namespace" -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null \
+    | sort
+)
+if ((${#clusters[@]} == 0)); then
+  echo "ERROR: no CNPG Cluster resources in ${namespace}" >&2
+  exit 1
+fi
+
 missing=0
+logical_backup_note=0

 for cluster in "${clusters[@]}"; do
  if ! kubectl get cluster "$cluster" -n "$namespace" >/dev/null 2>&1; then
@ -41,6 +49,10 @@ for cluster in "${clusters[@]}"; do
  if [[ -z "$scheduled" ]]; then
    echo "${cluster}: ScheduledBackup = none"
    missing=$((missing + 1))
+    if [[ "$cluster" == "apps-pg" || "$cluster" == "gitea-db" || "$cluster" == "forgejo-db" ]]; then
+      echo "${cluster}: Phase 1 fallback = logical pg_dump (see make apps-pg-backup-dry-run or platform forgejo-backup)"
+      logical_backup_note=1
+    fi
  else
    echo "${cluster}: ScheduledBackup = ${scheduled}"
  fi
@ -60,6 +72,10 @@ done
 if (( missing > 0 )); then
  echo "RESULT: degraded — ${missing} cluster(s) lack ScheduledBackup coverage"
  echo "See docs/app-data-backup-restore-handoff.md and manifests/cnpg-backup-readiness.yaml"
+  if (( logical_backup_note )); then
+    echo "Phase 1: logical pg_dump lane may still run when OpenBao offsite-lane is reachable"
+    echo "See docs/credential-routing-railiance-apps.md"
+  fi
  exit 1
 fi

--- a/workplans/RAILIANCE-WP-0013-cnpg-backup-wiring-restore-drill.md
+++ b/workplans/RAILIANCE-WP-0013-cnpg-backup-wiring-restore-drill.md
@ -16,7 +16,7 @@ state_hub_workstream_id: "764c2447-0713-45d5-a14d-810a13027b8b"

 Follow-up to `RAILIANCE-WP-0012-T04` and custodian inbox `a1957cce`. Observability
 and manifest templates exist; production clusters on railiance01 still report
-degraded backup posture via `make cnpg-backup-status`.
+degraded backup posture via `make cnpg-backup-status` on CoulombCore production.

 Evidence baseline:
 `the-custodian/docs/evidence/vergabe-teilnahme-delivery-lane-20260710.json`
@ -31,15 +31,18 @@ the longer-term target in `manifests/cnpg-backup-readiness.yaml`.

 ```task
 id: RAILIANCE-WP-0013-T01
-status: todo
+status: progress
 priority: high
+needs_human: true
+intervention_note: "OpenBao sealed at bao.coulomb.social — operator unseal + OIDC login (railiance-backup-workload-kv-read) required"
 state_hub_task_id: "f2051213-0a9f-418e-82ad-9bd7c79fd0dc"
 ```

-Record the approved backup mechanism per cluster: Phase 1 logical dump vs barman
-`ObjectStore` + `ScheduledBackup`. Identify OpenBao/operator paths for
-`cnpg-backup-s3` or equivalent without copying secret values into Git,
-workplans, or State Hub notes.
+Credential routing in `docs/credential-routing-railiance-apps.md`. `warden access`
+reaches the platform broker; KV fetch fails while OpenBao is sealed. Phase 1 lane:
+`platform/workloads/railiance/backup/offsite-lane`. CoulombCore clusters:
+`apps-pg`, `gitea-db`, `net-kingdom-pg`, `state-hub-db`. `make apps-pg-backup-dry-run`
+succeeded (pg_dump + age encrypt; upload blocked on offsite-lane).

 ## Wire apps-pg backup coverage

--- a/workplans/RAILIANCE-WP-0014-core-hub-helm-cutover-vergabe-refresh.md
+++ b/workplans/RAILIANCE-WP-0014-core-hub-helm-cutover-vergabe-refresh.md
@ -24,18 +24,19 @@ deployed at `f95de14` while the source repo head is `1061f3e` (14-commit drift).
 Production-touching deploy steps require explicit image tags and operator
 approval. Do not store secrets in workplans.

-## Core Hub server dry-run on Railiance01
+## Core Hub server dry-run on CoulombCore

 ```task
 id: RAILIANCE-WP-0014-T01
-status: todo
+status: progress
 priority: high
 state_hub_task_id: "b9dea9f3-f0c0-41c1-b9c4-ec4c44d43d55"
 ```

-Run `CORE_HUB_IMAGE_TAG=<sha> make check-core-hub-image`,
-`core-hub-server-dry-run`, and `core-hub-dry-run` against Railiance01. Record the
-chosen image tag and any schema or secret prerequisites before apply.
+Production on CoulombCore (`PRODUCTION_KUBECONFIG=~/.kube/config`). Live image:
+`gitea.coulomb.social/coulomb/core-hub:a33191a`. `core-hub-prod-env` exists.
+`core-hub-server-dry-run` blocked: existing Service/Deployment lack Helm ownership
+metadata — needs adopt/patch before `helm upgrade --install`.

 ## Adopt Helm-managed Core Hub release