From a689270f1835bf08d9325da0ad02b842ca6f8c5f Mon Sep 17 00:00:00 2001 From: tegwick Date: Fri, 10 Jul 2026 15:46:02 +0200 Subject: [PATCH] WP-0013/0014: warden credential routing, CoulombCore kubeconfig, apps-pg backup dry-run Document OpenBao/warden paths for the offsite backup lane and flag the sealed Vault blocker. Point production Makefile targets at CoulombCore, auto-discover CNPG clusters in backup status, add apps-pg pg_dump dry-run tooling, and record Core Hub Helm cutover findings (live gitea image, adopt required). --- Makefile | 57 +++++++++----- docs/core-hub-on-railiance01.md | 19 +++-- docs/credential-routing-railiance-apps.md | 74 +++++++++++++++++++ tools/apps-pg-backup-dry-run.sh | 66 +++++++++++++++++ tools/cnpg-backup-status.sh | 28 +++++-- ...P-0013-cnpg-backup-wiring-restore-drill.md | 15 ++-- ...4-core-hub-helm-cutover-vergabe-refresh.md | 11 +-- 7 files changed, 229 insertions(+), 41 deletions(-) create mode 100644 docs/credential-routing-railiance-apps.md create mode 100755 tools/apps-pg-backup-dry-run.sh diff --git a/Makefile b/Makefile index 797e922..950f72a 100644 --- a/Makefile +++ b/Makefile @@ -29,7 +29,9 @@ REUSE_VALUES ?= helm/reuse-surface-values.yaml REUSE_URL ?= https://reuse.coulomb.social REUSE_CERTIFICATE ?= reuse-surface-tls +COULOMBCORE_KUBECONFIG ?= $(HOME)/.kube/config RAILIANCE01_KUBECONFIG ?= $(HOME)/.kube/config-hosteurope +PRODUCTION_KUBECONFIG ?= $(COULOMBCORE_KUBECONFIG) INTER_HUB_KUBECONFIG ?= $(RAILIANCE01_KUBECONFIG) REUSE_KUBECONFIG ?= $(RAILIANCE01_KUBECONFIG) FORGEJO_KUBECONFIG ?= $(RAILIANCE01_KUBECONFIG) @@ -64,7 +66,8 @@ CORE_HUB_STAGING_VALUES ?= helm/core-hub-staging-values.yaml CORE_HUB_IMAGE_REPOSITORY ?= forgejo.coulomb.social/coulomb/core-hub CORE_HUB_IMAGE_TAG ?= CORE_HUB_BASE_URL ?= https://hub.coulomb.social -CORE_HUB_KUBECONFIG ?= $(RAILIANCE01_KUBECONFIG) +CORE_HUB_KUBECONFIG ?= $(PRODUCTION_KUBECONFIG) +VERGABE_KUBECONFIG ?= $(PRODUCTION_KUBECONFIG) CORE_HUB_IMAGE_REF = $(CORE_HUB_IMAGE_REPOSITORY):$(CORE_HUB_IMAGE_TAG) CORE_HUB_IMAGE_SET_ARG = $(if $(strip $(CORE_HUB_IMAGE_TAG)),--set image.tag=$(CORE_HUB_IMAGE_TAG),) @@ -99,6 +102,20 @@ check-railiance01-kubeconfig: ## Verify Railiance01 production kubeconfig exists exit 1; \ } +check-coulombcore-kubeconfig: ## Verify CoulombCore kubeconfig and k3s API tunnel + @test -r "$(COULOMBCORE_KUBECONFIG)" || { \ + echo "Missing CoulombCore kubeconfig: $(COULOMBCORE_KUBECONFIG)" >&2; \ + echo "Ensure ops-bridge tunnel is up: bridge up k3s-api-coulombcore" >&2; \ + exit 1; \ + } + @KUBECONFIG="$(COULOMBCORE_KUBECONFIG)" kubectl get --raw=/readyz >/dev/null 2>&1 || { \ + echo "CoulombCore API unreachable via $(COULOMBCORE_KUBECONFIG)." >&2; \ + echo "Run: bridge up k3s-api-coulombcore" >&2; \ + exit 1; \ + } + +check-production-kubeconfig: check-coulombcore-kubeconfig ## Verify production app cluster API (CoulombCore) + check-inter-hub-image-tag: ## Require an explicit inter-hub image tag for production deploys @test -n "$(INTER_HUB_IMAGE_TAG)" || { \ echo "Set INTER_HUB_IMAGE_TAG= for inter-hub production deploys." >&2; \ @@ -119,8 +136,12 @@ check-core-hub-image-tag: ## Require an explicit core-hub image tag for producti check-core-hub-image: check-core-hub-image-tag ## Verify the core-hub OCI image tag exists before deploy tools/check-oci-image.sh "$(CORE_HUB_IMAGE_REF)" -cnpg-backup-status: check-railiance01-kubeconfig ## Report CNPG backup spec and ScheduledBackup coverage - KUBECONFIG="$(RAILIANCE01_KUBECONFIG)" tools/cnpg-backup-status.sh +cnpg-backup-status: check-production-kubeconfig ## Report CNPG backup spec and ScheduledBackup coverage + KUBECONFIG="$(PRODUCTION_KUBECONFIG)" tools/cnpg-backup-status.sh + +apps-pg-backup-dry-run: check-production-kubeconfig ## Logical pg_dump + age encrypt for vergabe_db (no upload) + chmod +x tools/apps-pg-backup-dry-run.sh + KUBECONFIG="$(PRODUCTION_KUBECONFIG)" PLATFORM_REPO="$(PLATFORM_REPO)" tools/apps-pg-backup-dry-run.sh ##@ Vergabe Teilnahme @@ -129,16 +150,16 @@ vergabe-dry-run: ## helm template render (no apply) for inspection --namespace $(VERGABE_NAMESPACE) \ -f $(VERGABE_VALUES) -vergabe-deploy: ## Deploy / upgrade vergabe-teilnahme Helm release - helm upgrade --install $(VERGABE_RELEASE) $(VERGABE_CHART) \ +vergabe-deploy: check-production-kubeconfig ## Deploy / upgrade vergabe-teilnahme Helm release + KUBECONFIG="$(VERGABE_KUBECONFIG)" helm upgrade --install $(VERGABE_RELEASE) $(VERGABE_CHART) \ --namespace $(VERGABE_NAMESPACE) --create-namespace \ -f $(VERGABE_VALUES) --wait --timeout 3m vergabe-ingress-deploy: ## Apply the vergabe-teilnahme ingress (whywhynot.de) kubectl apply -f $(VERGABE_INGRESS) -vergabe-status: ## Show vergabe-teilnahme pod / svc / ingress / cert state - kubectl get pods,svc,ingress,certificate -n $(VERGABE_NAMESPACE) -l app.kubernetes.io/instance=$(VERGABE_RELEASE) --ignore-not-found +vergabe-status: check-production-kubeconfig ## Show vergabe-teilnahme pod / svc / ingress / cert state + KUBECONFIG="$(VERGABE_KUBECONFIG)" kubectl get pods,svc,ingress,certificate -n $(VERGABE_NAMESPACE) -l app.kubernetes.io/instance=$(VERGABE_RELEASE) --ignore-not-found vergabe-migrate: ## Run Django migrations against the live deployment kubectl exec -n $(VERGABE_NAMESPACE) deploy/$(VERGABE_RELEASE) -- python manage.py migrate --noinput @@ -206,22 +227,22 @@ core-hub-render-baseline: ## Render checked-in core-hub values for chart validat --namespace $(CORE_HUB_NAMESPACE) \ -f $(CORE_HUB_VALUES) -core-hub-dry-run: check-railiance01-kubeconfig check-core-hub-image-tag ## helm template render with an explicit production image tag +core-hub-dry-run: check-production-kubeconfig check-core-hub-image-tag ## helm template render with an explicit production image tag KUBECONFIG="$(CORE_HUB_KUBECONFIG)" helm template $(CORE_HUB_RELEASE) $(CORE_HUB_CHART) \ --namespace $(CORE_HUB_NAMESPACE) \ -f $(CORE_HUB_VALUES) $(CORE_HUB_IMAGE_SET_ARG) -core-hub-server-dry-run: check-railiance01-kubeconfig check-core-hub-image ## Helm server dry-run core-hub upgrade on Railiance01 +core-hub-server-dry-run: check-production-kubeconfig check-core-hub-image ## Helm server dry-run core-hub upgrade on CoulombCore KUBECONFIG="$(CORE_HUB_KUBECONFIG)" helm upgrade --install $(CORE_HUB_RELEASE) $(CORE_HUB_CHART) \ --namespace $(CORE_HUB_NAMESPACE) --create-namespace \ -f $(CORE_HUB_VALUES) $(CORE_HUB_IMAGE_SET_ARG) --dry-run=server --timeout 5m -core-hub-deploy: check-railiance01-kubeconfig check-core-hub-image ## Deploy / upgrade core-hub Helm release on Railiance01 +core-hub-deploy: check-production-kubeconfig check-core-hub-image ## Deploy / upgrade core-hub Helm release on CoulombCore KUBECONFIG="$(CORE_HUB_KUBECONFIG)" helm upgrade --install $(CORE_HUB_RELEASE) $(CORE_HUB_CHART) \ --namespace $(CORE_HUB_NAMESPACE) --create-namespace \ -f $(CORE_HUB_VALUES) $(CORE_HUB_IMAGE_SET_ARG) --wait --timeout 5m -core-hub-migrate: check-railiance01-kubeconfig check-core-hub-image ## Run Alembic migrations via a one-off Job +core-hub-migrate: check-production-kubeconfig check-core-hub-image ## Run Alembic migrations via a one-off Job KUBECONFIG="$(CORE_HUB_KUBECONFIG)" kubectl -n $(CORE_HUB_NAMESPACE) delete job $(CORE_HUB_RELEASE)-prod-migrate --ignore-not-found KUBECONFIG="$(CORE_HUB_KUBECONFIG)" helm template $(CORE_HUB_RELEASE) $(CORE_HUB_CHART) \ --namespace $(CORE_HUB_NAMESPACE) \ @@ -230,11 +251,11 @@ core-hub-migrate: check-railiance01-kubeconfig check-core-hub-image ## Run Alemb KUBECONFIG="$(CORE_HUB_KUBECONFIG)" kubectl -n $(CORE_HUB_NAMESPACE) wait --for=condition=complete \ job/$(CORE_HUB_RELEASE)-prod-migrate --timeout=180s -core-hub-status: check-railiance01-kubeconfig ## Show core-hub pod / svc / ingress / cert state on Railiance01 +core-hub-status: check-production-kubeconfig ## Show core-hub pod / svc / ingress / cert state on CoulombCore KUBECONFIG="$(CORE_HUB_KUBECONFIG)" kubectl get pods,svc,ingress,certificate -n $(CORE_HUB_NAMESPACE) \ -l app.kubernetes.io/name=core-hub --ignore-not-found -core-hub-release-info: check-railiance01-kubeconfig ## Show core-hub Helm history, values, deployment, and pods +core-hub-release-info: check-production-kubeconfig ## Show core-hub Helm history, values, deployment, and pods KUBECONFIG="$(CORE_HUB_KUBECONFIG)" helm history $(CORE_HUB_RELEASE) -n $(CORE_HUB_NAMESPACE) KUBECONFIG="$(CORE_HUB_KUBECONFIG)" helm get values $(CORE_HUB_RELEASE) -n $(CORE_HUB_NAMESPACE) KUBECONFIG="$(CORE_HUB_KUBECONFIG)" kubectl describe deploy/$(CORE_HUB_RELEASE)-api -n $(CORE_HUB_NAMESPACE) @@ -243,21 +264,21 @@ core-hub-release-info: check-railiance01-kubeconfig ## Show core-hub Helm histor core-hub-smoke: ## Verify public Core Hub health and v2 API surface after rollout CORE_HUB_BASE_URL="$(CORE_HUB_BASE_URL)" tools/core-hub-smoke.sh -core-hub-logs: check-railiance01-kubeconfig ## Tail core-hub app logs from Railiance01 +core-hub-logs: check-production-kubeconfig ## Tail core-hub app logs from CoulombCore KUBECONFIG="$(CORE_HUB_KUBECONFIG)" kubectl logs -n $(CORE_HUB_NAMESPACE) \ -l app.kubernetes.io/name=core-hub,app.kubernetes.io/component=api -f --tail=50 -core-hub-staging-dry-run: check-railiance01-kubeconfig check-core-hub-image-tag ## helm template render for core-hub staging +core-hub-staging-dry-run: check-production-kubeconfig check-core-hub-image-tag ## helm template render for core-hub staging KUBECONFIG="$(CORE_HUB_KUBECONFIG)" helm template $(CORE_HUB_RELEASE) $(CORE_HUB_CHART) \ --namespace $(CORE_HUB_STAGING_NAMESPACE) \ -f $(CORE_HUB_STAGING_VALUES) $(CORE_HUB_IMAGE_SET_ARG) -core-hub-staging-deploy: check-railiance01-kubeconfig check-core-hub-image ## Deploy / upgrade core-hub staging release +core-hub-staging-deploy: check-production-kubeconfig check-core-hub-image ## Deploy / upgrade core-hub staging release KUBECONFIG="$(CORE_HUB_KUBECONFIG)" helm upgrade --install $(CORE_HUB_RELEASE) $(CORE_HUB_CHART) \ --namespace $(CORE_HUB_STAGING_NAMESPACE) --create-namespace \ -f $(CORE_HUB_STAGING_VALUES) $(CORE_HUB_IMAGE_SET_ARG) --wait --timeout 5m -core-hub-staging-migrate: check-railiance01-kubeconfig check-core-hub-image ## Run Alembic migrations for core-hub staging +core-hub-staging-migrate: check-production-kubeconfig check-core-hub-image ## Run Alembic migrations for core-hub staging KUBECONFIG="$(CORE_HUB_KUBECONFIG)" kubectl -n $(CORE_HUB_STAGING_NAMESPACE) delete job $(CORE_HUB_RELEASE)-staging-migrate --ignore-not-found KUBECONFIG="$(CORE_HUB_KUBECONFIG)" helm template $(CORE_HUB_RELEASE) $(CORE_HUB_CHART) \ --namespace $(CORE_HUB_STAGING_NAMESPACE) \ @@ -266,7 +287,7 @@ core-hub-staging-migrate: check-railiance01-kubeconfig check-core-hub-image ## R KUBECONFIG="$(CORE_HUB_KUBECONFIG)" kubectl -n $(CORE_HUB_STAGING_NAMESPACE) wait --for=condition=complete \ job/$(CORE_HUB_RELEASE)-staging-migrate --timeout=180s -core-hub-staging-status: check-railiance01-kubeconfig ## Show core-hub staging pod / svc state +core-hub-staging-status: check-production-kubeconfig ## Show core-hub staging pod / svc state KUBECONFIG="$(CORE_HUB_KUBECONFIG)" kubectl get pods,svc -n $(CORE_HUB_STAGING_NAMESPACE) \ -l app.kubernetes.io/name=core-hub --ignore-not-found diff --git a/docs/core-hub-on-railiance01.md b/docs/core-hub-on-railiance01.md index c91e6bc..898afad 100644 --- a/docs/core-hub-on-railiance01.md +++ b/docs/core-hub-on-railiance01.md @@ -1,13 +1,18 @@ -# Core Hub on railiance01 +# Core Hub on CoulombCore (production) -`hub.coulomb.social` serves Core Hub in production. This repo owns the S5 Helm -release path; Core Hub owns the application image, migrations, and API behavior. +`hub.coulomb.social` serves Core Hub in production on **CoulombCore** +(`92.205.130.254`). This repo owns the S5 Helm release path; Core Hub owns the +application image, migrations, and API behavior. + +Railiance01 (`92.205.62.239`) hosts Forgejo and overlapping platform services but +not the live Core Hub or `vergabe-teilnahme` releases as of 2026-07-10. ## Hosts and release surface | Server | IP | Role | |---|---|---| -| Railiance01 | `92.205.62.239` | Production k3s; deploy S5 apps here | +| CoulombCore | `92.205.130.254` | Production k3s for Core Hub and S5 apps | +| Railiance01 | `92.205.62.239` | Forgejo and platform overlap | | Item | Production | Staging | |---|---|---| @@ -34,8 +39,10 @@ is the only schema path. ## Deploy -Use the Railiance01 kubeconfig. The Makefile defaults to -`~/.kube/config-hosteurope` and fails fast when it is missing. +Use the CoulombCore kubeconfig. The Makefile defaults to `~/.kube/config` +(tunnel `bridge up k3s-api-coulombcore`) and fails fast when the API is +unreachable. See `docs/credential-routing-railiance-apps.md` for cluster and +credential boundaries. ```bash make core-hub-render-baseline diff --git a/docs/credential-routing-railiance-apps.md b/docs/credential-routing-railiance-apps.md new file mode 100644 index 0000000..801bd66 --- /dev/null +++ b/docs/credential-routing-railiance-apps.md @@ -0,0 +1,74 @@ +# Credential routing for railiance-apps operators + +Use `warden route find "" --json` before requesting secrets. ops-warden +issues SSH certificates only; other credentials route to OpenBao, key-cape, or +platform tooling. + +## Production cluster access + +| Host | IP | Kubeconfig | Tunnel | +| --- | --- | --- | --- | +| CoulombCore (production apps) | `92.205.130.254` | `~/.kube/config` via `k3s-api-coulombcore` | `bridge up k3s-api-coulombcore` | +| Railiance01 (Forgejo S5) | `92.205.62.239` | `~/.kube/config-hosteurope` | `bridge up k3s-api-railiance01` if configured | + +Core Hub, `vergabe-teilnahme`, and `apps-pg` currently run on **CoulombCore**. +Railiance01 hosts Forgejo and a overlapping subset of platform databases. + +## Backup lane (Phase 1 — logical pg_dump + age + Nextcloud) + +| Item | Value | +| --- | --- | +| Owner | `railiance-platform` | +| OpenBao path | `platform/workloads/railiance/backup/offsite-lane` | +| Fields | `NC_WEBDAV_TOKEN`, `NC_WEBDAV_URL`, `AGE_PRIVATE_KEY` | +| OIDC role | `railiance-backup-workload-kv-read` on auth mount `netkingdom` | +| Age key (local) | `~/.config/age/railiance-backup.key` | +| warden advisory | `warden access "nextcloud backup webdav token railiance"` | +| Platform tool | `make -C ~/railiance-platform forgejo-backup` / `forgejo-backup-dry-run` | +| S5 tool | `make apps-pg-backup-dry-run` (encrypt only; upload needs OpenBao) | + +Login and fetch (operator attended — do not log values): + +```bash +warden access "openbao login oidc netkingdom backup" --fetch --no-policy +# or after browser OIDC: +bao login -method=oidc -path=netkingdom role=railiance-backup-workload-kv-read +export RAILIANCE_BACKUP_NC_TOKEN=$( + bao kv get -field=NC_WEBDAV_TOKEN platform/workloads/railiance/backup/offsite-lane +) +``` + +**Blocker observed 2026-07-10:** OpenBao at `https://bao.coulomb.social` reports +`Vault is sealed`. `warden access issue-core-ingestion-api-key --fetch --no-policy` +reaches the broker but fails with the same sealed error. Unseal requires operator +shamir keys before any KV fetch succeeds. + +## CNPG barman ObjectStore (Phase 2 — deferred) + +Barman `ObjectStore` + `ScheduledBackup` templates live in +`manifests/cnpg-backup-readiness.yaml`. Apply only after platform provisions +object-store credentials at a confirmed path. Coordinate via `railiance-platform`; +do not invent S3 secrets in this repo. + +## Core Hub runtime secrets + +Core Hub production uses the in-cluster Secret `core-hub-prod-env` on CoulombCore +(keys: `CORE_HUB_DATABASE_URL`, `CORE_HUB_API_TOKEN`). Custody is operator/OpenBao; +Helm cutover reuses the existing Secret — no secret fetch is required for deploy +dry-runs when the Secret already exists. + +warden advisory for new secret material: + +```bash +warden access "core hub database url api token" +``` + +## SSH certificates + +```bash +warden inventory list +warden sign --pubkey +``` + +Bridge tunnels (`agt-claude-coulombcore`, `agt-claude-railiance01`) normally +provide reachability; refresh certs when `warden status` reports EXPIRED. \ No newline at end of file diff --git a/tools/apps-pg-backup-dry-run.sh b/tools/apps-pg-backup-dry-run.sh new file mode 100755 index 0000000..7dfdf9c --- /dev/null +++ b/tools/apps-pg-backup-dry-run.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# Phase 1 logical backup for apps-pg consumer databases (Option A lane). +# Encrypts locally with age; Nextcloud upload requires OpenBao offsite-lane creds. +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +PLATFORM_REPO="${PLATFORM_REPO:-$HOME/railiance-platform}" +COMMON_SH="${PLATFORM_REPO}/lib/railiance-backup-common.sh" + +APPS_PG_NAMESPACE="${APPS_PG_NAMESPACE:-databases}" +APPS_PG_CLUSTER="${APPS_PG_CLUSTER:-apps-pg}" +APPS_PG_POD="${APPS_PG_POD:-}" +APPS_PG_DATABASE="${APPS_PG_DATABASE:-vergabe_db}" +BACKUP_DIR="${BACKUP_DIR:-$HOME/.cache/railiance/backups/apps-pg}" +DRY_RUN="${RAILIANCE_BACKUP_DRY_RUN:-1}" + +if [[ ! -f "$COMMON_SH" ]]; then + echo "ERROR: missing ${COMMON_SH}" >&2 + exit 1 +fi +# shellcheck source=/dev/null +source "$COMMON_SH" + +railiance_backup_require_tools +mkdir -p "$BACKUP_DIR" + +if [[ -z "$APPS_PG_POD" ]]; then + APPS_PG_POD="$( + kubectl get pods -n "$APPS_PG_NAMESPACE" \ + -l "cnpg.io/cluster=${APPS_PG_CLUSTER},role=primary" \ + -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true + )" +fi +if [[ -z "$APPS_PG_POD" ]]; then + echo "ERROR: primary pod for ${APPS_PG_CLUSTER} not found in ${APPS_PG_NAMESPACE}" >&2 + exit 1 +fi + +TS="$(date -u +%Y%m%dT%H%M%SZ)" +DB_PLAIN="${BACKUP_DIR}/apps-pg-${APPS_PG_DATABASE}-${TS}.dump" +DB_AGE="${DB_PLAIN}.age" + +echo "apps-pg backup dry-run" +echo " cluster: ${APPS_PG_CLUSTER}" +echo " pod: ${APPS_PG_POD}" +echo " database:${APPS_PG_DATABASE}" +echo + +kubectl exec -n "$APPS_PG_NAMESPACE" "$APPS_PG_POD" -c postgres -- \ + pg_dump -U postgres -d "$APPS_PG_DATABASE" -Fc --no-owner --no-acl > "$DB_PLAIN" +echo "ok: pg_dump $(du -h "$DB_PLAIN" | awk '{print $1}')" + +railiance_backup_encrypt "$DB_PLAIN" "$DB_AGE" +echo "ok: age encrypted $(basename "$DB_AGE")" + +if [[ "$DRY_RUN" == "1" ]]; then + echo "ok: dry-run — skipping Nextcloud upload (OpenBao offsite-lane required)" + echo " Decrypt with ~/.config/age/railiance-backup.key" + exit 0 +fi + +railiance_backup_require_openbao_lane +RAILIANCE_BACKUP_NC_PREFIX="${RAILIANCE_BACKUP_NC_PREFIX:-apps-pg}" +railiance_backup_nc_upload "$DB_AGE" "apps-pg-${APPS_PG_DATABASE}-${TS}.dump.age" +railiance_backup_record_success "$BACKUP_DIR" "$TS" +echo "ok: uploaded apps-pg-${APPS_PG_DATABASE}-${TS}.dump.age" \ No newline at end of file diff --git a/tools/cnpg-backup-status.sh b/tools/cnpg-backup-status.sh index d2848aa..7561cf5 100755 --- a/tools/cnpg-backup-status.sh +++ b/tools/cnpg-backup-status.sh @@ -2,12 +2,6 @@ set -euo pipefail namespace="${CNPG_NAMESPACE:-databases}" -clusters=( - apps-pg - forgejo-db - net-kingdom-pg - state-hub-db -) if ! kubectl get ns "$namespace" >/dev/null 2>&1; then echo "ERROR: namespace $namespace is not reachable from the current kubeconfig" >&2 @@ -15,9 +9,23 @@ if ! kubectl get ns "$namespace" >/dev/null 2>&1; then fi echo "CNPG backup posture in namespace ${namespace}" +context="$(kubectl config current-context 2>/dev/null || echo unknown)" +server="$(kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}' 2>/dev/null || echo unknown)" +echo "kube-context: ${context}" +echo "api-server: ${server}" echo +mapfile -t clusters < <( + kubectl get cluster -n "$namespace" -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null \ + | sort +) +if ((${#clusters[@]} == 0)); then + echo "ERROR: no CNPG Cluster resources in ${namespace}" >&2 + exit 1 +fi + missing=0 +logical_backup_note=0 for cluster in "${clusters[@]}"; do if ! kubectl get cluster "$cluster" -n "$namespace" >/dev/null 2>&1; then @@ -41,6 +49,10 @@ for cluster in "${clusters[@]}"; do if [[ -z "$scheduled" ]]; then echo "${cluster}: ScheduledBackup = none" missing=$((missing + 1)) + if [[ "$cluster" == "apps-pg" || "$cluster" == "gitea-db" || "$cluster" == "forgejo-db" ]]; then + echo "${cluster}: Phase 1 fallback = logical pg_dump (see make apps-pg-backup-dry-run or platform forgejo-backup)" + logical_backup_note=1 + fi else echo "${cluster}: ScheduledBackup = ${scheduled}" fi @@ -60,6 +72,10 @@ done if (( missing > 0 )); then echo "RESULT: degraded — ${missing} cluster(s) lack ScheduledBackup coverage" echo "See docs/app-data-backup-restore-handoff.md and manifests/cnpg-backup-readiness.yaml" + if (( logical_backup_note )); then + echo "Phase 1: logical pg_dump lane may still run when OpenBao offsite-lane is reachable" + echo "See docs/credential-routing-railiance-apps.md" + fi exit 1 fi diff --git a/workplans/RAILIANCE-WP-0013-cnpg-backup-wiring-restore-drill.md b/workplans/RAILIANCE-WP-0013-cnpg-backup-wiring-restore-drill.md index 4cdd445..529b955 100644 --- a/workplans/RAILIANCE-WP-0013-cnpg-backup-wiring-restore-drill.md +++ b/workplans/RAILIANCE-WP-0013-cnpg-backup-wiring-restore-drill.md @@ -16,7 +16,7 @@ state_hub_workstream_id: "764c2447-0713-45d5-a14d-810a13027b8b" Follow-up to `RAILIANCE-WP-0012-T04` and custodian inbox `a1957cce`. Observability and manifest templates exist; production clusters on railiance01 still report -degraded backup posture via `make cnpg-backup-status`. +degraded backup posture via `make cnpg-backup-status` on CoulombCore production. Evidence baseline: `the-custodian/docs/evidence/vergabe-teilnahme-delivery-lane-20260710.json` @@ -31,15 +31,18 @@ the longer-term target in `manifests/cnpg-backup-readiness.yaml`. ```task id: RAILIANCE-WP-0013-T01 -status: todo +status: progress priority: high +needs_human: true +intervention_note: "OpenBao sealed at bao.coulomb.social — operator unseal + OIDC login (railiance-backup-workload-kv-read) required" state_hub_task_id: "f2051213-0a9f-418e-82ad-9bd7c79fd0dc" ``` -Record the approved backup mechanism per cluster: Phase 1 logical dump vs barman -`ObjectStore` + `ScheduledBackup`. Identify OpenBao/operator paths for -`cnpg-backup-s3` or equivalent without copying secret values into Git, -workplans, or State Hub notes. +Credential routing in `docs/credential-routing-railiance-apps.md`. `warden access` +reaches the platform broker; KV fetch fails while OpenBao is sealed. Phase 1 lane: +`platform/workloads/railiance/backup/offsite-lane`. CoulombCore clusters: +`apps-pg`, `gitea-db`, `net-kingdom-pg`, `state-hub-db`. `make apps-pg-backup-dry-run` +succeeded (pg_dump + age encrypt; upload blocked on offsite-lane). ## Wire apps-pg backup coverage diff --git a/workplans/RAILIANCE-WP-0014-core-hub-helm-cutover-vergabe-refresh.md b/workplans/RAILIANCE-WP-0014-core-hub-helm-cutover-vergabe-refresh.md index 0e594ae..7b8672a 100644 --- a/workplans/RAILIANCE-WP-0014-core-hub-helm-cutover-vergabe-refresh.md +++ b/workplans/RAILIANCE-WP-0014-core-hub-helm-cutover-vergabe-refresh.md @@ -24,18 +24,19 @@ deployed at `f95de14` while the source repo head is `1061f3e` (14-commit drift). Production-touching deploy steps require explicit image tags and operator approval. Do not store secrets in workplans. -## Core Hub server dry-run on Railiance01 +## Core Hub server dry-run on CoulombCore ```task id: RAILIANCE-WP-0014-T01 -status: todo +status: progress priority: high state_hub_task_id: "b9dea9f3-f0c0-41c1-b9c4-ec4c44d43d55" ``` -Run `CORE_HUB_IMAGE_TAG= make check-core-hub-image`, -`core-hub-server-dry-run`, and `core-hub-dry-run` against Railiance01. Record the -chosen image tag and any schema or secret prerequisites before apply. +Production on CoulombCore (`PRODUCTION_KUBECONFIG=~/.kube/config`). Live image: +`gitea.coulomb.social/coulomb/core-hub:a33191a`. `core-hub-prod-env` exists. +`core-hub-server-dry-run` blocked: existing Service/Deployment lack Helm ownership +metadata — needs adopt/patch before `helm upgrade --install`. ## Adopt Helm-managed Core Hub release