feat(backup): multi-host CNPG Option A CLI for activity-core

Add cnpg-option-a-backup JSON runner, vendored static age, kubectl install
helper, and ESO policy path for offsite lane so railiance01 workers can
upload without workstation OIDC (RAILIANCE-WP-0016).

.gitignore

@@ -24,3 +24,4 @@ argocd/repositories/*.repository.sops.yaml
 # Python bytecode caches
 __pycache__/
 *.pyc
+tools/vendor/bin/kubectl

docs/cnpg-option-a-backup.md

@@ -0,0 +1,56 @@
+# CNPG Option A multi-host backup
+
+**Workplan:** RAILIANCE-WP-0016  
+**CLI:** `tools/cmd/cnpg-option-a-backup`  
+**Scheduler:** activity-core `daily-cnpg-option-a-backup` (railiance01)
+
+## What it does
+
+Logical `pg_dump -Fc` → age encrypt → Nextcloud WebDAV for production-of-record
+CNPG databases on **CoulombCore** and **railiance01**. Emits JSON for
+activity-core evidence sinks.
+
+Inventory: `railiance-apps/docs/cnpg-backup-topology-inventory.md`.
+
+## Operator commands
+
+```bash
+# One-time vendor tools (age committed; kubectl downloaded)
+tools/cmd/install-cnpg-backup-vendor-tools
+
+# Dry-run (no upload)
+RAILIANCE_BACKUP_DRY_RUN=1 tools/cmd/cnpg-option-a-backup
+
+# Live upload (env or bao OIDC)
+export RAILIANCE_BACKUP_NC_TOKEN=…   # or bao login + auto-fetch
+tools/cmd/cnpg-option-a-backup
+```
+
+From railiance-apps:
+
+```bash
+make cnpg-option-a-backup-dry-run
+make cnpg-option-a-backup
+```
+
+## activity-core enable
+
+1. Re-mint ESO token with backup policy:
+   `OPENBAO_ACTIVITY_CORE_POLICIES="… workload-kv-read-railiance-backup-offsite-lane" \
+    scripts/openbao-eso-token-apply.sh`
+2. Apply `k8s/railiance/15-externalsecret-backup-offsite.yaml`
+3. Ensure `/home/tegwick/railiance-platform` and `~/.kube` on railiance01 host
+   (worker hostPath). Place CoulombCore kubeconfig as `~/.kube/config`.
+4. Rebuild/restart worker with resolver code; sync ActivityDefinition;
+   set `enabled: true` on `daily-cnpg-option-a-backup`.
+5. Forced run; check State Hub progress + `make cnpg-backup-status`.
+
+## Credentials
+
+| Field | Where |
+| --- | --- |
+| `NC_WEBDAV_TOKEN` / `NC_WEBDAV_URL` | OpenBao offsite-lane → ESO → `actcore-runtime-secret` |
+| `AGE_PUBLIC_KEY` | public constant / env (not private key) |
+| `AGE_PRIVATE_KEY` | OpenBao only — restore drills, never on worker |
+
+Catalog: `warden route show railiance-backup-offsite-lane`.

lib/railiance-backup-common.sh

@@ -27,9 +27,19 @@ railiance_backup_require_openbao_lane() {
 }
 
 railiance_backup_require_tools() {
+  # Prefer vendored static binaries (activity-core hostPath / offline runners).
+  local root vendor
+  root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+  vendor="${root}/tools/vendor/bin"
+  if [[ -d "$vendor" ]]; then
+    export PATH="${vendor}:${PATH}"
+  fi
+  if [[ -n "${RAILIANCE_BACKUP_AGE_BIN:-}" && -x "${RAILIANCE_BACKUP_AGE_BIN}" ]]; then
+    export PATH="$(dirname "${RAILIANCE_BACKUP_AGE_BIN}"):${PATH}"
+  fi
   local t
   for t in age curl kubectl; do
-    command -v "$t" >/dev/null 2>&1 || { echo "ERROR: missing required tool: $t" >&2; exit 1; }
+    command -v "$t" >/dev/null 2>&1 || { echo "ERROR: missing required tool: $t (run tools/cmd/install-cnpg-backup-vendor-tools)" >&2; exit 1; }
   done
 }
 

openbao/policies/external-secrets-activity-core.hcl

@@ -33,3 +33,12 @@ path "platform/data/workloads/forgejo/forgejo-admin" {
 path "platform/metadata/workloads/forgejo/forgejo-admin" {
   capabilities = ["read"]
 }
+
+# Offsite backup lane (NC WebDAV + age pubkey material) for Option A uploads
+path "platform/data/workloads/railiance/backup/offsite-lane" {
+  capabilities = ["read"]
+}
+
+path "platform/metadata/workloads/railiance/backup/offsite-lane" {
+  capabilities = ["read"]
+}

tools/cmd/cnpg-option-a-backup

@@ -0,0 +1,238 @@
+#!/usr/bin/env bash
+# tools/cmd/cnpg-option-a-backup — multi-host CNPG Option A backup (JSON)
+#
+# Used by activity-core shell context (RAILIANCE-WP-0016) and operator Make.
+# Emits one JSON object on stdout; logs on stderr.
+#
+# Env (non-interactive preferred for activity-core):
+#   RAILIANCE_BACKUP_NC_TOKEN       required for upload (unless dry-run)
+#   RAILIANCE_BACKUP_NC_WEBDAV_URL  optional; defaults to filesdrop form
+#   RAILIANCE_BACKUP_AGE_PUBLIC_KEY optional
+#   RAILIANCE_BACKUP_DRY_RUN=1      encrypt only, no upload
+#   KUBECONFIG / KUBECONFIG_CORE / KUBECONFIG_R01
+#   CNPG_BACKUP_TARGETS             optional override CSV of target ids
+#   CNPG_UPDATE_STATUS_CM=1          patch last-success ConfigMap (default 1)
+#
+# Targets (default set = production-of-record inventory 2026-07-22):
+#   core:apps-pg, core:gitea-db, r01:forgejo-db
+# Dual-hosted NK/state-hub: backup both until T01 reconfirms single POR
+#   core:net-kingdom-pg, core:state-hub-db, r01:net-kingdom-pg, r01:state-hub-db
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+export PATH="${ROOT}/tools/vendor/bin:${PATH}"
+
+# shellcheck source=lib/railiance-backup-common.sh
+source "${ROOT}/lib/railiance-backup-common.sh"
+
+DRY_RUN="${RAILIANCE_BACKUP_DRY_RUN:-0}"
+UPDATE_STATUS="${CNPG_UPDATE_STATUS_CM:-1}"
+STATUS_CM="${CNPG_OPTION_A_STATUS_CM:-cnpg-option-a-status}"
+BACKUP_ROOT="${BACKUP_DIR:-${XDG_CACHE_HOME:-$HOME/.cache}/railiance/backups/cnpg}"
+JSON_ONLY="${CNPG_BACKUP_JSON:-1}"
+
+KUBECONFIG_CORE="${KUBECONFIG_CORE:-${KUBECONFIG_COULOMBCORE:-$HOME/.kube/config}}"
+KUBECONFIG_R01="${KUBECONFIG_R01:-${KUBECONFIG_RAILIANCE01:-$HOME/.kube/config-hosteurope}}"
+
+# target_id|kubeconfig_env|namespace|cluster|db1,db2,...
+DEFAULT_TARGETS=(
+  "core-apps-pg|KUBECONFIG_CORE|databases|apps-pg|apps_meta,core_hub,core_hub_staging,vergabe_db"
+  "core-gitea-db|KUBECONFIG_CORE|databases|gitea-db|gitea"
+  "core-net-kingdom-pg|KUBECONFIG_CORE|databases|net-kingdom-pg|interhub,privacyidea_db"
+  "core-state-hub-db|KUBECONFIG_CORE|databases|state-hub-db|state_hub"
+  "r01-forgejo-db|KUBECONFIG_R01|databases|forgejo-db|forgejo"
+  "r01-net-kingdom-pg|KUBECONFIG_R01|databases|net-kingdom-pg|interhub,privacyidea_db"
+  "r01-state-hub-db|KUBECONFIG_R01|databases|state-hub-db|state_hub"
+)
+
+if [[ -n "${CNPG_BACKUP_TARGETS:-}" ]]; then
+  FILTER=",${CNPG_BACKUP_TARGETS},"
+else
+  FILTER=""
+fi
+
+TS="$(date -u +%Y%m%dT%H%M%SZ)"
+declare -a RESULTS=()
+FAILED=0
+UPLOADED=0
+DUMPED=0
+
+json_escape() {
+  python3 -c 'import json,sys; print(json.dumps(sys.stdin.read()[:-1] if False else sys.argv[1]))' "$1"
+}
+
+log() { echo "$*" >&2; }
+
+resolve_kubeconfig() {
+  local env_name="$1"
+  case "$env_name" in
+    KUBECONFIG_CORE) echo "${KUBECONFIG_CORE}" ;;
+    KUBECONFIG_R01) echo "${KUBECONFIG_R01}" ;;
+    *) echo "${KUBECONFIG:-}" ;;
+  esac
+}
+
+primary_pod() {
+  local kc="$1" ns="$2" cluster="$3"
+  KUBECONFIG="$kc" kubectl get pods -n "$ns" \
+    -l "cnpg.io/cluster=${cluster},role=primary" \
+    -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true
+}
+
+record_status_cm() {
+  local kc="$1" ns="$2" key="$3" ts="$4"
+  [[ "$UPDATE_STATUS" == "1" ]] || return 0
+  if ! KUBECONFIG="$kc" kubectl get ns "$ns" >/dev/null 2>&1; then
+    return 0
+  fi
+  if ! KUBECONFIG="$kc" kubectl get configmap "$STATUS_CM" -n "$ns" >/dev/null 2>&1; then
+    KUBECONFIG="$kc" kubectl create configmap "$STATUS_CM" -n "$ns" \
+      --from-literal="${key}=${ts}" \
+      --from-literal=lane=option-a \
+      --from-literal=mode=activity-core \
+      --from-literal=updated="${ts}" >/dev/null 2>&1 || true
+  else
+    KUBECONFIG="$kc" kubectl patch configmap "$STATUS_CM" -n "$ns" --type merge \
+      -p "{\"data\":{\"${key}\":\"${ts}\",\"updated\":\"${ts}\",\"lane\":\"option-a\",\"mode\":\"activity-core\"}}" \
+      >/dev/null 2>&1 || true
+  fi
+  KUBECONFIG="$kc" kubectl label configmap "$STATUS_CM" -n "$ns" \
+    railiance.apps/backup-lane=option-a --overwrite >/dev/null 2>&1 || true
+}
+
+# Prefer env credentials (activity-core); bao only if present
+if [[ "$DRY_RUN" != "1" ]]; then
+  railiance_backup_require_openbao_lane
+fi
+railiance_backup_require_tools
+
+for entry in "${DEFAULT_TARGETS[@]}"; do
+  IFS='|' read -r tid kenv ns cluster dbs_csv <<<"$entry"
+  if [[ -n "$FILTER" && "$FILTER" != *",${tid},"* && "$FILTER" != *",${cluster},"* ]]; then
+    continue
+  fi
+
+  kc="$(resolve_kubeconfig "$kenv")"
+  result_status="ok"
+  err=""
+  artifacts=()
+
+  if [[ ! -r "$kc" ]]; then
+    result_status="skip"
+    err="kubeconfig_missing:${kc}"
+    log "SKIP ${tid}: ${err}"
+    RESULTS+=("{\"id\":$(json_escape "$tid"),\"cluster\":$(json_escape "$cluster"),\"status\":$(json_escape "$result_status"),\"error\":$(json_escape "$err")}")
+    continue
+  fi
+
+  pod="$(primary_pod "$kc" "$ns" "$cluster")"
+  if [[ -z "$pod" ]]; then
+    result_status="error"
+    err="primary_pod_missing"
+    FAILED=$((FAILED + 1))
+    log "ERROR ${tid}: ${err}"
+    RESULTS+=("{\"id\":$(json_escape "$tid"),\"cluster\":$(json_escape "$cluster"),\"status\":$(json_escape "$result_status"),\"error\":$(json_escape "$err")}")
+    continue
+  fi
+
+  cluster_dir="${BACKUP_ROOT}/${tid}"
+  mkdir -p "$cluster_dir"
+  cluster_ok=1
+  IFS=',' read -r -a dbs <<<"$dbs_csv"
+  for db in "${dbs[@]}"; do
+    plain="${cluster_dir}/${cluster}-${db}-${TS}.dump"
+    aged="${plain}.age"
+    log "dump ${tid}/${db} via ${pod}"
+    if ! KUBECONFIG="$kc" kubectl exec -n "$ns" "$pod" -c postgres -- \
+      pg_dump -U postgres -d "$db" -Fc --no-owner --no-acl >"$plain" 2>/tmp/cnpg-backup-err.$$; then
+      cluster_ok=0
+      FAILED=$((FAILED + 1))
+      err="pg_dump_failed:${db}:$(head -c 200 /tmp/cnpg-backup-err.$$ | tr '\n' ' ')"
+      rm -f "$plain" /tmp/cnpg-backup-err.$$
+      log "ERROR ${err}"
+      continue
+    fi
+    rm -f /tmp/cnpg-backup-err.$$
+    railiance_backup_encrypt "$plain" "$aged"
+    rm -f "$plain"
+    DUMPED=$((DUMPED + 1))
+    artifacts+=("${cluster}-${db}-${TS}.dump.age")
+
+    if [[ "$DRY_RUN" == "1" ]]; then
+      log "dry-run skip upload ${tid}/${db}"
+    else
+      RAILIANCE_BACKUP_NC_PREFIX="${tid}"
+      if railiance_backup_nc_upload "$aged" "${cluster}-${db}-${TS}.dump.age"; then
+        UPLOADED=$((UPLOADED + 1))
+        if [[ "$(date -u +%u)" == "7" ]]; then
+          railiance_backup_nc_upload "$aged" "${cluster}-${db}-weekly-${TS}.dump.age" || true
+        fi
+      else
+        cluster_ok=0
+        FAILED=$((FAILED + 1))
+        err="upload_failed:${db}"
+      fi
+    fi
+    railiance_backup_prune_local "$cluster_dir" "${cluster}-${db}-*.dump.age" 7
+  done
+
+  if ((cluster_ok)); then
+    railiance_backup_record_success "$cluster_dir" "$TS"
+    # status key uses cluster name for cnpg-backup-status compatibility on that host
+    record_status_cm "$kc" "$ns" "$cluster" "$TS"
+    result_status="ok"
+  else
+    result_status="error"
+  fi
+
+  arts_json="["
+  first=1
+  for a in "${artifacts[@]+"${artifacts[@]}"}"; do
+    [[ $first -eq 1 ]] || arts_json+=","
+    first=0
+    arts_json+=$(json_escape "$a")
+  done
+  arts_json+="]"
+
+  RESULTS+=("{\"id\":$(json_escape "$tid"),\"cluster\":$(json_escape "$cluster"),\"host_key\":$(json_escape "$kenv"),\"status\":$(json_escape "$result_status"),\"artifacts\":${arts_json},\"error\":$(json_escape "${err}")}")
+done
+
+overall="ok"
+if ((FAILED > 0)); then
+  overall="degraded"
+fi
+# If everything skipped (no kubeconfigs), fail hard
+if ((DUMPED == 0 && FAILED == 0)); then
+  overall="error"
+  FAILED=1
+fi
+
+items="["
+first=1
+for r in "${RESULTS[@]+"${RESULTS[@]}"}"; do
+  [[ $first -eq 1 ]] || items+=","
+  first=0
+  items+="$r"
+done
+items+="]"
+
+cat <<EOF
+{
+  "kind": "cnpg_option_a_backup",
+  "overall": "${overall}",
+  "dry_run": $([[ "$DRY_RUN" == "1" ]] && echo true || echo false),
+  "ts": "${TS}",
+  "dumped": ${DUMPED},
+  "uploaded": ${UPLOADED},
+  "failed": ${FAILED},
+  "targets": ${items}
+}
+EOF
+
+if [[ "$overall" == "error" ]]; then
+  exit 1
+fi
+if [[ "$overall" == "degraded" ]]; then
+  exit 2
+fi
+exit 0

tools/cmd/install-cnpg-backup-vendor-tools

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+# Install static age + kubectl into tools/vendor/bin for activity-core hostPath.
+# Run on railiance01 (or any backup runner host) once; kubectl is not committed.
+set -euo pipefail
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+BIN="${ROOT}/tools/vendor/bin"
+mkdir -p "$BIN"
+
+AGE_VER="${AGE_VERSION:-v1.2.1}"
+KUBECTL_VER="${KUBECTL_VERSION:-v1.31.12}"
+
+if [[ ! -x "${BIN}/age" ]]; then
+  echo "fetch age ${AGE_VER}"
+  tmp="$(mktemp -d)"
+  curl -fsSL "https://github.com/FiloSottile/age/releases/download/${AGE_VER}/age-${AGE_VER}-linux-amd64.tar.gz" \
+    | tar -xz -C "$tmp"
+  install -m 0755 "${tmp}/age/age" "${BIN}/age"
+  rm -rf "$tmp"
+fi
+
+if [[ ! -x "${BIN}/kubectl" ]]; then
+  echo "fetch kubectl ${KUBECTL_VER}"
+  curl -fsSLo "${BIN}/kubectl" \
+    "https://dl.k8s.io/release/${KUBECTL_VER}/bin/linux/amd64/kubectl"
+  chmod 0755 "${BIN}/kubectl"
+fi
+
+# curl is usually present; verify
+for t in age kubectl curl; do
+  if ! command -v "$t" >/dev/null 2>&1 && [[ ! -x "${BIN}/$t" ]]; then
+    echo "ERROR: missing $t" >&2
+    exit 1
+  fi
+done
+
+echo "ok: vendor tools in ${BIN}"
+ls -la "$BIN"