diff --git a/.gitignore b/.gitignore index 8d6f3ac..fc4cfea 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,4 @@ argocd/repositories/*.repository.sops.yaml # Python bytecode caches __pycache__/ *.pyc +tools/vendor/bin/kubectl diff --git a/docs/cnpg-option-a-backup.md b/docs/cnpg-option-a-backup.md new file mode 100644 index 0000000..fc201b9 --- /dev/null +++ b/docs/cnpg-option-a-backup.md @@ -0,0 +1,56 @@ +# CNPG Option A multi-host backup + +**Workplan:** RAILIANCE-WP-0016 +**CLI:** `tools/cmd/cnpg-option-a-backup` +**Scheduler:** activity-core `daily-cnpg-option-a-backup` (railiance01) + +## What it does + +Logical `pg_dump -Fc` → age encrypt → Nextcloud WebDAV for production-of-record +CNPG databases on **CoulombCore** and **railiance01**. Emits JSON for +activity-core evidence sinks. + +Inventory: `railiance-apps/docs/cnpg-backup-topology-inventory.md`. + +## Operator commands + +```bash +# One-time vendor tools (age committed; kubectl downloaded) +tools/cmd/install-cnpg-backup-vendor-tools + +# Dry-run (no upload) +RAILIANCE_BACKUP_DRY_RUN=1 tools/cmd/cnpg-option-a-backup + +# Live upload (env or bao OIDC) +export RAILIANCE_BACKUP_NC_TOKEN=… # or bao login + auto-fetch +tools/cmd/cnpg-option-a-backup +``` + +From railiance-apps: + +```bash +make cnpg-option-a-backup-dry-run +make cnpg-option-a-backup +``` + +## activity-core enable + +1. Re-mint ESO token with backup policy: + `OPENBAO_ACTIVITY_CORE_POLICIES="… workload-kv-read-railiance-backup-offsite-lane" \ + scripts/openbao-eso-token-apply.sh` +2. Apply `k8s/railiance/15-externalsecret-backup-offsite.yaml` +3. Ensure `/home/tegwick/railiance-platform` and `~/.kube` on railiance01 host + (worker hostPath). Place CoulombCore kubeconfig as `~/.kube/config`. +4. Rebuild/restart worker with resolver code; sync ActivityDefinition; + set `enabled: true` on `daily-cnpg-option-a-backup`. +5. Forced run; check State Hub progress + `make cnpg-backup-status`. + +## Credentials + +| Field | Where | +| --- | --- | +| `NC_WEBDAV_TOKEN` / `NC_WEBDAV_URL` | OpenBao offsite-lane → ESO → `actcore-runtime-secret` | +| `AGE_PUBLIC_KEY` | public constant / env (not private key) | +| `AGE_PRIVATE_KEY` | OpenBao only — restore drills, never on worker | + +Catalog: `warden route show railiance-backup-offsite-lane`. diff --git a/lib/railiance-backup-common.sh b/lib/railiance-backup-common.sh index db1aa90..6a70205 100644 --- a/lib/railiance-backup-common.sh +++ b/lib/railiance-backup-common.sh @@ -27,9 +27,19 @@ railiance_backup_require_openbao_lane() { } railiance_backup_require_tools() { + # Prefer vendored static binaries (activity-core hostPath / offline runners). + local root vendor + root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + vendor="${root}/tools/vendor/bin" + if [[ -d "$vendor" ]]; then + export PATH="${vendor}:${PATH}" + fi + if [[ -n "${RAILIANCE_BACKUP_AGE_BIN:-}" && -x "${RAILIANCE_BACKUP_AGE_BIN}" ]]; then + export PATH="$(dirname "${RAILIANCE_BACKUP_AGE_BIN}"):${PATH}" + fi local t for t in age curl kubectl; do - command -v "$t" >/dev/null 2>&1 || { echo "ERROR: missing required tool: $t" >&2; exit 1; } + command -v "$t" >/dev/null 2>&1 || { echo "ERROR: missing required tool: $t (run tools/cmd/install-cnpg-backup-vendor-tools)" >&2; exit 1; } done } diff --git a/openbao/policies/external-secrets-activity-core.hcl b/openbao/policies/external-secrets-activity-core.hcl index 7955037..5f61018 100644 --- a/openbao/policies/external-secrets-activity-core.hcl +++ b/openbao/policies/external-secrets-activity-core.hcl @@ -32,4 +32,13 @@ path "platform/data/workloads/forgejo/forgejo-admin" { path "platform/metadata/workloads/forgejo/forgejo-admin" { capabilities = ["read"] +} + +# Offsite backup lane (NC WebDAV + age pubkey material) for Option A uploads +path "platform/data/workloads/railiance/backup/offsite-lane" { + capabilities = ["read"] +} + +path "platform/metadata/workloads/railiance/backup/offsite-lane" { + capabilities = ["read"] } \ No newline at end of file diff --git a/tools/cmd/cnpg-option-a-backup b/tools/cmd/cnpg-option-a-backup new file mode 100755 index 0000000..d10018a --- /dev/null +++ b/tools/cmd/cnpg-option-a-backup @@ -0,0 +1,238 @@ +#!/usr/bin/env bash +# tools/cmd/cnpg-option-a-backup — multi-host CNPG Option A backup (JSON) +# +# Used by activity-core shell context (RAILIANCE-WP-0016) and operator Make. +# Emits one JSON object on stdout; logs on stderr. +# +# Env (non-interactive preferred for activity-core): +# RAILIANCE_BACKUP_NC_TOKEN required for upload (unless dry-run) +# RAILIANCE_BACKUP_NC_WEBDAV_URL optional; defaults to filesdrop form +# RAILIANCE_BACKUP_AGE_PUBLIC_KEY optional +# RAILIANCE_BACKUP_DRY_RUN=1 encrypt only, no upload +# KUBECONFIG / KUBECONFIG_CORE / KUBECONFIG_R01 +# CNPG_BACKUP_TARGETS optional override CSV of target ids +# CNPG_UPDATE_STATUS_CM=1 patch last-success ConfigMap (default 1) +# +# Targets (default set = production-of-record inventory 2026-07-22): +# core:apps-pg, core:gitea-db, r01:forgejo-db +# Dual-hosted NK/state-hub: backup both until T01 reconfirms single POR +# core:net-kingdom-pg, core:state-hub-db, r01:net-kingdom-pg, r01:state-hub-db +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +export PATH="${ROOT}/tools/vendor/bin:${PATH}" + +# shellcheck source=lib/railiance-backup-common.sh +source "${ROOT}/lib/railiance-backup-common.sh" + +DRY_RUN="${RAILIANCE_BACKUP_DRY_RUN:-0}" +UPDATE_STATUS="${CNPG_UPDATE_STATUS_CM:-1}" +STATUS_CM="${CNPG_OPTION_A_STATUS_CM:-cnpg-option-a-status}" +BACKUP_ROOT="${BACKUP_DIR:-${XDG_CACHE_HOME:-$HOME/.cache}/railiance/backups/cnpg}" +JSON_ONLY="${CNPG_BACKUP_JSON:-1}" + +KUBECONFIG_CORE="${KUBECONFIG_CORE:-${KUBECONFIG_COULOMBCORE:-$HOME/.kube/config}}" +KUBECONFIG_R01="${KUBECONFIG_R01:-${KUBECONFIG_RAILIANCE01:-$HOME/.kube/config-hosteurope}}" + +# target_id|kubeconfig_env|namespace|cluster|db1,db2,... +DEFAULT_TARGETS=( + "core-apps-pg|KUBECONFIG_CORE|databases|apps-pg|apps_meta,core_hub,core_hub_staging,vergabe_db" + "core-gitea-db|KUBECONFIG_CORE|databases|gitea-db|gitea" + "core-net-kingdom-pg|KUBECONFIG_CORE|databases|net-kingdom-pg|interhub,privacyidea_db" + "core-state-hub-db|KUBECONFIG_CORE|databases|state-hub-db|state_hub" + "r01-forgejo-db|KUBECONFIG_R01|databases|forgejo-db|forgejo" + "r01-net-kingdom-pg|KUBECONFIG_R01|databases|net-kingdom-pg|interhub,privacyidea_db" + "r01-state-hub-db|KUBECONFIG_R01|databases|state-hub-db|state_hub" +) + +if [[ -n "${CNPG_BACKUP_TARGETS:-}" ]]; then + FILTER=",${CNPG_BACKUP_TARGETS}," +else + FILTER="" +fi + +TS="$(date -u +%Y%m%dT%H%M%SZ)" +declare -a RESULTS=() +FAILED=0 +UPLOADED=0 +DUMPED=0 + +json_escape() { + python3 -c 'import json,sys; print(json.dumps(sys.stdin.read()[:-1] if False else sys.argv[1]))' "$1" +} + +log() { echo "$*" >&2; } + +resolve_kubeconfig() { + local env_name="$1" + case "$env_name" in + KUBECONFIG_CORE) echo "${KUBECONFIG_CORE}" ;; + KUBECONFIG_R01) echo "${KUBECONFIG_R01}" ;; + *) echo "${KUBECONFIG:-}" ;; + esac +} + +primary_pod() { + local kc="$1" ns="$2" cluster="$3" + KUBECONFIG="$kc" kubectl get pods -n "$ns" \ + -l "cnpg.io/cluster=${cluster},role=primary" \ + -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true +} + +record_status_cm() { + local kc="$1" ns="$2" key="$3" ts="$4" + [[ "$UPDATE_STATUS" == "1" ]] || return 0 + if ! KUBECONFIG="$kc" kubectl get ns "$ns" >/dev/null 2>&1; then + return 0 + fi + if ! KUBECONFIG="$kc" kubectl get configmap "$STATUS_CM" -n "$ns" >/dev/null 2>&1; then + KUBECONFIG="$kc" kubectl create configmap "$STATUS_CM" -n "$ns" \ + --from-literal="${key}=${ts}" \ + --from-literal=lane=option-a \ + --from-literal=mode=activity-core \ + --from-literal=updated="${ts}" >/dev/null 2>&1 || true + else + KUBECONFIG="$kc" kubectl patch configmap "$STATUS_CM" -n "$ns" --type merge \ + -p "{\"data\":{\"${key}\":\"${ts}\",\"updated\":\"${ts}\",\"lane\":\"option-a\",\"mode\":\"activity-core\"}}" \ + >/dev/null 2>&1 || true + fi + KUBECONFIG="$kc" kubectl label configmap "$STATUS_CM" -n "$ns" \ + railiance.apps/backup-lane=option-a --overwrite >/dev/null 2>&1 || true +} + +# Prefer env credentials (activity-core); bao only if present +if [[ "$DRY_RUN" != "1" ]]; then + railiance_backup_require_openbao_lane +fi +railiance_backup_require_tools + +for entry in "${DEFAULT_TARGETS[@]}"; do + IFS='|' read -r tid kenv ns cluster dbs_csv <<<"$entry" + if [[ -n "$FILTER" && "$FILTER" != *",${tid},"* && "$FILTER" != *",${cluster},"* ]]; then + continue + fi + + kc="$(resolve_kubeconfig "$kenv")" + result_status="ok" + err="" + artifacts=() + + if [[ ! -r "$kc" ]]; then + result_status="skip" + err="kubeconfig_missing:${kc}" + log "SKIP ${tid}: ${err}" + RESULTS+=("{\"id\":$(json_escape "$tid"),\"cluster\":$(json_escape "$cluster"),\"status\":$(json_escape "$result_status"),\"error\":$(json_escape "$err")}") + continue + fi + + pod="$(primary_pod "$kc" "$ns" "$cluster")" + if [[ -z "$pod" ]]; then + result_status="error" + err="primary_pod_missing" + FAILED=$((FAILED + 1)) + log "ERROR ${tid}: ${err}" + RESULTS+=("{\"id\":$(json_escape "$tid"),\"cluster\":$(json_escape "$cluster"),\"status\":$(json_escape "$result_status"),\"error\":$(json_escape "$err")}") + continue + fi + + cluster_dir="${BACKUP_ROOT}/${tid}" + mkdir -p "$cluster_dir" + cluster_ok=1 + IFS=',' read -r -a dbs <<<"$dbs_csv" + for db in "${dbs[@]}"; do + plain="${cluster_dir}/${cluster}-${db}-${TS}.dump" + aged="${plain}.age" + log "dump ${tid}/${db} via ${pod}" + if ! KUBECONFIG="$kc" kubectl exec -n "$ns" "$pod" -c postgres -- \ + pg_dump -U postgres -d "$db" -Fc --no-owner --no-acl >"$plain" 2>/tmp/cnpg-backup-err.$$; then + cluster_ok=0 + FAILED=$((FAILED + 1)) + err="pg_dump_failed:${db}:$(head -c 200 /tmp/cnpg-backup-err.$$ | tr '\n' ' ')" + rm -f "$plain" /tmp/cnpg-backup-err.$$ + log "ERROR ${err}" + continue + fi + rm -f /tmp/cnpg-backup-err.$$ + railiance_backup_encrypt "$plain" "$aged" + rm -f "$plain" + DUMPED=$((DUMPED + 1)) + artifacts+=("${cluster}-${db}-${TS}.dump.age") + + if [[ "$DRY_RUN" == "1" ]]; then + log "dry-run skip upload ${tid}/${db}" + else + RAILIANCE_BACKUP_NC_PREFIX="${tid}" + if railiance_backup_nc_upload "$aged" "${cluster}-${db}-${TS}.dump.age"; then + UPLOADED=$((UPLOADED + 1)) + if [[ "$(date -u +%u)" == "7" ]]; then + railiance_backup_nc_upload "$aged" "${cluster}-${db}-weekly-${TS}.dump.age" || true + fi + else + cluster_ok=0 + FAILED=$((FAILED + 1)) + err="upload_failed:${db}" + fi + fi + railiance_backup_prune_local "$cluster_dir" "${cluster}-${db}-*.dump.age" 7 + done + + if ((cluster_ok)); then + railiance_backup_record_success "$cluster_dir" "$TS" + # status key uses cluster name for cnpg-backup-status compatibility on that host + record_status_cm "$kc" "$ns" "$cluster" "$TS" + result_status="ok" + else + result_status="error" + fi + + arts_json="[" + first=1 + for a in "${artifacts[@]+"${artifacts[@]}"}"; do + [[ $first -eq 1 ]] || arts_json+="," + first=0 + arts_json+=$(json_escape "$a") + done + arts_json+="]" + + RESULTS+=("{\"id\":$(json_escape "$tid"),\"cluster\":$(json_escape "$cluster"),\"host_key\":$(json_escape "$kenv"),\"status\":$(json_escape "$result_status"),\"artifacts\":${arts_json},\"error\":$(json_escape "${err}")}") +done + +overall="ok" +if ((FAILED > 0)); then + overall="degraded" +fi +# If everything skipped (no kubeconfigs), fail hard +if ((DUMPED == 0 && FAILED == 0)); then + overall="error" + FAILED=1 +fi + +items="[" +first=1 +for r in "${RESULTS[@]+"${RESULTS[@]}"}"; do + [[ $first -eq 1 ]] || items+="," + first=0 + items+="$r" +done +items+="]" + +cat </dev/null 2>&1 && [[ ! -x "${BIN}/$t" ]]; then + echo "ERROR: missing $t" >&2 + exit 1 + fi +done + +echo "ok: vendor tools in ${BIN}" +ls -la "$BIN" diff --git a/tools/vendor/bin/age b/tools/vendor/bin/age new file mode 100755 index 0000000..018e76a Binary files /dev/null and b/tools/vendor/bin/age differ