feat(backup): multi-host CNPG Option A CLI for activity-core
Add cnpg-option-a-backup JSON runner, vendored static age, kubectl install helper, and ESO policy path for offsite lane so railiance01 workers can upload without workstation OIDC (RAILIANCE-WP-0016).
This commit is contained in:
parent
b11855f64c
commit
16a93b8e5c
7 changed files with 352 additions and 1 deletions
238
tools/cmd/cnpg-option-a-backup
Executable file
238
tools/cmd/cnpg-option-a-backup
Executable file
|
|
@ -0,0 +1,238 @@
|
|||
#!/usr/bin/env bash
|
||||
# tools/cmd/cnpg-option-a-backup — multi-host CNPG Option A backup (JSON)
|
||||
#
|
||||
# Used by activity-core shell context (RAILIANCE-WP-0016) and operator Make.
|
||||
# Emits one JSON object on stdout; logs on stderr.
|
||||
#
|
||||
# Env (non-interactive preferred for activity-core):
|
||||
# RAILIANCE_BACKUP_NC_TOKEN required for upload (unless dry-run)
|
||||
# RAILIANCE_BACKUP_NC_WEBDAV_URL optional; defaults to filesdrop form
|
||||
# RAILIANCE_BACKUP_AGE_PUBLIC_KEY optional
|
||||
# RAILIANCE_BACKUP_DRY_RUN=1 encrypt only, no upload
|
||||
# KUBECONFIG / KUBECONFIG_CORE / KUBECONFIG_R01
|
||||
# CNPG_BACKUP_TARGETS optional override CSV of target ids
|
||||
# CNPG_UPDATE_STATUS_CM=1 patch last-success ConfigMap (default 1)
|
||||
#
|
||||
# Targets (default set = production-of-record inventory 2026-07-22):
|
||||
# core:apps-pg, core:gitea-db, r01:forgejo-db
|
||||
# Dual-hosted NK/state-hub: backup both until T01 reconfirms single POR
|
||||
# core:net-kingdom-pg, core:state-hub-db, r01:net-kingdom-pg, r01:state-hub-db
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
export PATH="${ROOT}/tools/vendor/bin:${PATH}"
|
||||
|
||||
# shellcheck source=lib/railiance-backup-common.sh
|
||||
source "${ROOT}/lib/railiance-backup-common.sh"
|
||||
|
||||
DRY_RUN="${RAILIANCE_BACKUP_DRY_RUN:-0}"
|
||||
UPDATE_STATUS="${CNPG_UPDATE_STATUS_CM:-1}"
|
||||
STATUS_CM="${CNPG_OPTION_A_STATUS_CM:-cnpg-option-a-status}"
|
||||
BACKUP_ROOT="${BACKUP_DIR:-${XDG_CACHE_HOME:-$HOME/.cache}/railiance/backups/cnpg}"
|
||||
JSON_ONLY="${CNPG_BACKUP_JSON:-1}"
|
||||
|
||||
KUBECONFIG_CORE="${KUBECONFIG_CORE:-${KUBECONFIG_COULOMBCORE:-$HOME/.kube/config}}"
|
||||
KUBECONFIG_R01="${KUBECONFIG_R01:-${KUBECONFIG_RAILIANCE01:-$HOME/.kube/config-hosteurope}}"
|
||||
|
||||
# target_id|kubeconfig_env|namespace|cluster|db1,db2,...
|
||||
DEFAULT_TARGETS=(
|
||||
"core-apps-pg|KUBECONFIG_CORE|databases|apps-pg|apps_meta,core_hub,core_hub_staging,vergabe_db"
|
||||
"core-gitea-db|KUBECONFIG_CORE|databases|gitea-db|gitea"
|
||||
"core-net-kingdom-pg|KUBECONFIG_CORE|databases|net-kingdom-pg|interhub,privacyidea_db"
|
||||
"core-state-hub-db|KUBECONFIG_CORE|databases|state-hub-db|state_hub"
|
||||
"r01-forgejo-db|KUBECONFIG_R01|databases|forgejo-db|forgejo"
|
||||
"r01-net-kingdom-pg|KUBECONFIG_R01|databases|net-kingdom-pg|interhub,privacyidea_db"
|
||||
"r01-state-hub-db|KUBECONFIG_R01|databases|state-hub-db|state_hub"
|
||||
)
|
||||
|
||||
if [[ -n "${CNPG_BACKUP_TARGETS:-}" ]]; then
|
||||
FILTER=",${CNPG_BACKUP_TARGETS},"
|
||||
else
|
||||
FILTER=""
|
||||
fi
|
||||
|
||||
TS="$(date -u +%Y%m%dT%H%M%SZ)"
|
||||
declare -a RESULTS=()
|
||||
FAILED=0
|
||||
UPLOADED=0
|
||||
DUMPED=0
|
||||
|
||||
json_escape() {
|
||||
python3 -c 'import json,sys; print(json.dumps(sys.stdin.read()[:-1] if False else sys.argv[1]))' "$1"
|
||||
}
|
||||
|
||||
log() { echo "$*" >&2; }
|
||||
|
||||
resolve_kubeconfig() {
|
||||
local env_name="$1"
|
||||
case "$env_name" in
|
||||
KUBECONFIG_CORE) echo "${KUBECONFIG_CORE}" ;;
|
||||
KUBECONFIG_R01) echo "${KUBECONFIG_R01}" ;;
|
||||
*) echo "${KUBECONFIG:-}" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
primary_pod() {
|
||||
local kc="$1" ns="$2" cluster="$3"
|
||||
KUBECONFIG="$kc" kubectl get pods -n "$ns" \
|
||||
-l "cnpg.io/cluster=${cluster},role=primary" \
|
||||
-o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true
|
||||
}
|
||||
|
||||
record_status_cm() {
|
||||
local kc="$1" ns="$2" key="$3" ts="$4"
|
||||
[[ "$UPDATE_STATUS" == "1" ]] || return 0
|
||||
if ! KUBECONFIG="$kc" kubectl get ns "$ns" >/dev/null 2>&1; then
|
||||
return 0
|
||||
fi
|
||||
if ! KUBECONFIG="$kc" kubectl get configmap "$STATUS_CM" -n "$ns" >/dev/null 2>&1; then
|
||||
KUBECONFIG="$kc" kubectl create configmap "$STATUS_CM" -n "$ns" \
|
||||
--from-literal="${key}=${ts}" \
|
||||
--from-literal=lane=option-a \
|
||||
--from-literal=mode=activity-core \
|
||||
--from-literal=updated="${ts}" >/dev/null 2>&1 || true
|
||||
else
|
||||
KUBECONFIG="$kc" kubectl patch configmap "$STATUS_CM" -n "$ns" --type merge \
|
||||
-p "{\"data\":{\"${key}\":\"${ts}\",\"updated\":\"${ts}\",\"lane\":\"option-a\",\"mode\":\"activity-core\"}}" \
|
||||
>/dev/null 2>&1 || true
|
||||
fi
|
||||
KUBECONFIG="$kc" kubectl label configmap "$STATUS_CM" -n "$ns" \
|
||||
railiance.apps/backup-lane=option-a --overwrite >/dev/null 2>&1 || true
|
||||
}
|
||||
|
||||
# Prefer env credentials (activity-core); bao only if present
|
||||
if [[ "$DRY_RUN" != "1" ]]; then
|
||||
railiance_backup_require_openbao_lane
|
||||
fi
|
||||
railiance_backup_require_tools
|
||||
|
||||
for entry in "${DEFAULT_TARGETS[@]}"; do
|
||||
IFS='|' read -r tid kenv ns cluster dbs_csv <<<"$entry"
|
||||
if [[ -n "$FILTER" && "$FILTER" != *",${tid},"* && "$FILTER" != *",${cluster},"* ]]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
kc="$(resolve_kubeconfig "$kenv")"
|
||||
result_status="ok"
|
||||
err=""
|
||||
artifacts=()
|
||||
|
||||
if [[ ! -r "$kc" ]]; then
|
||||
result_status="skip"
|
||||
err="kubeconfig_missing:${kc}"
|
||||
log "SKIP ${tid}: ${err}"
|
||||
RESULTS+=("{\"id\":$(json_escape "$tid"),\"cluster\":$(json_escape "$cluster"),\"status\":$(json_escape "$result_status"),\"error\":$(json_escape "$err")}")
|
||||
continue
|
||||
fi
|
||||
|
||||
pod="$(primary_pod "$kc" "$ns" "$cluster")"
|
||||
if [[ -z "$pod" ]]; then
|
||||
result_status="error"
|
||||
err="primary_pod_missing"
|
||||
FAILED=$((FAILED + 1))
|
||||
log "ERROR ${tid}: ${err}"
|
||||
RESULTS+=("{\"id\":$(json_escape "$tid"),\"cluster\":$(json_escape "$cluster"),\"status\":$(json_escape "$result_status"),\"error\":$(json_escape "$err")}")
|
||||
continue
|
||||
fi
|
||||
|
||||
cluster_dir="${BACKUP_ROOT}/${tid}"
|
||||
mkdir -p "$cluster_dir"
|
||||
cluster_ok=1
|
||||
IFS=',' read -r -a dbs <<<"$dbs_csv"
|
||||
for db in "${dbs[@]}"; do
|
||||
plain="${cluster_dir}/${cluster}-${db}-${TS}.dump"
|
||||
aged="${plain}.age"
|
||||
log "dump ${tid}/${db} via ${pod}"
|
||||
if ! KUBECONFIG="$kc" kubectl exec -n "$ns" "$pod" -c postgres -- \
|
||||
pg_dump -U postgres -d "$db" -Fc --no-owner --no-acl >"$plain" 2>/tmp/cnpg-backup-err.$$; then
|
||||
cluster_ok=0
|
||||
FAILED=$((FAILED + 1))
|
||||
err="pg_dump_failed:${db}:$(head -c 200 /tmp/cnpg-backup-err.$$ | tr '\n' ' ')"
|
||||
rm -f "$plain" /tmp/cnpg-backup-err.$$
|
||||
log "ERROR ${err}"
|
||||
continue
|
||||
fi
|
||||
rm -f /tmp/cnpg-backup-err.$$
|
||||
railiance_backup_encrypt "$plain" "$aged"
|
||||
rm -f "$plain"
|
||||
DUMPED=$((DUMPED + 1))
|
||||
artifacts+=("${cluster}-${db}-${TS}.dump.age")
|
||||
|
||||
if [[ "$DRY_RUN" == "1" ]]; then
|
||||
log "dry-run skip upload ${tid}/${db}"
|
||||
else
|
||||
RAILIANCE_BACKUP_NC_PREFIX="${tid}"
|
||||
if railiance_backup_nc_upload "$aged" "${cluster}-${db}-${TS}.dump.age"; then
|
||||
UPLOADED=$((UPLOADED + 1))
|
||||
if [[ "$(date -u +%u)" == "7" ]]; then
|
||||
railiance_backup_nc_upload "$aged" "${cluster}-${db}-weekly-${TS}.dump.age" || true
|
||||
fi
|
||||
else
|
||||
cluster_ok=0
|
||||
FAILED=$((FAILED + 1))
|
||||
err="upload_failed:${db}"
|
||||
fi
|
||||
fi
|
||||
railiance_backup_prune_local "$cluster_dir" "${cluster}-${db}-*.dump.age" 7
|
||||
done
|
||||
|
||||
if ((cluster_ok)); then
|
||||
railiance_backup_record_success "$cluster_dir" "$TS"
|
||||
# status key uses cluster name for cnpg-backup-status compatibility on that host
|
||||
record_status_cm "$kc" "$ns" "$cluster" "$TS"
|
||||
result_status="ok"
|
||||
else
|
||||
result_status="error"
|
||||
fi
|
||||
|
||||
arts_json="["
|
||||
first=1
|
||||
for a in "${artifacts[@]+"${artifacts[@]}"}"; do
|
||||
[[ $first -eq 1 ]] || arts_json+=","
|
||||
first=0
|
||||
arts_json+=$(json_escape "$a")
|
||||
done
|
||||
arts_json+="]"
|
||||
|
||||
RESULTS+=("{\"id\":$(json_escape "$tid"),\"cluster\":$(json_escape "$cluster"),\"host_key\":$(json_escape "$kenv"),\"status\":$(json_escape "$result_status"),\"artifacts\":${arts_json},\"error\":$(json_escape "${err}")}")
|
||||
done
|
||||
|
||||
overall="ok"
|
||||
if ((FAILED > 0)); then
|
||||
overall="degraded"
|
||||
fi
|
||||
# If everything skipped (no kubeconfigs), fail hard
|
||||
if ((DUMPED == 0 && FAILED == 0)); then
|
||||
overall="error"
|
||||
FAILED=1
|
||||
fi
|
||||
|
||||
items="["
|
||||
first=1
|
||||
for r in "${RESULTS[@]+"${RESULTS[@]}"}"; do
|
||||
[[ $first -eq 1 ]] || items+=","
|
||||
first=0
|
||||
items+="$r"
|
||||
done
|
||||
items+="]"
|
||||
|
||||
cat <<EOF
|
||||
{
|
||||
"kind": "cnpg_option_a_backup",
|
||||
"overall": "${overall}",
|
||||
"dry_run": $([[ "$DRY_RUN" == "1" ]] && echo true || echo false),
|
||||
"ts": "${TS}",
|
||||
"dumped": ${DUMPED},
|
||||
"uploaded": ${UPLOADED},
|
||||
"failed": ${FAILED},
|
||||
"targets": ${items}
|
||||
}
|
||||
EOF
|
||||
|
||||
if [[ "$overall" == "error" ]]; then
|
||||
exit 1
|
||||
fi
|
||||
if [[ "$overall" == "degraded" ]]; then
|
||||
exit 2
|
||||
fi
|
||||
exit 0
|
||||
37
tools/cmd/install-cnpg-backup-vendor-tools
Executable file
37
tools/cmd/install-cnpg-backup-vendor-tools
Executable file
|
|
@ -0,0 +1,37 @@
|
|||
#!/usr/bin/env bash
|
||||
# Install static age + kubectl into tools/vendor/bin for activity-core hostPath.
|
||||
# Run on railiance01 (or any backup runner host) once; kubectl is not committed.
|
||||
set -euo pipefail
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
BIN="${ROOT}/tools/vendor/bin"
|
||||
mkdir -p "$BIN"
|
||||
|
||||
AGE_VER="${AGE_VERSION:-v1.2.1}"
|
||||
KUBECTL_VER="${KUBECTL_VERSION:-v1.31.12}"
|
||||
|
||||
if [[ ! -x "${BIN}/age" ]]; then
|
||||
echo "fetch age ${AGE_VER}"
|
||||
tmp="$(mktemp -d)"
|
||||
curl -fsSL "https://github.com/FiloSottile/age/releases/download/${AGE_VER}/age-${AGE_VER}-linux-amd64.tar.gz" \
|
||||
| tar -xz -C "$tmp"
|
||||
install -m 0755 "${tmp}/age/age" "${BIN}/age"
|
||||
rm -rf "$tmp"
|
||||
fi
|
||||
|
||||
if [[ ! -x "${BIN}/kubectl" ]]; then
|
||||
echo "fetch kubectl ${KUBECTL_VER}"
|
||||
curl -fsSLo "${BIN}/kubectl" \
|
||||
"https://dl.k8s.io/release/${KUBECTL_VER}/bin/linux/amd64/kubectl"
|
||||
chmod 0755 "${BIN}/kubectl"
|
||||
fi
|
||||
|
||||
# curl is usually present; verify
|
||||
for t in age kubectl curl; do
|
||||
if ! command -v "$t" >/dev/null 2>&1 && [[ ! -x "${BIN}/$t" ]]; then
|
||||
echo "ERROR: missing $t" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
echo "ok: vendor tools in ${BIN}"
|
||||
ls -la "$BIN"
|
||||
BIN
tools/vendor/bin/age
vendored
Executable file
BIN
tools/vendor/bin/age
vendored
Executable file
Binary file not shown.
Loading…
Add table
Add a link
Reference in a new issue