#!/usr/bin/env bash # CNPG backup posture for Option A (logical dumps) and legacy barman ScheduledBackup. set -euo pipefail namespace="${CNPG_NAMESPACE:-databases}" # RPO target is 24h; allow 36h before last-success is considered stale. rpo_hours="${CNPG_BACKUP_RPO_HOURS:-36}" status_cm="${CNPG_OPTION_A_STATUS_CM:-cnpg-option-a-status}" schedule_cm="${CNPG_OPTION_A_SCHEDULE_CM:-cnpg-option-a-schedule}" expected_clusters="${CNPG_EXPECTED_CLUSTERS:-apps-pg,gitea-db,net-kingdom-pg,state-hub-db}" if ! kubectl get ns "$namespace" >/dev/null 2>&1; then echo "ERROR: namespace $namespace is not reachable from the current kubeconfig" >&2 exit 1 fi echo "CNPG backup posture in namespace ${namespace}" context="$(kubectl config current-context 2>/dev/null || echo unknown)" server="$(kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}' 2>/dev/null || echo unknown)" echo "kube-context: ${context}" echo "api-server: ${server}" echo "lane: Option A (logical dump → age → Nextcloud); barman ScheduledBackup also accepted" echo "RPO window: ${rpo_hours}h (last-success freshness)" echo mapfile -t clusters < <( kubectl get cluster -n "$namespace" -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null \ | sort ) if ((${#clusters[@]} == 0)); then echo "ERROR: no CNPG Cluster resources in ${namespace}" >&2 exit 1 fi # Schedule declaration (workstation-cron or in-cluster) schedule_mode="" if kubectl get configmap "$schedule_cm" -n "$namespace" >/dev/null 2>&1; then schedule_mode="$(kubectl get configmap "$schedule_cm" -n "$namespace" -o jsonpath='{.data.mode}' 2>/dev/null || true)" schedule_expr="$(kubectl get configmap "$schedule_cm" -n "$namespace" -o jsonpath='{.data.schedule}' 2>/dev/null || true)" echo "schedule ConfigMap: ${schedule_cm} mode=${schedule_mode:-?} schedule=${schedule_expr:-?}" else echo "schedule ConfigMap: ${schedule_cm} (missing)" fi # Load status ConfigMap timestamps (cluster -> ISO-ish stamp YYYYMMDDTHHMMSSZ) declare -A last_success=() if kubectl get configmap "$status_cm" -n "$namespace" >/dev/null 2>&1; then while IFS= read -r line; do [[ -z "$line" ]] && continue key="${line%%=*}" val="${line#*=}" case "$key" in lane|updated|schedule_source|mode) continue ;; *) last_success["$key"]="$val" ;; esac done < <( kubectl get configmap "$status_cm" -n "$namespace" -o json 2>/dev/null \ | python3 -c 'import sys,json d=json.load(sys.stdin).get("data") or {} for k,v in sorted(d.items()): print(f"{k}={v}")' || true ) echo "status ConfigMap: ${status_cm} (present)" else echo "status ConfigMap: ${status_cm} (missing)" fi echo fresh_enough() { local ts="$1" [[ -z "$ts" ]] && return 1 if [[ ! "$ts" =~ ^[0-9]{8}T[0-9]{6}Z$ ]]; then return 1 fi local epoch now delta max_delta epoch="$(date -u -d "${ts:0:4}-${ts:4:2}-${ts:6:2} ${ts:9:2}:${ts:11:2}:${ts:13:2}" +%s 2>/dev/null || true)" if [[ -z "$epoch" ]]; then return 1 fi now="$(date -u +%s)" delta=$((now - epoch)) max_delta=$((rpo_hours * 3600)) ((delta >= 0 && delta <= max_delta)) } missing=0 degraded_detail=() IFS=',' read -r -a expected <<<"$expected_clusters" # Declared unattended plan modes workstation_schedule=0 activity_core_schedule=0 in_cluster_schedule=0 if [[ "$schedule_mode" == "workstation-cron" ]]; then workstation_schedule=1 fi if [[ "$schedule_mode" == "activity-core" ]]; then activity_core_schedule=1 fi if [[ "$schedule_mode" == "in-cluster-cron" || "$schedule_mode" == activity-core* ]]; then # in-cluster-cron is primary for CoulombCore; activity-core* covers hybrid labels in_cluster_schedule=1 fi for cluster in "${clusters[@]}"; do if ! kubectl get cluster "$cluster" -n "$namespace" >/dev/null 2>&1; then echo "${cluster}: MISSING (Cluster CR not found)" missing=$((missing + 1)) echo continue fi backup_spec="$(kubectl get cluster "$cluster" -n "$namespace" \ -o jsonpath='{.spec.backup}' 2>/dev/null || true)" if [[ -z "$backup_spec" || "$backup_spec" == "null" ]]; then echo "${cluster}: backup spec = null (expected for Option A)" else echo "${cluster}: backup spec configured (barman path)" fi scheduled="$(kubectl get scheduledbackup -n "$namespace" \ -o jsonpath="{range .items[?(@.spec.cluster.name=='${cluster}')]}{.metadata.name}{' '}{end}" 2>/dev/null || true)" scheduled="${scheduled%" "}" option_a_cron="$(kubectl get cronjob -n "$namespace" \ -l "railiance.apps/backup-lane=option-a,railiance.apps/backup-cluster=${cluster}" \ -o jsonpath='{range .items[*]}{.metadata.name}{"|"}{.spec.suspend}{" "}{end}' 2>/dev/null || true)" schedule_ok=0 schedule_note="" if [[ -n "$scheduled" ]]; then schedule_ok=1 schedule_note="ScheduledBackup=${scheduled}" echo "${cluster}: ScheduledBackup = ${scheduled}" else echo "${cluster}: ScheduledBackup = none" fi if [[ -n "$option_a_cron" ]]; then cj_name="${option_a_cron%%|*}" rest="${option_a_cron#*|}" cj_suspend="${rest%% *}" if [[ "${cj_suspend}" == "true" ]]; then echo "${cluster}: Option A CronJob = ${cj_name} (suspended — workstation path expected)" schedule_note="${schedule_note:+$schedule_note; }OptionA=${cj_name}:suspended" else schedule_ok=1 schedule_note="${schedule_note:+$schedule_note; }OptionA=${cj_name}" echo "${cluster}: Option A CronJob = ${cj_name}" fi else echo "${cluster}: Option A CronJob = none" fi if ((workstation_schedule)); then schedule_ok=1 schedule_note="${schedule_note:+$schedule_note; }workstation-cron" echo "${cluster}: schedule declaration = workstation-cron (stopgap)" fi if ((activity_core_schedule)); then schedule_ok=1 schedule_note="${schedule_note:+$schedule_note; }activity-core" echo "${cluster}: schedule declaration = activity-core" fi if ((in_cluster_schedule)); then schedule_ok=1 schedule_note="${schedule_note:+$schedule_note; }in-cluster-cron" echo "${cluster}: schedule declaration = in-cluster-cron" fi ts="${last_success[$cluster]:-}" success_ok=0 if [[ -n "$ts" ]]; then if fresh_enough "$ts"; then success_ok=1 echo "${cluster}: last-success = ${ts} (fresh ≤${rpo_hours}h)" else echo "${cluster}: last-success = ${ts} (STALE >${rpo_hours}h)" degraded_detail+=("${cluster}:stale-success") fi else echo "${cluster}: last-success = none" degraded_detail+=("${cluster}:no-success-yet") fi phase="$(kubectl get cluster "$cluster" -n "$namespace" -o jsonpath='{.status.phase}' 2>/dev/null || true)" instances="$(kubectl get cluster "$cluster" -n "$namespace" -o jsonpath='{.status.instances}' 2>/dev/null || true)" echo "${cluster}: phase=${phase:-unknown} instances=${instances:-unknown}" if ((schedule_ok && success_ok)); then echo "${cluster}: coverage = ok (${schedule_note})" else missing=$((missing + 1)) echo "${cluster}: coverage = MISSING (schedule_ok=${schedule_ok} success_ok=${success_ok})" fi echo done for exp in "${expected[@]}"; do found=0 for c in "${clusters[@]}"; do [[ "$c" == "$exp" ]] && found=1 && break done if ((found == 0)); then echo "WARN: expected cluster ${exp} not found in ${namespace}" missing=$((missing + 1)) fi done if ((missing > 0)); then echo "RESULT: degraded — ${missing} cluster(s) lack healthy Option A / ScheduledBackup coverage" if ((${#degraded_detail[@]} > 0)); then echo "detail: ${degraded_detail[*]}" fi echo "See docs/app-data-backup-restore-handoff.md and manifests/cnpg-option-a-backup.yaml" echo "Operator: make cnpg-backup-offsite-secret-apply && make cnpg-option-a-apply" echo "Daily: make cnpg-logical-backup # or workstation cron at 02:30 UTC" exit 1 fi echo "RESULT: ok — all tracked clusters have healthy Option A (or barman) backup coverage"