railiance-apps/tools/cnpg-backup-status.sh

#!/usr/bin/env bash
# CNPG backup posture for Option A (logical dumps) and legacy barman ScheduledBackup.
set -euo pipefail

namespace="${CNPG_NAMESPACE:-databases}"
# RPO target is 24h; allow 36h before last-success is considered stale.
rpo_hours="${CNPG_BACKUP_RPO_HOURS:-36}"
status_cm="${CNPG_OPTION_A_STATUS_CM:-cnpg-option-a-status}"
schedule_cm="${CNPG_OPTION_A_SCHEDULE_CM:-cnpg-option-a-schedule}"
expected_clusters="${CNPG_EXPECTED_CLUSTERS:-apps-pg,gitea-db,net-kingdom-pg,state-hub-db}"

if ! kubectl get ns "$namespace" >/dev/null 2>&1; then
  echo "ERROR: namespace $namespace is not reachable from the current kubeconfig" >&2
  exit 1
fi

echo "CNPG backup posture in namespace ${namespace}"
context="$(kubectl config current-context 2>/dev/null || echo unknown)"
server="$(kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}' 2>/dev/null || echo unknown)"
echo "kube-context: ${context}"
echo "api-server:   ${server}"
echo "lane:         Option A (logical dump → age → Nextcloud); barman ScheduledBackup also accepted"
echo "RPO window:   ${rpo_hours}h (last-success freshness)"
echo

mapfile -t clusters < <(
  kubectl get cluster -n "$namespace" -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null \
    | sort
)
if ((${#clusters[@]} == 0)); then
  echo "ERROR: no CNPG Cluster resources in ${namespace}" >&2
  exit 1
fi

# Schedule declaration (workstation-cron or in-cluster)
schedule_mode=""
if kubectl get configmap "$schedule_cm" -n "$namespace" >/dev/null 2>&1; then
  schedule_mode="$(kubectl get configmap "$schedule_cm" -n "$namespace" -o jsonpath='{.data.mode}' 2>/dev/null || true)"
  schedule_expr="$(kubectl get configmap "$schedule_cm" -n "$namespace" -o jsonpath='{.data.schedule}' 2>/dev/null || true)"
  echo "schedule ConfigMap: ${schedule_cm} mode=${schedule_mode:-?} schedule=${schedule_expr:-?}"
else
  echo "schedule ConfigMap: ${schedule_cm} (missing)"
fi

# Load status ConfigMap timestamps (cluster -> ISO-ish stamp YYYYMMDDTHHMMSSZ)
declare -A last_success=()
if kubectl get configmap "$status_cm" -n "$namespace" >/dev/null 2>&1; then
  while IFS= read -r line; do
    [[ -z "$line" ]] && continue
    key="${line%%=*}"
    val="${line#*=}"
    case "$key" in
      lane|updated|schedule_source|mode) continue ;;
      *) last_success["$key"]="$val" ;;
    esac
  done < <(
    kubectl get configmap "$status_cm" -n "$namespace" -o json 2>/dev/null \
      | python3 -c 'import sys,json
d=json.load(sys.stdin).get("data") or {}
for k,v in sorted(d.items()):
  print(f"{k}={v}")' || true
  )
  echo "status ConfigMap: ${status_cm} (present)"
else
  echo "status ConfigMap: ${status_cm} (missing)"
fi
echo

fresh_enough() {
  local ts="$1"
  [[ -z "$ts" ]] && return 1
  if [[ ! "$ts" =~ ^[0-9]{8}T[0-9]{6}Z$ ]]; then
    return 1
  fi
  local epoch now delta max_delta
  epoch="$(date -u -d "${ts:0:4}-${ts:4:2}-${ts:6:2} ${ts:9:2}:${ts:11:2}:${ts:13:2}" +%s 2>/dev/null || true)"
  if [[ -z "$epoch" ]]; then
    return 1
  fi
  now="$(date -u +%s)"
  delta=$((now - epoch))
  max_delta=$((rpo_hours * 3600))
  ((delta >= 0 && delta <= max_delta))
}

missing=0
degraded_detail=()

IFS=',' read -r -a expected <<<"$expected_clusters"

# Workstation schedule counts as a declared unattended plan for all expected clusters
workstation_schedule=0
if [[ "$schedule_mode" == "workstation-cron" ]]; then
  workstation_schedule=1
fi

for cluster in "${clusters[@]}"; do
  if ! kubectl get cluster "$cluster" -n "$namespace" >/dev/null 2>&1; then
    echo "${cluster}: MISSING (Cluster CR not found)"
    missing=$((missing + 1))
    echo
    continue
  fi

  backup_spec="$(kubectl get cluster "$cluster" -n "$namespace" \
    -o jsonpath='{.spec.backup}' 2>/dev/null || true)"
  if [[ -z "$backup_spec" || "$backup_spec" == "null" ]]; then
    echo "${cluster}: backup spec = null (expected for Option A)"
  else
    echo "${cluster}: backup spec configured (barman path)"
  fi

  scheduled="$(kubectl get scheduledbackup -n "$namespace" \
    -o jsonpath="{range .items[?(@.spec.cluster.name=='${cluster}')]}{.metadata.name}{' '}{end}" 2>/dev/null || true)"
  scheduled="${scheduled%" "}"

  option_a_cron="$(kubectl get cronjob -n "$namespace" \
    -l "railiance.apps/backup-lane=option-a,railiance.apps/backup-cluster=${cluster}" \
    -o jsonpath='{range .items[*]}{.metadata.name}{"|"}{.spec.suspend}{" "}{end}' 2>/dev/null || true)"

  schedule_ok=0
  schedule_note=""

  if [[ -n "$scheduled" ]]; then
    schedule_ok=1
    schedule_note="ScheduledBackup=${scheduled}"
    echo "${cluster}: ScheduledBackup = ${scheduled}"
  else
    echo "${cluster}: ScheduledBackup = none"
  fi

  if [[ -n "$option_a_cron" ]]; then
    cj_name="${option_a_cron%%|*}"
    rest="${option_a_cron#*|}"
    cj_suspend="${rest%% *}"
    if [[ "${cj_suspend}" == "true" ]]; then
      echo "${cluster}: Option A CronJob = ${cj_name} (suspended — workstation path expected)"
      schedule_note="${schedule_note:+$schedule_note; }OptionA=${cj_name}:suspended"
    else
      schedule_ok=1
      schedule_note="${schedule_note:+$schedule_note; }OptionA=${cj_name}"
      echo "${cluster}: Option A CronJob = ${cj_name}"
    fi
  else
    echo "${cluster}: Option A CronJob = none"
  fi

  if ((workstation_schedule)); then
    schedule_ok=1
    schedule_note="${schedule_note:+$schedule_note; }workstation-cron"
    echo "${cluster}: schedule declaration = workstation-cron"
  fi

  ts="${last_success[$cluster]:-}"
  success_ok=0
  if [[ -n "$ts" ]]; then
    if fresh_enough "$ts"; then
      success_ok=1
      echo "${cluster}: last-success = ${ts} (fresh ≤${rpo_hours}h)"
    else
      echo "${cluster}: last-success = ${ts} (STALE >${rpo_hours}h)"
      degraded_detail+=("${cluster}:stale-success")
    fi
  else
    echo "${cluster}: last-success = none"
    degraded_detail+=("${cluster}:no-success-yet")
  fi

  phase="$(kubectl get cluster "$cluster" -n "$namespace" -o jsonpath='{.status.phase}' 2>/dev/null || true)"
  instances="$(kubectl get cluster "$cluster" -n "$namespace" -o jsonpath='{.status.instances}' 2>/dev/null || true)"
  echo "${cluster}: phase=${phase:-unknown} instances=${instances:-unknown}"

  if ((schedule_ok && success_ok)); then
    echo "${cluster}: coverage = ok (${schedule_note})"
  else
    missing=$((missing + 1))
    echo "${cluster}: coverage = MISSING (schedule_ok=${schedule_ok} success_ok=${success_ok})"
  fi
  echo
done

for exp in "${expected[@]}"; do
  found=0
  for c in "${clusters[@]}"; do
    [[ "$c" == "$exp" ]] && found=1 && break
  done
  if ((found == 0)); then
    echo "WARN: expected cluster ${exp} not found in ${namespace}"
    missing=$((missing + 1))
  fi
done

if ((missing > 0)); then
  echo "RESULT: degraded — ${missing} cluster(s) lack healthy Option A / ScheduledBackup coverage"
  if ((${#degraded_detail[@]} > 0)); then
    echo "detail: ${degraded_detail[*]}"
  fi
  echo "See docs/app-data-backup-restore-handoff.md and manifests/cnpg-option-a-backup.yaml"
  echo "Operator: make cnpg-backup-offsite-secret-apply && make cnpg-option-a-apply"
  echo "Daily:    make cnpg-logical-backup   # or workstation cron at 02:30 UTC"
  exit 1
fi

echo "RESULT: ok — all tracked clusters have healthy Option A (or barman) backup coverage"