RAILIANCE-WP-0015: use workstation cron; suspend in-cluster CronJobs
Cluster egress blocks age installs (github + Alpine CDN). Declare workstation-cron schedule ConfigMap as the unattended path and keep Option A CronJobs suspended until a prebuilt image is available.
This commit is contained in:
parent
cf742a0868
commit
1376b4f34c
4 changed files with 96 additions and 44 deletions
|
|
@ -1,11 +1,12 @@
|
|||
#!/usr/bin/env bash
|
||||
# CNPG backup posture for Option A (logical CronJobs) and legacy barman ScheduledBackup.
|
||||
# CNPG backup posture for Option A (logical dumps) and legacy barman ScheduledBackup.
|
||||
set -euo pipefail
|
||||
|
||||
namespace="${CNPG_NAMESPACE:-databases}"
|
||||
# RPO target is 24h; allow 36h before last-success is considered stale.
|
||||
rpo_hours="${CNPG_BACKUP_RPO_HOURS:-36}"
|
||||
status_cm="${CNPG_OPTION_A_STATUS_CM:-cnpg-option-a-status}"
|
||||
schedule_cm="${CNPG_OPTION_A_SCHEDULE_CM:-cnpg-option-a-schedule}"
|
||||
expected_clusters="${CNPG_EXPECTED_CLUSTERS:-apps-pg,gitea-db,net-kingdom-pg,state-hub-db}"
|
||||
|
||||
if ! kubectl get ns "$namespace" >/dev/null 2>&1; then
|
||||
|
|
@ -18,7 +19,7 @@ context="$(kubectl config current-context 2>/dev/null || echo unknown)"
|
|||
server="$(kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}' 2>/dev/null || echo unknown)"
|
||||
echo "kube-context: ${context}"
|
||||
echo "api-server: ${server}"
|
||||
echo "lane: Option A (logical dump → age → Nextcloud) preferred; barman ScheduledBackup also accepted"
|
||||
echo "lane: Option A (logical dump → age → Nextcloud); barman ScheduledBackup also accepted"
|
||||
echo "RPO window: ${rpo_hours}h (last-success freshness)"
|
||||
echo
|
||||
|
||||
|
|
@ -31,6 +32,16 @@ if ((${#clusters[@]} == 0)); then
|
|||
exit 1
|
||||
fi
|
||||
|
||||
# Schedule declaration (workstation-cron or in-cluster)
|
||||
schedule_mode=""
|
||||
if kubectl get configmap "$schedule_cm" -n "$namespace" >/dev/null 2>&1; then
|
||||
schedule_mode="$(kubectl get configmap "$schedule_cm" -n "$namespace" -o jsonpath='{.data.mode}' 2>/dev/null || true)"
|
||||
schedule_expr="$(kubectl get configmap "$schedule_cm" -n "$namespace" -o jsonpath='{.data.schedule}' 2>/dev/null || true)"
|
||||
echo "schedule ConfigMap: ${schedule_cm} mode=${schedule_mode:-?} schedule=${schedule_expr:-?}"
|
||||
else
|
||||
echo "schedule ConfigMap: ${schedule_cm} (missing)"
|
||||
fi
|
||||
|
||||
# Load status ConfigMap timestamps (cluster -> ISO-ish stamp YYYYMMDDTHHMMSSZ)
|
||||
declare -A last_success=()
|
||||
if kubectl get configmap "$status_cm" -n "$namespace" >/dev/null 2>&1; then
|
||||
|
|
@ -39,7 +50,7 @@ if kubectl get configmap "$status_cm" -n "$namespace" >/dev/null 2>&1; then
|
|||
key="${line%%=*}"
|
||||
val="${line#*=}"
|
||||
case "$key" in
|
||||
lane|updated) continue ;;
|
||||
lane|updated|schedule_source|mode) continue ;;
|
||||
*) last_success["$key"]="$val" ;;
|
||||
esac
|
||||
done < <(
|
||||
|
|
@ -58,7 +69,6 @@ echo
|
|||
fresh_enough() {
|
||||
local ts="$1"
|
||||
[[ -z "$ts" ]] && return 1
|
||||
# Accept YYYYMMDDTHHMMSSZ
|
||||
if [[ ! "$ts" =~ ^[0-9]{8}T[0-9]{6}Z$ ]]; then
|
||||
return 1
|
||||
fi
|
||||
|
|
@ -78,6 +88,12 @@ degraded_detail=()
|
|||
|
||||
IFS=',' read -r -a expected <<<"$expected_clusters"
|
||||
|
||||
# Workstation schedule counts as a declared unattended plan for all expected clusters
|
||||
workstation_schedule=0
|
||||
if [[ "$schedule_mode" == "workstation-cron" ]]; then
|
||||
workstation_schedule=1
|
||||
fi
|
||||
|
||||
for cluster in "${clusters[@]}"; do
|
||||
if ! kubectl get cluster "$cluster" -n "$namespace" >/dev/null 2>&1; then
|
||||
echo "${cluster}: MISSING (Cluster CR not found)"
|
||||
|
|
@ -100,24 +116,28 @@ for cluster in "${clusters[@]}"; do
|
|||
|
||||
option_a_cron="$(kubectl get cronjob -n "$namespace" \
|
||||
-l "railiance.apps/backup-lane=option-a,railiance.apps/backup-cluster=${cluster}" \
|
||||
-o jsonpath='{range .items[*]}{.metadata.name}{" "}{.spec.suspend}{" "}{end}' 2>/dev/null || true)"
|
||||
-o jsonpath='{range .items[*]}{.metadata.name}{"|"}{.spec.suspend}{" "}{end}' 2>/dev/null || true)"
|
||||
|
||||
covered=0
|
||||
schedule_ok=0
|
||||
schedule_note=""
|
||||
|
||||
if [[ -n "$scheduled" ]]; then
|
||||
covered=1
|
||||
schedule_ok=1
|
||||
schedule_note="ScheduledBackup=${scheduled}"
|
||||
echo "${cluster}: ScheduledBackup = ${scheduled}"
|
||||
else
|
||||
echo "${cluster}: ScheduledBackup = none"
|
||||
fi
|
||||
|
||||
if [[ -n "$option_a_cron" ]]; then
|
||||
# format: name suspend name suspend ...
|
||||
read -r cj_name cj_suspend _ <<<"$option_a_cron"
|
||||
cj_name="${option_a_cron%%|*}"
|
||||
rest="${option_a_cron#*|}"
|
||||
cj_suspend="${rest%% *}"
|
||||
if [[ "${cj_suspend}" == "true" ]]; then
|
||||
echo "${cluster}: Option A CronJob = ${cj_name} (SUSPENDED)"
|
||||
echo "${cluster}: Option A CronJob = ${cj_name} (suspended — workstation path expected)"
|
||||
schedule_note="${schedule_note:+$schedule_note; }OptionA=${cj_name}:suspended"
|
||||
else
|
||||
covered=1
|
||||
schedule_ok=1
|
||||
schedule_note="${schedule_note:+$schedule_note; }OptionA=${cj_name}"
|
||||
echo "${cluster}: Option A CronJob = ${cj_name}"
|
||||
fi
|
||||
|
|
@ -125,48 +145,40 @@ for cluster in "${clusters[@]}"; do
|
|||
echo "${cluster}: Option A CronJob = none"
|
||||
fi
|
||||
|
||||
if [[ -n "$scheduled" ]]; then
|
||||
echo "${cluster}: ScheduledBackup = ${scheduled}"
|
||||
else
|
||||
echo "${cluster}: ScheduledBackup = none"
|
||||
if ((workstation_schedule)); then
|
||||
schedule_ok=1
|
||||
schedule_note="${schedule_note:+$schedule_note; }workstation-cron"
|
||||
echo "${cluster}: schedule declaration = workstation-cron"
|
||||
fi
|
||||
|
||||
ts="${last_success[$cluster]:-}"
|
||||
success_ok=0
|
||||
if [[ -n "$ts" ]]; then
|
||||
if fresh_enough "$ts"; then
|
||||
success_ok=1
|
||||
echo "${cluster}: last-success = ${ts} (fresh ≤${rpo_hours}h)"
|
||||
else
|
||||
echo "${cluster}: last-success = ${ts} (STALE >${rpo_hours}h)"
|
||||
if ((covered)); then
|
||||
degraded_detail+=("${cluster}:stale-success")
|
||||
# scheduled but stale still counts as coverage with warning; mark degraded
|
||||
covered=0
|
||||
fi
|
||||
degraded_detail+=("${cluster}:stale-success")
|
||||
fi
|
||||
else
|
||||
echo "${cluster}: last-success = none"
|
||||
# Schedule present but never succeeded: still incomplete for healthy
|
||||
if ((covered)); then
|
||||
echo "${cluster}: note = schedule present but no recorded success yet"
|
||||
covered=0
|
||||
degraded_detail+=("${cluster}:no-success-yet")
|
||||
fi
|
||||
degraded_detail+=("${cluster}:no-success-yet")
|
||||
fi
|
||||
|
||||
phase="$(kubectl get cluster "$cluster" -n "$namespace" -o jsonpath='{.status.phase}' 2>/dev/null || true)"
|
||||
instances="$(kubectl get cluster "$cluster" -n "$namespace" -o jsonpath='{.status.instances}' 2>/dev/null || true)"
|
||||
echo "${cluster}: phase=${phase:-unknown} instances=${instances:-unknown}"
|
||||
|
||||
if ((covered == 0)); then
|
||||
missing=$((missing + 1))
|
||||
echo "${cluster}: coverage = MISSING"
|
||||
else
|
||||
if ((schedule_ok && success_ok)); then
|
||||
echo "${cluster}: coverage = ok (${schedule_note})"
|
||||
else
|
||||
missing=$((missing + 1))
|
||||
echo "${cluster}: coverage = MISSING (schedule_ok=${schedule_ok} success_ok=${success_ok})"
|
||||
fi
|
||||
echo
|
||||
done
|
||||
|
||||
# Warn if expected clusters are absent from the cluster list
|
||||
for exp in "${expected[@]}"; do
|
||||
found=0
|
||||
for c in "${clusters[@]}"; do
|
||||
|
|
@ -185,7 +197,7 @@ if ((missing > 0)); then
|
|||
fi
|
||||
echo "See docs/app-data-backup-restore-handoff.md and manifests/cnpg-option-a-backup.yaml"
|
||||
echo "Operator: make cnpg-backup-offsite-secret-apply && make cnpg-option-a-apply"
|
||||
echo "Manual run: make cnpg-logical-backup (or DRY_RUN=1 make cnpg-logical-backup-dry-run)"
|
||||
echo "Daily: make cnpg-logical-backup # or workstation cron at 02:30 UTC"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue