From 1376b4f34cf30ae78fe05b6999f9cc62363f7faf Mon Sep 17 00:00:00 2001 From: tegwick Date: Wed, 22 Jul 2026 18:06:06 +0200 Subject: [PATCH] RAILIANCE-WP-0015: use workstation cron; suspend in-cluster CronJobs Cluster egress blocks age installs (github + Alpine CDN). Declare workstation-cron schedule ConfigMap as the unattended path and keep Option A CronJobs suspended until a prebuilt image is available. --- docs/app-data-backup-restore-handoff.md | 12 ++- .../cnpg-option-a-backup-20260722.json | 4 +- manifests/cnpg-option-a-backup.yaml | 48 +++++++++--- tools/cnpg-backup-status.sh | 76 +++++++++++-------- 4 files changed, 96 insertions(+), 44 deletions(-) diff --git a/docs/app-data-backup-restore-handoff.md b/docs/app-data-backup-restore-handoff.md index d3f17eb..4763afe 100644 --- a/docs/app-data-backup-restore-handoff.md +++ b/docs/app-data-backup-restore-handoff.md @@ -71,11 +71,19 @@ Nextcloud WebDAV via CronJobs in `manifests/cnpg-option-a-backup.yaml`. ```bash bao login -method=oidc -path=netkingdom role=railiance-backup-workload-kv-read make cnpg-backup-offsite-secret-apply # Secret from OpenBao (no private key) -make cnpg-option-a-apply # CronJobs + RBAC -make cnpg-logical-backup # optional immediate workstation run +make cnpg-option-a-apply # schedule CM + RBAC + suspended CronJobs +make cnpg-logical-backup # immediate / daily workstation run make cnpg-backup-status ``` +**Unattended schedule (primary):** workstation cron (cluster pods lack egress to +install `age`; in-cluster CronJobs ship suspended until a prebuilt image exists): + +```cron +# Daily 02:30 UTC — Option A multi-cluster logical backup (RPO 24h) +30 2 * * * cd $HOME/railiance-apps && make cnpg-logical-backup >>$HOME/.cache/railiance/backups/cnpg/cron.log 2>&1 +``` + Barman `ObjectStore` / CNPG `ScheduledBackup` remain deferred (Phase 2 stubs in `manifests/cnpg-backup-readiness.yaml`). diff --git a/docs/evidence/cnpg-option-a-backup-20260722.json b/docs/evidence/cnpg-option-a-backup-20260722.json index 918c5af..7516dd8 100644 --- a/docs/evidence/cnpg-option-a-backup-20260722.json +++ b/docs/evidence/cnpg-option-a-backup-20260722.json @@ -43,7 +43,9 @@ "rto_note": "decrypt + isolated restore completed under 5 minutes on operator workstation" }, "notes": [ - "First success wave seeded via workstation tools/cnpg-logical-backup.sh; CronJobs schedule daily 02:30–02:45 UTC.", + "First success wave seeded via workstation tools/cnpg-logical-backup.sh.", + "In-cluster CronJobs are suspended: cluster egress blocks github.com and Alpine CDN age installs.", + "Unattended path is workstation cron at 02:30 UTC (ConfigMap cnpg-option-a-schedule mode=workstation-cron).", "AGE_PRIVATE_KEY remains OpenBao recovery escrow only — not stored in-cluster.", "No secret values recorded in this evidence file." ] diff --git a/manifests/cnpg-option-a-backup.yaml b/manifests/cnpg-option-a-backup.yaml index d9147b4..dcb119d 100644 --- a/manifests/cnpg-option-a-backup.yaml +++ b/manifests/cnpg-option-a-backup.yaml @@ -3,17 +3,41 @@ # Decided approach (not barman ScheduledBackup). Applies to CoulombCore # databases namespace clusters: apps-pg, gitea-db, net-kingdom-pg, state-hub-db. # +# Unattended path (primary): workstation cron (same pattern as forgejo-backup). +# CoulombCore pods cannot reach github.com / Alpine CDN to fetch `age`, so +# in-cluster CronJobs are shipped **suspended**. Enable only after an image +# with kubectl+age+curl is available in-cluster. +# # Prereqs: # 1. OpenBao lane CCR-2026-0004 resolvable # 2. make cnpg-backup-offsite-secret-apply # Secret cnpg-backup-offsite # 3. kubectl apply -f manifests/cnpg-option-a-backup.yaml +# 4. Install workstation cron (see schedule ConfigMap / docs) # # Schedule: daily 02:30 UTC (RPO 24h). Retention target 14 daily + 4 weekly -# is enforced on the Nextcloud side / operator prune; local Job pods are ephemeral. +# is enforced on the Nextcloud side / operator prune. # # Health: make cnpg-backup-status --- apiVersion: v1 +kind: ConfigMap +metadata: + name: cnpg-option-a-schedule + namespace: databases + labels: + app.kubernetes.io/part-of: railiance-apps + railiance.apps/backup-lane: option-a +data: + mode: workstation-cron + schedule: "30 2 * * *" + command: "cd $HOME/railiance-apps && make cnpg-logical-backup" + clusters: "apps-pg,gitea-db,net-kingdom-pg,state-hub-db" + rpo_hours: "24" + rto_hours: "4" + retention: "14 daily + 4 weekly (Nextcloud operator prune)" + note: "In-cluster CronJobs suspended until image with age is available without egress installs" +--- +apiVersion: v1 kind: ServiceAccount metadata: name: cnpg-logical-backup @@ -79,17 +103,19 @@ data: DATABASES="${DATABASES:?DATABASES required}" NAMESPACE="${NAMESPACE:-databases}" STATUS_CM="${STATUS_CM:-cnpg-option-a-status}" - AGE_BIN="${AGE_BIN:-/tmp/age/age}" WORK=/tmp/backup-work - mkdir -p "$WORK" /tmp/age + mkdir -p "$WORK" - if [ ! -x "$AGE_BIN" ]; then - echo "fetch age static binary" - curl -fsSL "https://github.com/FiloSottile/age/releases/download/v1.2.1/age-v1.2.1-linux-amd64.tar.gz" \ - | tar -xz -C /tmp - # tarball extracts to /tmp/age/age - AGE_BIN=/tmp/age/age + # Prefer distro package (cluster often cannot reach github.com). + if ! command -v age >/dev/null 2>&1; then + echo "install age via apk" + apk add --no-cache age >/dev/null fi + if ! command -v age >/dev/null 2>&1; then + echo "ERROR: age not available (apk install failed; github fallback disabled)" >&2 + exit 1 + fi + AGE_BIN="$(command -v age)" NC_TOKEN="$(cat /secrets/offsite/NC_WEBDAV_TOKEN)" NC_URL="$(cat /secrets/offsite/NC_WEBDAV_URL)" @@ -160,6 +186,7 @@ metadata: railiance.apps/backup-cluster: apps-pg spec: schedule: "30 2 * * *" + suspend: true concurrencyPolicy: Forbid successfulJobsHistoryLimit: 3 failedJobsHistoryLimit: 3 @@ -221,6 +248,7 @@ metadata: railiance.apps/backup-cluster: gitea-db spec: schedule: "35 2 * * *" + suspend: true concurrencyPolicy: Forbid successfulJobsHistoryLimit: 3 failedJobsHistoryLimit: 3 @@ -282,6 +310,7 @@ metadata: railiance.apps/backup-cluster: net-kingdom-pg spec: schedule: "40 2 * * *" + suspend: true concurrencyPolicy: Forbid successfulJobsHistoryLimit: 3 failedJobsHistoryLimit: 3 @@ -343,6 +372,7 @@ metadata: railiance.apps/backup-cluster: state-hub-db spec: schedule: "45 2 * * *" + suspend: true concurrencyPolicy: Forbid successfulJobsHistoryLimit: 3 failedJobsHistoryLimit: 3 diff --git a/tools/cnpg-backup-status.sh b/tools/cnpg-backup-status.sh index 42ec006..e90f785 100755 --- a/tools/cnpg-backup-status.sh +++ b/tools/cnpg-backup-status.sh @@ -1,11 +1,12 @@ #!/usr/bin/env bash -# CNPG backup posture for Option A (logical CronJobs) and legacy barman ScheduledBackup. +# CNPG backup posture for Option A (logical dumps) and legacy barman ScheduledBackup. set -euo pipefail namespace="${CNPG_NAMESPACE:-databases}" # RPO target is 24h; allow 36h before last-success is considered stale. rpo_hours="${CNPG_BACKUP_RPO_HOURS:-36}" status_cm="${CNPG_OPTION_A_STATUS_CM:-cnpg-option-a-status}" +schedule_cm="${CNPG_OPTION_A_SCHEDULE_CM:-cnpg-option-a-schedule}" expected_clusters="${CNPG_EXPECTED_CLUSTERS:-apps-pg,gitea-db,net-kingdom-pg,state-hub-db}" if ! kubectl get ns "$namespace" >/dev/null 2>&1; then @@ -18,7 +19,7 @@ context="$(kubectl config current-context 2>/dev/null || echo unknown)" server="$(kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}' 2>/dev/null || echo unknown)" echo "kube-context: ${context}" echo "api-server: ${server}" -echo "lane: Option A (logical dump → age → Nextcloud) preferred; barman ScheduledBackup also accepted" +echo "lane: Option A (logical dump → age → Nextcloud); barman ScheduledBackup also accepted" echo "RPO window: ${rpo_hours}h (last-success freshness)" echo @@ -31,6 +32,16 @@ if ((${#clusters[@]} == 0)); then exit 1 fi +# Schedule declaration (workstation-cron or in-cluster) +schedule_mode="" +if kubectl get configmap "$schedule_cm" -n "$namespace" >/dev/null 2>&1; then + schedule_mode="$(kubectl get configmap "$schedule_cm" -n "$namespace" -o jsonpath='{.data.mode}' 2>/dev/null || true)" + schedule_expr="$(kubectl get configmap "$schedule_cm" -n "$namespace" -o jsonpath='{.data.schedule}' 2>/dev/null || true)" + echo "schedule ConfigMap: ${schedule_cm} mode=${schedule_mode:-?} schedule=${schedule_expr:-?}" +else + echo "schedule ConfigMap: ${schedule_cm} (missing)" +fi + # Load status ConfigMap timestamps (cluster -> ISO-ish stamp YYYYMMDDTHHMMSSZ) declare -A last_success=() if kubectl get configmap "$status_cm" -n "$namespace" >/dev/null 2>&1; then @@ -39,7 +50,7 @@ if kubectl get configmap "$status_cm" -n "$namespace" >/dev/null 2>&1; then key="${line%%=*}" val="${line#*=}" case "$key" in - lane|updated) continue ;; + lane|updated|schedule_source|mode) continue ;; *) last_success["$key"]="$val" ;; esac done < <( @@ -58,7 +69,6 @@ echo fresh_enough() { local ts="$1" [[ -z "$ts" ]] && return 1 - # Accept YYYYMMDDTHHMMSSZ if [[ ! "$ts" =~ ^[0-9]{8}T[0-9]{6}Z$ ]]; then return 1 fi @@ -78,6 +88,12 @@ degraded_detail=() IFS=',' read -r -a expected <<<"$expected_clusters" +# Workstation schedule counts as a declared unattended plan for all expected clusters +workstation_schedule=0 +if [[ "$schedule_mode" == "workstation-cron" ]]; then + workstation_schedule=1 +fi + for cluster in "${clusters[@]}"; do if ! kubectl get cluster "$cluster" -n "$namespace" >/dev/null 2>&1; then echo "${cluster}: MISSING (Cluster CR not found)" @@ -100,24 +116,28 @@ for cluster in "${clusters[@]}"; do option_a_cron="$(kubectl get cronjob -n "$namespace" \ -l "railiance.apps/backup-lane=option-a,railiance.apps/backup-cluster=${cluster}" \ - -o jsonpath='{range .items[*]}{.metadata.name}{" "}{.spec.suspend}{" "}{end}' 2>/dev/null || true)" + -o jsonpath='{range .items[*]}{.metadata.name}{"|"}{.spec.suspend}{" "}{end}' 2>/dev/null || true)" - covered=0 + schedule_ok=0 schedule_note="" if [[ -n "$scheduled" ]]; then - covered=1 + schedule_ok=1 schedule_note="ScheduledBackup=${scheduled}" + echo "${cluster}: ScheduledBackup = ${scheduled}" + else + echo "${cluster}: ScheduledBackup = none" fi if [[ -n "$option_a_cron" ]]; then - # format: name suspend name suspend ... - read -r cj_name cj_suspend _ <<<"$option_a_cron" + cj_name="${option_a_cron%%|*}" + rest="${option_a_cron#*|}" + cj_suspend="${rest%% *}" if [[ "${cj_suspend}" == "true" ]]; then - echo "${cluster}: Option A CronJob = ${cj_name} (SUSPENDED)" + echo "${cluster}: Option A CronJob = ${cj_name} (suspended — workstation path expected)" schedule_note="${schedule_note:+$schedule_note; }OptionA=${cj_name}:suspended" else - covered=1 + schedule_ok=1 schedule_note="${schedule_note:+$schedule_note; }OptionA=${cj_name}" echo "${cluster}: Option A CronJob = ${cj_name}" fi @@ -125,48 +145,40 @@ for cluster in "${clusters[@]}"; do echo "${cluster}: Option A CronJob = none" fi - if [[ -n "$scheduled" ]]; then - echo "${cluster}: ScheduledBackup = ${scheduled}" - else - echo "${cluster}: ScheduledBackup = none" + if ((workstation_schedule)); then + schedule_ok=1 + schedule_note="${schedule_note:+$schedule_note; }workstation-cron" + echo "${cluster}: schedule declaration = workstation-cron" fi ts="${last_success[$cluster]:-}" + success_ok=0 if [[ -n "$ts" ]]; then if fresh_enough "$ts"; then + success_ok=1 echo "${cluster}: last-success = ${ts} (fresh ≤${rpo_hours}h)" else echo "${cluster}: last-success = ${ts} (STALE >${rpo_hours}h)" - if ((covered)); then - degraded_detail+=("${cluster}:stale-success") - # scheduled but stale still counts as coverage with warning; mark degraded - covered=0 - fi + degraded_detail+=("${cluster}:stale-success") fi else echo "${cluster}: last-success = none" - # Schedule present but never succeeded: still incomplete for healthy - if ((covered)); then - echo "${cluster}: note = schedule present but no recorded success yet" - covered=0 - degraded_detail+=("${cluster}:no-success-yet") - fi + degraded_detail+=("${cluster}:no-success-yet") fi phase="$(kubectl get cluster "$cluster" -n "$namespace" -o jsonpath='{.status.phase}' 2>/dev/null || true)" instances="$(kubectl get cluster "$cluster" -n "$namespace" -o jsonpath='{.status.instances}' 2>/dev/null || true)" echo "${cluster}: phase=${phase:-unknown} instances=${instances:-unknown}" - if ((covered == 0)); then - missing=$((missing + 1)) - echo "${cluster}: coverage = MISSING" - else + if ((schedule_ok && success_ok)); then echo "${cluster}: coverage = ok (${schedule_note})" + else + missing=$((missing + 1)) + echo "${cluster}: coverage = MISSING (schedule_ok=${schedule_ok} success_ok=${success_ok})" fi echo done -# Warn if expected clusters are absent from the cluster list for exp in "${expected[@]}"; do found=0 for c in "${clusters[@]}"; do @@ -185,7 +197,7 @@ if ((missing > 0)); then fi echo "See docs/app-data-backup-restore-handoff.md and manifests/cnpg-option-a-backup.yaml" echo "Operator: make cnpg-backup-offsite-secret-apply && make cnpg-option-a-apply" - echo "Manual run: make cnpg-logical-backup (or DRY_RUN=1 make cnpg-logical-backup-dry-run)" + echo "Daily: make cnpg-logical-backup # or workstation cron at 02:30 UTC" exit 1 fi