RAILIANCE-WP-0015: use workstation cron; suspend in-cluster CronJobs

Cluster egress blocks age installs (github + Alpine CDN). Declare workstation-cron schedule ConfigMap as the unattended path and keep Option A CronJobs suspended until a prebuilt image is available.
2026-07-22 18:06:06 +02:00 · 2026-07-22 18:06:06 +02:00 · 1376b4f34c
commit 1376b4f34c
parent cf742a0868
4 changed files with 96 additions and 44 deletions
--- a/docs/app-data-backup-restore-handoff.md
+++ b/docs/app-data-backup-restore-handoff.md
@ -71,11 +71,19 @@ Nextcloud WebDAV via CronJobs in `manifests/cnpg-option-a-backup.yaml`.
 ```bash
 bao login -method=oidc -path=netkingdom role=railiance-backup-workload-kv-read
 make cnpg-backup-offsite-secret-apply   # Secret from OpenBao (no private key)
-make cnpg-option-a-apply                # CronJobs + RBAC
-make cnpg-logical-backup                # optional immediate workstation run
+make cnpg-option-a-apply                # schedule CM + RBAC + suspended CronJobs
+make cnpg-logical-backup                # immediate / daily workstation run
 make cnpg-backup-status
 ```

+**Unattended schedule (primary):** workstation cron (cluster pods lack egress to
+install `age`; in-cluster CronJobs ship suspended until a prebuilt image exists):
+
+```cron
+# Daily 02:30 UTC — Option A multi-cluster logical backup (RPO 24h)
+30 2 * * * cd $HOME/railiance-apps && make cnpg-logical-backup >>$HOME/.cache/railiance/backups/cnpg/cron.log 2>&1
+```
+
 Barman `ObjectStore` / CNPG `ScheduledBackup` remain deferred (Phase 2 stubs in
 `manifests/cnpg-backup-readiness.yaml`).

--- a/docs/evidence/cnpg-option-a-backup-20260722.json
+++ b/docs/evidence/cnpg-option-a-backup-20260722.json
@ -43,7 +43,9 @@
    "rto_note": "decrypt + isolated restore completed under 5 minutes on operator workstation"
  },
  "notes": [
-    "First success wave seeded via workstation tools/cnpg-logical-backup.sh; CronJobs schedule daily 02:30–02:45 UTC.",
+    "First success wave seeded via workstation tools/cnpg-logical-backup.sh.",
+    "In-cluster CronJobs are suspended: cluster egress blocks github.com and Alpine CDN age installs.",
+    "Unattended path is workstation cron at 02:30 UTC (ConfigMap cnpg-option-a-schedule mode=workstation-cron).",
    "AGE_PRIVATE_KEY remains OpenBao recovery escrow only — not stored in-cluster.",
    "No secret values recorded in this evidence file."
  ]
--- a/manifests/cnpg-option-a-backup.yaml
+++ b/manifests/cnpg-option-a-backup.yaml
@ -3,17 +3,41 @@
 # Decided approach (not barman ScheduledBackup). Applies to CoulombCore
 # databases namespace clusters: apps-pg, gitea-db, net-kingdom-pg, state-hub-db.
 #
+# Unattended path (primary): workstation cron (same pattern as forgejo-backup).
+# CoulombCore pods cannot reach github.com / Alpine CDN to fetch `age`, so
+# in-cluster CronJobs are shipped **suspended**. Enable only after an image
+# with kubectl+age+curl is available in-cluster.
+#
 # Prereqs:
 #   1. OpenBao lane CCR-2026-0004 resolvable
 #   2. make cnpg-backup-offsite-secret-apply   # Secret cnpg-backup-offsite
 #   3. kubectl apply -f manifests/cnpg-option-a-backup.yaml
+#   4. Install workstation cron (see schedule ConfigMap / docs)
 #
 # Schedule: daily 02:30 UTC (RPO 24h). Retention target 14 daily + 4 weekly
-# is enforced on the Nextcloud side / operator prune; local Job pods are ephemeral.
+# is enforced on the Nextcloud side / operator prune.
 #
 # Health: make cnpg-backup-status
 ---
 apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: cnpg-option-a-schedule
+  namespace: databases
+  labels:
+    app.kubernetes.io/part-of: railiance-apps
+    railiance.apps/backup-lane: option-a
+data:
+  mode: workstation-cron
+  schedule: "30 2 * * *"
+  command: "cd $HOME/railiance-apps && make cnpg-logical-backup"
+  clusters: "apps-pg,gitea-db,net-kingdom-pg,state-hub-db"
+  rpo_hours: "24"
+  rto_hours: "4"
+  retention: "14 daily + 4 weekly (Nextcloud operator prune)"
+  note: "In-cluster CronJobs suspended until image with age is available without egress installs"
+---
+apiVersion: v1
 kind: ServiceAccount
 metadata:
  name: cnpg-logical-backup
@ -79,17 +103,19 @@ data:
    DATABASES="${DATABASES:?DATABASES required}"
    NAMESPACE="${NAMESPACE:-databases}"
    STATUS_CM="${STATUS_CM:-cnpg-option-a-status}"
-    AGE_BIN="${AGE_BIN:-/tmp/age/age}"
    WORK=/tmp/backup-work
-    mkdir -p "$WORK" /tmp/age
+    mkdir -p "$WORK"

-    if [ ! -x "$AGE_BIN" ]; then
-      echo "fetch age static binary"
-      curl -fsSL "https://github.com/FiloSottile/age/releases/download/v1.2.1/age-v1.2.1-linux-amd64.tar.gz" \
-        | tar -xz -C /tmp
-      # tarball extracts to /tmp/age/age
-      AGE_BIN=/tmp/age/age
+    # Prefer distro package (cluster often cannot reach github.com).
+    if ! command -v age >/dev/null 2>&1; then
+      echo "install age via apk"
+      apk add --no-cache age >/dev/null
    fi
+    if ! command -v age >/dev/null 2>&1; then
+      echo "ERROR: age not available (apk install failed; github fallback disabled)" >&2
+      exit 1
+    fi
+    AGE_BIN="$(command -v age)"

    NC_TOKEN="$(cat /secrets/offsite/NC_WEBDAV_TOKEN)"
    NC_URL="$(cat /secrets/offsite/NC_WEBDAV_URL)"
@ -160,6 +186,7 @@ metadata:
    railiance.apps/backup-cluster: apps-pg
 spec:
  schedule: "30 2 * * *"
+  suspend: true
  concurrencyPolicy: Forbid
  successfulJobsHistoryLimit: 3
  failedJobsHistoryLimit: 3
@ -221,6 +248,7 @@ metadata:
    railiance.apps/backup-cluster: gitea-db
 spec:
  schedule: "35 2 * * *"
+  suspend: true
  concurrencyPolicy: Forbid
  successfulJobsHistoryLimit: 3
  failedJobsHistoryLimit: 3
@ -282,6 +310,7 @@ metadata:
    railiance.apps/backup-cluster: net-kingdom-pg
 spec:
  schedule: "40 2 * * *"
+  suspend: true
  concurrencyPolicy: Forbid
  successfulJobsHistoryLimit: 3
  failedJobsHistoryLimit: 3
@ -343,6 +372,7 @@ metadata:
    railiance.apps/backup-cluster: state-hub-db
 spec:
  schedule: "45 2 * * *"
+  suspend: true
  concurrencyPolicy: Forbid
  successfulJobsHistoryLimit: 3
  failedJobsHistoryLimit: 3
--- a/tools/cnpg-backup-status.sh
+++ b/tools/cnpg-backup-status.sh
@ -1,11 +1,12 @@
 #!/usr/bin/env bash
-# CNPG backup posture for Option A (logical CronJobs) and legacy barman ScheduledBackup.
+# CNPG backup posture for Option A (logical dumps) and legacy barman ScheduledBackup.
 set -euo pipefail

 namespace="${CNPG_NAMESPACE:-databases}"
 # RPO target is 24h; allow 36h before last-success is considered stale.
 rpo_hours="${CNPG_BACKUP_RPO_HOURS:-36}"
 status_cm="${CNPG_OPTION_A_STATUS_CM:-cnpg-option-a-status}"
+schedule_cm="${CNPG_OPTION_A_SCHEDULE_CM:-cnpg-option-a-schedule}"
 expected_clusters="${CNPG_EXPECTED_CLUSTERS:-apps-pg,gitea-db,net-kingdom-pg,state-hub-db}"

 if ! kubectl get ns "$namespace" >/dev/null 2>&1; then
@ -18,7 +19,7 @@ context="$(kubectl config current-context 2>/dev/null || echo unknown)"
 server="$(kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}' 2>/dev/null || echo unknown)"
 echo "kube-context: ${context}"
 echo "api-server:   ${server}"
-echo "lane:         Option A (logical dump → age → Nextcloud) preferred; barman ScheduledBackup also accepted"
+echo "lane:         Option A (logical dump → age → Nextcloud); barman ScheduledBackup also accepted"
 echo "RPO window:   ${rpo_hours}h (last-success freshness)"
 echo

@ -31,6 +32,16 @@ if ((${#clusters[@]} == 0)); then
  exit 1
 fi

+# Schedule declaration (workstation-cron or in-cluster)
+schedule_mode=""
+if kubectl get configmap "$schedule_cm" -n "$namespace" >/dev/null 2>&1; then
+  schedule_mode="$(kubectl get configmap "$schedule_cm" -n "$namespace" -o jsonpath='{.data.mode}' 2>/dev/null || true)"
+  schedule_expr="$(kubectl get configmap "$schedule_cm" -n "$namespace" -o jsonpath='{.data.schedule}' 2>/dev/null || true)"
+  echo "schedule ConfigMap: ${schedule_cm} mode=${schedule_mode:-?} schedule=${schedule_expr:-?}"
+else
+  echo "schedule ConfigMap: ${schedule_cm} (missing)"
+fi
+
 # Load status ConfigMap timestamps (cluster -> ISO-ish stamp YYYYMMDDTHHMMSSZ)
 declare -A last_success=()
 if kubectl get configmap "$status_cm" -n "$namespace" >/dev/null 2>&1; then
@ -39,7 +50,7 @@ if kubectl get configmap "$status_cm" -n "$namespace" >/dev/null 2>&1; then
    key="${line%%=*}"
    val="${line#*=}"
    case "$key" in
-      lane|updated) continue ;;
+      lane|updated|schedule_source|mode) continue ;;
      *) last_success["$key"]="$val" ;;
    esac
  done < <(
@ -58,7 +69,6 @@ echo
 fresh_enough() {
  local ts="$1"
  [[ -z "$ts" ]] && return 1
-  # Accept YYYYMMDDTHHMMSSZ
  if [[ ! "$ts" =~ ^[0-9]{8}T[0-9]{6}Z$ ]]; then
    return 1
  fi
@ -78,6 +88,12 @@ degraded_detail=()

 IFS=',' read -r -a expected <<<"$expected_clusters"

+# Workstation schedule counts as a declared unattended plan for all expected clusters
+workstation_schedule=0
+if [[ "$schedule_mode" == "workstation-cron" ]]; then
+  workstation_schedule=1
+fi
+
 for cluster in "${clusters[@]}"; do
  if ! kubectl get cluster "$cluster" -n "$namespace" >/dev/null 2>&1; then
    echo "${cluster}: MISSING (Cluster CR not found)"
@ -100,24 +116,28 @@ for cluster in "${clusters[@]}"; do

  option_a_cron="$(kubectl get cronjob -n "$namespace" \
    -l "railiance.apps/backup-lane=option-a,railiance.apps/backup-cluster=${cluster}" \
-    -o jsonpath='{range .items[*]}{.metadata.name}{" "}{.spec.suspend}{" "}{end}' 2>/dev/null || true)"
+    -o jsonpath='{range .items[*]}{.metadata.name}{"|"}{.spec.suspend}{" "}{end}' 2>/dev/null || true)"

-  covered=0
+  schedule_ok=0
  schedule_note=""

  if [[ -n "$scheduled" ]]; then
-    covered=1
+    schedule_ok=1
    schedule_note="ScheduledBackup=${scheduled}"
+    echo "${cluster}: ScheduledBackup = ${scheduled}"
+  else
+    echo "${cluster}: ScheduledBackup = none"
  fi

  if [[ -n "$option_a_cron" ]]; then
-    # format: name suspend name suspend ...
-    read -r cj_name cj_suspend _ <<<"$option_a_cron"
+    cj_name="${option_a_cron%%|*}"
+    rest="${option_a_cron#*|}"
+    cj_suspend="${rest%% *}"
    if [[ "${cj_suspend}" == "true" ]]; then
-      echo "${cluster}: Option A CronJob = ${cj_name} (SUSPENDED)"
+      echo "${cluster}: Option A CronJob = ${cj_name} (suspended — workstation path expected)"
      schedule_note="${schedule_note:+$schedule_note; }OptionA=${cj_name}:suspended"
    else
-      covered=1
+      schedule_ok=1
      schedule_note="${schedule_note:+$schedule_note; }OptionA=${cj_name}"
      echo "${cluster}: Option A CronJob = ${cj_name}"
    fi
@ -125,48 +145,40 @@ for cluster in "${clusters[@]}"; do
    echo "${cluster}: Option A CronJob = none"
  fi

-  if [[ -n "$scheduled" ]]; then
-    echo "${cluster}: ScheduledBackup = ${scheduled}"
-  else
-    echo "${cluster}: ScheduledBackup = none"
+  if ((workstation_schedule)); then
+    schedule_ok=1
+    schedule_note="${schedule_note:+$schedule_note; }workstation-cron"
+    echo "${cluster}: schedule declaration = workstation-cron"
  fi

  ts="${last_success[$cluster]:-}"
+  success_ok=0
  if [[ -n "$ts" ]]; then
    if fresh_enough "$ts"; then
+      success_ok=1
      echo "${cluster}: last-success = ${ts} (fresh ≤${rpo_hours}h)"
    else
      echo "${cluster}: last-success = ${ts} (STALE >${rpo_hours}h)"
-      if ((covered)); then
-        degraded_detail+=("${cluster}:stale-success")
-        # scheduled but stale still counts as coverage with warning; mark degraded
-        covered=0
-      fi
+      degraded_detail+=("${cluster}:stale-success")
    fi
  else
    echo "${cluster}: last-success = none"
-    # Schedule present but never succeeded: still incomplete for healthy
-    if ((covered)); then
-      echo "${cluster}: note = schedule present but no recorded success yet"
-      covered=0
-      degraded_detail+=("${cluster}:no-success-yet")
-    fi
+    degraded_detail+=("${cluster}:no-success-yet")
  fi

  phase="$(kubectl get cluster "$cluster" -n "$namespace" -o jsonpath='{.status.phase}' 2>/dev/null || true)"
  instances="$(kubectl get cluster "$cluster" -n "$namespace" -o jsonpath='{.status.instances}' 2>/dev/null || true)"
  echo "${cluster}: phase=${phase:-unknown} instances=${instances:-unknown}"

-  if ((covered == 0)); then
-    missing=$((missing + 1))
-    echo "${cluster}: coverage = MISSING"
-  else
+  if ((schedule_ok && success_ok)); then
    echo "${cluster}: coverage = ok (${schedule_note})"
+  else
+    missing=$((missing + 1))
+    echo "${cluster}: coverage = MISSING (schedule_ok=${schedule_ok} success_ok=${success_ok})"
  fi
  echo
 done

-# Warn if expected clusters are absent from the cluster list
 for exp in "${expected[@]}"; do
  found=0
  for c in "${clusters[@]}"; do
@ -185,7 +197,7 @@ if ((missing > 0)); then
  fi
  echo "See docs/app-data-backup-restore-handoff.md and manifests/cnpg-option-a-backup.yaml"
  echo "Operator: make cnpg-backup-offsite-secret-apply && make cnpg-option-a-apply"
-  echo "Manual run: make cnpg-logical-backup (or DRY_RUN=1 make cnpg-logical-backup-dry-run)"
+  echo "Daily:    make cnpg-logical-backup   # or workstation cron at 02:30 UTC"
  exit 1
 fi