RAILIANCE-WP-0015: Option A CNPG logical backup coverage healthy

Materialize offsite Secret from OpenBao, deploy per-cluster CronJobs, generalize multi-cluster logical backup + status health for Option A, seed encrypted uploads and restore-drill evidence; workplan finished.
2026-07-22 18:00:48 +02:00 · 2026-07-22 18:00:48 +02:00 · 6635fdc976
commit 6635fdc976
parent 374ebed349
11 changed files with 929 additions and 69 deletions
--- a/manifests/cnpg-option-a-backup.yaml
+++ b/manifests/cnpg-option-a-backup.yaml
@ -0,0 +1,394 @@
+# Option A — age-encrypted logical dumps → Nextcloud WebDAV (RAILIANCE-WP-0015).
+#
+# Decided approach (not barman ScheduledBackup). Applies to CoulombCore
+# databases namespace clusters: apps-pg, gitea-db, net-kingdom-pg, state-hub-db.
+#
+# Prereqs:
+#   1. OpenBao lane CCR-2026-0004 resolvable
+#   2. make cnpg-backup-offsite-secret-apply   # Secret cnpg-backup-offsite
+#   3. kubectl apply -f manifests/cnpg-option-a-backup.yaml
+#
+# Schedule: daily 02:30 UTC (RPO 24h). Retention target 14 daily + 4 weekly
+# is enforced on the Nextcloud side / operator prune; local Job pods are ephemeral.
+#
+# Health: make cnpg-backup-status
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: cnpg-logical-backup
+  namespace: databases
+  labels:
+    app.kubernetes.io/part-of: railiance-apps
+    railiance.apps/backup-lane: option-a
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: cnpg-logical-backup
+  namespace: databases
+  labels:
+    app.kubernetes.io/part-of: railiance-apps
+    railiance.apps/backup-lane: option-a
+rules:
+  - apiGroups: [""]
+    resources: ["pods"]
+    verbs: ["get", "list"]
+  - apiGroups: [""]
+    resources: ["pods/exec"]
+    verbs: ["create"]
+  - apiGroups: [""]
+    resources: ["configmaps"]
+    verbs: ["get", "create", "update", "patch"]
+  - apiGroups: [""]
+    resources: ["secrets"]
+    resourceNames: ["cnpg-backup-offsite"]
+    verbs: ["get"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: cnpg-logical-backup
+  namespace: databases
+  labels:
+    app.kubernetes.io/part-of: railiance-apps
+    railiance.apps/backup-lane: option-a
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: cnpg-logical-backup
+subjects:
+  - kind: ServiceAccount
+    name: cnpg-logical-backup
+    namespace: databases
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: cnpg-option-a-runner
+  namespace: databases
+  labels:
+    app.kubernetes.io/part-of: railiance-apps
+    railiance.apps/backup-lane: option-a
+data:
+  run-backup.sh: |
+    #!/bin/sh
+    # In-cluster Option A runner. Env: CLUSTER, DATABASES (comma-separated).
+    set -eu
+    CLUSTER="${CLUSTER:?CLUSTER required}"
+    DATABASES="${DATABASES:?DATABASES required}"
+    NAMESPACE="${NAMESPACE:-databases}"
+    STATUS_CM="${STATUS_CM:-cnpg-option-a-status}"
+    AGE_BIN="${AGE_BIN:-/tmp/age/age}"
+    WORK=/tmp/backup-work
+    mkdir -p "$WORK" /tmp/age
+
+    if [ ! -x "$AGE_BIN" ]; then
+      echo "fetch age static binary"
+      curl -fsSL "https://github.com/FiloSottile/age/releases/download/v1.2.1/age-v1.2.1-linux-amd64.tar.gz" \
+        | tar -xz -C /tmp
+      # tarball extracts to /tmp/age/age
+      AGE_BIN=/tmp/age/age
+    fi
+
+    NC_TOKEN="$(cat /secrets/offsite/NC_WEBDAV_TOKEN)"
+    NC_URL="$(cat /secrets/offsite/NC_WEBDAV_URL)"
+    AGE_PUB="$(cat /secrets/offsite/AGE_PUBLIC_KEY)"
+    # filesdrop shares often need token-only URL form
+    case "$NC_URL" in
+      */) ;;
+      *) NC_URL="${NC_URL}/" ;;
+    esac
+
+    POD="$(kubectl get pods -n "$NAMESPACE" \
+      -l "cnpg.io/cluster=${CLUSTER},role=primary" \
+      -o jsonpath='{.items[0].metadata.name}')"
+    if [ -z "$POD" ]; then
+      echo "ERROR: primary pod not found for ${CLUSTER}" >&2
+      exit 1
+    fi
+    echo "cluster=${CLUSTER} pod=${POD}"
+
+    TS="$(date -u +%Y%m%dT%H%M%SZ)"
+    OLD_IFS=$IFS
+    IFS=,
+    for db in $DATABASES; do
+      IFS=$OLD_IFS
+      plain="${WORK}/${CLUSTER}-${db}-${TS}.dump"
+      aged="${plain}.age"
+      echo "dump ${db}"
+      kubectl exec -n "$NAMESPACE" "$POD" -c postgres -- \
+        pg_dump -U postgres -d "$db" -Fc --no-owner --no-acl >"$plain"
+      "$AGE_BIN" -r "$AGE_PUB" -o "$aged" "$plain"
+      rm -f "$plain"
+      remote="${CLUSTER}/${CLUSTER}-${db}-${TS}.dump.age"
+      dest="${NC_URL}${remote}"
+      echo "upload ${remote}"
+      curl -sf -u "${NC_TOKEN}:" -T "$aged" "$dest"
+      # weekly on Sunday UTC
+      if [ "$(date -u +%u)" = "7" ]; then
+        curl -sf -u "${NC_TOKEN}:" -T "$aged" \
+          "${NC_URL}${CLUSTER}/${CLUSTER}-${db}-weekly-${TS}.dump.age" || true
+      fi
+      rm -f "$aged"
+    done
+    IFS=$OLD_IFS
+
+    # Record last success for cnpg-backup-status
+    if kubectl get configmap "$STATUS_CM" -n "$NAMESPACE" >/dev/null 2>&1; then
+      kubectl patch configmap "$STATUS_CM" -n "$NAMESPACE" --type merge \
+        -p "{\"data\":{\"${CLUSTER}\":\"${TS}\",\"updated\":\"${TS}\",\"lane\":\"option-a\"}}"
+    else
+      kubectl create configmap "$STATUS_CM" -n "$NAMESPACE" \
+        --from-literal="${CLUSTER}=${TS}" \
+        --from-literal=updated="${TS}" \
+        --from-literal=lane=option-a
+    fi
+    kubectl label configmap "$STATUS_CM" -n "$NAMESPACE" \
+      railiance.apps/backup-lane=option-a --overwrite >/dev/null
+    echo "ok: ${CLUSTER} backup ${TS}"
+---
+# Shared pod template fields are repeated per CronJob for readability (no Kustomize).
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: cnpg-logical-backup-apps-pg
+  namespace: databases
+  labels:
+    app.kubernetes.io/part-of: railiance-apps
+    railiance.apps/backup-lane: option-a
+    railiance.apps/backup-cluster: apps-pg
+spec:
+  schedule: "30 2 * * *"
+  concurrencyPolicy: Forbid
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 3
+  jobTemplate:
+    spec:
+      backoffLimit: 1
+      activeDeadlineSeconds: 7200
+      template:
+        metadata:
+          labels:
+            railiance.apps/backup-lane: option-a
+            railiance.apps/backup-cluster: apps-pg
+        spec:
+          serviceAccountName: cnpg-logical-backup
+          restartPolicy: Never
+          containers:
+            - name: backup
+              image: alpine/k8s:1.31.12
+              imagePullPolicy: IfNotPresent
+              env:
+                - name: CLUSTER
+                  value: apps-pg
+                - name: DATABASES
+                  value: apps_meta,core_hub,core_hub_staging,vergabe_db
+                - name: NAMESPACE
+                  value: databases
+              command: ["/bin/sh", "/runner/run-backup.sh"]
+              volumeMounts:
+                - name: runner
+                  mountPath: /runner
+                  readOnly: true
+                - name: offsite
+                  mountPath: /secrets/offsite
+                  readOnly: true
+              resources:
+                requests:
+                  cpu: 100m
+                  memory: 256Mi
+                limits:
+                  cpu: "1"
+                  memory: 1Gi
+          volumes:
+            - name: runner
+              configMap:
+                name: cnpg-option-a-runner
+                defaultMode: 0755
+            - name: offsite
+              secret:
+                secretName: cnpg-backup-offsite
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: cnpg-logical-backup-gitea-db
+  namespace: databases
+  labels:
+    app.kubernetes.io/part-of: railiance-apps
+    railiance.apps/backup-lane: option-a
+    railiance.apps/backup-cluster: gitea-db
+spec:
+  schedule: "35 2 * * *"
+  concurrencyPolicy: Forbid
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 3
+  jobTemplate:
+    spec:
+      backoffLimit: 1
+      activeDeadlineSeconds: 7200
+      template:
+        metadata:
+          labels:
+            railiance.apps/backup-lane: option-a
+            railiance.apps/backup-cluster: gitea-db
+        spec:
+          serviceAccountName: cnpg-logical-backup
+          restartPolicy: Never
+          containers:
+            - name: backup
+              image: alpine/k8s:1.31.12
+              imagePullPolicy: IfNotPresent
+              env:
+                - name: CLUSTER
+                  value: gitea-db
+                - name: DATABASES
+                  value: gitea
+                - name: NAMESPACE
+                  value: databases
+              command: ["/bin/sh", "/runner/run-backup.sh"]
+              volumeMounts:
+                - name: runner
+                  mountPath: /runner
+                  readOnly: true
+                - name: offsite
+                  mountPath: /secrets/offsite
+                  readOnly: true
+              resources:
+                requests:
+                  cpu: 100m
+                  memory: 256Mi
+                limits:
+                  cpu: "1"
+                  memory: 1Gi
+          volumes:
+            - name: runner
+              configMap:
+                name: cnpg-option-a-runner
+                defaultMode: 0755
+            - name: offsite
+              secret:
+                secretName: cnpg-backup-offsite
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: cnpg-logical-backup-net-kingdom-pg
+  namespace: databases
+  labels:
+    app.kubernetes.io/part-of: railiance-apps
+    railiance.apps/backup-lane: option-a
+    railiance.apps/backup-cluster: net-kingdom-pg
+spec:
+  schedule: "40 2 * * *"
+  concurrencyPolicy: Forbid
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 3
+  jobTemplate:
+    spec:
+      backoffLimit: 1
+      activeDeadlineSeconds: 7200
+      template:
+        metadata:
+          labels:
+            railiance.apps/backup-lane: option-a
+            railiance.apps/backup-cluster: net-kingdom-pg
+        spec:
+          serviceAccountName: cnpg-logical-backup
+          restartPolicy: Never
+          containers:
+            - name: backup
+              image: alpine/k8s:1.31.12
+              imagePullPolicy: IfNotPresent
+              env:
+                - name: CLUSTER
+                  value: net-kingdom-pg
+                - name: DATABASES
+                  value: interhub,privacyidea_db
+                - name: NAMESPACE
+                  value: databases
+              command: ["/bin/sh", "/runner/run-backup.sh"]
+              volumeMounts:
+                - name: runner
+                  mountPath: /runner
+                  readOnly: true
+                - name: offsite
+                  mountPath: /secrets/offsite
+                  readOnly: true
+              resources:
+                requests:
+                  cpu: 100m
+                  memory: 256Mi
+                limits:
+                  cpu: "1"
+                  memory: 1Gi
+          volumes:
+            - name: runner
+              configMap:
+                name: cnpg-option-a-runner
+                defaultMode: 0755
+            - name: offsite
+              secret:
+                secretName: cnpg-backup-offsite
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: cnpg-logical-backup-state-hub-db
+  namespace: databases
+  labels:
+    app.kubernetes.io/part-of: railiance-apps
+    railiance.apps/backup-lane: option-a
+    railiance.apps/backup-cluster: state-hub-db
+spec:
+  schedule: "45 2 * * *"
+  concurrencyPolicy: Forbid
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 3
+  jobTemplate:
+    spec:
+      backoffLimit: 1
+      activeDeadlineSeconds: 7200
+      template:
+        metadata:
+          labels:
+            railiance.apps/backup-lane: option-a
+            railiance.apps/backup-cluster: state-hub-db
+        spec:
+          serviceAccountName: cnpg-logical-backup
+          restartPolicy: Never
+          containers:
+            - name: backup
+              image: alpine/k8s:1.31.12
+              imagePullPolicy: IfNotPresent
+              env:
+                - name: CLUSTER
+                  value: state-hub-db
+                - name: DATABASES
+                  value: state_hub
+                - name: NAMESPACE
+                  value: databases
+              command: ["/bin/sh", "/runner/run-backup.sh"]
+              volumeMounts:
+                - name: runner
+                  mountPath: /runner
+                  readOnly: true
+                - name: offsite
+                  mountPath: /secrets/offsite
+                  readOnly: true
+              resources:
+                requests:
+                  cpu: 100m
+                  memory: 256Mi
+                limits:
+                  cpu: "1"
+                  memory: 1Gi
+          volumes:
+            - name: runner
+              configMap:
+                name: cnpg-option-a-runner
+                defaultMode: 0755
+            - name: offsite
+              secret:
+                secretName: cnpg-backup-offsite