Add user-engine backup and rollback verification

docs/evidence/user-engine-operability-2026-07-29.json

@@ -0,0 +1,24 @@
+{
+  "captured_at": "2026-07-29T21:56:00Z",
+  "checks": {
+    "backup_archive_catalog_valid": true,
+    "backup_checksum_valid": true,
+    "backup_schedule_present": true,
+    "database_ready": true,
+    "metrics_authorized": true,
+    "metrics_unauthorized_denied": true,
+    "restore_drill_complete": true,
+    "restore_migration_rows": 1,
+    "restore_public_tables": 4,
+    "rollback_exercised": true,
+    "rollforward_restored": true
+  },
+  "declared_image": "user-engine:portal-2bcda7f",
+  "residual_risks": [
+    "Backup PVC uses the reef's single local node and is not an independent disaster-recovery copy.",
+    "The directly imported image still needs publication through the approved registry lane."
+  ],
+  "rollback_image": "user-engine:portal-746bf21",
+  "schema_version": "user-engine-operability-evidence/v1",
+  "secret_values_observed": false
+}

sso-mfa/k8s/user-engine/backup.yaml

@@ -0,0 +1,121 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: user-engine-backups
+  namespace: user-engine
+  labels:
+    app.kubernetes.io/name: user-engine-backup
+    app.kubernetes.io/part-of: user-engine
+spec:
+  accessModes: [ReadWriteOnce]
+  resources:
+    requests: {storage: 1Gi}
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: user-engine-backup
+  namespace: user-engine
+  labels:
+    app.kubernetes.io/name: user-engine-backup
+    app.kubernetes.io/part-of: user-engine
+spec:
+  schedule: "0 4 * * *"
+  concurrencyPolicy: Forbid
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 3
+  jobTemplate:
+    spec:
+      backoffLimit: 2
+      template:
+        metadata:
+          labels:
+            app.kubernetes.io/name: user-engine-backup
+            app.kubernetes.io/part-of: user-engine
+        spec:
+          restartPolicy: Never
+          automountServiceAccountToken: false
+          securityContext:
+            runAsNonRoot: true
+            runAsUser: 26
+            runAsGroup: 26
+            fsGroup: 26
+            seccompProfile: {type: RuntimeDefault}
+          containers:
+            - name: backup
+              image: ghcr.io/cloudnative-pg/postgresql:17.5
+              imagePullPolicy: IfNotPresent
+              env:
+                - name: DATABASE_URL
+                  valueFrom:
+                    secretKeyRef: {name: user-engine-pg-app, key: uri}
+              command:
+                - /bin/bash
+                - -ec
+                - |
+                  umask 077
+                  stamp="$(date -u +%Y%m%dT%H%M%SZ)"
+                  target="/backup/user-engine-${stamp}.dump"
+                  for attempt in $(seq 1 30); do
+                    if pg_isready --dbname="${DATABASE_URL}" >/dev/null 2>&1; then
+                      break
+                    fi
+                    test "${attempt}" -lt 30
+                    sleep 2
+                  done
+                  pg_dump --format=custom --no-owner --no-acl \
+                    --file="${target}.partial" "${DATABASE_URL}"
+                  pg_restore --list "${target}.partial" >/dev/null
+                  mv "${target}.partial" "${target}"
+                  sha256sum "${target}" >"${target}.sha256"
+                  find /backup -type f -name 'user-engine-*.dump*' -mtime +7 -delete
+                  echo "backup_complete file=$(basename "${target}")"
+              volumeMounts:
+                - {name: backup, mountPath: /backup}
+              securityContext:
+                allowPrivilegeEscalation: false
+                capabilities: {drop: ["ALL"]}
+              resources:
+                requests: {cpu: 25m, memory: 64Mi}
+                limits: {cpu: 500m, memory: 256Mi}
+          volumes:
+            - name: backup
+              persistentVolumeClaim: {claimName: user-engine-backups}
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: user-engine-backup
+  namespace: user-engine
+spec:
+  podSelector:
+    matchLabels:
+      app.kubernetes.io/name: user-engine-backup
+      app.kubernetes.io/part-of: user-engine
+  policyTypes: [Egress]
+  egress:
+    - to:
+        - podSelector:
+            matchLabels: {cnpg.io/cluster: user-engine-pg}
+      ports: [{protocol: TCP, port: 5432}]
+    - to:
+        - namespaceSelector:
+            matchLabels: {kubernetes.io/metadata.name: kube-system}
+      ports: [{protocol: UDP, port: 53}, {protocol: TCP, port: 53}]
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: user-engine-postgres-backup-ingress
+  namespace: user-engine
+spec:
+  podSelector:
+    matchLabels: {cnpg.io/cluster: user-engine-pg}
+  policyTypes: [Ingress]
+  ingress:
+    - from:
+        - podSelector:
+            matchLabels:
+              app.kubernetes.io/name: user-engine-backup
+              app.kubernetes.io/part-of: user-engine
+      ports: [{protocol: TCP, port: 5432}]

sso-mfa/k8s/user-engine/restore-drill.yaml

@@ -0,0 +1,62 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: user-engine-restore-drill
+  namespace: user-engine
+  labels:
+    app.kubernetes.io/name: user-engine-restore-drill
+    app.kubernetes.io/part-of: user-engine
+spec:
+  backoffLimit: 0
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: user-engine-restore-drill
+        app.kubernetes.io/part-of: user-engine
+    spec:
+      restartPolicy: Never
+      automountServiceAccountToken: false
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 26
+        runAsGroup: 26
+        fsGroup: 26
+        seccompProfile: {type: RuntimeDefault}
+      containers:
+        - name: restore
+          image: ghcr.io/cloudnative-pg/postgresql:17.5
+          imagePullPolicy: IfNotPresent
+          command:
+            - /bin/bash
+            - -ec
+            - |
+              latest="$(find /backup -type f -name 'user-engine-*.dump' | sort | tail -1)"
+              test -n "${latest}"
+              sha256sum --check "${latest}.sha256"
+              initdb --no-locale --encoding=UTF8 -D /restore/data >/dev/null
+              pg_ctl -D /restore/data -o '-k /tmp -p 55432' -w start >/dev/null
+              trap 'pg_ctl -D /restore/data -m immediate stop >/dev/null' EXIT
+              createdb -h /tmp -p 55432 user_engine_restore
+              pg_restore --exit-on-error --no-owner --no-acl \
+                -h /tmp -p 55432 -d user_engine_restore "${latest}"
+              tables="$(psql -At -h /tmp -p 55432 -d user_engine_restore \
+                -c "select count(*) from pg_catalog.pg_tables where schemaname='public'")"
+              migrations="$(psql -At -h /tmp -p 55432 -d user_engine_restore \
+                -c 'select count(*) from user_engine_schema_versions')"
+              test "${tables}" -gt 0
+              test "${migrations}" -gt 0
+              echo "restore_complete tables=${tables} migrations=${migrations}"
+          volumeMounts:
+            - {name: backup, mountPath: /backup, readOnly: true}
+            - {name: restore, mountPath: /restore}
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities: {drop: ["ALL"]}
+          resources:
+            requests: {cpu: 50m, memory: 128Mi}
+            limits: {cpu: "1", memory: 512Mi}
+      volumes:
+        - name: backup
+          persistentVolumeClaim: {claimName: user-engine-backups}
+        - name: restore
+          emptyDir: {}

sso-mfa/k8s/user-engine/runtime.yaml

@@ -45,7 +45,7 @@ spec:
         seccompProfile: {type: RuntimeDefault}
       containers:
         - name: portal
-          image: user-engine:portal-746bf21
+          image: user-engine:portal-2bcda7f
           imagePullPolicy: Never
           ports: [{name: http, containerPort: 8080}]
           env:
@@ -151,6 +151,12 @@ spec:
         - {protocol: TCP, port: 5432}
         - {protocol: TCP, port: 8000}
         - {protocol: TCP, port: 9187}
+    - from:
+        - podSelector:
+            matchLabels:
+              app.kubernetes.io/name: user-engine-backup
+              app.kubernetes.io/part-of: user-engine
+      ports: [{protocol: TCP, port: 5432}]
   egress:
     # CNPG instance manager must read its Cluster resource during bootstrap.
     # K3s network policy evaluates the API service after DNAT on port 6443.

sso-mfa/k8s/user-engine/verify-operability.sh

@@ -0,0 +1,81 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+namespace=user-engine
+deployment=user-engine
+manifest="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/runtime.yaml"
+exercise_rollback=false
+if [[ "${1:-}" == "--exercise-rollback" ]]; then
+  exercise_rollback=true
+elif [[ $# -gt 0 ]]; then
+  echo "usage: $0 [--exercise-rollback]" >&2
+  exit 2
+fi
+
+current_image="$(kubectl -n "$namespace" get deployment "$deployment" \
+  -o jsonpath='{.spec.template.spec.containers[0].image}')"
+
+restore_current() {
+  if $exercise_rollback; then
+    kubectl apply -f "$manifest" >/dev/null
+    kubectl -n "$namespace" rollout status deployment/"$deployment" \
+      --timeout=180s >/dev/null
+  fi
+}
+trap restore_current EXIT
+
+kubectl -n "$namespace" wait --for=condition=Ready \
+  cluster/user-engine-pg --timeout=60s >/dev/null
+kubectl -n "$namespace" get cronjob user-engine-backup >/dev/null
+kubectl -n "$namespace" wait --for=condition=complete \
+  job/user-engine-backup-manual-20260729b --timeout=30s >/dev/null
+kubectl -n "$namespace" wait --for=condition=complete \
+  job/user-engine-restore-drill --timeout=30s >/dev/null
+
+metrics="$(
+  kubectl -n "$namespace" exec deployment/"$deployment" -- python3 -c \
+    'import os,urllib.error,urllib.request
+u="http://127.0.0.1:8080/metrics"; denied=0
+try: urllib.request.urlopen(u)
+except urllib.error.HTTPError as error: denied=error.code
+request=urllib.request.Request(u,headers={"X-User-Engine-Proxy-Secret":os.environ["USER_ENGINE_PROXY_SECRET"].strip()})
+response=urllib.request.urlopen(request); body=response.read().decode()
+print("%s %s %s" % (denied, response.status, int("user_engine_ready 1" in body)))'
+)"
+[[ "$metrics" == "403 200 1" ]]
+
+rollback_image=""
+rollback_restored=true
+if $exercise_rollback; then
+  kubectl -n "$namespace" rollout undo deployment/"$deployment" >/dev/null
+  kubectl -n "$namespace" rollout status deployment/"$deployment" \
+    --timeout=180s >/dev/null
+  rollback_image="$(kubectl -n "$namespace" get deployment "$deployment" \
+    -o jsonpath='{.spec.template.spec.containers[0].image}')"
+  [[ "$rollback_image" != "$current_image" ]]
+  kubectl apply -f "$manifest" >/dev/null
+  kubectl -n "$namespace" rollout status deployment/"$deployment" \
+    --timeout=180s >/dev/null
+  restored_image="$(kubectl -n "$namespace" get deployment "$deployment" \
+    -o jsonpath='{.spec.template.spec.containers[0].image}')"
+  [[ "$restored_image" == "$current_image" ]]
+fi
+
+python3 - "$current_image" "$rollback_image" "$rollback_restored" <<'PY'
+import json
+import sys
+
+print(json.dumps({
+    "backup_schedule_present": True,
+    "database_ready": True,
+    "latest_backup_complete": True,
+    "metrics_authorized": True,
+    "metrics_unauthorized_denied": True,
+    "restore_drill_complete": True,
+    "rollback_exercised": bool(sys.argv[2]),
+    "rollback_image": sys.argv[2] or None,
+    "rollforward_image": sys.argv[1],
+    "rollforward_restored": sys.argv[3] == "true",
+    "secret_values_observed": False,
+}, sort_keys=True))
+PY

workplans/NK-WP-0023-user-engine-portal-platform-integration.md

@@ -178,6 +178,22 @@ cluster, and the companion `identity-provisioner:dbf7cfd` deployment is
 healthy behind namespace-scoped credentials and default-deny policy. Registry
 publication, backup/restore evidence, metrics, and automated rollback remain.
 
+2026-07-29 operability increment: deployed protected Prometheus metrics in
+`user-engine:portal-2bcda7f`; an unmarked request is denied with 403 while the
+trusted workload marker receives readiness plus 18 aggregate record series
+without identity values. Added a daily atomic `pg_dump` schedule using the
+existing CNPG application Secret, a restricted backup PVC, checksum/catalog
+validation, and seven-day retention. The isolated restore drill verified the
+latest archive in an ephemeral PostgreSQL instance with four public tables and
+one migration row. `verify-operability.sh --exercise-rollback` rolled back to
+`portal-746bf21`, verified readiness, and restored the declared image.
+Machine-readable evidence is in
+`docs/evidence/user-engine-operability-2026-07-29.json`.
+
+The remaining production gaps are registry publication and an independently
+custodied/off-node backup copy; the current PVC shares reef-railiance's single
+failure domain, so T05 remains in progress.
+
 ## T06 - Prove role-scoped administration and failure safety
 
 ```task
@@ -203,6 +219,11 @@ machine-readable drift detection, automated repair, compensation semantics,
 transient retry, and replay-safe deletion. The broader browser, authorization,
 provider-outage, and restore matrix remains open.
 
+2026-07-29 operability evidence adds an isolated database restore and a
+reversible deployment rollback/roll-forward cycle. Backup and metrics access
+are secret-free in recorded evidence. Independent disaster-recovery custody
+and the remaining browser/provider-outage matrix keep T06 open.
+
 ## T07 - Complete KEY-WP-0004 through the reusable portal
 
 ```task