Add user-engine backup and rollback verification

sso-mfa/k8s/user-engine/verify-operability.sh

@@ -0,0 +1,81 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+namespace=user-engine
+deployment=user-engine
+manifest="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/runtime.yaml"
+exercise_rollback=false
+if [[ "${1:-}" == "--exercise-rollback" ]]; then
+  exercise_rollback=true
+elif [[ $# -gt 0 ]]; then
+  echo "usage: $0 [--exercise-rollback]" >&2
+  exit 2
+fi
+
+current_image="$(kubectl -n "$namespace" get deployment "$deployment" \
+  -o jsonpath='{.spec.template.spec.containers[0].image}')"
+
+restore_current() {
+  if $exercise_rollback; then
+    kubectl apply -f "$manifest" >/dev/null
+    kubectl -n "$namespace" rollout status deployment/"$deployment" \
+      --timeout=180s >/dev/null
+  fi
+}
+trap restore_current EXIT
+
+kubectl -n "$namespace" wait --for=condition=Ready \
+  cluster/user-engine-pg --timeout=60s >/dev/null
+kubectl -n "$namespace" get cronjob user-engine-backup >/dev/null
+kubectl -n "$namespace" wait --for=condition=complete \
+  job/user-engine-backup-manual-20260729b --timeout=30s >/dev/null
+kubectl -n "$namespace" wait --for=condition=complete \
+  job/user-engine-restore-drill --timeout=30s >/dev/null
+
+metrics="$(
+  kubectl -n "$namespace" exec deployment/"$deployment" -- python3 -c \
+    'import os,urllib.error,urllib.request
+u="http://127.0.0.1:8080/metrics"; denied=0
+try: urllib.request.urlopen(u)
+except urllib.error.HTTPError as error: denied=error.code
+request=urllib.request.Request(u,headers={"X-User-Engine-Proxy-Secret":os.environ["USER_ENGINE_PROXY_SECRET"].strip()})
+response=urllib.request.urlopen(request); body=response.read().decode()
+print("%s %s %s" % (denied, response.status, int("user_engine_ready 1" in body)))'
+)"
+[[ "$metrics" == "403 200 1" ]]
+
+rollback_image=""
+rollback_restored=true
+if $exercise_rollback; then
+  kubectl -n "$namespace" rollout undo deployment/"$deployment" >/dev/null
+  kubectl -n "$namespace" rollout status deployment/"$deployment" \
+    --timeout=180s >/dev/null
+  rollback_image="$(kubectl -n "$namespace" get deployment "$deployment" \
+    -o jsonpath='{.spec.template.spec.containers[0].image}')"
+  [[ "$rollback_image" != "$current_image" ]]
+  kubectl apply -f "$manifest" >/dev/null
+  kubectl -n "$namespace" rollout status deployment/"$deployment" \
+    --timeout=180s >/dev/null
+  restored_image="$(kubectl -n "$namespace" get deployment "$deployment" \
+    -o jsonpath='{.spec.template.spec.containers[0].image}')"
+  [[ "$restored_image" == "$current_image" ]]
+fi
+
+python3 - "$current_image" "$rollback_image" "$rollback_restored" <<'PY'
+import json
+import sys
+
+print(json.dumps({
+    "backup_schedule_present": True,
+    "database_ready": True,
+    "latest_backup_complete": True,
+    "metrics_authorized": True,
+    "metrics_unauthorized_denied": True,
+    "restore_drill_complete": True,
+    "rollback_exercised": bool(sys.argv[2]),
+    "rollback_image": sys.argv[2] or None,
+    "rollforward_image": sys.argv[1],
+    "rollforward_restored": sys.argv[3] == "true",
+    "secret_values_observed": False,
+}, sort_keys=True))
+PY