diff --git a/docs/evidence/user-engine-operability-2026-07-29.json b/docs/evidence/user-engine-operability-2026-07-29.json new file mode 100644 index 0000000..0a51358 --- /dev/null +++ b/docs/evidence/user-engine-operability-2026-07-29.json @@ -0,0 +1,24 @@ +{ + "captured_at": "2026-07-29T21:56:00Z", + "checks": { + "backup_archive_catalog_valid": true, + "backup_checksum_valid": true, + "backup_schedule_present": true, + "database_ready": true, + "metrics_authorized": true, + "metrics_unauthorized_denied": true, + "restore_drill_complete": true, + "restore_migration_rows": 1, + "restore_public_tables": 4, + "rollback_exercised": true, + "rollforward_restored": true + }, + "declared_image": "user-engine:portal-2bcda7f", + "residual_risks": [ + "Backup PVC uses the reef's single local node and is not an independent disaster-recovery copy.", + "The directly imported image still needs publication through the approved registry lane." + ], + "rollback_image": "user-engine:portal-746bf21", + "schema_version": "user-engine-operability-evidence/v1", + "secret_values_observed": false +} diff --git a/sso-mfa/k8s/user-engine/backup.yaml b/sso-mfa/k8s/user-engine/backup.yaml new file mode 100644 index 0000000..9e00fbc --- /dev/null +++ b/sso-mfa/k8s/user-engine/backup.yaml @@ -0,0 +1,121 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: user-engine-backups + namespace: user-engine + labels: + app.kubernetes.io/name: user-engine-backup + app.kubernetes.io/part-of: user-engine +spec: + accessModes: [ReadWriteOnce] + resources: + requests: {storage: 1Gi} +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: user-engine-backup + namespace: user-engine + labels: + app.kubernetes.io/name: user-engine-backup + app.kubernetes.io/part-of: user-engine +spec: + schedule: "0 4 * * *" + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + backoffLimit: 2 + template: + metadata: + labels: + app.kubernetes.io/name: user-engine-backup + app.kubernetes.io/part-of: user-engine + spec: + restartPolicy: Never + automountServiceAccountToken: false + securityContext: + runAsNonRoot: true + runAsUser: 26 + runAsGroup: 26 + fsGroup: 26 + seccompProfile: {type: RuntimeDefault} + containers: + - name: backup + image: ghcr.io/cloudnative-pg/postgresql:17.5 + imagePullPolicy: IfNotPresent + env: + - name: DATABASE_URL + valueFrom: + secretKeyRef: {name: user-engine-pg-app, key: uri} + command: + - /bin/bash + - -ec + - | + umask 077 + stamp="$(date -u +%Y%m%dT%H%M%SZ)" + target="/backup/user-engine-${stamp}.dump" + for attempt in $(seq 1 30); do + if pg_isready --dbname="${DATABASE_URL}" >/dev/null 2>&1; then + break + fi + test "${attempt}" -lt 30 + sleep 2 + done + pg_dump --format=custom --no-owner --no-acl \ + --file="${target}.partial" "${DATABASE_URL}" + pg_restore --list "${target}.partial" >/dev/null + mv "${target}.partial" "${target}" + sha256sum "${target}" >"${target}.sha256" + find /backup -type f -name 'user-engine-*.dump*' -mtime +7 -delete + echo "backup_complete file=$(basename "${target}")" + volumeMounts: + - {name: backup, mountPath: /backup} + securityContext: + allowPrivilegeEscalation: false + capabilities: {drop: ["ALL"]} + resources: + requests: {cpu: 25m, memory: 64Mi} + limits: {cpu: 500m, memory: 256Mi} + volumes: + - name: backup + persistentVolumeClaim: {claimName: user-engine-backups} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: user-engine-backup + namespace: user-engine +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: user-engine-backup + app.kubernetes.io/part-of: user-engine + policyTypes: [Egress] + egress: + - to: + - podSelector: + matchLabels: {cnpg.io/cluster: user-engine-pg} + ports: [{protocol: TCP, port: 5432}] + - to: + - namespaceSelector: + matchLabels: {kubernetes.io/metadata.name: kube-system} + ports: [{protocol: UDP, port: 53}, {protocol: TCP, port: 53}] +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: user-engine-postgres-backup-ingress + namespace: user-engine +spec: + podSelector: + matchLabels: {cnpg.io/cluster: user-engine-pg} + policyTypes: [Ingress] + ingress: + - from: + - podSelector: + matchLabels: + app.kubernetes.io/name: user-engine-backup + app.kubernetes.io/part-of: user-engine + ports: [{protocol: TCP, port: 5432}] diff --git a/sso-mfa/k8s/user-engine/restore-drill.yaml b/sso-mfa/k8s/user-engine/restore-drill.yaml new file mode 100644 index 0000000..bf0d862 --- /dev/null +++ b/sso-mfa/k8s/user-engine/restore-drill.yaml @@ -0,0 +1,62 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: user-engine-restore-drill + namespace: user-engine + labels: + app.kubernetes.io/name: user-engine-restore-drill + app.kubernetes.io/part-of: user-engine +spec: + backoffLimit: 0 + template: + metadata: + labels: + app.kubernetes.io/name: user-engine-restore-drill + app.kubernetes.io/part-of: user-engine + spec: + restartPolicy: Never + automountServiceAccountToken: false + securityContext: + runAsNonRoot: true + runAsUser: 26 + runAsGroup: 26 + fsGroup: 26 + seccompProfile: {type: RuntimeDefault} + containers: + - name: restore + image: ghcr.io/cloudnative-pg/postgresql:17.5 + imagePullPolicy: IfNotPresent + command: + - /bin/bash + - -ec + - | + latest="$(find /backup -type f -name 'user-engine-*.dump' | sort | tail -1)" + test -n "${latest}" + sha256sum --check "${latest}.sha256" + initdb --no-locale --encoding=UTF8 -D /restore/data >/dev/null + pg_ctl -D /restore/data -o '-k /tmp -p 55432' -w start >/dev/null + trap 'pg_ctl -D /restore/data -m immediate stop >/dev/null' EXIT + createdb -h /tmp -p 55432 user_engine_restore + pg_restore --exit-on-error --no-owner --no-acl \ + -h /tmp -p 55432 -d user_engine_restore "${latest}" + tables="$(psql -At -h /tmp -p 55432 -d user_engine_restore \ + -c "select count(*) from pg_catalog.pg_tables where schemaname='public'")" + migrations="$(psql -At -h /tmp -p 55432 -d user_engine_restore \ + -c 'select count(*) from user_engine_schema_versions')" + test "${tables}" -gt 0 + test "${migrations}" -gt 0 + echo "restore_complete tables=${tables} migrations=${migrations}" + volumeMounts: + - {name: backup, mountPath: /backup, readOnly: true} + - {name: restore, mountPath: /restore} + securityContext: + allowPrivilegeEscalation: false + capabilities: {drop: ["ALL"]} + resources: + requests: {cpu: 50m, memory: 128Mi} + limits: {cpu: "1", memory: 512Mi} + volumes: + - name: backup + persistentVolumeClaim: {claimName: user-engine-backups} + - name: restore + emptyDir: {} diff --git a/sso-mfa/k8s/user-engine/runtime.yaml b/sso-mfa/k8s/user-engine/runtime.yaml index 16480f1..082295e 100644 --- a/sso-mfa/k8s/user-engine/runtime.yaml +++ b/sso-mfa/k8s/user-engine/runtime.yaml @@ -45,7 +45,7 @@ spec: seccompProfile: {type: RuntimeDefault} containers: - name: portal - image: user-engine:portal-746bf21 + image: user-engine:portal-2bcda7f imagePullPolicy: Never ports: [{name: http, containerPort: 8080}] env: @@ -151,6 +151,12 @@ spec: - {protocol: TCP, port: 5432} - {protocol: TCP, port: 8000} - {protocol: TCP, port: 9187} + - from: + - podSelector: + matchLabels: + app.kubernetes.io/name: user-engine-backup + app.kubernetes.io/part-of: user-engine + ports: [{protocol: TCP, port: 5432}] egress: # CNPG instance manager must read its Cluster resource during bootstrap. # K3s network policy evaluates the API service after DNAT on port 6443. diff --git a/sso-mfa/k8s/user-engine/verify-operability.sh b/sso-mfa/k8s/user-engine/verify-operability.sh new file mode 100755 index 0000000..e138ccb --- /dev/null +++ b/sso-mfa/k8s/user-engine/verify-operability.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +set -euo pipefail + +namespace=user-engine +deployment=user-engine +manifest="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/runtime.yaml" +exercise_rollback=false +if [[ "${1:-}" == "--exercise-rollback" ]]; then + exercise_rollback=true +elif [[ $# -gt 0 ]]; then + echo "usage: $0 [--exercise-rollback]" >&2 + exit 2 +fi + +current_image="$(kubectl -n "$namespace" get deployment "$deployment" \ + -o jsonpath='{.spec.template.spec.containers[0].image}')" + +restore_current() { + if $exercise_rollback; then + kubectl apply -f "$manifest" >/dev/null + kubectl -n "$namespace" rollout status deployment/"$deployment" \ + --timeout=180s >/dev/null + fi +} +trap restore_current EXIT + +kubectl -n "$namespace" wait --for=condition=Ready \ + cluster/user-engine-pg --timeout=60s >/dev/null +kubectl -n "$namespace" get cronjob user-engine-backup >/dev/null +kubectl -n "$namespace" wait --for=condition=complete \ + job/user-engine-backup-manual-20260729b --timeout=30s >/dev/null +kubectl -n "$namespace" wait --for=condition=complete \ + job/user-engine-restore-drill --timeout=30s >/dev/null + +metrics="$( + kubectl -n "$namespace" exec deployment/"$deployment" -- python3 -c \ + 'import os,urllib.error,urllib.request +u="http://127.0.0.1:8080/metrics"; denied=0 +try: urllib.request.urlopen(u) +except urllib.error.HTTPError as error: denied=error.code +request=urllib.request.Request(u,headers={"X-User-Engine-Proxy-Secret":os.environ["USER_ENGINE_PROXY_SECRET"].strip()}) +response=urllib.request.urlopen(request); body=response.read().decode() +print("%s %s %s" % (denied, response.status, int("user_engine_ready 1" in body)))' +)" +[[ "$metrics" == "403 200 1" ]] + +rollback_image="" +rollback_restored=true +if $exercise_rollback; then + kubectl -n "$namespace" rollout undo deployment/"$deployment" >/dev/null + kubectl -n "$namespace" rollout status deployment/"$deployment" \ + --timeout=180s >/dev/null + rollback_image="$(kubectl -n "$namespace" get deployment "$deployment" \ + -o jsonpath='{.spec.template.spec.containers[0].image}')" + [[ "$rollback_image" != "$current_image" ]] + kubectl apply -f "$manifest" >/dev/null + kubectl -n "$namespace" rollout status deployment/"$deployment" \ + --timeout=180s >/dev/null + restored_image="$(kubectl -n "$namespace" get deployment "$deployment" \ + -o jsonpath='{.spec.template.spec.containers[0].image}')" + [[ "$restored_image" == "$current_image" ]] +fi + +python3 - "$current_image" "$rollback_image" "$rollback_restored" <<'PY' +import json +import sys + +print(json.dumps({ + "backup_schedule_present": True, + "database_ready": True, + "latest_backup_complete": True, + "metrics_authorized": True, + "metrics_unauthorized_denied": True, + "restore_drill_complete": True, + "rollback_exercised": bool(sys.argv[2]), + "rollback_image": sys.argv[2] or None, + "rollforward_image": sys.argv[1], + "rollforward_restored": sys.argv[3] == "true", + "secret_values_observed": False, +}, sort_keys=True)) +PY diff --git a/workplans/NK-WP-0023-user-engine-portal-platform-integration.md b/workplans/NK-WP-0023-user-engine-portal-platform-integration.md index 72b04d5..4c3797e 100644 --- a/workplans/NK-WP-0023-user-engine-portal-platform-integration.md +++ b/workplans/NK-WP-0023-user-engine-portal-platform-integration.md @@ -178,6 +178,22 @@ cluster, and the companion `identity-provisioner:dbf7cfd` deployment is healthy behind namespace-scoped credentials and default-deny policy. Registry publication, backup/restore evidence, metrics, and automated rollback remain. +2026-07-29 operability increment: deployed protected Prometheus metrics in +`user-engine:portal-2bcda7f`; an unmarked request is denied with 403 while the +trusted workload marker receives readiness plus 18 aggregate record series +without identity values. Added a daily atomic `pg_dump` schedule using the +existing CNPG application Secret, a restricted backup PVC, checksum/catalog +validation, and seven-day retention. The isolated restore drill verified the +latest archive in an ephemeral PostgreSQL instance with four public tables and +one migration row. `verify-operability.sh --exercise-rollback` rolled back to +`portal-746bf21`, verified readiness, and restored the declared image. +Machine-readable evidence is in +`docs/evidence/user-engine-operability-2026-07-29.json`. + +The remaining production gaps are registry publication and an independently +custodied/off-node backup copy; the current PVC shares reef-railiance's single +failure domain, so T05 remains in progress. + ## T06 - Prove role-scoped administration and failure safety ```task @@ -203,6 +219,11 @@ machine-readable drift detection, automated repair, compensation semantics, transient retry, and replay-safe deletion. The broader browser, authorization, provider-outage, and restore matrix remains open. +2026-07-29 operability evidence adds an isolated database restore and a +reversible deployment rollback/roll-forward cycle. Backup and metrics access +are secret-free in recorded evidence. Independent disaster-recovery custody +and the remaining browser/provider-outage matrix keep T06 open. + ## T07 - Complete KEY-WP-0004 through the reusable portal ```task