Add user-engine backup and rollback verification
This commit is contained in:
parent
92ae9103c7
commit
52b57c99f7
6 changed files with 316 additions and 1 deletions
121
sso-mfa/k8s/user-engine/backup.yaml
Normal file
121
sso-mfa/k8s/user-engine/backup.yaml
Normal file
|
|
@ -0,0 +1,121 @@
|
|||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: user-engine-backups
|
||||
namespace: user-engine
|
||||
labels:
|
||||
app.kubernetes.io/name: user-engine-backup
|
||||
app.kubernetes.io/part-of: user-engine
|
||||
spec:
|
||||
accessModes: [ReadWriteOnce]
|
||||
resources:
|
||||
requests: {storage: 1Gi}
|
||||
---
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: user-engine-backup
|
||||
namespace: user-engine
|
||||
labels:
|
||||
app.kubernetes.io/name: user-engine-backup
|
||||
app.kubernetes.io/part-of: user-engine
|
||||
spec:
|
||||
schedule: "0 4 * * *"
|
||||
concurrencyPolicy: Forbid
|
||||
successfulJobsHistoryLimit: 3
|
||||
failedJobsHistoryLimit: 3
|
||||
jobTemplate:
|
||||
spec:
|
||||
backoffLimit: 2
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: user-engine-backup
|
||||
app.kubernetes.io/part-of: user-engine
|
||||
spec:
|
||||
restartPolicy: Never
|
||||
automountServiceAccountToken: false
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 26
|
||||
runAsGroup: 26
|
||||
fsGroup: 26
|
||||
seccompProfile: {type: RuntimeDefault}
|
||||
containers:
|
||||
- name: backup
|
||||
image: ghcr.io/cloudnative-pg/postgresql:17.5
|
||||
imagePullPolicy: IfNotPresent
|
||||
env:
|
||||
- name: DATABASE_URL
|
||||
valueFrom:
|
||||
secretKeyRef: {name: user-engine-pg-app, key: uri}
|
||||
command:
|
||||
- /bin/bash
|
||||
- -ec
|
||||
- |
|
||||
umask 077
|
||||
stamp="$(date -u +%Y%m%dT%H%M%SZ)"
|
||||
target="/backup/user-engine-${stamp}.dump"
|
||||
for attempt in $(seq 1 30); do
|
||||
if pg_isready --dbname="${DATABASE_URL}" >/dev/null 2>&1; then
|
||||
break
|
||||
fi
|
||||
test "${attempt}" -lt 30
|
||||
sleep 2
|
||||
done
|
||||
pg_dump --format=custom --no-owner --no-acl \
|
||||
--file="${target}.partial" "${DATABASE_URL}"
|
||||
pg_restore --list "${target}.partial" >/dev/null
|
||||
mv "${target}.partial" "${target}"
|
||||
sha256sum "${target}" >"${target}.sha256"
|
||||
find /backup -type f -name 'user-engine-*.dump*' -mtime +7 -delete
|
||||
echo "backup_complete file=$(basename "${target}")"
|
||||
volumeMounts:
|
||||
- {name: backup, mountPath: /backup}
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities: {drop: ["ALL"]}
|
||||
resources:
|
||||
requests: {cpu: 25m, memory: 64Mi}
|
||||
limits: {cpu: 500m, memory: 256Mi}
|
||||
volumes:
|
||||
- name: backup
|
||||
persistentVolumeClaim: {claimName: user-engine-backups}
|
||||
---
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: user-engine-backup
|
||||
namespace: user-engine
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: user-engine-backup
|
||||
app.kubernetes.io/part-of: user-engine
|
||||
policyTypes: [Egress]
|
||||
egress:
|
||||
- to:
|
||||
- podSelector:
|
||||
matchLabels: {cnpg.io/cluster: user-engine-pg}
|
||||
ports: [{protocol: TCP, port: 5432}]
|
||||
- to:
|
||||
- namespaceSelector:
|
||||
matchLabels: {kubernetes.io/metadata.name: kube-system}
|
||||
ports: [{protocol: UDP, port: 53}, {protocol: TCP, port: 53}]
|
||||
---
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: user-engine-postgres-backup-ingress
|
||||
namespace: user-engine
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels: {cnpg.io/cluster: user-engine-pg}
|
||||
policyTypes: [Ingress]
|
||||
ingress:
|
||||
- from:
|
||||
- podSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: user-engine-backup
|
||||
app.kubernetes.io/part-of: user-engine
|
||||
ports: [{protocol: TCP, port: 5432}]
|
||||
62
sso-mfa/k8s/user-engine/restore-drill.yaml
Normal file
62
sso-mfa/k8s/user-engine/restore-drill.yaml
Normal file
|
|
@ -0,0 +1,62 @@
|
|||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: user-engine-restore-drill
|
||||
namespace: user-engine
|
||||
labels:
|
||||
app.kubernetes.io/name: user-engine-restore-drill
|
||||
app.kubernetes.io/part-of: user-engine
|
||||
spec:
|
||||
backoffLimit: 0
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: user-engine-restore-drill
|
||||
app.kubernetes.io/part-of: user-engine
|
||||
spec:
|
||||
restartPolicy: Never
|
||||
automountServiceAccountToken: false
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 26
|
||||
runAsGroup: 26
|
||||
fsGroup: 26
|
||||
seccompProfile: {type: RuntimeDefault}
|
||||
containers:
|
||||
- name: restore
|
||||
image: ghcr.io/cloudnative-pg/postgresql:17.5
|
||||
imagePullPolicy: IfNotPresent
|
||||
command:
|
||||
- /bin/bash
|
||||
- -ec
|
||||
- |
|
||||
latest="$(find /backup -type f -name 'user-engine-*.dump' | sort | tail -1)"
|
||||
test -n "${latest}"
|
||||
sha256sum --check "${latest}.sha256"
|
||||
initdb --no-locale --encoding=UTF8 -D /restore/data >/dev/null
|
||||
pg_ctl -D /restore/data -o '-k /tmp -p 55432' -w start >/dev/null
|
||||
trap 'pg_ctl -D /restore/data -m immediate stop >/dev/null' EXIT
|
||||
createdb -h /tmp -p 55432 user_engine_restore
|
||||
pg_restore --exit-on-error --no-owner --no-acl \
|
||||
-h /tmp -p 55432 -d user_engine_restore "${latest}"
|
||||
tables="$(psql -At -h /tmp -p 55432 -d user_engine_restore \
|
||||
-c "select count(*) from pg_catalog.pg_tables where schemaname='public'")"
|
||||
migrations="$(psql -At -h /tmp -p 55432 -d user_engine_restore \
|
||||
-c 'select count(*) from user_engine_schema_versions')"
|
||||
test "${tables}" -gt 0
|
||||
test "${migrations}" -gt 0
|
||||
echo "restore_complete tables=${tables} migrations=${migrations}"
|
||||
volumeMounts:
|
||||
- {name: backup, mountPath: /backup, readOnly: true}
|
||||
- {name: restore, mountPath: /restore}
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities: {drop: ["ALL"]}
|
||||
resources:
|
||||
requests: {cpu: 50m, memory: 128Mi}
|
||||
limits: {cpu: "1", memory: 512Mi}
|
||||
volumes:
|
||||
- name: backup
|
||||
persistentVolumeClaim: {claimName: user-engine-backups}
|
||||
- name: restore
|
||||
emptyDir: {}
|
||||
|
|
@ -45,7 +45,7 @@ spec:
|
|||
seccompProfile: {type: RuntimeDefault}
|
||||
containers:
|
||||
- name: portal
|
||||
image: user-engine:portal-746bf21
|
||||
image: user-engine:portal-2bcda7f
|
||||
imagePullPolicy: Never
|
||||
ports: [{name: http, containerPort: 8080}]
|
||||
env:
|
||||
|
|
@ -151,6 +151,12 @@ spec:
|
|||
- {protocol: TCP, port: 5432}
|
||||
- {protocol: TCP, port: 8000}
|
||||
- {protocol: TCP, port: 9187}
|
||||
- from:
|
||||
- podSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: user-engine-backup
|
||||
app.kubernetes.io/part-of: user-engine
|
||||
ports: [{protocol: TCP, port: 5432}]
|
||||
egress:
|
||||
# CNPG instance manager must read its Cluster resource during bootstrap.
|
||||
# K3s network policy evaluates the API service after DNAT on port 6443.
|
||||
|
|
|
|||
81
sso-mfa/k8s/user-engine/verify-operability.sh
Executable file
81
sso-mfa/k8s/user-engine/verify-operability.sh
Executable file
|
|
@ -0,0 +1,81 @@
|
|||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
namespace=user-engine
|
||||
deployment=user-engine
|
||||
manifest="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/runtime.yaml"
|
||||
exercise_rollback=false
|
||||
if [[ "${1:-}" == "--exercise-rollback" ]]; then
|
||||
exercise_rollback=true
|
||||
elif [[ $# -gt 0 ]]; then
|
||||
echo "usage: $0 [--exercise-rollback]" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
current_image="$(kubectl -n "$namespace" get deployment "$deployment" \
|
||||
-o jsonpath='{.spec.template.spec.containers[0].image}')"
|
||||
|
||||
restore_current() {
|
||||
if $exercise_rollback; then
|
||||
kubectl apply -f "$manifest" >/dev/null
|
||||
kubectl -n "$namespace" rollout status deployment/"$deployment" \
|
||||
--timeout=180s >/dev/null
|
||||
fi
|
||||
}
|
||||
trap restore_current EXIT
|
||||
|
||||
kubectl -n "$namespace" wait --for=condition=Ready \
|
||||
cluster/user-engine-pg --timeout=60s >/dev/null
|
||||
kubectl -n "$namespace" get cronjob user-engine-backup >/dev/null
|
||||
kubectl -n "$namespace" wait --for=condition=complete \
|
||||
job/user-engine-backup-manual-20260729b --timeout=30s >/dev/null
|
||||
kubectl -n "$namespace" wait --for=condition=complete \
|
||||
job/user-engine-restore-drill --timeout=30s >/dev/null
|
||||
|
||||
metrics="$(
|
||||
kubectl -n "$namespace" exec deployment/"$deployment" -- python3 -c \
|
||||
'import os,urllib.error,urllib.request
|
||||
u="http://127.0.0.1:8080/metrics"; denied=0
|
||||
try: urllib.request.urlopen(u)
|
||||
except urllib.error.HTTPError as error: denied=error.code
|
||||
request=urllib.request.Request(u,headers={"X-User-Engine-Proxy-Secret":os.environ["USER_ENGINE_PROXY_SECRET"].strip()})
|
||||
response=urllib.request.urlopen(request); body=response.read().decode()
|
||||
print("%s %s %s" % (denied, response.status, int("user_engine_ready 1" in body)))'
|
||||
)"
|
||||
[[ "$metrics" == "403 200 1" ]]
|
||||
|
||||
rollback_image=""
|
||||
rollback_restored=true
|
||||
if $exercise_rollback; then
|
||||
kubectl -n "$namespace" rollout undo deployment/"$deployment" >/dev/null
|
||||
kubectl -n "$namespace" rollout status deployment/"$deployment" \
|
||||
--timeout=180s >/dev/null
|
||||
rollback_image="$(kubectl -n "$namespace" get deployment "$deployment" \
|
||||
-o jsonpath='{.spec.template.spec.containers[0].image}')"
|
||||
[[ "$rollback_image" != "$current_image" ]]
|
||||
kubectl apply -f "$manifest" >/dev/null
|
||||
kubectl -n "$namespace" rollout status deployment/"$deployment" \
|
||||
--timeout=180s >/dev/null
|
||||
restored_image="$(kubectl -n "$namespace" get deployment "$deployment" \
|
||||
-o jsonpath='{.spec.template.spec.containers[0].image}')"
|
||||
[[ "$restored_image" == "$current_image" ]]
|
||||
fi
|
||||
|
||||
python3 - "$current_image" "$rollback_image" "$rollback_restored" <<'PY'
|
||||
import json
|
||||
import sys
|
||||
|
||||
print(json.dumps({
|
||||
"backup_schedule_present": True,
|
||||
"database_ready": True,
|
||||
"latest_backup_complete": True,
|
||||
"metrics_authorized": True,
|
||||
"metrics_unauthorized_denied": True,
|
||||
"restore_drill_complete": True,
|
||||
"rollback_exercised": bool(sys.argv[2]),
|
||||
"rollback_image": sys.argv[2] or None,
|
||||
"rollforward_image": sys.argv[1],
|
||||
"rollforward_restored": sys.argv[3] == "true",
|
||||
"secret_values_observed": False,
|
||||
}, sort_keys=True))
|
||||
PY
|
||||
Loading…
Add table
Add a link
Reference in a new issue