83 lines
3.1 KiB
Bash
Executable file
83 lines
3.1 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
namespace=user-engine
|
|
deployment=user-engine
|
|
manifest="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/runtime.yaml"
|
|
rollback_target="${USER_ENGINE_ROLLBACK_IMAGE:-forgejo.coulomb.social/coulomb/user-engine@sha256:d25037e61c0c71dd9fbf89f653797932061b0be1fc5b4b23223821e5c2e3d6d0}"
|
|
exercise_rollback=false
|
|
if [[ "${1:-}" == "--exercise-rollback" ]]; then
|
|
exercise_rollback=true
|
|
elif [[ $# -gt 0 ]]; then
|
|
echo "usage: $0 [--exercise-rollback]" >&2
|
|
exit 2
|
|
fi
|
|
|
|
current_image="$(kubectl -n "$namespace" get deployment "$deployment" \
|
|
-o jsonpath='{.spec.template.spec.containers[0].image}')"
|
|
|
|
restore_current() {
|
|
if $exercise_rollback; then
|
|
kubectl apply -f "$manifest" >/dev/null
|
|
kubectl -n "$namespace" rollout status deployment/"$deployment" \
|
|
--timeout=180s >/dev/null
|
|
fi
|
|
}
|
|
trap restore_current EXIT
|
|
|
|
kubectl -n "$namespace" wait --for=condition=Ready \
|
|
cluster/user-engine-pg --timeout=60s >/dev/null
|
|
kubectl -n "$namespace" get cronjob user-engine-backup >/dev/null
|
|
kubectl -n "$namespace" wait --for=condition=complete \
|
|
job/user-engine-backup-manual-20260729b --timeout=30s >/dev/null
|
|
kubectl -n "$namespace" wait --for=condition=complete \
|
|
job/user-engine-restore-drill --timeout=30s >/dev/null
|
|
|
|
metrics="$(
|
|
kubectl -n "$namespace" exec deployment/"$deployment" -- python3 -c \
|
|
'import os,urllib.error,urllib.request
|
|
u="http://127.0.0.1:8080/metrics"; denied=0
|
|
try: urllib.request.urlopen(u)
|
|
except urllib.error.HTTPError as error: denied=error.code
|
|
request=urllib.request.Request(u,headers={"X-User-Engine-Proxy-Secret":os.environ["USER_ENGINE_PROXY_SECRET"].strip()})
|
|
response=urllib.request.urlopen(request); body=response.read().decode()
|
|
print("%s %s %s" % (denied, response.status, int("user_engine_ready 1" in body)))'
|
|
)"
|
|
[[ "$metrics" == "403 200 1" ]]
|
|
|
|
rollback_image=""
|
|
rollback_restored=true
|
|
if $exercise_rollback; then
|
|
kubectl -n "$namespace" set image deployment/"$deployment" \
|
|
"portal=${rollback_target}" >/dev/null
|
|
kubectl -n "$namespace" rollout status deployment/"$deployment" \
|
|
--timeout=180s >/dev/null
|
|
rollback_image="$(kubectl -n "$namespace" get deployment "$deployment" \
|
|
-o jsonpath='{.spec.template.spec.containers[0].image}')"
|
|
[[ "$rollback_image" == "$rollback_target" ]]
|
|
kubectl apply -f "$manifest" >/dev/null
|
|
kubectl -n "$namespace" rollout status deployment/"$deployment" \
|
|
--timeout=180s >/dev/null
|
|
restored_image="$(kubectl -n "$namespace" get deployment "$deployment" \
|
|
-o jsonpath='{.spec.template.spec.containers[0].image}')"
|
|
[[ "$restored_image" == "$current_image" ]]
|
|
fi
|
|
|
|
python3 - "$current_image" "$rollback_image" "$rollback_restored" <<'PY'
|
|
import json
|
|
import sys
|
|
|
|
print(json.dumps({
|
|
"backup_schedule_present": True,
|
|
"database_ready": True,
|
|
"latest_backup_complete": True,
|
|
"metrics_authorized": True,
|
|
"metrics_unauthorized_denied": True,
|
|
"restore_drill_complete": True,
|
|
"rollback_exercised": bool(sys.argv[2]),
|
|
"rollback_image": sys.argv[2] or None,
|
|
"rollforward_image": sys.argv[1],
|
|
"rollforward_restored": sys.argv[3] == "true",
|
|
"secret_values_observed": False,
|
|
}, sort_keys=True))
|
|
PY
|