#!/usr/bin/env bash # Rotate every credential class formerly embedded in sso/keycape-config without # rendering a Secret value. This is an emergency, deliberate-invalidation path. set -euo pipefail MODE="${1:-}" if [[ "$MODE" != "--preflight" && "$MODE" != "--execute" ]]; then echo "usage: $0 --preflight|--execute" >&2 exit 2 fi if [[ "$MODE" == "--execute" && "${KEYCAPE_ACKNOWLEDGE_SESSION_INVALIDATION:-}" != "1" ]]; then echo "refusing execution without KEYCAPE_ACKNOWLEDGE_SESSION_INVALIDATION=1" >&2 exit 2 fi for tool in age base64 curl jq kubectl openssl python3 sha256sum tar; do command -v "$tool" >/dev/null || { echo "missing required tool: $tool" >&2 exit 1 } done python3 -c 'import bcrypt' 2>/dev/null || { echo "missing required Python bcrypt module" >&2 exit 1 } ROOT_DIR="$(cd "$(dirname "$0")/.." && pwd)" RECIPIENT_FILE="${NETKINGDOM_AGE_RECIPIENT_FILE:-/home/worsch/net-kingdom/keys/age.pub}" ARCHIVE_FILE="${KEYCAPE_RECOVERY_ARCHIVE:-$ROOT_DIR/history/KEY-WP-0011-rotated-credentials-2026-08-23.tar.age}" [[ -s "$RECIPIENT_FILE" ]] || { echo "age recipient file not found: $RECIPIENT_FILE" >&2 exit 1 } required_secrets=( "sso/keycape-config" "sso/keycape-pi-token" "sso/authelia-secrets" "sso/lldap-secrets" "mfa/privacyidea-config" ) required_deployments=( "sso/keycape" "sso/authelia" "sso/lldap" "mfa/privacyidea" ) for ref in "${required_secrets[@]}"; do namespace="${ref%%/*}" name="${ref#*/}" kubectl get secret "$name" -n "$namespace" -o name >/dev/null done for ref in "${required_deployments[@]}"; do namespace="${ref%%/*}" name="${ref#*/}" kubectl get deployment "$name" -n "$namespace" -o name >/dev/null done jwks_fingerprint() { jwks_uri="$(curl -fsS https://kc.coulomb.social/.well-known/openid-configuration | jq -er '.jwks_uri')" curl -fsS "$jwks_uri" | jq -cS '.keys | map({alg,e,kid,kty,n,use})' | sha256sum | awk '{print $1}' } before_jwks="$(jwks_fingerprint)" before_keycape_rv="$(kubectl get secret keycape-config -n sso -o jsonpath='{.metadata.resourceVersion}')" before_pi_rv="$(kubectl get secret privacyidea-config -n mfa -o jsonpath='{.metadata.resourceVersion}')" before_lldap_rv="$(kubectl get secret lldap-secrets -n sso -o jsonpath='{.metadata.resourceVersion}')" before_authelia_rv="$(kubectl get secret authelia-secrets -n sso -o jsonpath='{.metadata.resourceVersion}')" echo "preflight: deployments and Secrets exist" echo "preflight: current public JWKS fingerprint: $before_jwks" echo "preflight: current resource versions: keycape=$before_keycape_rv privacyidea=$before_pi_rv lldap=$before_lldap_rv authelia=$before_authelia_rv" if [[ "$MODE" == "--preflight" ]]; then exit 0 fi if [[ -e "$ARCHIVE_FILE" ]]; then echo "refusing to overwrite recovery archive: $ARCHIVE_FILE" >&2 exit 1 fi umask 077 work_dir="$(mktemp -d /tmp/keycape-recovery.XXXXXX)" port_forward_pids=() lldap_force_reset_active=false cleanup() { status=$? for pid in "${port_forward_pids[@]:-}"; do kill "$pid" 2>/dev/null || true wait "$pid" 2>/dev/null || true done if [[ "$lldap_force_reset_active" == "true" ]]; then kubectl set env deployment/lldap -n sso LLDAP_FORCE_LDAP_USER_PASS_RESET- >/dev/null 2>&1 || true fi case "$work_dir" in /tmp/keycape-recovery.*) find "$work_dir" -type f -exec shred -u -- {} + 2>/dev/null || true rm -rf -- "$work_dir" ;; esac exit "$status" } trap cleanup EXIT INT TERM openssl rand -hex 32 >"$work_dir/lldap-bind-password" openssl rand -hex 32 >"$work_dir/authelia-keycape-client-secret" openssl rand -hex 32 >"$work_dir/privacyidea-secret-key" openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 -out "$work_dir/key.pem" 2>/dev/null python3 -c ' import bcrypt import sys secret = sys.stdin.buffer.read().rstrip(b"\n") sys.stdout.buffer.write(bcrypt.hashpw(secret, bcrypt.gensalt(rounds=12))) ' <"$work_dir/authelia-keycape-client-secret" >"$work_dir/authelia-keycape-client-secret.bcrypt" lldap_password="$(tr -d '\n' <"$work_dir/lldap-bind-password")" authelia_secret="$(tr -d '\n' <"$work_dir/authelia-keycape-client-secret")" cat >"$work_dir/config.yaml" <"$work_dir/README.txt" tar -C "$work_dir" -cf "$work_dir/recovery.tar" README.txt config.yaml key.pem lldap-bind-password authelia-keycape-client-secret privacyidea-secret-key age -R "$RECIPIENT_FILE" -o "$ARCHIVE_FILE" "$work_dir/recovery.tar" chmod 600 "$ARCHIVE_FILE" echo "custody: encrypted recovery overlay created at $ARCHIVE_FILE" patch_field() { namespace="$1" secret_name="$2" field_name="$3" value_file="$4" patch_file="$work_dir/patch-${namespace}-${secret_name}-${field_name}.json" encoded="$(base64 -w0 <"$value_file")" printf '{"data":{"%s":"%s"}}' "$field_name" "$encoded" >"$patch_file" unset encoded kubectl patch secret "$secret_name" -n "$namespace" --type=merge --patch-file "$patch_file" >/dev/null shred -u -- "$patch_file" } echo "rotation: invalidating the exposed privacyIDEA admin JWT" patch_field mfa privacyidea-config PI_SECRET_KEY "$work_dir/privacyidea-secret-key" printf '%s' 'ROTATED_DISABLED_ADMIN_TOKEN' >"$work_dir/pi-token-disabled" patch_field sso keycape-pi-token token "$work_dir/pi-token-disabled" kubectl rollout restart deployment/privacyidea -n mfa >/dev/null kubectl rollout status deployment/privacyidea -n mfa --timeout=180s echo "rotation: resetting the LLDAP admin/bind credential" patch_field sso lldap-secrets LLDAP_LDAP_USER_PASS "$work_dir/lldap-bind-password" kubectl set env deployment/lldap -n sso LLDAP_FORCE_LDAP_USER_PASS_RESET=always >/dev/null lldap_force_reset_active=true kubectl rollout status deployment/lldap -n sso --timeout=180s kubectl port-forward -n sso service/lldap 11717:17170 >"$work_dir/lldap-port-forward.log" 2>&1 & port_forward_pids+=("$!") for _ in {1..30}; do curl -fsS -o /dev/null http://127.0.0.1:11717/health >/dev/null 2>&1 && break sleep 1 done printf '{"username":"admin","password":"%s"}' "$(tr -d '\n' <"$work_dir/lldap-bind-password")" >"$work_dir/lldap-login.json" lldap_status="$(curl -sS -o /dev/null -w '%{http_code}' -H 'Content-Type: application/json' --data-binary @"$work_dir/lldap-login.json" http://127.0.0.1:11717/auth/simple/login)" [[ "$lldap_status" == "200" ]] || { echo "LLDAP new-credential verification failed with HTTP $lldap_status" >&2 exit 1 } echo "verification: LLDAP accepted the replacement bind credential" kill "${port_forward_pids[-1]}" 2>/dev/null || true wait "${port_forward_pids[-1]}" 2>/dev/null || true unset 'port_forward_pids[-1]' kubectl set env deployment/lldap -n sso LLDAP_FORCE_LDAP_USER_PASS_RESET- >/dev/null lldap_force_reset_active=false kubectl rollout status deployment/lldap -n sso --timeout=180s echo "rotation: restarting identity-provisioner consumer" kubectl rollout restart deployment/identity-provisioner -n sso >/dev/null kubectl rollout status deployment/identity-provisioner -n sso --timeout=180s echo "rotation: updating Authelia and KeyCape consumers" authelia_patch="$work_dir/patch-authelia.json" ldap_b64="$(base64 -w0 <"$work_dir/lldap-bind-password")" client_hash_b64="$(base64 -w0 <"$work_dir/authelia-keycape-client-secret.bcrypt")" printf '{"data":{"ldap_password":"%s","keycape_client_secret_hash":"%s"}}' "$ldap_b64" "$client_hash_b64" >"$authelia_patch" unset ldap_b64 client_hash_b64 kubectl patch secret authelia-secrets -n sso --type=merge --patch-file "$authelia_patch" >/dev/null shred -u -- "$authelia_patch" # The current Authelia deployment injects most secret fields from files, but # its OIDC client verifier is still code-defined in authelia-config. Keep the # Secret copy for the intended file-backed migration and update the active # ConfigMap atomically for the live 4.38 deployment. kubectl get configmap authelia-config -n sso -o jsonpath='{.data.configuration\.yml}' >"$work_dir/authelia-configuration.yml" AUTHELIA_CLIENT_HASH="$(tr -d '\n' <"$work_dir/authelia-keycape-client-secret.bcrypt")" python3 -c ' import os import re import sys source = sys.stdin.read() updated, count = re.subn( r"(?m)^(\s+secret:\s*).*$", lambda match: match.group(1) + chr(34) + os.environ["AUTHELIA_CLIENT_HASH"] + chr(34), source, ) if count != 1: raise SystemExit(f"expected one Authelia OIDC client secret field, found {count}") sys.stdout.write(updated) ' <"$work_dir/authelia-configuration.yml" >"$work_dir/authelia-configuration.updated.yml" kubectl create configmap authelia-config -n sso --from-file=configuration.yml="$work_dir/authelia-configuration.updated.yml" --dry-run=client -o yaml | kubectl apply -f - >/dev/null kubectl create secret generic keycape-config -n sso --from-file=config.yaml="$work_dir/config.yaml" --from-file=key.pem="$work_dir/key.pem" --dry-run=client -o yaml | kubectl apply -f - >/dev/null kubectl rollout restart deployment/authelia deployment/keycape -n sso >/dev/null kubectl rollout status deployment/authelia -n sso --timeout=180s kubectl rollout status deployment/keycape -n sso --timeout=180s echo "verification: checking replacement Authelia client credential" kubectl port-forward -n sso service/authelia 19091:9091 >"$work_dir/authelia-port-forward.log" 2>&1 & port_forward_pids+=("$!") for _ in {1..30}; do curl -fsS -o /dev/null http://127.0.0.1:19091/api/health >/dev/null 2>&1 && break sleep 1 done basic="$(printf 'keycape:%s' "$(tr -d '\n' <"$work_dir/authelia-keycape-client-secret")" | base64 -w0)" printf 'Authorization: Basic %s\nContent-Type: application/x-www-form-urlencoded\n' "$basic" >"$work_dir/authelia-new.headers" unset basic printf '%s' 'grant_type=authorization_code&code=definitely-invalid&redirect_uri=https%3A%2F%2Fkc.coulomb.social%2Fauthorize%2Fcallback&client_id=keycape' >"$work_dir/authelia-token.body" new_client_status="$(curl -sS -o "$work_dir/authelia-new.response" -w '%{http_code}' -H @"$work_dir/authelia-new.headers" --data-binary @"$work_dir/authelia-token.body" http://127.0.0.1:19091/api/oidc/token)" new_client_error="$(jq -r '.error // empty' "$work_dir/authelia-new.response" 2>/dev/null || true)" printf 'Authorization: Basic a2V5Y2FwZTp3cm9uZy1yZWNvdmVyeS1wcm9iZQ==\nContent-Type: application/x-www-form-urlencoded\n' >"$work_dir/authelia-wrong.headers" wrong_client_status="$(curl -sS -o "$work_dir/authelia-wrong.response" -w '%{http_code}' -H @"$work_dir/authelia-wrong.headers" --data-binary @"$work_dir/authelia-token.body" http://127.0.0.1:19091/api/oidc/token)" wrong_client_error="$(jq -r '.error // empty' "$work_dir/authelia-wrong.response" 2>/dev/null || true)" if [[ "$new_client_error" == "invalid_client" || "$new_client_status" == "401" ]]; then echo "replacement Authelia client credential was rejected" >&2 exit 1 fi if [[ "$wrong_client_error" != "invalid_client" && "$wrong_client_status" != "401" ]]; then echo "Authelia wrong-client-secret negative check did not reject authentication" >&2 exit 1 fi echo "verification: Authelia accepted the replacement client and rejected a wrong secret" kill "${port_forward_pids[-1]}" 2>/dev/null || true wait "${port_forward_pids[-1]}" 2>/dev/null || true unset 'port_forward_pids[-1]' echo "verification: checking fail-closed privacyIDEA behavior" kubectl port-forward -n mfa service/privacyidea 18081:8080 >"$work_dir/privacyidea-port-forward.log" 2>&1 & port_forward_pids+=("$!") for _ in {1..30}; do curl -sS -o /dev/null http://127.0.0.1:18081/ >/dev/null 2>&1 && break sleep 1 done printf 'Authorization: Bearer ROTATED_DISABLED_ADMIN_TOKEN\nContent-Type: application/x-www-form-urlencoded\n' >"$work_dir/privacyidea.headers" printf '%s' 'user=__keycape_recovery_probe__&pass=000000&realm=coulomb' >"$work_dir/privacyidea.body" pi_status="$(curl -sS -o "$work_dir/privacyidea.response" -w '%{http_code}' -H @"$work_dir/privacyidea.headers" --data-binary @"$work_dir/privacyidea.body" http://127.0.0.1:18081/validate/check)" pi_result="$(jq -r 'if .result | has("value") then (.result.value | tostring) else empty end' "$work_dir/privacyidea.response" 2>/dev/null || true)" pi_api_status="$(jq -r 'if .result | has("status") then (.result.status | tostring) else empty end' "$work_dir/privacyidea.response" 2>/dev/null || true)" if [[ "$pi_result" == "true" ]] || ! { [[ "$pi_status" == "200" && "$pi_result" == "false" ]] || [[ ( "$pi_status" == "400" || "$pi_status" == "401" ) && "$pi_api_status" == "false" ]]; }; then echo "privacyIDEA fail-closed probe failed (HTTP $pi_status, result=$pi_result)" >&2 exit 1 fi echo "verification: privacyIDEA rejected the invalid MFA proof" kill "${port_forward_pids[-1]}" 2>/dev/null || true wait "${port_forward_pids[-1]}" 2>/dev/null || true unset 'port_forward_pids[-1]' issuer="$(curl -fsS https://kc.coulomb.social/.well-known/openid-configuration | jq -r '.issuer')" [[ "$issuer" == "https://kc.coulomb.social" ]] || { echo "unexpected live issuer: $issuer" >&2 exit 1 } authorize_status="$(curl -sS -o /dev/null -w '%{http_code}' 'https://kc.coulomb.social/authorize?client_id=openbao-admin&redirect_uri=http%3A%2F%2F127.0.0.1%3A18200%2Fui%2Fvault%2Fauth%2Fnetkingdom%2Foidc%2Fcallback&response_type=code&scope=openid%20profile%20email%20groups&state=keycape-recovery&code_challenge=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA&code_challenge_method=S256')" [[ "$authorize_status" == "302" ]] || { echo "OpenBao authorization handoff returned HTTP $authorize_status" >&2 exit 1 } after_jwks="$(jwks_fingerprint)" after_keycape_rv="$(kubectl get secret keycape-config -n sso -o jsonpath='{.metadata.resourceVersion}')" after_pi_rv="$(kubectl get secret privacyidea-config -n mfa -o jsonpath='{.metadata.resourceVersion}')" after_lldap_rv="$(kubectl get secret lldap-secrets -n sso -o jsonpath='{.metadata.resourceVersion}')" after_authelia_rv="$(kubectl get secret authelia-secrets -n sso -o jsonpath='{.metadata.resourceVersion}')" [[ "$after_jwks" != "$before_jwks" ]] || { echo "public JWKS fingerprint did not change" >&2 exit 1 } [[ "$after_keycape_rv" != "$before_keycape_rv" && "$after_pi_rv" != "$before_pi_rv" && "$after_lldap_rv" != "$before_lldap_rv" && "$after_authelia_rv" != "$before_authelia_rv" ]] || { echo "one or more Secret resource versions did not change" >&2 exit 1 } echo "verification: discovery, OpenBao callback admission, readiness, and JWKS rollover passed" echo "verification: new public JWKS fingerprint: $after_jwks" echo "verification: new resource versions: keycape=$after_keycape_rv privacyidea=$after_pi_rv lldap=$after_lldap_rv authelia=$after_authelia_rv" echo "recovery complete"