key-cape/tools/rotate-live-exposed-secret.sh

#!/usr/bin/env bash
# Rotate every credential class formerly embedded in sso/keycape-config without
# rendering a Secret value. This is an emergency, deliberate-invalidation path.

set -euo pipefail

MODE="${1:-}"
if [[ "$MODE" != "--preflight" && "$MODE" != "--execute" ]]; then
    echo "usage: $0 --preflight|--execute" >&2
    exit 2
fi
if [[ "$MODE" == "--execute" && "${KEYCAPE_ACKNOWLEDGE_SESSION_INVALIDATION:-}" != "1" ]]; then
    echo "refusing execution without KEYCAPE_ACKNOWLEDGE_SESSION_INVALIDATION=1" >&2
    exit 2
fi

for tool in age base64 curl jq kubectl openssl python3 sha256sum tar; do
    command -v "$tool" >/dev/null || {
        echo "missing required tool: $tool" >&2
        exit 1
    }
done
python3 -c 'import bcrypt' 2>/dev/null || {
    echo "missing required Python bcrypt module" >&2
    exit 1
}

ROOT_DIR="$(cd "$(dirname "$0")/.." && pwd)"
RECIPIENT_FILE="${NETKINGDOM_AGE_RECIPIENT_FILE:-/home/worsch/net-kingdom/keys/age.pub}"
ARCHIVE_FILE="${KEYCAPE_RECOVERY_ARCHIVE:-$ROOT_DIR/history/KEY-WP-0011-rotated-credentials-2026-08-23.tar.age}"

[[ -s "$RECIPIENT_FILE" ]] || {
    echo "age recipient file not found: $RECIPIENT_FILE" >&2
    exit 1
}

required_secrets=(
    "sso/keycape-config"
    "sso/keycape-pi-token"
    "sso/authelia-secrets"
    "sso/lldap-secrets"
    "mfa/privacyidea-config"
)
required_deployments=(
    "sso/keycape"
    "sso/authelia"
    "sso/lldap"
    "mfa/privacyidea"
)

for ref in "${required_secrets[@]}"; do
    namespace="${ref%%/*}"
    name="${ref#*/}"
    kubectl get secret "$name" -n "$namespace" -o name >/dev/null
done
for ref in "${required_deployments[@]}"; do
    namespace="${ref%%/*}"
    name="${ref#*/}"
    kubectl get deployment "$name" -n "$namespace" -o name >/dev/null
done

jwks_fingerprint() {
    jwks_uri="$(curl -fsS https://kc.coulomb.social/.well-known/openid-configuration | jq -er '.jwks_uri')"
    curl -fsS "$jwks_uri" |
        jq -cS '.keys | map({alg,e,kid,kty,n,use})' |
        sha256sum |
        awk '{print $1}'
}

before_jwks="$(jwks_fingerprint)"
before_keycape_rv="$(kubectl get secret keycape-config -n sso -o jsonpath='{.metadata.resourceVersion}')"
before_pi_rv="$(kubectl get secret privacyidea-config -n mfa -o jsonpath='{.metadata.resourceVersion}')"
before_lldap_rv="$(kubectl get secret lldap-secrets -n sso -o jsonpath='{.metadata.resourceVersion}')"
before_authelia_rv="$(kubectl get secret authelia-secrets -n sso -o jsonpath='{.metadata.resourceVersion}')"

echo "preflight: deployments and Secrets exist"
echo "preflight: current public JWKS fingerprint: $before_jwks"
echo "preflight: current resource versions: keycape=$before_keycape_rv privacyidea=$before_pi_rv lldap=$before_lldap_rv authelia=$before_authelia_rv"

if [[ "$MODE" == "--preflight" ]]; then
    exit 0
fi
if [[ -e "$ARCHIVE_FILE" ]]; then
    echo "refusing to overwrite recovery archive: $ARCHIVE_FILE" >&2
    exit 1
fi

umask 077
work_dir="$(mktemp -d /tmp/keycape-recovery.XXXXXX)"
port_forward_pids=()
lldap_force_reset_active=false

cleanup() {
    status=$?
    for pid in "${port_forward_pids[@]:-}"; do
        kill "$pid" 2>/dev/null || true
        wait "$pid" 2>/dev/null || true
    done
    if [[ "$lldap_force_reset_active" == "true" ]]; then
        kubectl set env deployment/lldap -n sso LLDAP_FORCE_LDAP_USER_PASS_RESET- >/dev/null 2>&1 || true
    fi
    case "$work_dir" in
        /tmp/keycape-recovery.*)
            find "$work_dir" -type f -exec shred -u -- {} + 2>/dev/null || true
            rm -rf -- "$work_dir"
            ;;
    esac
    exit "$status"
}
trap cleanup EXIT INT TERM

openssl rand -hex 32 >"$work_dir/lldap-bind-password"
openssl rand -hex 32 >"$work_dir/authelia-keycape-client-secret"
openssl rand -hex 32 >"$work_dir/privacyidea-secret-key"
openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 -out "$work_dir/key.pem" 2>/dev/null

python3 -c '
import bcrypt
import sys
secret = sys.stdin.buffer.read().rstrip(b"\n")
sys.stdout.buffer.write(bcrypt.hashpw(secret, bcrypt.gensalt(rounds=12)))
' <"$work_dir/authelia-keycape-client-secret" >"$work_dir/authelia-keycape-client-secret.bcrypt"

lldap_password="$(tr -d '\n' <"$work_dir/lldap-bind-password")"
authelia_secret="$(tr -d '\n' <"$work_dir/authelia-keycape-client-secret")"

cat >"$work_dir/config.yaml" <<EOF
issuer: "https://kc.coulomb.social"
port: 8080
tokenLifetime: "15m"
privateKeyPem: "/etc/keycape/key.pem"
environment: "production"

lldap:
  url: "ldap://lldap.sso.svc.cluster.local:3890"
  bindDN: "uid=admin,ou=people,dc=netkingdom,dc=local"
  bindPW: "$lldap_password"
  baseDN: "dc=netkingdom,dc=local"
  userOU: "ou=people"
  groupOU: "ou=groups"

authelia:
  baseURL: "http://authelia.sso.svc.cluster.local:9091"
  browserBaseURL: "https://auth.coulomb.social"
  tokenBaseURL: "http://authelia.sso.svc.cluster.local:9091"
  clientId: "keycape"
  clientSecret: "$authelia_secret"
  redirectURI: "https://kc.coulomb.social/authorize/callback"

privacyidea:
  baseURL: "http://privacyidea.mfa.svc.cluster.local:8080"
  adminToken: "ROTATED_DISABLED_ADMIN_TOKEN"
  realm: "coulomb"
  requireForAll: true

clients:
  - clientId: "demo-app"
    displayName: "Demo Application"
    redirectUris:
      - "http://localhost:3000/callback"
      - "http://127.0.0.1:8876/oidc/callback"
      - "http://localhost:8876/oidc/callback"
      - "https://demo.coulomb.social/callback"
    allowedScopes: ["openid", "profile", "email", "groups"]
    grantTypes: ["authorization_code"]
    clientType: "public"
  - clientId: "netkingdom-bootstrap-console"
    displayName: "NetKingdom Bootstrap Console"
    redirectUris:
      - "http://127.0.0.1:8876/oidc/callback"
      - "http://localhost:8876/oidc/callback"
    allowedScopes: ["openid", "profile", "email", "groups"]
    grantTypes: ["authorization_code"]
    clientType: "public"
  - clientId: "openbao-admin"
    displayName: "Railiance OpenBao Admin"
    redirectUris:
      - "http://localhost:8250/oidc/callback"
      - "http://127.0.0.1:8250/oidc/callback"
      - "http://127.0.0.1:18200/ui/vault/auth/netkingdom/oidc/callback"
      - "https://bao.coulomb.social/ui/vault/auth/netkingdom/oidc/callback"
      - "https://bao.coulomb.social/ui/vault/auth/keycape/oidc/callback"
    allowedScopes: ["openid", "profile", "email", "groups"]
    grantTypes: ["authorization_code"]
    clientType: "public"
  - clientId: "rapp-qonto-client"
    displayName: "rapp-qonto workload"
    allowedScopes: ["qonto:read"]
    grantTypes: ["client_credentials"]
    clientType: "confidential"
    secretRef: "env:KEYCAPE_RAPP_QONTO_CLIENT_SECRET"
    serviceSubject: "rapp-qonto"
    tenant: "tenant:friendly:binky"
    roles: ["qonto-reader"]
EOF
unset lldap_password authelia_secret

printf '%s\n' 'KEY-WP-0011 emergency rotation overlay.' 'Contains only post-exposure credential material; decrypt only through approved NetKingdom age custody.' >"$work_dir/README.txt"
tar -C "$work_dir" -cf "$work_dir/recovery.tar" README.txt config.yaml key.pem lldap-bind-password authelia-keycape-client-secret privacyidea-secret-key
age -R "$RECIPIENT_FILE" -o "$ARCHIVE_FILE" "$work_dir/recovery.tar"
chmod 600 "$ARCHIVE_FILE"
echo "custody: encrypted recovery overlay created at $ARCHIVE_FILE"

patch_field() {
    namespace="$1"
    secret_name="$2"
    field_name="$3"
    value_file="$4"
    patch_file="$work_dir/patch-${namespace}-${secret_name}-${field_name}.json"
    encoded="$(base64 -w0 <"$value_file")"
    printf '{"data":{"%s":"%s"}}' "$field_name" "$encoded" >"$patch_file"
    unset encoded
    kubectl patch secret "$secret_name" -n "$namespace" --type=merge --patch-file "$patch_file" >/dev/null
    shred -u -- "$patch_file"
}

echo "rotation: invalidating the exposed privacyIDEA admin JWT"
patch_field mfa privacyidea-config PI_SECRET_KEY "$work_dir/privacyidea-secret-key"
printf '%s' 'ROTATED_DISABLED_ADMIN_TOKEN' >"$work_dir/pi-token-disabled"
patch_field sso keycape-pi-token token "$work_dir/pi-token-disabled"
kubectl rollout restart deployment/privacyidea -n mfa >/dev/null
kubectl rollout status deployment/privacyidea -n mfa --timeout=180s

echo "rotation: resetting the LLDAP admin/bind credential"
patch_field sso lldap-secrets LLDAP_LDAP_USER_PASS "$work_dir/lldap-bind-password"
kubectl set env deployment/lldap -n sso LLDAP_FORCE_LDAP_USER_PASS_RESET=always >/dev/null
lldap_force_reset_active=true
kubectl rollout status deployment/lldap -n sso --timeout=180s

kubectl port-forward -n sso service/lldap 11717:17170 >"$work_dir/lldap-port-forward.log" 2>&1 &
port_forward_pids+=("$!")
for _ in {1..30}; do
    curl -fsS -o /dev/null http://127.0.0.1:11717/health >/dev/null 2>&1 && break
    sleep 1
done
printf '{"username":"admin","password":"%s"}' "$(tr -d '\n' <"$work_dir/lldap-bind-password")" >"$work_dir/lldap-login.json"
lldap_status="$(curl -sS -o /dev/null -w '%{http_code}' -H 'Content-Type: application/json' --data-binary @"$work_dir/lldap-login.json" http://127.0.0.1:11717/auth/simple/login)"
[[ "$lldap_status" == "200" ]] || {
    echo "LLDAP new-credential verification failed with HTTP $lldap_status" >&2
    exit 1
}
echo "verification: LLDAP accepted the replacement bind credential"
kill "${port_forward_pids[-1]}" 2>/dev/null || true
wait "${port_forward_pids[-1]}" 2>/dev/null || true
unset 'port_forward_pids[-1]'

kubectl set env deployment/lldap -n sso LLDAP_FORCE_LDAP_USER_PASS_RESET- >/dev/null
lldap_force_reset_active=false
kubectl rollout status deployment/lldap -n sso --timeout=180s

echo "rotation: restarting identity-provisioner consumer"
kubectl rollout restart deployment/identity-provisioner -n sso >/dev/null
kubectl rollout status deployment/identity-provisioner -n sso --timeout=180s

echo "rotation: updating Authelia and KeyCape consumers"
authelia_patch="$work_dir/patch-authelia.json"
ldap_b64="$(base64 -w0 <"$work_dir/lldap-bind-password")"
client_hash_b64="$(base64 -w0 <"$work_dir/authelia-keycape-client-secret.bcrypt")"
printf '{"data":{"ldap_password":"%s","keycape_client_secret_hash":"%s"}}' "$ldap_b64" "$client_hash_b64" >"$authelia_patch"
unset ldap_b64 client_hash_b64
kubectl patch secret authelia-secrets -n sso --type=merge --patch-file "$authelia_patch" >/dev/null
shred -u -- "$authelia_patch"

# The current Authelia deployment injects most secret fields from files, but
# its OIDC client verifier is still code-defined in authelia-config. Keep the
# Secret copy for the intended file-backed migration and update the active
# ConfigMap atomically for the live 4.38 deployment.
kubectl get configmap authelia-config -n sso -o jsonpath='{.data.configuration\.yml}' >"$work_dir/authelia-configuration.yml"
AUTHELIA_CLIENT_HASH="$(tr -d '\n' <"$work_dir/authelia-keycape-client-secret.bcrypt")" python3 -c '
import os
import re
import sys
source = sys.stdin.read()
updated, count = re.subn(
    r"(?m)^(\s+secret:\s*).*$",
    lambda match: match.group(1) + chr(34) + os.environ["AUTHELIA_CLIENT_HASH"] + chr(34),
    source,
)
if count != 1:
    raise SystemExit(f"expected one Authelia OIDC client secret field, found {count}")
sys.stdout.write(updated)
' <"$work_dir/authelia-configuration.yml" >"$work_dir/authelia-configuration.updated.yml"
kubectl create configmap authelia-config -n sso --from-file=configuration.yml="$work_dir/authelia-configuration.updated.yml" --dry-run=client -o yaml |
    kubectl apply -f - >/dev/null

kubectl create secret generic keycape-config -n sso --from-file=config.yaml="$work_dir/config.yaml" --from-file=key.pem="$work_dir/key.pem" --dry-run=client -o yaml |
    kubectl apply -f - >/dev/null

kubectl rollout restart deployment/authelia deployment/keycape -n sso >/dev/null
kubectl rollout status deployment/authelia -n sso --timeout=180s
kubectl rollout status deployment/keycape -n sso --timeout=180s

echo "verification: checking replacement Authelia client credential"
kubectl port-forward -n sso service/authelia 19091:9091 >"$work_dir/authelia-port-forward.log" 2>&1 &
port_forward_pids+=("$!")
for _ in {1..30}; do
    curl -fsS -o /dev/null http://127.0.0.1:19091/api/health >/dev/null 2>&1 && break
    sleep 1
done
basic="$(printf 'keycape:%s' "$(tr -d '\n' <"$work_dir/authelia-keycape-client-secret")" | base64 -w0)"
printf 'Authorization: Basic %s\nContent-Type: application/x-www-form-urlencoded\n' "$basic" >"$work_dir/authelia-new.headers"
unset basic
printf '%s' 'grant_type=authorization_code&code=definitely-invalid&redirect_uri=https%3A%2F%2Fkc.coulomb.social%2Fauthorize%2Fcallback&client_id=keycape' >"$work_dir/authelia-token.body"
new_client_status="$(curl -sS -o "$work_dir/authelia-new.response" -w '%{http_code}' -H @"$work_dir/authelia-new.headers" --data-binary @"$work_dir/authelia-token.body" http://127.0.0.1:19091/api/oidc/token)"
new_client_error="$(jq -r '.error // empty' "$work_dir/authelia-new.response" 2>/dev/null || true)"
printf 'Authorization: Basic a2V5Y2FwZTp3cm9uZy1yZWNvdmVyeS1wcm9iZQ==\nContent-Type: application/x-www-form-urlencoded\n' >"$work_dir/authelia-wrong.headers"
wrong_client_status="$(curl -sS -o "$work_dir/authelia-wrong.response" -w '%{http_code}' -H @"$work_dir/authelia-wrong.headers" --data-binary @"$work_dir/authelia-token.body" http://127.0.0.1:19091/api/oidc/token)"
wrong_client_error="$(jq -r '.error // empty' "$work_dir/authelia-wrong.response" 2>/dev/null || true)"
if [[ "$new_client_error" == "invalid_client" || "$new_client_status" == "401" ]]; then
    echo "replacement Authelia client credential was rejected" >&2
    exit 1
fi
if [[ "$wrong_client_error" != "invalid_client" && "$wrong_client_status" != "401" ]]; then
    echo "Authelia wrong-client-secret negative check did not reject authentication" >&2
    exit 1
fi
echo "verification: Authelia accepted the replacement client and rejected a wrong secret"
kill "${port_forward_pids[-1]}" 2>/dev/null || true
wait "${port_forward_pids[-1]}" 2>/dev/null || true
unset 'port_forward_pids[-1]'

echo "verification: checking fail-closed privacyIDEA behavior"
kubectl port-forward -n mfa service/privacyidea 18081:8080 >"$work_dir/privacyidea-port-forward.log" 2>&1 &
port_forward_pids+=("$!")
for _ in {1..30}; do
    curl -sS -o /dev/null http://127.0.0.1:18081/ >/dev/null 2>&1 && break
    sleep 1
done
printf 'Authorization: Bearer ROTATED_DISABLED_ADMIN_TOKEN\nContent-Type: application/x-www-form-urlencoded\n' >"$work_dir/privacyidea.headers"
printf '%s' 'user=__keycape_recovery_probe__&pass=000000&realm=coulomb' >"$work_dir/privacyidea.body"
pi_status="$(curl -sS -o "$work_dir/privacyidea.response" -w '%{http_code}' -H @"$work_dir/privacyidea.headers" --data-binary @"$work_dir/privacyidea.body" http://127.0.0.1:18081/validate/check)"
pi_result="$(jq -r 'if .result | has("value") then (.result.value | tostring) else empty end' "$work_dir/privacyidea.response" 2>/dev/null || true)"
pi_api_status="$(jq -r 'if .result | has("status") then (.result.status | tostring) else empty end' "$work_dir/privacyidea.response" 2>/dev/null || true)"
if [[ "$pi_result" == "true" ]] ||
   ! { [[ "$pi_status" == "200" && "$pi_result" == "false" ]] ||
       [[ ( "$pi_status" == "400" || "$pi_status" == "401" ) && "$pi_api_status" == "false" ]]; }; then
    echo "privacyIDEA fail-closed probe failed (HTTP $pi_status, result=$pi_result)" >&2
    exit 1
fi
echo "verification: privacyIDEA rejected the invalid MFA proof"
kill "${port_forward_pids[-1]}" 2>/dev/null || true
wait "${port_forward_pids[-1]}" 2>/dev/null || true
unset 'port_forward_pids[-1]'

issuer="$(curl -fsS https://kc.coulomb.social/.well-known/openid-configuration | jq -r '.issuer')"
[[ "$issuer" == "https://kc.coulomb.social" ]] || {
    echo "unexpected live issuer: $issuer" >&2
    exit 1
}
authorize_status="$(curl -sS -o /dev/null -w '%{http_code}' 'https://kc.coulomb.social/authorize?client_id=openbao-admin&redirect_uri=http%3A%2F%2F127.0.0.1%3A18200%2Fui%2Fvault%2Fauth%2Fnetkingdom%2Foidc%2Fcallback&response_type=code&scope=openid%20profile%20email%20groups&state=keycape-recovery&code_challenge=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA&code_challenge_method=S256')"
[[ "$authorize_status" == "302" ]] || {
    echo "OpenBao authorization handoff returned HTTP $authorize_status" >&2
    exit 1
}

after_jwks="$(jwks_fingerprint)"
after_keycape_rv="$(kubectl get secret keycape-config -n sso -o jsonpath='{.metadata.resourceVersion}')"
after_pi_rv="$(kubectl get secret privacyidea-config -n mfa -o jsonpath='{.metadata.resourceVersion}')"
after_lldap_rv="$(kubectl get secret lldap-secrets -n sso -o jsonpath='{.metadata.resourceVersion}')"
after_authelia_rv="$(kubectl get secret authelia-secrets -n sso -o jsonpath='{.metadata.resourceVersion}')"

[[ "$after_jwks" != "$before_jwks" ]] || {
    echo "public JWKS fingerprint did not change" >&2
    exit 1
}
[[ "$after_keycape_rv" != "$before_keycape_rv" &&
   "$after_pi_rv" != "$before_pi_rv" &&
   "$after_lldap_rv" != "$before_lldap_rv" &&
   "$after_authelia_rv" != "$before_authelia_rv" ]] || {
    echo "one or more Secret resource versions did not change" >&2
    exit 1
}

echo "verification: discovery, OpenBao callback admission, readiness, and JWKS rollover passed"
echo "verification: new public JWKS fingerprint: $after_jwks"
echo "verification: new resource versions: keycape=$after_keycape_rv privacyidea=$after_pi_rv lldap=$after_lldap_rv authelia=$after_authelia_rv"
echo "recovery complete"