net-kingdom/sso-mfa/bootstrap/creds-bootstrap-agent.sh

#!/usr/bin/env bash
# creds-bootstrap-agent.sh — attended bootstrap with public-recipient encryption
#
# Usage:
#   bash sso-mfa/bootstrap/creds-bootstrap-agent.sh [--dry-run] [--resume] [--initialize-audit-keys]
#   make creds-agent-init
#
# Runs in the operator terminal and ends at the encrypted recovery handoff
# acknowledgment. Each phase updates creds-state.yaml so interrupted runs resume
# automatically from where they left off.
#
# Prerequisites:
#   - age  (apt install age)
#   - kubectl with a reachable cluster (KUBECONFIG set or ~/.kube/config)
#   - git  (configured with commit access)
#   - openssl
#   - keys/age.pub — independently registered public recipient
#   - temporary NETKINGDOM_CUSTODY_AGE_KEY_FILE only when resuming encrypted input

set +x
set -euo pipefail
umask 077

SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
K8S_DIR="$REPO_ROOT/sso-mfa/k8s"
SECRETS_DIR="$SCRIPT_DIR/secrets"
STATE_FILE="$SCRIPT_DIR/creds-state.yaml"
AGE_KEY="${NETKINGDOM_CUSTODY_AGE_KEY_FILE:-}"

DRY_RUN=false
AUDIT_ARGS=()
for arg in "$@"; do
    case "$arg" in
        --dry-run) DRY_RUN=true ;;
        --initialize-audit-keys) AUDIT_ARGS=(--initialize-audit-keys) ;;
        --resume) ;; # Phase state already controls resume.
        *) printf '%s\n' 'ERROR: unsupported bootstrap argument.' >&2; exit 2 ;;
    esac
done

# A plan must neither discover credentials nor contact a cluster or change state.
if [[ "$DRY_RUN" == true ]]; then
    printf '%s\n' 'Plan only: validate registered public recipient; generate or attended-decrypt service files; encrypt; apply/verify; create encrypted ops bundle; acknowledge off-host recovery; clean plaintext.'
    printf '%s\n' 'No files, credentials, cluster state or historical custody receipts were inspected or changed.'
    exit 0
fi
[[ -t 0 && -t 1 ]] || { printf '%s\n' 'ERROR: bootstrap requires an attended terminal; use --dry-run for the non-mutating plan.' >&2; exit 2; }

# ── Helpers ───────────────────────────────────────────────────────────────────

log()  { echo "  [bootstrap] $*"; }
step() { echo ""; echo "══════════════════════════════════════════════════════"; echo "  Phase $*"; echo "══════════════════════════════════════════════════════"; }
ok()   { echo "  ✔ $*"; }
warn() { echo "  ⚠  $*"; }
die()  { echo ""; echo "  ERROR: $*" >&2; exit 1; }

dry_run_guard() {
    if [[ "$DRY_RUN" == true ]]; then
        echo "  [dry-run] would run: $*"
        return 0
    fi
    "$@"
}

# Read a top-level value from creds-state.yaml
state_get() { grep -E "^$1:" "$STATE_FILE" | sed 's/^[^:]*: *//' | sed 's/ *#.*//' | tr -d '"'; }
state_get_nested() { grep -E "^  $1:" "$STATE_FILE" | sed 's/^[^:]*: *//' | sed 's/ *#.*//' | tr -d '"'; }

# Update a top-level key in creds-state.yaml
state_set() {
    local key="$1" value="$2"
    if [[ -f "$STATE_FILE" ]]; then
        if grep -q "^$key:" "$STATE_FILE"; then
            sed -i "s|^$key: .*|$key: $value|" "$STATE_FILE"
        else
            printf '%s: %s\n' "$key" "$value" >> "$STATE_FILE"
        fi
    fi
}

# Update a nested (2-space indent) key
state_set_nested() {
    local key="$1" value="$2"
    if [[ -f "$STATE_FILE" ]]; then
        sed -i "s|^  $key: .*|  $key: $value|" "$STATE_FILE"
    fi
}

# ── Pre-flight ────────────────────────────────────────────────────────────────

step "0 — Pre-flight"

command -v age     >/dev/null 2>&1 || die "age not installed (apt install age)"
command -v kubectl >/dev/null 2>&1 || die "kubectl not found — install it and configure KUBECONFIG"
command -v git     >/dev/null 2>&1 || die "git not found"
command -v openssl >/dev/null 2>&1 || die "openssl not found"
ok "required tools present"

# Encryption needs only the registered public recipient. Never auto-generate
# or discover a private key in HOME. Custody source is independently held.
[[ -f "$REPO_ROOT/keys/age.pub" ]] || die "registered keys/age.pub is missing"
AGE_PUBKEY=$(grep -m1 '^age1' "$REPO_ROOT/keys/age.pub" || true)
[[ -n "$AGE_PUBKEY" ]] || die "registered public age recipient is missing"
printf '' | age -r "$AGE_PUBKEY" >/dev/null 2>&1 || die "registered public age recipient is invalid"

# Refuse an encrypted resume before cluster activity unless the owner explicitly
# supplied a temporary key through the custody helper. No ambient key fallback.
if [[ "$(state_get secrets_generated)" == "true" && ! -d "$SECRETS_DIR" ]]; then
    [[ -n "$AGE_KEY" && -f "$AGE_KEY" ]] || die "encrypted resume requires the attended custody unlock helper"
    DERIVED_RECIPIENT=$(age-keygen -y "$AGE_KEY" 2>/dev/null) || die "custody key validation failed"
    [[ "$DERIVED_RECIPIENT" == "$AGE_PUBKEY" ]] || die "custody key does not match the registered recipient"
    unset DERIVED_RECIPIENT
fi

# Cluster reachability
if ! kubectl cluster-info &>/dev/null; then
    die "Cannot reach the Kubernetes cluster. Check KUBECONFIG / cluster status."
fi
KUBE_CTX=$(kubectl config current-context 2>/dev/null || echo '(unknown)')
ok "cluster reachable: $KUBE_CTX"
state_set "age_public_recipient_ready" "true"
state_set "bootstrap_complete" "false"

# ── Phase 1: Generate secrets ─────────────────────────────────────────────────

step "1 — Generate secrets"

if [[ "$(state_get secrets_generated)" == "true" ]]; then
    ok "secrets already generated — skipping"
else
    # Clean up any partial generation from a failed prior run
    if [[ -d "$SECRETS_DIR" ]]; then
        warn "leftover secrets/ found from previous run — removing"
        find "$SECRETS_DIR" -type f -exec shred -u {} \; 2>/dev/null || true
        rm -rf "$SECRETS_DIR"
    fi

    log "running gen-secrets.sh..."
    if [[ "$DRY_RUN" == false ]]; then
        (cd "$SCRIPT_DIR" && bash gen-secrets.sh "$SECRETS_DIR")
        ok "secrets generated in $SECRETS_DIR"
        state_set "secrets_generated" "true"
    else
        echo "  [dry-run] would run: bash gen-secrets.sh $SECRETS_DIR"
    fi
fi

# ── Phase 2: Encrypt + commit ─────────────────────────────────────────────────

step "2 — Encrypt secrets to secrets.enc/ and commit"

# Re-check: if secrets_generated is true but secrets/ is gone, decrypt first
if [[ "$(state_get secrets_generated)" == "true" && ! -d "$SECRETS_DIR" ]]; then
    log "secrets/ absent (was shredded) — decrypting from secrets.enc/ for re-apply..."
    if [[ "$DRY_RUN" == false ]]; then
        (cd "$SCRIPT_DIR" && bash decrypt-secrets.sh "$SECRETS_DIR" "$AGE_KEY")
    else
        echo "  [dry-run] would decrypt secrets.enc/ → secrets/"
    fi
fi

# Always (re-)encrypt in case secrets were just regenerated
if [[ "$DRY_RUN" == false ]]; then
    log "encrypting secrets → secrets.enc/ ..."
    (cd "$SCRIPT_DIR" && bash encrypt-secrets.sh "$SECRETS_DIR" "$AGE_PUBKEY" --no-shred)
    ok "secrets encrypted to secrets.enc/"

    # Commit the encrypted secrets
    cd "$REPO_ROOT"
    if git diff --quiet HEAD sso-mfa/bootstrap/secrets.enc/ 2>/dev/null && \
       git diff --cached --quiet sso-mfa/bootstrap/secrets.enc/ 2>/dev/null; then
        ok "secrets.enc/ already committed — no changes"
    else
        git add sso-mfa/bootstrap/secrets.enc/ sso-mfa/bootstrap/creds-state.yaml
        git commit -m "chore(creds): encrypted secrets [agent NK-WP-0005]"
        ok "encrypted secrets committed"
    fi
else
    echo "  [dry-run] would encrypt + git commit secrets.enc/"
fi

# ── Phase 3: Inject into cluster ──────────────────────────────────────────────

step "3 — Inject secrets into cluster (postgres → lldap → authelia → privacyidea)"

# Check if all non-keycape components are already applied
PG=$(state_get_nested postgres); LLDAP=$(state_get_nested lldap)
AUTH=$(state_get_nested authelia); PI=$(state_get_nested privacyidea)
if [[ "$PG" == "true" && "$LLDAP" == "true" && "$AUTH" == "true" && "$PI" == "true" ]]; then
    ok "all cluster secrets already applied — skipping (avoids CNPG connection disruption)"
else
    if [[ "$DRY_RUN" == false ]]; then
        (cd "$SCRIPT_DIR" && bash creds-apply.sh "$SECRETS_DIR")
        ok "secrets applied to cluster"
        _PHASE3_RAN=true
    else
        echo "  [dry-run] would run: bash creds-apply.sh $SECRETS_DIR"
    fi
fi

# ── Phase 4: Verify initial secrets ───────────────────────────────────────────

step "4 — Verify K8s secrets (pre-bootstrap)"

if [[ "$DRY_RUN" == false ]]; then
    # Verify postgres, lldap, authelia, privacyidea (keycape not yet applied)
    ALL_OK=true
    for ns_secret in "databases/net-kingdom-pg-privacyidea-app" "sso/lldap-secrets" "sso/authelia-secrets" "mfa/privacyidea-config"; do
        ns="${ns_secret%%/*}"
        name="${ns_secret##*/}"
        if kubectl get secret "$name" --namespace="$ns" --ignore-not-found -o name 2>/dev/null | grep -q .; then
            ok "secret $ns/$name exists"
        else
            warn "secret $ns/$name is missing"
            ALL_OK=false
        fi
    done
    [[ "$ALL_OK" == true ]] || die "One or more required secrets are missing — check creds-apply output above"

    # Restart privacyIDEA ONLY if secrets were just applied this run.
    # Re-applying an unchanged secret still marks it "configured", so we track
    # whether Phase 3 actually ran to avoid spurious restarts on resume.
    if [[ "${_PHASE3_RAN:-false}" == "true" ]] && \
       kubectl get deployment privacyidea -n mfa &>/dev/null 2>&1; then
        log "restarting privacyIDEA deployment to pick up new secrets..."
        kubectl rollout restart deployment/privacyidea -n mfa
        ok "privacyIDEA restart triggered"
        log "waiting for rollout to complete..."
        kubectl rollout status deployment/privacyidea -n mfa --timeout=300s
        ok "privacyIDEA rollout complete"
    fi
else
    echo "  [dry-run] would verify K8s secrets"
    echo "  [dry-run] would restart privacyIDEA if deployment exists"
fi

# ── Phase 5: Post-apply bootstrap — wait for privacyIDEA ──────────────────────

step "5 — Post-apply bootstrap (privacyIDEA enckey + admin)"

NAMESPACE="mfa"
MAX_WAIT=300  # 5 minutes

if [[ "$(state_get enckey_bootstrapped)" == "true" && "$(state_get pi_admin_created)" == "true" ]]; then
    ok "privacyIDEA already bootstrapped — skipping"
else
    log "waiting for privacyIDEA pod to be Ready (max ${MAX_WAIT}s)..."
    if [[ "$DRY_RUN" == false ]]; then
        # Wait for the deployment rollout to fully complete — this ensures any
        # previous pod from a rollout restart is terminated and the new pod is Ready.
        if ! kubectl rollout status deployment/privacyidea -n "$NAMESPACE" \
                --timeout="${MAX_WAIT}s" 2>/dev/null; then
            die "privacyIDEA deployment did not roll out within ${MAX_WAIT}s — check: kubectl get pods -n $NAMESPACE"
        fi

        # Find the ready pod (rollout status guarantees exactly one ready pod)
        PI_POD=$(kubectl get pod -n "$NAMESPACE" \
            -l app.kubernetes.io/name=privacyidea \
            --field-selector=status.phase=Running \
            -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || echo "")

        [[ -z "$PI_POD" ]] && die "privacyIDEA pod not found after rollout — check: kubectl get pods -n $NAMESPACE"
        ok "privacyIDEA pod ready: $PI_POD"

        # Run enckey bootstrap
        if [[ "$(state_get enckey_bootstrapped)" != "true" ]]; then
            log "running enckey-bootstrap.sh..."
            (cd "$K8S_DIR/privacyidea" && bash enckey-bootstrap.sh "$SECRETS_DIR" "${AUDIT_ARGS[@]}")
            state_set "enckey_bootstrapped" "true"
            ok "enckey bootstrapped"
        else
            ok "enckey already bootstrapped"
        fi

        # Run pi-admin bootstrap
        if [[ "$(state_get pi_admin_created)" != "true" ]]; then
            log "running bootstrap-admin.sh..."
            (cd "$K8S_DIR/privacyidea" && bash bootstrap-admin.sh "$SECRETS_DIR")
            state_set "pi_admin_created" "true"
            ok "pi-admin created"
        else
            ok "pi-admin already created"
        fi
    else
        echo "  [dry-run] would wait for privacyIDEA pod, run enckey-bootstrap.sh and bootstrap-admin.sh"
    fi
fi

# ── Phase 6: Apply keycape secrets ────────────────────────────────────────────

step "6 — Create KeyCape bootstrap secrets (requires scoped factor delivery)"

if [[ "$(state_get_nested keycape)" == "true" ]]; then
    ok "keycape secrets already applied — skipping"
else
    if [[ "$DRY_RUN" == false ]]; then
        log "applying keycape secrets..."
        (cd "$K8S_DIR/keycape" && bash create-secrets.sh "$SECRETS_DIR")
        state_set_nested "keycape" "true"
        ok "keycape secrets applied"
    else
        echo "  [dry-run] would verify RPF-WP-0040 delivery and create first-bootstrap keycape-config"
    fi
fi

# ── Phase 7: Final verification ────────────────────────────────────────────────

step "7 — Final verification (all components)"

if [[ "$DRY_RUN" == false ]]; then
    (cd "$SCRIPT_DIR" && bash creds-verify.sh)
    ok "all secrets verified"
else
    echo "  [dry-run] would run: bash creds-verify.sh"
fi

# ── Phase 7b: OpenBao init/unseal (sops-held-automation, optional) ───────────

step "7b — OpenBao init/unseal (sops-held-automation, optional)"

# NET-WP-0020 T2: greenfield-rebuild hook. Runs only when the openbao
# namespace exists AND the console has selected sops-held-automation
# (the helper enforces that gate itself and refuses attended-ceremony /
# auto-unseal-transit). Skipped silently on clusters without OpenBao.
if kubectl get namespace openbao &>/dev/null; then
    if [[ "$(state_get openbao_post_unseal_verified)" == "true" ]]; then
        ok "OpenBao already verified — skipping"
    elif [[ "$DRY_RUN" == false ]]; then
        if (cd "$SCRIPT_DIR" && bash openbao-init-unseal.sh "$SECRETS_DIR"); then
            state_set "openbao_initialized" "true"
            state_set "openbao_post_unseal_verified" "true"
            ok "OpenBao initialized/unsealed and verified"

            # New init material must reach age custody before cleanup.
            if [[ -d "$SECRETS_DIR/openbao" ]]; then
                log "encrypting OpenBao init material → secrets.enc/ ..."
                (cd "$SCRIPT_DIR" && bash encrypt-secrets.sh \
                    "$SECRETS_DIR" "$AGE_PUBKEY" --no-shred)
                cd "$REPO_ROOT"
                git add sso-mfa/bootstrap/secrets.enc/ \
                    sso-mfa/bootstrap/creds-state.yaml
                git diff --cached --quiet || git commit -m \
                    "chore(creds): encrypted OpenBao init material [agent]"
            fi
        else
            warn "OpenBao init/unseal did not complete — see output"
            warn "(gate unselected or pod not ready; bootstrap continues)"
        fi
    else
        echo "  [dry-run] would run: openbao-init-unseal.sh $SECRETS_DIR"
    fi
else
    ok "no openbao namespace in this cluster — skipping"
fi

# ── Phase 8: Ops bundle ────────────────────────────────────────────────────────

step "8 — Create ops bundle (age-encrypted snapshot)"

BUNDLE_PATH=$(mktemp "$REPO_ROOT/ops-bundle-$(date +%Y%m%dT%H%M%S)-XXXXXXXX.tar.age")

# Always snapshot the current service files after apply. An old phase boolean
# must not cause a newly generated/rotated file to be omitted from recovery.
[[ -d "$SECRETS_DIR" ]] || die "service files absent; cannot create recovery bundle"
(cd "$SCRIPT_DIR" && bash pack-bundle.sh "$SECRETS_DIR" "$AGE_PUBKEY" "$BUNDLE_PATH")
BUNDLE_DIGEST=$(sha256sum "$BUNDLE_PATH" | cut -d ' ' -f 1)
state_set "ops_bundle_created" "true"
state_set "ops_bundle_location" "\"$BUNDLE_PATH\""
state_set "recovery_handoff_confirmed" "false"

# ── Phase 9: Encrypted recovery handoff ──────────────────────────────────────
step "9 — Encrypted recovery handoff (operator acknowledgment required)"
printf 'Encrypted bundle: %s\nSHA256: %s\n' "$BUNDLE_PATH" "$BUNDLE_DIGEST"
bash "$SCRIPT_DIR/emergency-bundle.sh" --confirm
state_set "recovery_handoff_confirmed" "true"
state_set "recovery_handoff_bundle_sha256" "\"$BUNDLE_DIGEST\""
state_set "recovery_handoff_confirmed_at" "\"$(date -Iseconds)\""
# Historical emergency_bundle_delivered fields are preserved, never reinterpreted.

# ── Phase 10: Cleanup + finalise ──────────────────────────────────────────────

step "10 — Cleanup and finalise"

if [[ "$DRY_RUN" == false ]]; then
    if [[ -d "$SECRETS_DIR" ]]; then
        log "shredding plaintext secrets..."
        find "$SECRETS_DIR" -type f -exec shred -u {} \;
        rm -rf "$SECRETS_DIR"
        ok "plaintext secrets shredded"
    fi

    state_set "bootstrap_complete" "true"

    # Commit final state
    cd "$REPO_ROOT"
    git add sso-mfa/bootstrap/creds-state.yaml
    if ! git diff --cached --quiet; then
        git commit -m "chore(creds): bootstrap complete [agent NK-WP-0005]"
        ok "final state committed"
    fi
else
    echo "  [dry-run] would shred secrets/ and set bootstrap_complete: true"
fi

# ── Done ──────────────────────────────────────────────────────────────────────

echo ""
echo "╔══════════════════════════════════════════════════════════════════╗"
echo "║          NET-KINGDOM CREDENTIAL BOOTSTRAP COMPLETE               ║"
echo "╚══════════════════════════════════════════════════════════════════╝"
echo ""
echo "  All service secrets have been generated, encrypted, committed,"
echo "  and injected into the cluster. The emergency bundle has been"
echo "  delivered to you for storage in your personal password manager."
echo ""
echo "  Run 'make creds-agent-status' to review the final state."
echo ""