175 lines
8 KiB
Bash
175 lines
8 KiB
Bash
|
#!/usr/bin/env bash
|
|||
|
|
# verify-t08.sh — verify NK-WP-0001-T08 done-criteria
|
|||
|
|
#
|
|||
|
|
# Checks backups, DR readiness, and break-glass account.
|
|||
|
|
#
|
|||
|
|
# Sections:
|
|||
|
|
# 1. Backup CronJobs exist (lldap-backup, authelia-backup, privacyidea-backup)
|
|||
|
|
# 2. backup-sa ServiceAccount and RBAC exist
|
|||
|
|
# 3. lldap-backup has run successfully at least once
|
|||
|
|
# 4. authelia-backup has run successfully at least once
|
|||
|
|
# 5. privacyidea-backup has run successfully at least once
|
|||
|
|
# 6. privacyIDEA enckey backup exists on PVC
|
|||
|
|
# 7. LLDAP SQLite backup exists on PVC
|
|||
|
|
# 8. DR-RUNBOOK.md present in repo
|
|||
|
|
# 9. KeePassXC ops bundle (pack-bundle.sh) — manual confirmation required
|
|||
|
|
#
|
|||
|
|
# Usage:
|
|||
|
|
# chmod +x verify-t08.sh
|
|||
|
|
# ./verify-t08.sh
|
|||
|
|
|
|||
|
|
set -euo pipefail
|
|||
|
|
|
|||
|
|
SSO_NAMESPACE="sso"
|
|||
|
|
MFA_NAMESPACE="mfa"
|
|||
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|||
|
|
|
|||
|
|
PASS=0
|
|||
|
|
FAIL=0
|
|||
|
|
WARN=0
|
|||
|
|
|
|||
|
|
pass() { echo " [PASS] $1"; ((PASS++)); }
|
|||
|
|
fail() { echo " [FAIL] $1"; ((FAIL++)); }
|
|||
|
|
warn() { echo " [WARN] $1"; ((WARN++)); }
|
|||
|
|
|
|||
|
|
section() { echo ""; echo "── $1 ──────────────────────────────────────"; }
|
|||
|
|
|
|||
|
|
check_cronjob() {
|
|||
|
|
local name="$1"; local ns="$2"
|
|||
|
|
if kubectl get cronjob "$name" -n "$ns" &>/dev/null; then
|
|||
|
|
pass "CronJob $name exists (namespace: $ns)"
|
|||
|
|
local schedule
|
|||
|
|
schedule=$(kubectl get cronjob "$name" -n "$ns" \
|
|||
|
|
-o jsonpath='{.spec.schedule}' 2>/dev/null || echo "?")
|
|||
|
|
pass " Schedule: $schedule"
|
|||
|
|
else
|
|||
|
|
fail "CronJob $name not found in namespace $ns — apply backup/cronjob-sqlite-backups.yaml"
|
|||
|
|
fi
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
check_last_job() {
|
|||
|
|
local cronjob="$1"; local ns="$2"
|
|||
|
|
# Find the most recent Job spawned by this CronJob
|
|||
|
|
LAST_JOB=$(kubectl get job -n "$ns" \
|
|||
|
|
-l "batch.kubernetes.io/controller-uid" \
|
|||
|
|
--sort-by=.metadata.creationTimestamp \
|
|||
|
|
-o jsonpath='{.items[-1].metadata.name}' 2>/dev/null || echo "")
|
|||
|
|
# Simpler: look for any completed job with the cronjob name prefix
|
|||
|
|
SUCCEEDED=$(kubectl get job -n "$ns" \
|
|||
|
|
-o jsonpath="{.items[?(@.metadata.ownerReferences[0].name==\"$cronjob\")].status.succeeded}" \
|
|||
|
|
2>/dev/null || echo "")
|
|||
|
|
if [[ "$SUCCEEDED" == *"1"* ]]; then
|
|||
|
|
pass "CronJob $cronjob has at least one successful run"
|
|||
|
|
else
|
|||
|
|
warn "CronJob $cronjob has no successful runs yet — trigger manually to test:"
|
|||
|
|
warn " kubectl create job -n $ns --from=cronjob/$cronjob ${cronjob}-manual-test"
|
|||
|
|
fi
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
# ── 1. Backup CronJobs ────────────────────────────────────────────────────────
|
|||
|
|
section "1. Backup CronJobs"
|
|||
|
|
check_cronjob "lldap-backup" "$SSO_NAMESPACE"
|
|||
|
|
check_cronjob "authelia-backup" "$SSO_NAMESPACE"
|
|||
|
|
check_cronjob "privacyidea-backup" "$MFA_NAMESPACE"
|
|||
|
|
|
|||
|
|
# ── 2. RBAC ───────────────────────────────────────────────────────────────────
|
|||
|
|
section "2. Backup ServiceAccount and RBAC (namespace: $SSO_NAMESPACE)"
|
|||
|
|
if kubectl get serviceaccount backup-sa -n "$SSO_NAMESPACE" &>/dev/null; then
|
|||
|
|
pass "ServiceAccount backup-sa exists"
|
|||
|
|
else
|
|||
|
|
fail "ServiceAccount backup-sa not found — apply backup/cronjob-sqlite-backups.yaml"
|
|||
|
|
fi
|
|||
|
|
if kubectl get role backup-scaler -n "$SSO_NAMESPACE" &>/dev/null; then
|
|||
|
|
pass "Role backup-scaler exists"
|
|||
|
|
else
|
|||
|
|
fail "Role backup-scaler not found"
|
|||
|
|
fi
|
|||
|
|
if kubectl get rolebinding backup-sa-scaler -n "$SSO_NAMESPACE" &>/dev/null; then
|
|||
|
|
pass "RoleBinding backup-sa-scaler exists"
|
|||
|
|
else
|
|||
|
|
fail "RoleBinding backup-sa-scaler not found"
|
|||
|
|
fi
|
|||
|
|
|
|||
|
|
# ── 3–5. CronJob run history ──────────────────────────────────────────────────
|
|||
|
|
section "3. lldap-backup run history"
|
|||
|
|
check_last_job "lldap-backup" "$SSO_NAMESPACE"
|
|||
|
|
|
|||
|
|
section "4. authelia-backup run history"
|
|||
|
|
check_last_job "authelia-backup" "$SSO_NAMESPACE"
|
|||
|
|
|
|||
|
|
section "5. privacyidea-backup run history"
|
|||
|
|
check_last_job "privacyidea-backup" "$MFA_NAMESPACE"
|
|||
|
|
|
|||
|
|
# ── 6. privacyIDEA enckey backup on PVC ──────────────────────────────────────
|
|||
|
|
section "6. privacyIDEA enckey backup on PVC"
|
|||
|
|
PI_POD=$(kubectl get pod -n "$MFA_NAMESPACE" \
|
|||
|
|
-l app.kubernetes.io/name=privacyidea \
|
|||
|
|
--field-selector=status.phase=Running \
|
|||
|
|
-o jsonpath='{.items[0].metadata.name}' 2>/dev/null || echo "")
|
|||
|
|
if [[ -n "$PI_POD" ]]; then
|
|||
|
|
BACKUP_COUNT=$(kubectl exec -n "$MFA_NAMESPACE" "$PI_POD" -- \
|
|||
|
|
sh -c 'ls /data/backups/enckey.backup.* 2>/dev/null | wc -l' 2>/dev/null || echo "0")
|
|||
|
|
BACKUP_COUNT="${BACKUP_COUNT// /}"
|
|||
|
|
if [[ "$BACKUP_COUNT" -gt 0 ]]; then
|
|||
|
|
pass "privacyIDEA enckey backups found on PVC ($BACKUP_COUNT file(s))"
|
|||
|
|
else
|
|||
|
|
warn "No enckey backup files on PVC yet — trigger privacyidea-backup CronJob to create one"
|
|||
|
|
warn " kubectl create job -n $MFA_NAMESPACE --from=cronjob/privacyidea-backup pi-backup-test"
|
|||
|
|
fi
|
|||
|
|
else
|
|||
|
|
warn "Skipping enckey backup check — no running privacyIDEA pod"
|
|||
|
|
fi
|
|||
|
|
|
|||
|
|
# ── 7. LLDAP SQLite backup on PVC ────────────────────────────────────────────
|
|||
|
|
section "7. LLDAP SQLite backup on PVC"
|
|||
|
|
LLDAP_POD=$(kubectl get pod -n "$SSO_NAMESPACE" \
|
|||
|
|
-l app.kubernetes.io/name=lldap \
|
|||
|
|
--field-selector=status.phase=Running \
|
|||
|
|
-o jsonpath='{.items[0].metadata.name}' 2>/dev/null || echo "")
|
|||
|
|
if [[ -n "$LLDAP_POD" ]]; then
|
|||
|
|
BACKUP_COUNT=$(kubectl exec -n "$SSO_NAMESPACE" "$LLDAP_POD" -- \
|
|||
|
|
sh -c 'ls /data/backups/users.backup.* 2>/dev/null | wc -l' 2>/dev/null || echo "0")
|
|||
|
|
BACKUP_COUNT="${BACKUP_COUNT// /}"
|
|||
|
|
if [[ "$BACKUP_COUNT" -gt 0 ]]; then
|
|||
|
|
pass "LLDAP SQLite backups found on PVC ($BACKUP_COUNT file(s))"
|
|||
|
|
else
|
|||
|
|
warn "No LLDAP backup files on PVC yet — trigger lldap-backup CronJob to create one"
|
|||
|
|
warn " kubectl create job -n $SSO_NAMESPACE --from=cronjob/lldap-backup lldap-backup-test"
|
|||
|
|
fi
|
|||
|
|
else
|
|||
|
|
warn "Skipping LLDAP backup check — no running LLDAP pod"
|
|||
|
|
fi
|
|||
|
|
|
|||
|
|
# ── 8. DR runbook present ─────────────────────────────────────────────────────
|
|||
|
|
section "8. DR runbook"
|
|||
|
|
RUNBOOK="$SCRIPT_DIR/backup/DR-RUNBOOK.md"
|
|||
|
|
if [[ -f "$RUNBOOK" ]]; then
|
|||
|
|
pass "DR-RUNBOOK.md present at $RUNBOOK"
|
|||
|
|
else
|
|||
|
|
fail "DR-RUNBOOK.md not found — it should be at sso-mfa/k8s/backup/DR-RUNBOOK.md"
|
|||
|
|
fi
|
|||
|
|
|
|||
|
|
# ── 9. Offsite backup (manual confirmation) ───────────────────────────────────
|
|||
|
|
section "9. Offsite backup (manual)"
|
|||
|
|
warn "Cannot verify offsite backup automatically — confirm manually:"
|
|||
|
|
warn " - pack-bundle.sh has been run with current secrets"
|
|||
|
|
warn " - ops-bundle.tar.age stored in a separate physical location"
|
|||
|
|
warn " - age decryption key stored separately (NOT in the same location as the bundle)"
|
|||
|
|
|
|||
|
|
# ── Summary ───────────────────────────────────────────────────────────────────
|
|||
|
|
echo ""
|
|||
|
|
echo "════════════════════════════════════════════════════════════"
|
|||
|
|
echo " T08 verification: PASS=$PASS WARN=$WARN FAIL=$FAIL"
|
|||
|
|
echo "════════════════════════════════════════════════════════════"
|
|||
|
|
|
|||
|
|
if [[ "$FAIL" -gt 0 ]]; then
|
|||
|
|
echo " Result: INCOMPLETE — resolve FAIL items before marking T08 done"
|
|||
|
|
exit 1
|
|||
|
|
elif [[ "$WARN" -gt 0 ]]; then
|
|||
|
|
echo " Result: PARTIAL — structure is in place; resolve WARN items (trigger CronJobs)"
|
|||
|
|
exit 0
|
|||
|
|
else
|
|||
|
|
echo " Result: COMPLETE — T08 done-criteria met; SSO/MFA platform workplan complete!"
|
|||
|
|
exit 0
|
|||
|
|
fi
|