Add OpenBao authenticated readiness verifier

Makefile

@@ -13,6 +13,7 @@ OPENBAO_CHART_VERSION ?= 0.28.2
 OPENBAO_NAMESPACE ?= openbao
 OPENBAO_RELEASE ?= openbao
 OPENBAO_VALUES ?= helm/openbao-values.yaml
+OPENBAO_VERIFY_AUTH_ARGS ?=
 
 ##@ CloudNative PG (cnpg) — primary database operator
 
@@ -121,6 +122,10 @@ openbao-configure-initial: ## Apply first post-unseal audit, auth, mounts, and p
 	KUBECTL='$(KUBECTL)' OPENBAO_NAMESPACE=$(OPENBAO_NAMESPACE) \
 	  OPENBAO_RELEASE=$(OPENBAO_RELEASE) scripts/openbao-apply-initial-config.sh
 
+openbao-verify-authenticated: ## Run authenticated non-mutating OpenBao audit/auth/mount checks
+	KUBECTL='$(KUBECTL)' OPENBAO_NAMESPACE=$(OPENBAO_NAMESPACE) \
+	  OPENBAO_RELEASE=$(OPENBAO_RELEASE) scripts/openbao-verify-authenticated.sh $(OPENBAO_VERIFY_AUTH_ARGS)
+
 ##@ Backup
 
 backup: ## Backup platform services (PostgreSQL logical dump) — age-encrypted to Nextcloud
@@ -133,4 +138,4 @@ help: ## Show this help
 	  /^[a-zA-Z_-]+:.*?##/ { printf "  \033[36m%-22s\033[0m %s\n", $$1, $$2 } \
 	  /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) }' $(MAKEFILE_LIST)
 
-.PHONY: db-deploy db-status db-shell db-logs apps-pg-deploy apps-pg-status apps-pg-shell apps-pg-logs pg-deploy pg-status pg-pgpool-check valkey-deploy valkey-status openbao-repo openbao-dry-run openbao-deploy openbao-status openbao-verify openbao-verify-post-unseal openbao-configure-initial backup help
+.PHONY: db-deploy db-status db-shell db-logs apps-pg-deploy apps-pg-status apps-pg-shell apps-pg-logs pg-deploy pg-status pg-pgpool-check valkey-deploy valkey-status openbao-repo openbao-dry-run openbao-deploy openbao-status openbao-verify openbao-verify-post-unseal openbao-configure-initial openbao-verify-authenticated backup help

docs/openbao.md

@@ -279,6 +279,33 @@ Before any live application secrets move into OpenBao:
    make openbao-verify-post-unseal
    ```
 
+Authenticated verification, after the KeyCape-backed `platform-admin` path or
+another approved operator token is available:
+
+```bash
+make openbao-verify-authenticated
+```
+
+The target prompts for the token without echoing it, never puts the token on
+the command line, and only runs non-mutating checks. It verifies that
+`bao audit list` shows `file/`, `bao secrets list` shows `platform/`,
+`bao auth list` shows both `kubernetes/` and `keycape/`, and that the file
+audit log is non-empty.
+
+If a previous attended OIDC login stored a still-valid token in the pod token
+helper, use:
+
+```bash
+make openbao-verify-authenticated OPENBAO_VERIFY_AUTH_ARGS=--use-token-helper
+```
+
+Current durable audit status: the file audit device writes to the audit PVC,
+which is necessary but not enough for production trust. Before application
+secrets move into OpenBao, choose and test a durable audit sink beyond that PVC
+such as an encrypted platform backup/export path or the future centralized
+logging stack. Do not treat non-secret hashes, screenshots, or State Hub notes
+as substitutes for retained audit log custody.
+
 Monitoring baseline:
 
 - pod readiness and liveness from Kubernetes probes

scripts/openbao-verify-authenticated.sh

@@ -0,0 +1,246 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+OPENBAO_NAMESPACE="${OPENBAO_NAMESPACE:-openbao}"
+OPENBAO_RELEASE="${OPENBAO_RELEASE:-openbao}"
+KUBECTL="${KUBECTL:-kubectl}"
+TOKEN_FILE="${OPENBAO_TOKEN_FILE:-}"
+DRY_RUN=0
+USE_TOKEN_HELPER=0
+
+usage() {
+  cat <<'USAGE'
+Usage: scripts/openbao-verify-authenticated.sh [--dry-run] [--use-token-helper]
+
+Runs authenticated, non-mutating OpenBao readiness checks:
+  - audit list includes file/
+  - secrets list includes platform/
+  - auth list includes kubernetes/ and keycape/
+  - audit log exists and is non-empty
+
+The token is read from OPENBAO_TOKEN_FILE or an interactive hidden prompt. The
+script does not print, store, or pass the token on the command line.
+
+With --use-token-helper, no token is read by this script. The checks rely on
+the OpenBao token helper inside the pod, if a previous attended login stored a
+still-valid token there.
+USAGE
+}
+
+while [ "$#" -gt 0 ]; do
+  case "$1" in
+    --dry-run)
+      DRY_RUN=1
+      shift
+      ;;
+    --use-token-helper)
+      USE_TOKEN_HELPER=1
+      shift
+      ;;
+    -h|--help)
+      usage
+      exit 0
+      ;;
+    *)
+      echo "ERROR: unknown argument: $1" >&2
+      usage >&2
+      exit 2
+      ;;
+  esac
+done
+
+pod="${OPENBAO_RELEASE}-0"
+FAILURES=0
+WARNINGS=0
+
+ok() { printf '[OK] %s\n' "$*"; }
+warn() { WARNINGS=$((WARNINGS + 1)); printf '[WARN] %s\n' "$*" >&2; }
+fail() { FAILURES=$((FAILURES + 1)); printf '[FAIL] %s\n' "$*" >&2; }
+step() { printf '\n==> %s\n' "$*"; }
+
+check_cmd() {
+  if ! command -v "${KUBECTL%% *}" >/dev/null 2>&1; then
+    echo "ERROR: kubectl command not found: $KUBECTL" >&2
+    exit 1
+  fi
+}
+
+run() {
+  # shellcheck disable=SC2086
+  $KUBECTL "$@"
+}
+
+read_token() {
+  if [ "$USE_TOKEN_HELPER" -eq 1 ]; then
+    printf '__USE_TOKEN_HELPER__\n'
+    return
+  fi
+
+  if [ "$DRY_RUN" -eq 1 ]; then
+    printf 'dry-run-token\n'
+    return
+  fi
+
+  if [ -n "$TOKEN_FILE" ]; then
+    if [ ! -f "$TOKEN_FILE" ]; then
+      echo "ERROR: OPENBAO_TOKEN_FILE does not exist: $TOKEN_FILE" >&2
+      exit 1
+    fi
+    head -n 1 "$TOKEN_FILE"
+    return
+  fi
+
+  local token
+  read -r -s -p "OpenBao token: " token
+  printf '\n' >&2
+  printf '%s\n' "$token"
+}
+
+remote_bao() {
+  local token="$1"
+  shift
+  if [ "$DRY_RUN" -eq 1 ]; then
+    case "$*" in
+      status)
+        cat <<'STATUS'
+Key        Value
+---        -----
+Sealed     false
+STATUS
+        ;;
+      "audit list")
+        cat <<'AUDIT'
+Path     Type    Description
+----     ----    -----------
+file/    file    Default file audit device on the OpenBao audit PVC.
+AUDIT
+        ;;
+      "secrets list")
+        cat <<'SECRETS'
+Path          Type
+----          ----
+cubbyhole/    cubbyhole
+identity/     identity
+platform/     kv
+SECRETS
+        ;;
+      "auth list")
+        cat <<'AUTH'
+Path           Type
+----           ----
+keycape/       oidc
+kubernetes/    kubernetes
+token/         token
+AUTH
+        ;;
+      *)
+        printf 'DRY-RUN: bao %s\n' "$*"
+        ;;
+    esac
+    return 0
+  fi
+  if [ "$USE_TOKEN_HELPER" -eq 1 ]; then
+    run exec -n "$OPENBAO_NAMESPACE" "$pod" -- bao "$@"
+    return $?
+  fi
+  printf '%s\n' "$token" | run exec -i -n "$OPENBAO_NAMESPACE" "$pod" -- \
+    sh -c 'read -r BAO_TOKEN; export BAO_TOKEN; exec bao "$@"' sh "$@"
+}
+
+audit_log_bytes() {
+  if [ "$DRY_RUN" -eq 1 ]; then
+    printf '1\n'
+    return 0
+  fi
+  run exec -n "$OPENBAO_NAMESPACE" "$pod" -- \
+    sh -c 'test -s /openbao/audit/openbao-audit.log && wc -c < /openbao/audit/openbao-audit.log' \
+    2>/dev/null | tr -d '[:space:]'
+}
+
+require_pattern() {
+  local label="$1"
+  local output="$2"
+  local pattern="$3"
+  if printf '%s\n' "$output" | grep -Eq "$pattern"; then
+    ok "$label"
+  else
+    fail "$label"
+  fi
+}
+
+check_cmd
+
+token="$(read_token)"
+if [ -z "$token" ]; then
+  echo "ERROR: empty OpenBao token" >&2
+  exit 1
+fi
+
+before_bytes="$(audit_log_bytes || true)"
+before_bytes="${before_bytes:-0}"
+
+step "OpenBao status"
+if status_output="$(remote_bao "$token" status 2>&1)"; then
+  printf '%s\n' "$status_output"
+  require_pattern "OpenBao is unsealed" "$status_output" '^Sealed[[:space:]]+false$'
+else
+  printf '%s\n' "$status_output" >&2
+  fail "bao status succeeded"
+fi
+
+step "Audit devices"
+if audit_output="$(remote_bao "$token" audit list 2>&1)"; then
+  printf '%s\n' "$audit_output"
+  require_pattern "file/ audit device is visible" "$audit_output" '(^|[[:space:]])file/'
+else
+  printf '%s\n' "$audit_output" >&2
+  fail "bao audit list succeeded"
+fi
+
+step "Secrets engines"
+if secrets_output="$(remote_bao "$token" secrets list 2>&1)"; then
+  printf '%s\n' "$secrets_output"
+  require_pattern "platform/ secrets engine is visible" "$secrets_output" '(^|[[:space:]])platform/'
+else
+  printf '%s\n' "$secrets_output" >&2
+  fail "bao secrets list succeeded"
+fi
+
+step "Auth methods"
+if auth_output="$(remote_bao "$token" auth list 2>&1)"; then
+  printf '%s\n' "$auth_output"
+  require_pattern "kubernetes/ auth method is visible" "$auth_output" '(^|[[:space:]])kubernetes/'
+  require_pattern "keycape/ auth method is visible" "$auth_output" '(^|[[:space:]])keycape/'
+else
+  printf '%s\n' "$auth_output" >&2
+  fail "bao auth list succeeded"
+fi
+
+after_bytes="$(audit_log_bytes || true)"
+after_bytes="${after_bytes:-0}"
+
+step "Audit log file"
+if [ "$after_bytes" -gt 0 ]; then
+  ok "audit log file is non-empty (${after_bytes} bytes)"
+else
+  fail "audit log file is non-empty"
+fi
+
+if [ "$DRY_RUN" -eq 1 ]; then
+  ok "audit log growth check skipped in dry-run"
+elif [ "$after_bytes" -gt "$before_bytes" ]; then
+  ok "audit log grew during authenticated checks (${before_bytes} -> ${after_bytes} bytes)"
+else
+  warn "audit log size did not increase during this run (${before_bytes} -> ${after_bytes} bytes)"
+fi
+
+if [ "$FAILURES" -gt 0 ]; then
+  printf '\nAuthenticated OpenBao verification failed with %s failure(s) and %s warning(s).\n' "$FAILURES" "$WARNINGS" >&2
+  exit 1
+fi
+
+printf '\nAuthenticated OpenBao verification passed'
+if [ "$WARNINGS" -gt 0 ]; then
+  printf ' with %s warning(s)' "$WARNINGS"
+fi
+printf '.\n'

workplans/RAIL-PL-WP-0002-openbao-platform-secrets-service.md

@@ -265,6 +265,17 @@ pin the live OpenBao image tag to `2.5.4`; Helm release revision 3 has the same
 explicit tag and the pod remained ready, so future chart upgrades do not
 implicitly change the runtime version while applying unrelated configuration.
 
+**2026-06-01:** Added `make openbao-verify-authenticated` as a non-mutating
+operator proof for the remaining OpenBao readiness checks that require an
+approved token. The helper prompts for the token without echoing it, verifies
+`file/` audit visibility, `platform/` secrets, `kubernetes/` and `keycape/`
+auth methods, and confirms the audit log file is non-empty. It can also use an
+already-valid pod token helper via
+`OPENBAO_VERIFY_AUTH_ARGS=--use-token-helper` so the token does not move
+through the local shell at all. Durable audit shipping beyond the audit PVC
+remains intentionally open until a tested sink is selected; State Hub notes and
+hashes are evidence, not retained audit custody.
+
 ### T07 - Cross-Repo Transition Tasks
 
 ```task