RPF-WP-0046: periodic ESO database tokens and a daily renewer

The five dynamic-database ClusterSecretStores use 768h static tokens that
nothing renews; expiry revoked their DB leases on 2026-09-23 and recurs
around 2026-10-25. Kubernetes auth is not a drop-in fix: ESO v0.16.1 revokes
its login token after each reconcile, which revokes the leases it created.

- eso-token-renewer CronJob (ArgoCD draft, no RBAC, mounted Secrets).
- Attended periodic mint script for all five lanes.
- CronJob added to the platform-addons AppProject in git (not yet applied).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

Assistant: claude-code
Assistant-Model: opus
Assistant-Process: 150322@bnt-lap001
Assistant-Session: 16a7b788-374e-4915-a1df-fc87ffd9a5e4

scripts/openbao-eso-db-token-periodic-attended.sh

@@ -0,0 +1,113 @@
+#!/bin/sh
+# RPF-WP-0046-T03: re-mint the dynamic-database ESO parent tokens as PERIODIC
+# tokens, so the eso-token-renewer CronJob can keep them alive indefinitely.
+# A 768h max-TTL token cannot be renewed past its max TTL, and when it expires
+# OpenBao revokes every database lease it created (the 2026-09-23 outages).
+#
+# Derived from rapp-postgres scripts/apply-eso-token-attended.sh (3ebd984):
+# same policies per lane, same silent contract, but -period instead of -ttl.
+#
+#   python3 /home/worsch/railiance-platform/scripts/openbao-attended-exec.py -- \
+#     /home/worsch/railiance-platform/scripts/openbao-eso-db-token-periodic-attended.sh \
+#     --confirm RPF-WP-0046-PERIODIC-ESO-TOKEN --status <absolute status file> \
+#     audit-core canned-prompts core-hub sbom-nexus tenant-engine
+#
+# - Silent: warden fails closed on any child output. The status file (absolute,
+#   created 0600) gets a non-secret trace only.
+# - Tokens move only through pipes on stdin: never in argv, output, or files.
+# - Secrets are written with server-side apply and without the last-applied
+#   annotation. The replaced token is left to expire; see RPF-WP-0046-T05 for
+#   the consumer restart that must precede its expiry.
+#
+# Exit codes: 0 all lanes applied and verified · 2 bad arguments or missing
+# tool · 3 mint failed · 4 token not periodic/renewable or wrong policies ·
+# 5 Kubernetes apply failed · 6 post-apply check failed.
+exec >/dev/null 2>&1
+set -u
+
+CONFIRM=RPF-WP-0046-PERIODIC-ESO-TOKEN
+TOKEN_PERIOD=${TOKEN_PERIOD:-168h}
+REMOTE_HOST=${REMOTE_HOST:-railiance01}
+SECRET_NAMESPACE=external-secrets
+STATUS=/dev/null
+ERR="$(mktemp)"
+trap 'rm -f "$ERR"; unset child_token token_json' EXIT
+
+note() { printf '%s\n' "$*" >>"$STATUS"; }
+fail() { note "exit $1 at $2"; sed 's/^/  err: /' "$ERR" | head -5 >>"$STATUS"; exit "$1"; }
+
+# lane -> "secret-name policy[,policy...]"; keep in step with the renewer mounts.
+lane_spec() {
+  case "$1" in
+    audit-core) echo "openbao-audit-core-eso-token external-secrets-audit-core" ;;
+    canned-prompts) echo "openbao-canned-prompts-eso-token external-secrets-canned-prompts-database" ;;
+    core-hub) echo "openbao-core-hub-eso-token external-secrets-core-hub-database" ;;
+    sbom-nexus) echo "openbao-sbom-nexus-eso-token external-secrets-sbom-nexus-database" ;;
+    tenant-engine) echo "openbao-tenant-engine-eso-token credential-broker-tenant-engine-runtime,credential-broker-tenant-engine-migration" ;;
+    *) return 1 ;;
+  esac
+}
+
+[ "${1:-}" = "--confirm" ] && [ "${2:-}" = "$CONFIRM" ] || exit 2
+shift 2
+if [ "${1:-}" = "--status" ]; then
+  case "${2:-}" in /*) ;; *) exit 2 ;; esac
+  STATUS=$2
+  shift 2
+  ( umask 077; : >"$STATUS" ) || exit 2
+fi
+[ "$#" -ge 1 ] || { note "usage: --confirm $CONFIRM [--status FILE] <lane>..."; exit 2; }
+for lane in "$@"; do
+  lane_spec "$lane" >/dev/null || { note "unknown lane: $lane"; exit 2; }
+done
+for tool in bao python3 ssh; do
+  command -v "$tool" || { note "missing tool: $tool"; exit 2; }
+done
+note "BAO_ADDR=${BAO_ADDR:-unset} remote=$REMOTE_HOST period=$TOKEN_PERIOD lanes=$*"
+
+for lane in "$@"; do
+  spec=$(lane_spec "$lane")
+  secret=${spec%% *}
+  policies=${spec#* }
+  policy_args=$(printf '%s' "$policies" | tr ',' '\n' | sed 's/^/-policy=/' | tr '\n' ' ')
+
+  # shellcheck disable=SC2086  # policy_args is a controlled word list
+  token_json=$(bao token create $policy_args -period="$TOKEN_PERIOD" \
+    -renewable=true -orphan -format=json 2>"$ERR") || fail 3 "$lane token create"
+  child_token=$(printf '%s' "$token_json" | python3 -c \
+    'import json,sys; print(json.load(sys.stdin)["auth"]["client_token"], end="")' 2>"$ERR") \
+    || fail 3 "$lane parse token"
+  [ "${#child_token}" -ge 8 ] || fail 3 "$lane empty token"
+
+  printf '%s' "$child_token" | bao write -format=json auth/token/lookup token=- 2>"$ERR" \
+    | EXPECTED="$policies" python3 -c '
+import json, os, sys
+d = json.load(sys.stdin)["data"]
+got = sorted(p for p in d.get("policies", []) if p != "default")
+want = sorted(os.environ["EXPECTED"].split(","))
+print("  renewable=%s period=%s policies=%s orphan=%s" % (
+    d.get("renewable"), d.get("period"), ",".join(d.get("policies", [])), d.get("orphan")))
+sys.exit(0 if d.get("renewable") is True and int(d.get("period") or 0) > 0 and got == want else 1)
+' >>"$STATUS" 2>"$ERR" || fail 4 "$lane token lookup/verify"
+  note "$lane: minted periodic and verified"
+
+  printf '%s' "$child_token" | ssh -o BatchMode=yes "$REMOTE_HOST" "
+    set -e
+    kubectl -n '$SECRET_NAMESPACE' create secret generic '$secret' \
+      --from-file=token=/dev/stdin --dry-run=client -o yaml \
+      | kubectl apply --server-side --force-conflicts \
+          --field-manager=railiance-platform-attended -f - >/dev/null
+    kubectl -n '$SECRET_NAMESPACE' annotate secret '$secret' \
+      kubectl.kubernetes.io/last-applied-configuration- >/dev/null 2>&1 || true
+  " 2>"$ERR" || fail 5 "$lane kubernetes apply"
+  note "$lane: secret $secret applied"
+
+  check=$(ssh -o BatchMode=yes "$REMOTE_HOST" \
+    "kubectl -n '$SECRET_NAMESPACE' get secret '$secret' -o go-template='{{ with .metadata.annotations }}{{ if index . \"kubectl.kubernetes.io/last-applied-configuration\" }}HAS-ANNOTATION{{ else }}clean{{ end }}{{ else }}clean{{ end }}'" 2>"$ERR") \
+    || fail 6 "$lane post-apply read"
+  note "$lane: annotation check $check"
+  [ "$check" = "clean" ] || fail 6 "$lane annotation still present"
+  unset child_token token_json
+done
+note "exit 0"
+exit 0