diff --git a/argocd/platform-addons/eso-token-renewer/cronjob.yaml b/argocd/platform-addons/eso-token-renewer/cronjob.yaml new file mode 100644 index 0000000..ff22fcd --- /dev/null +++ b/argocd/platform-addons/eso-token-renewer/cronjob.yaml @@ -0,0 +1,90 @@ +# RPF-WP-0046: daily renew-self for the ESO parent tokens of the five +# dynamic-database ClusterSecretStores. The Secrets are mounted, not read via +# the API, so the ServiceAccount has no RBAC and no API token. +apiVersion: v1 +kind: ServiceAccount +metadata: + name: eso-token-renewer + namespace: external-secrets + labels: + app.kubernetes.io/name: eso-token-renewer + app.kubernetes.io/part-of: railiance-platform +automountServiceAccountToken: false +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: eso-token-renewer + namespace: external-secrets + labels: + app.kubernetes.io/name: eso-token-renewer + app.kubernetes.io/part-of: railiance-platform +spec: + schedule: '40 2 * * *' + timeZone: Etc/UTC + suspend: false + concurrencyPolicy: Forbid + startingDeadlineSeconds: 3600 + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 5 + jobTemplate: + spec: + activeDeadlineSeconds: 300 + backoffLimit: 2 + ttlSecondsAfterFinished: 604800 + template: + metadata: + labels: + app.kubernetes.io/name: eso-token-renewer + spec: + serviceAccountName: eso-token-renewer + automountServiceAccountToken: false + restartPolicy: Never + securityContext: + runAsNonRoot: true + runAsUser: 65534 + runAsGroup: 65534 + fsGroup: 65534 + seccompProfile: + type: RuntimeDefault + containers: + - name: renew + image: python:3.12-slim@sha256:d764629ce0ddd8c71fd371e9901efb324a95789d2315a47db7e4d27e78f1b0e9 + command: [python3, /worker/renew.py] + env: + - name: PYTHONDONTWRITEBYTECODE + value: '1' + - name: BAO_ADDR + value: http://openbao.openbao.svc:8200 + - name: ESO_TOKEN_ROOT + value: /var/run/eso-tokens + - name: MIN_TTL_SECONDS + value: '259200' + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: [ALL] + resources: + requests: {cpu: 10m, memory: 24Mi} + limits: {cpu: 100m, memory: 64Mi} + volumeMounts: + - {name: worker, mountPath: /worker, readOnly: true} + - {name: audit-core, mountPath: /var/run/eso-tokens/audit-core, readOnly: true} + - {name: canned-prompts, mountPath: /var/run/eso-tokens/canned-prompts, readOnly: true} + - {name: core-hub, mountPath: /var/run/eso-tokens/core-hub, readOnly: true} + - {name: sbom-nexus, mountPath: /var/run/eso-tokens/sbom-nexus, readOnly: true} + - {name: tenant-engine, mountPath: /var/run/eso-tokens/tenant-engine, readOnly: true} + volumes: + - name: worker + configMap: {name: eso-token-renewer-worker, defaultMode: 0444} + - name: audit-core + secret: {secretName: openbao-audit-core-eso-token, defaultMode: 0440, items: [{key: token, path: token}]} + - name: canned-prompts + secret: {secretName: openbao-canned-prompts-eso-token, defaultMode: 0440, items: [{key: token, path: token}]} + - name: core-hub + secret: {secretName: openbao-core-hub-eso-token, defaultMode: 0440, items: [{key: token, path: token}]} + - name: sbom-nexus + secret: {secretName: openbao-sbom-nexus-eso-token, defaultMode: 0440, items: [{key: token, path: token}]} + - name: tenant-engine + secret: {secretName: openbao-tenant-engine-eso-token, defaultMode: 0440, items: [{key: token, path: token}]} diff --git a/argocd/platform-addons/eso-token-renewer/kustomization.yaml b/argocd/platform-addons/eso-token-renewer/kustomization.yaml new file mode 100644 index 0000000..7551a35 --- /dev/null +++ b/argocd/platform-addons/eso-token-renewer/kustomization.yaml @@ -0,0 +1,16 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +# RPF-WP-0046. Read only by the railiance01 Application in +# argocd/railiance01/drafts/eso-token-renewer.application.yaml. No coulombcore +# Application points here. + +namespace: external-secrets + +resources: + - cronjob.yaml + +configMapGenerator: + - name: eso-token-renewer-worker + files: + - renew.py diff --git a/argocd/platform-addons/eso-token-renewer/renew.py b/argocd/platform-addons/eso-token-renewer/renew.py new file mode 100644 index 0000000..6914ee0 --- /dev/null +++ b/argocd/platform-addons/eso-token-renewer/renew.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +"""Renew the ESO parent tokens that project OpenBao dynamic database leases. + +ESO never renews a tokenSecretRef token, and OpenBao revokes every lease a +token created when that token expires (RPF-WP-0046). Each token Secret is +mounted read-only at //token. For each lane this calls +auth/token/renew-self and prints one JSON line with the lane, the resulting +TTL and an error class. It never prints a token. + +Exit 1 if any lane fails or ends below MIN_TTL_SECONDS, so the Job is marked +failed. A non-periodic token cannot be renewed past its max TTL, so it trips +the threshold as it nears expiry instead of failing silently. +""" +import json +import os +from pathlib import Path +import sys +import urllib.error +import urllib.request + +ADDR = os.environ.get('BAO_ADDR', 'http://openbao.openbao.svc:8200').rstrip('/') +ROOT = Path(os.environ.get('ESO_TOKEN_ROOT', '/var/run/eso-tokens')) +MIN_TTL = int(os.environ.get('MIN_TTL_SECONDS', str(72 * 3600))) + + +def renew(token, opener=urllib.request.urlopen): + request = urllib.request.Request( + ADDR + '/v1/auth/token/renew-self', data=b'{}', method='POST', + headers={'X-Vault-Token': token, 'Content-Type': 'application/json'}) + with opener(request, timeout=15) as response: + auth = json.load(response).get('auth') or {} + return {'ttl': auth.get('lease_duration'), 'renewable': auth.get('renewable')} + + +def check_lane(lane_dir, opener=urllib.request.urlopen): + result = {'lane': lane_dir.name, 'ok': False} + try: + token = (lane_dir / 'token').read_text(encoding='utf-8').strip() + if not token: + return dict(result, error='empty_token') + result.update(renew(token, opener)) + except urllib.error.HTTPError as error: + return dict(result, error='http_%s' % error.code) + except (urllib.error.URLError, OSError, ValueError) as error: + return dict(result, error=type(error).__name__) + ttl = result.get('ttl') + if not isinstance(ttl, int) or ttl < MIN_TTL: + return dict(result, error='ttl_below_threshold') + return dict(result, ok=True) + + +def main(opener=urllib.request.urlopen): + lanes = sorted(p for p in ROOT.iterdir() if p.is_dir() and not p.name.startswith('.')) + if not lanes: + print(json.dumps({'ok': False, 'error': 'no_lanes_mounted'})) + return 1 + results = [check_lane(lane, opener) for lane in lanes] + for item in results: + print(json.dumps(item, sort_keys=True)) + return 0 if all(item['ok'] for item in results) else 1 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/argocd/railiance01/bootstrap/02-railiance-platform-addons-project.yaml b/argocd/railiance01/bootstrap/02-railiance-platform-addons-project.yaml index 182b110..e61f072 100644 --- a/argocd/railiance01/bootstrap/02-railiance-platform-addons-project.yaml +++ b/argocd/railiance01/bootstrap/02-railiance-platform-addons-project.yaml @@ -40,6 +40,8 @@ spec: kind: ServiceAccount - group: apps kind: Deployment + - group: batch + kind: CronJob - group: rbac.authorization.k8s.io kind: Role - group: rbac.authorization.k8s.io diff --git a/argocd/railiance01/drafts/eso-token-renewer.application.yaml b/argocd/railiance01/drafts/eso-token-renewer.application.yaml new file mode 100644 index 0000000..6e57b14 --- /dev/null +++ b/argocd/railiance01/drafts/eso-token-renewer.application.yaml @@ -0,0 +1,27 @@ +# DRAFT for railiance01 (RPF-WP-0046-T04). Not synced by any root: move to +# ../applications/ only in T04, with the founder's go-ahead, after the +# platform-addons AppProject carries batch/CronJob. No automated sync, no +# finalizer. Set targetRevision to the reviewed commit at adoption. +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: eso-token-renewer + namespace: argocd + labels: + app.kubernetes.io/part-of: railiance-gitops + railiance-platform/component: external-secrets + annotations: + argocd.argoproj.io/sync-wave: "2" +spec: + project: railiance-platform-addons + source: + repoURL: https://forgejo.coulomb.social/coulomb/railiance-platform.git + targetRevision: PIN-AT-ADOPTION + path: argocd/platform-addons/eso-token-renewer + destination: + server: https://kubernetes.default.svc + namespace: external-secrets + syncPolicy: + syncOptions: + - ApplyOutOfSyncOnly=true + - PruneLast=true diff --git a/scripts/openbao-eso-db-token-periodic-attended.sh b/scripts/openbao-eso-db-token-periodic-attended.sh new file mode 100755 index 0000000..7459d5a --- /dev/null +++ b/scripts/openbao-eso-db-token-periodic-attended.sh @@ -0,0 +1,113 @@ +#!/bin/sh +# RPF-WP-0046-T03: re-mint the dynamic-database ESO parent tokens as PERIODIC +# tokens, so the eso-token-renewer CronJob can keep them alive indefinitely. +# A 768h max-TTL token cannot be renewed past its max TTL, and when it expires +# OpenBao revokes every database lease it created (the 2026-09-23 outages). +# +# Derived from rapp-postgres scripts/apply-eso-token-attended.sh (3ebd984): +# same policies per lane, same silent contract, but -period instead of -ttl. +# +# python3 /home/worsch/railiance-platform/scripts/openbao-attended-exec.py -- \ +# /home/worsch/railiance-platform/scripts/openbao-eso-db-token-periodic-attended.sh \ +# --confirm RPF-WP-0046-PERIODIC-ESO-TOKEN --status \ +# audit-core canned-prompts core-hub sbom-nexus tenant-engine +# +# - Silent: warden fails closed on any child output. The status file (absolute, +# created 0600) gets a non-secret trace only. +# - Tokens move only through pipes on stdin: never in argv, output, or files. +# - Secrets are written with server-side apply and without the last-applied +# annotation. The replaced token is left to expire; see RPF-WP-0046-T05 for +# the consumer restart that must precede its expiry. +# +# Exit codes: 0 all lanes applied and verified · 2 bad arguments or missing +# tool · 3 mint failed · 4 token not periodic/renewable or wrong policies · +# 5 Kubernetes apply failed · 6 post-apply check failed. +exec >/dev/null 2>&1 +set -u + +CONFIRM=RPF-WP-0046-PERIODIC-ESO-TOKEN +TOKEN_PERIOD=${TOKEN_PERIOD:-168h} +REMOTE_HOST=${REMOTE_HOST:-railiance01} +SECRET_NAMESPACE=external-secrets +STATUS=/dev/null +ERR="$(mktemp)" +trap 'rm -f "$ERR"; unset child_token token_json' EXIT + +note() { printf '%s\n' "$*" >>"$STATUS"; } +fail() { note "exit $1 at $2"; sed 's/^/ err: /' "$ERR" | head -5 >>"$STATUS"; exit "$1"; } + +# lane -> "secret-name policy[,policy...]"; keep in step with the renewer mounts. +lane_spec() { + case "$1" in + audit-core) echo "openbao-audit-core-eso-token external-secrets-audit-core" ;; + canned-prompts) echo "openbao-canned-prompts-eso-token external-secrets-canned-prompts-database" ;; + core-hub) echo "openbao-core-hub-eso-token external-secrets-core-hub-database" ;; + sbom-nexus) echo "openbao-sbom-nexus-eso-token external-secrets-sbom-nexus-database" ;; + tenant-engine) echo "openbao-tenant-engine-eso-token credential-broker-tenant-engine-runtime,credential-broker-tenant-engine-migration" ;; + *) return 1 ;; + esac +} + +[ "${1:-}" = "--confirm" ] && [ "${2:-}" = "$CONFIRM" ] || exit 2 +shift 2 +if [ "${1:-}" = "--status" ]; then + case "${2:-}" in /*) ;; *) exit 2 ;; esac + STATUS=$2 + shift 2 + ( umask 077; : >"$STATUS" ) || exit 2 +fi +[ "$#" -ge 1 ] || { note "usage: --confirm $CONFIRM [--status FILE] ..."; exit 2; } +for lane in "$@"; do + lane_spec "$lane" >/dev/null || { note "unknown lane: $lane"; exit 2; } +done +for tool in bao python3 ssh; do + command -v "$tool" || { note "missing tool: $tool"; exit 2; } +done +note "BAO_ADDR=${BAO_ADDR:-unset} remote=$REMOTE_HOST period=$TOKEN_PERIOD lanes=$*" + +for lane in "$@"; do + spec=$(lane_spec "$lane") + secret=${spec%% *} + policies=${spec#* } + policy_args=$(printf '%s' "$policies" | tr ',' '\n' | sed 's/^/-policy=/' | tr '\n' ' ') + + # shellcheck disable=SC2086 # policy_args is a controlled word list + token_json=$(bao token create $policy_args -period="$TOKEN_PERIOD" \ + -renewable=true -orphan -format=json 2>"$ERR") || fail 3 "$lane token create" + child_token=$(printf '%s' "$token_json" | python3 -c \ + 'import json,sys; print(json.load(sys.stdin)["auth"]["client_token"], end="")' 2>"$ERR") \ + || fail 3 "$lane parse token" + [ "${#child_token}" -ge 8 ] || fail 3 "$lane empty token" + + printf '%s' "$child_token" | bao write -format=json auth/token/lookup token=- 2>"$ERR" \ + | EXPECTED="$policies" python3 -c ' +import json, os, sys +d = json.load(sys.stdin)["data"] +got = sorted(p for p in d.get("policies", []) if p != "default") +want = sorted(os.environ["EXPECTED"].split(",")) +print(" renewable=%s period=%s policies=%s orphan=%s" % ( + d.get("renewable"), d.get("period"), ",".join(d.get("policies", [])), d.get("orphan"))) +sys.exit(0 if d.get("renewable") is True and int(d.get("period") or 0) > 0 and got == want else 1) +' >>"$STATUS" 2>"$ERR" || fail 4 "$lane token lookup/verify" + note "$lane: minted periodic and verified" + + printf '%s' "$child_token" | ssh -o BatchMode=yes "$REMOTE_HOST" " + set -e + kubectl -n '$SECRET_NAMESPACE' create secret generic '$secret' \ + --from-file=token=/dev/stdin --dry-run=client -o yaml \ + | kubectl apply --server-side --force-conflicts \ + --field-manager=railiance-platform-attended -f - >/dev/null + kubectl -n '$SECRET_NAMESPACE' annotate secret '$secret' \ + kubectl.kubernetes.io/last-applied-configuration- >/dev/null 2>&1 || true + " 2>"$ERR" || fail 5 "$lane kubernetes apply" + note "$lane: secret $secret applied" + + check=$(ssh -o BatchMode=yes "$REMOTE_HOST" \ + "kubectl -n '$SECRET_NAMESPACE' get secret '$secret' -o go-template='{{ with .metadata.annotations }}{{ if index . \"kubectl.kubernetes.io/last-applied-configuration\" }}HAS-ANNOTATION{{ else }}clean{{ end }}{{ else }}clean{{ end }}'" 2>"$ERR") \ + || fail 6 "$lane post-apply read" + note "$lane: annotation check $check" + [ "$check" = "clean" ] || fail 6 "$lane annotation still present" + unset child_token token_json +done +note "exit 0" +exit 0 diff --git a/tests/test_eso_token_renewer.py b/tests/test_eso_token_renewer.py new file mode 100644 index 0000000..5fa5631 --- /dev/null +++ b/tests/test_eso_token_renewer.py @@ -0,0 +1,102 @@ +import importlib.util +import io +import json +from pathlib import Path +import re +import subprocess +import urllib.error + +import yaml + +ROOT = Path(__file__).resolve().parents[1] +ADDON = ROOT / 'argocd/platform-addons/eso-token-renewer' +MINT = ROOT / 'scripts/openbao-eso-db-token-periodic-attended.sh' +spec = importlib.util.spec_from_file_location('renew', ADDON / 'renew.py') +m = importlib.util.module_from_spec(spec) +spec.loader.exec_module(m) + + +class Response(io.BytesIO): + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + +def opener_for(ttls): + seen = [] + + def opener(request, timeout): + token = request.headers['X-vault-token'] + seen.append((request.full_url, request.get_method(), token)) + ttl = ttls[token] + if isinstance(ttl, Exception): + raise ttl + return Response(json.dumps({'auth': {'lease_duration': ttl, 'renewable': True}}).encode()) + return opener, seen + + +def lanes(tmp_path, tokens): + for lane, token in tokens.items(): + (tmp_path / lane).mkdir() + (tmp_path / lane / 'token').write_text(token + '\n') + m.ROOT = tmp_path + + +def test_all_lanes_renewed_and_no_token_printed(tmp_path, capsys): + lanes(tmp_path, {'core-hub': 'tok-a', 'tenant-engine': 'tok-b'}) + opener, seen = opener_for({'tok-a': 604800, 'tok-b': 604800}) + assert m.main(opener) == 0 + assert [s[1] for s in seen] == ['POST', 'POST'] + assert all(s[0].endswith('/v1/auth/token/renew-self') for s in seen) + out = capsys.readouterr().out + assert 'tok-a' not in out and 'tok-b' not in out + assert [json.loads(line)['lane'] for line in out.splitlines()] == ['core-hub', 'tenant-engine'] + + +def test_capped_ttl_and_http_errors_fail_the_job(tmp_path, capsys): + lanes(tmp_path, {'a': 't1', 'b': 't2', 'c': 't3'}) + denied = urllib.error.HTTPError('u', 403, 'denied', {}, None) + opener, _ = opener_for({'t1': 604800, 't2': 3600, 't3': denied}) + assert m.main(opener) == 1 + rows = {r['lane']: r for r in map(json.loads, capsys.readouterr().out.splitlines())} + assert rows['a']['ok'] and rows['b']['error'] == 'ttl_below_threshold' + assert rows['c']['error'] == 'http_403' + + +def test_no_lanes_is_a_failure(tmp_path): + m.ROOT = tmp_path + assert m.main() == 1 + + +def test_cronjob_mounts_match_mint_script_lanes(): + docs = list(yaml.safe_load_all((ADDON / 'cronjob.yaml').read_text())) + cron = next(d for d in docs if d['kind'] == 'CronJob') + pod = cron['spec']['jobTemplate']['spec']['template']['spec'] + mounted = {v['name']: v['secret']['secretName'] for v in pod['volumes'] if 'secret' in v} + script = MINT.read_text() + minted = dict(re.findall(r'^\s+([a-z-]+)\) echo "(openbao-[a-z-]+-eso-token) ', script, re.M)) + assert mounted == minted + assert pod['automountServiceAccountToken'] is False + assert all(m_['readOnly'] for m_ in pod['containers'][0]['volumeMounts']) + + +def test_mint_script_uses_period_and_is_silent(): + script = MINT.read_text() + assert 'exec >/dev/null 2>&1' in script + assert '-period="$TOKEN_PERIOD"' in script and '-ttl=' not in script + assert subprocess.run(['sh', '-n', str(MINT)]).returncode == 0 + assert subprocess.run(['sh', str(MINT), '--confirm', 'wrong', 'core-hub']).returncode == 2 + + +def test_mint_policies_exist_or_are_known_external(): + script = MINT.read_text() + policies = set() + for group in re.findall(r'-eso-token ([a-z0-9,-]+)"', script): + policies.update(group.split(',')) + declared = {p.stem for p in (ROOT / 'openbao/policies').glob('*.hcl')} + # Configured by rapp-postgres scripts (configure-*-openbao.sh), not as .hcl here. + external = {'external-secrets-canned-prompts-database', 'external-secrets-sbom-nexus-database', + 'credential-broker-tenant-engine-runtime', 'credential-broker-tenant-engine-migration'} + assert policies - declared <= external diff --git a/workplans/RPF-WP-0046-eso-database-token-renewal.md b/workplans/RPF-WP-0046-eso-database-token-renewal.md new file mode 100644 index 0000000..2c930d8 --- /dev/null +++ b/workplans/RPF-WP-0046-eso-database-token-renewal.md @@ -0,0 +1,172 @@ +--- +id: RPF-WP-0046 +type: workplan +title: "Keep the dynamic-database ESO parent tokens alive: periodic tokens and a renewer" +domain: financials +repo: railiance-platform +status: active +owner: railiance-platform +topic_slug: railiance +created: "2026-09-23" +updated: "2026-09-23" +related: [RPF-WP-0045, RPF-WP-0037, RPF-WP-0036] +--- + +## Problem + +Five ClusterSecretStores read OpenBao dynamic database credentials +(`path: database`) with a static `tokenSecretRef`. The stores are +`openbao-audit-core-database`, `-canned-prompts-database`, +`-core-hub-database`, `-sbom-nexus-database` and `-tenant-engine-database`. +Each token is a 768h renewable orphan. Nothing renews it, and ESO never renews +a `tokenSecretRef` token. When a token reaches its max TTL, OpenBao revokes +every lease the token created. On 2026-09-23 this took down sbom-nexus +(State Hub `/repos` returned 502), tenant-engine (742 restarts) and +core-hub-api-candidate (43h unready). activity-core restored all three with +fresh 768h tokens (rapp-postgres `3ebd984`, hub messages `200e2aae` and +`42b0365c`). **Those tokens expire again around 2026-10-25T17:00Z.** + +## Decision (founder, 2026-09-23) + +Auto-renew, rather than switching to Kubernetes auth or static DB roles. + +Why Kubernetes auth is not a drop-in fix here: ESO v0.16.1 +(`pkg/provider/vault/client.go`, `Close()`) revokes a login-obtained token after +each reconcile unless token caching is enabled. Revoking that token revokes +the dynamic leases it just created, so the delivered DB password would die +within seconds. RPF-WP-0045 was safe only because its stores read KV. Static +database roles would remove the parent-token dependency. That change crosses +rapp-postgres and four consumer repos, and remains a possible later plan. + +Design: + +- Re-mint each token once as a **periodic** token (`-period=168h`, renewable, + orphan, same policies). A periodic token has no max TTL; each renewal resets + it to the period. +- A daily CronJob `external-secrets/eso-token-renewer` mounts the five token + Secrets read-only and calls `auth/token/renew-self`. The job fails if any + lane fails or ends below 72h. A token that was not re-minted as periodic + trips that threshold near its max TTL, so it cannot expire silently. +- The CronJob ships through ArgoCD (`railiance-platform-addons`), not by + direct apply, per the change gate. + +## T01 Renewer worker and manifests + +```task +id: RPF-WP-0046-T01 +status: done +priority: high +``` + +`argocd/platform-addons/eso-token-renewer/` holds `renew.py`, the +ServiceAccount (no RBAC, no API token), the CronJob and the kustomization. The +kustomization generates the worker ConfigMap. The draft Application is +`argocd/railiance01/drafts/eso-token-renewer.application.yaml`. The +`railiance-platform-addons` AppProject gains `batch/CronJob` in git. That +AppProject is applied by hand, so the git change alone is not live. Tests: +`tests/test_eso_token_renewer.py`. A server-side dry run on railiance01 on +2026-09-23 passed for all three objects. + +## T02 Periodic attended mint script + +```task +id: RPF-WP-0046-T02 +status: done +priority: high +``` + +The script is `scripts/openbao-eso-db-token-periodic-attended.sh`. It is +rapp-postgres's reviewed silent contract, changed only to use `-period` and +to cover all five lanes. It verifies `period > 0`, `renewable` and the exact +policy set before it writes the Secret with server-side apply. Lane-to-Secret +parity with the CronJob mounts is tested. + +## T03 Re-mint the five tokens as periodic (live, founder) + +```task +id: RPF-WP-0046-T03 +status: wait +priority: high +``` + +This step is `ADMINISTER @ realm:kubernetes/railiance01`, `activation=APPROVED`, +and needs an attended OIDC/MFA login: + +```sh +python3 /home/worsch/railiance-platform/scripts/openbao-attended-exec.py -- \ + /home/worsch/railiance-platform/scripts/openbao-eso-db-token-periodic-attended.sh \ + --confirm RPF-WP-0046-PERIODIC-ESO-TOKEN \ + --status /home/worsch/railiance-platform/docs/evidence/-eso-db-token-periodic.status \ + audit-core canned-prompts core-hub sbom-nexus tenant-engine +``` + +Then force a refresh and check that all five stores are `Valid` and their +ExternalSecrets are `SecretSynced`. + +## T04 Adopt the renewer through ArgoCD (live, founder) + +```task +id: RPF-WP-0046-T04 +status: wait +priority: high +``` + +1. Apply `argocd/railiance01/bootstrap/02-railiance-platform-addons-project.yaml` + by hand, which adds `batch/CronJob`. +2. Pin `targetRevision` in the draft to the reviewed commit, move the draft to + `argocd/railiance01/applications/`, merge, and sync the root by hand. +3. Sync `eso-token-renewer` manually. Then run one Job by hand: + `kubectl -n external-secrets create job --from=cronjob/eso-token-renewer eso-token-renewer-first`. + Every lane must print `ok: true` with a TTL of 604800. + +## T05 Move consumers off leases from the old tokens before 2026-10-25 + +```task +id: RPF-WP-0046-T05 +status: wait +priority: high +``` + +The replaced 768h tokens still expire around 2026-10-25T17:00Z, and they +revoke their leases when they do. ESO refreshes every 5 minutes, so within +minutes of T03 each target Secret holds credentials from the new token. The +mounted files update, but a process that reads its password only at startup +keeps the old one. core-hub runs a credential watcher. Restart the others once +after T03, at least 10 minutes later and well before 2026-10-25: +`audit-core/audit-core`, `canned-prompts/canned-prompts`, +`sbom-nexus/sbom-nexus`, `tenant-engine/tenant-engine`. Check readiness after +each restart. Then confirm that core-hub-api and core-hub-api-candidate stay +ready. + +## T06 Hand-offs and assurance + +```task +id: RPF-WP-0046-T06 +status: todo +priority: medium +``` + +- rapp-postgres: the 768h mint scripts (`apply-*-eso-token*.sh`) would undo + T03 if rerun. Point them at this plan's periodic script, or change them to + `-period`. +- audit-core: `scripts/renew-runtime-lease.sh` re-mints a 768h token. It needs + the same alignment. +- railiance-telemetry: alert on a failed `eso-token-renewer` Job, or on no + successful run in 48h. +- RPF-WP-0036: add the renewer's last successful run to `assurance-capture`. + +## Risks + +- **Renewer outage.** Seven days of slack at the 168h period, and the job + fails loudly. The alert (T06) is what makes it safe. +- **Lease max TTL is unchanged.** Each dynamic lease is still capped by its + role or mount max TTL. A pod that reads its password once and runs longer + than that loses the lease, whatever the parent token does. This is a + consumer reload issue, noted in the T06 hand-offs to the consumer owners. +- **Credential lifetime.** A periodic token lives as long as it is renewed. + The scope stays exact per lane, and the Secrets carry no last-applied + annotation. Revocation is `bao token revoke` on the accessor, or deleting + the Secret and letting the period lapse. +- **Rollback.** Suspend the CronJob, or delete the Application with prune. + Re-minting 768h tokens with rapp-postgres `apply-eso-token-attended.sh` + restores today's state.