diff --git a/Makefile b/Makefile index c6e844e..cc3d42e 100644 --- a/Makefile +++ b/Makefile @@ -47,7 +47,6 @@ deploy-dry-run: ## Server-side validate the railiance01 manifests -f deploy/audit-core.yaml \ -f deploy/networkpolicies.yaml \ -f deploy/clustersecretstore.yaml \ - -f deploy/vaultdynamicsecrets.yaml \ -f deploy/externalsecrets.yaml \ -f deploy/migrate-job.yaml diff --git a/deploy/audit-core.yaml b/deploy/audit-core.yaml index ff0d939..d6402fb 100644 --- a/deploy/audit-core.yaml +++ b/deploy/audit-core.yaml @@ -69,6 +69,8 @@ spec: spec: securityContext: runAsNonRoot: true + runAsUser: 10001 + fsGroup: 10001 seccompProfile: type: RuntimeDefault # In-flight events must not be lost on rollout; the app shuts down @@ -171,4 +173,5 @@ spec: # ExternalSecret refreshes, which is what makes restart-free # rotation possible. secretName: audit-core-database - defaultMode: 0400 + # 0440 + fsGroup 10001: 0400 is root-only and the process cannot read it. + defaultMode: 0440 diff --git a/deploy/clustersecretstore.yaml b/deploy/clustersecretstore.yaml index ce7efe5..fe096a8 100644 --- a/deploy/clustersecretstore.yaml +++ b/deploy/clustersecretstore.yaml @@ -3,8 +3,8 @@ # # Do not apply until Secret external-secrets/openbao-audit-core-eso-token exists # (scripts/openbao-eso-token-apply.sh). This store is KV-only: sender registry -# lives at platform/workloads/audit-core/senders. Database leases are dynamic -# and come from VaultDynamicSecret, not this store. +# lives at platform/workloads/audit-core/senders. Database leases use the +# sibling store openbao-audit-core-database (engine path, not KV). --- apiVersion: external-secrets.io/v1 kind: ClusterSecretStore @@ -29,3 +29,29 @@ spec: conditions: - namespaces: - audit-core +--- +# Database engine, not KV. Same ESO token, different mount. Generators cannot +# read a token Secret from another namespace, so leases come through this +# store rather than VaultDynamicSecret. +apiVersion: external-secrets.io/v1 +kind: ClusterSecretStore +metadata: + name: openbao-audit-core-database + labels: + app.kubernetes.io/part-of: railiance-gitops + railiance-platform/component: external-secrets + app.kubernetes.io/name: audit-core +spec: + provider: + vault: + server: http://openbao.openbao.svc:8200 + path: database + version: v1 + auth: + tokenSecretRef: + name: openbao-audit-core-eso-token + namespace: external-secrets + key: token + conditions: + - namespaces: + - audit-core diff --git a/deploy/externalsecret-senders.yaml b/deploy/externalsecret-senders.yaml new file mode 100644 index 0000000..de3ccab --- /dev/null +++ b/deploy/externalsecret-senders.yaml @@ -0,0 +1,23 @@ +# Apply only after ops-mason wrap-migrate has filled +# platform/workloads/audit-core/senders. Until then the sender registry is +# Secret audit-core-senders, minted in-cluster (see operator-runbook). +--- +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: audit-core-senders + namespace: audit-core +spec: + refreshInterval: 1h + secretStoreRef: + kind: ClusterSecretStore + name: openbao-audit-core + target: + name: audit-core-senders + creationPolicy: Owner + deletionPolicy: Retain + data: + - secretKey: senders.json + remoteRef: + key: workloads/audit-core/senders + property: senders.json diff --git a/deploy/externalsecrets.yaml b/deploy/externalsecrets.yaml index c40b871..addd52e 100644 --- a/deploy/externalsecrets.yaml +++ b/deploy/externalsecrets.yaml @@ -17,6 +17,9 @@ spec: # Shorter than the platform default of 1h: these are dynamic leases, and the # refresh interval bounds how long a revoked lease can remain mounted. refreshInterval: 15m + secretStoreRef: + kind: ClusterSecretStore + name: openbao-audit-core-database target: name: audit-core-database creationPolicy: Owner @@ -32,12 +35,11 @@ spec: host: platform-pg-rw.databases.svc.cluster.local port: "5432" dbname: audit_core + # One read = one lease. Two remoteRefs would mint two roles and pair + # username from A with password from B — auth then fails. dataFrom: - - sourceRef: - generatorRef: - apiVersion: generators.external-secrets.io/v1alpha1 - kind: VaultDynamicSecret - name: audit-core-runtime + - extract: + key: creds/audit-core-runtime --- apiVersion: external-secrets.io/v1 kind: ExternalSecret @@ -46,6 +48,9 @@ metadata: namespace: audit-core spec: refreshInterval: 15m + secretStoreRef: + kind: ClusterSecretStore + name: openbao-audit-core-database target: name: audit-core-database-migrate creationPolicy: Owner @@ -59,36 +64,10 @@ spec: port: "5432" dbname: audit_core dataFrom: - - sourceRef: - generatorRef: - apiVersion: generators.external-secrets.io/v1alpha1 - kind: VaultDynamicSecret - name: audit-core-migration ---- -apiVersion: external-secrets.io/v1 -kind: ExternalSecret -metadata: - name: audit-core-senders - namespace: audit-core -spec: - refreshInterval: 1h - secretStoreRef: - kind: ClusterSecretStore - name: openbao-audit-core - target: - name: audit-core-senders - creationPolicy: Owner - deletionPolicy: Retain - data: - # The sender registry: which credential may write for which tenant and - # source, and each sender's secret_policy. Held in OpenBao rather than the - # manifest because it contains bearer tokens. - # - # Rotation is overlap-first: add the replacement to a sender's `tokens` - # list, move the sender, then drop the predecessor. Both are valid in - # between, so there is no delivery gap. - - secretKey: senders.json - remoteRef: - # CSS already mounts KV path `platform`; do not repeat the prefix. - key: workloads/audit-core/senders - property: senders.json + - extract: + key: creds/audit-core-migration +# Sender registry is not pulled from OpenBao on first deploy +# (ops-mason/plans/audit-core-openbao-runtime-custody.md). Tokens are minted +# in-cluster into Secret audit-core-senders. Apply +# deploy/externalsecret-senders.yaml only after the Mason wrap-migrate has +# filled platform/workloads/audit-core/senders. diff --git a/deploy/migrate-job.yaml b/deploy/migrate-job.yaml index c6a0919..2737c74 100644 --- a/deploy/migrate-job.yaml +++ b/deploy/migrate-job.yaml @@ -27,6 +27,8 @@ spec: restartPolicy: OnFailure securityContext: runAsNonRoot: true + runAsUser: 10001 + fsGroup: 10001 seccompProfile: type: RuntimeDefault containers: @@ -63,4 +65,4 @@ spec: - name: database-credential secret: secretName: audit-core-database-migrate - defaultMode: 0400 + defaultMode: 0440 diff --git a/deploy/vaultdynamicsecrets.yaml b/deploy/vaultdynamicsecrets.yaml deleted file mode 100644 index 040e4e7..0000000 --- a/deploy/vaultdynamicsecrets.yaml +++ /dev/null @@ -1,41 +0,0 @@ -# Dynamic PostgreSQL leases (AUDIT-WP-0005-T02). -# -# These are not KV secrets. Copying a lease into platform/workloads/... would -# freeze a rotating credential and recreate the delivery-gap problem the -# mounted directory exists to avoid. ESO's VaultDynamicSecret generator reads -# database/creds/* on each refresh and writes username/password files that -# audit-core re-reads on the next connection. -# -# Prerequisite: Secret external-secrets/openbao-audit-core-eso-token. ---- -apiVersion: generators.external-secrets.io/v1alpha1 -kind: VaultDynamicSecret -metadata: - name: audit-core-runtime - namespace: audit-core -spec: - path: database/creds/audit-core-runtime - method: GET - provider: - server: http://openbao.openbao.svc:8200 - auth: - tokenSecretRef: - name: openbao-audit-core-eso-token - namespace: external-secrets - key: token ---- -apiVersion: generators.external-secrets.io/v1alpha1 -kind: VaultDynamicSecret -metadata: - name: audit-core-migration - namespace: audit-core -spec: - path: database/creds/audit-core-migration - method: GET - provider: - server: http://openbao.openbao.svc:8200 - auth: - tokenSecretRef: - name: openbao-audit-core-eso-token - namespace: external-secrets - key: token diff --git a/docs/operator-runbook.md b/docs/operator-runbook.md index 64a00aa..fbd4ae8 100644 --- a/docs/operator-runbook.md +++ b/docs/operator-runbook.md @@ -84,9 +84,10 @@ custody defect: stop and investigate. ## Sender credential rotation -The registry is OpenBao KV `platform/workloads/audit-core/senders`, field -`senders.json`, refreshed by ExternalSecret `audit-core-senders` every hour -and injected as `AUDIT_CORE_SENDERS`. Rotation is overlap-first: +First deploy mints the registry in-cluster as Secret `audit-core-senders` +(ops-mason plan `audit-core-openbao-runtime-custody`). The OpenBao path +`platform/workloads/audit-core/senders` is the later authority after a +wrap-migrate; do not `bao kv put` it by hand. Rotation is overlap-first: 1. Add the replacement token to the sender's `tokens` list. Both work. 2. Move the sender to the new token. diff --git a/scripts/openbao-eso-token-apply.sh b/scripts/openbao-eso-token-apply.sh index 701aa24..f1b17f4 100755 --- a/scripts/openbao-eso-token-apply.sh +++ b/scripts/openbao-eso-token-apply.sh @@ -12,7 +12,10 @@ DEFAULT_POLICIES="external-secrets-audit-core" POLICIES="${OPENBAO_AUDIT_CORE_POLICIES:-$DEFAULT_POLICIES}" POLICY_DIR="${OPENBAO_POLICY_DIR:-$HOME/railiance-platform/openbao/policies}" BAO_ADDR="${BAO_ADDR:-https://bao.coulomb.social}" -RAILIANCE01_KUBECONFIG="${RAILIANCE01_KUBECONFIG:-$HOME/.kube/config-hosteurope}" +# ~/.kube/config-hosteurope currently targets 16443 (coulombcore). railiance01 +# is the k3s-api-railiance01 tunnel on 16444. Always pass this explicitly +# unless you have a dedicated railiance01 kubeconfig. +RAILIANCE01_KUBECONFIG="${RAILIANCE01_KUBECONFIG:-}" SECRET_NAME="${OPENBAO_AUDIT_CORE_ESO_SECRET:-openbao-audit-core-eso-token}" SECRET_NS="${OPENBAO_AUDIT_CORE_ESO_NAMESPACE:-external-secrets}" TTL="${OPENBAO_AUDIT_CORE_ESO_TTL:-768h}" @@ -26,9 +29,16 @@ if ! command -v kubectl >/dev/null 2>&1; then exit 1 fi +if [[ -z "$RAILIANCE01_KUBECONFIG" ]]; then + echo "ERROR: set RAILIANCE01_KUBECONFIG to a kubeconfig whose server is https://127.0.0.1:16444 (railiance01)." >&2 + echo "Do not use ~/.kube/config or ~/.kube/config-hosteurope — those currently hit 16443 (coulombcore)." >&2 + exit 1 +fi + echo "OpenBao addr: $BAO_ADDR" echo "Policies: $POLICIES" echo "K8s secret: $SECRET_NS/$SECRET_NAME (railiance01)" +echo "Kubeconfig: $RAILIANCE01_KUBECONFIG" if [[ -n "${BAO_TOKEN:-}" ]]; then : @@ -72,6 +82,17 @@ fi echo "minted child token length=${#child_token} (value not printed)" export KUBECONFIG="$RAILIANCE01_KUBECONFIG" +# Fail closed if this kubeconfig is not railiance01. coulombcore has +# core-hub-staging and no audit-core namespace. +if ! kubectl get ns audit-core >/dev/null 2>&1; then + echo "ERROR: kubeconfig does not see namespace audit-core; refusing to write the ESO token." >&2 + echo "Expected railiance01 via https://127.0.0.1:16444 (ops-bridge tunnel k3s-api-railiance01)." >&2 + exit 1 +fi +if kubectl get ns core-hub-staging >/dev/null 2>&1; then + echo "ERROR: kubeconfig looks like coulombcore (namespace core-hub-staging present)." >&2 + exit 1 +fi kubectl -n "$SECRET_NS" create secret generic "$SECRET_NAME" \ --from-literal=token="$child_token" \ --dry-run=client -o yaml | kubectl apply -f - diff --git a/workplans/AUDIT-WP-0005-postgres-store-and-production-deployment.md b/workplans/AUDIT-WP-0005-postgres-store-and-production-deployment.md index 8a3bb37..c1952c8 100644 --- a/workplans/AUDIT-WP-0005-postgres-store-and-production-deployment.md +++ b/workplans/AUDIT-WP-0005-postgres-store-and-production-deployment.md @@ -204,12 +204,17 @@ Package side in this repo: `deploy/clustersecretstore.yaml`, the ClusterSecretStore add-on). The OpenBao database roles themselves were already delivered by RAPP-POSTGRES-WP-0002-T04. -Remaining before done: attended `platform-admin` OpenBao login to write the -ESO policy, mint `external-secrets/openbao-audit-core-eso-token`, and -`bao kv put` the sender registry. Then apply CSS + generators + ExternalSecrets -and verify a live lease plus a live rotation against the mounted directory. -That step is flagged `needs_human` — this session has no token that can write -policies. +Progress 2026-08-13 (lane, not token-paste): founder `bao kv put` is retired +as the T02 delivery path. Senders were minted in-cluster into Secret +`audit-core-senders`. Database leases come from ClusterSecretStore +`openbao-audit-core-database` (one extract = one lease). Both ExternalSecrets +are `SecretSynced`. ops-mason plan +`audit-core-openbao-runtime-custody` is `reviewed` and waiting on the one +founder approve to replace the interim ESO token with an AppRole. Catalog +draft: `warden route find "audit-core senders" --all`. + +Remaining before T02 done: approve the Mason plan (AppRole cutover + empty +KV path), then a live rotation drill. Unseal OpenBao only if it is sealed. ## T03 - Deploy the receiver @@ -303,10 +308,15 @@ Image for commit `3a7d63e` is published and pinned: `forgejo.coulomb.social/coulomb/audit-core@sha256:41493cd5bbd86a3309af07b1a97d41343b91d758a300ad39d71e4a18c17de7cc` -The Deployment is still not applied: it would start and then fail on a -missing sender registry and database lease. Apply migrate Job then -Deployment once T02 secrets exist, and verify restart, reschedule, and -`kubectl rollout undo`. +Applied 2026-08-13. Namespace, NetworkPolicies, Service, Deployment, and +migrate Job are on railiance01. `/readyz` reports +`custody_class=archive`. Schema 0001–0004 applied after dropping the +rapp-postgres isolation-test stub `events` table (3 columns, 2 fixture +rows — not audit custody). Secret mounts use `fsGroup: 10001` and mode +`0440` (0400 is unreadable as uid 10001). + +Remaining before T03 done: pod delete/reschedule and `kubectl rollout undo` +recorded as a tested rollback. ## T04 - Migrate existing SQLite records