From 098c2c7810ad182a3e0782ab1fe5bc5d0f101b76 Mon Sep 17 00:00:00 2001 From: tegwick Date: Fri, 30 Oct 2026 19:54:33 +0100 Subject: [PATCH] NK-WP-0054-T05: live LLDAP bind split passed (2026-10-30); runbook setup fixes Founder-run steps 1-10, verify --old-admin-retired pass. Evidence and deviations in docs/evidence/2026-10-30-nk54-bind-split/. Runbook: export the admin KUBECONFIG and decrypt the sealed custody copy first. Finding: custody AUTHELIA_KEYCAPE_CLIENT_SECRET is stale against the live hash. Co-Authored-By: Claude Opus 5.5 Assistant: claude-code Assistant-Model: opus Assistant-Process: 620478@bnt-lap001 Assistant-Session: 944e8353-7102-4bc8-affe-84586a10f2d0 --- .../01-prepare.json | 17 ++++++ .../2026-10-30-nk54-bind-split/02-plan.json | 36 +++++++++++ .../03a-accounts-preview.json | 37 ++++++++++++ .../03b-accounts.json | 59 ++++++++++++++++++ .../2026-10-30-nk54-bind-split/04-verify.json | 59 ++++++++++++++++++ .../2026-10-30-nk54-bind-split/10-verify.json | 60 +++++++++++++++++++ .../2026-10-30-nk54-bind-split/README.md | 40 +++++++++++++ sso-mfa/k8s/lldap/BIND-SPLIT-RUNBOOK.md | 12 ++++ .../NK-WP-0054-service-identity-policy.md | 10 ++++ 9 files changed, 330 insertions(+) create mode 100644 docs/evidence/2026-10-30-nk54-bind-split/01-prepare.json create mode 100644 docs/evidence/2026-10-30-nk54-bind-split/02-plan.json create mode 100644 docs/evidence/2026-10-30-nk54-bind-split/03a-accounts-preview.json create mode 100644 docs/evidence/2026-10-30-nk54-bind-split/03b-accounts.json create mode 100644 docs/evidence/2026-10-30-nk54-bind-split/04-verify.json create mode 100644 docs/evidence/2026-10-30-nk54-bind-split/10-verify.json create mode 100644 docs/evidence/2026-10-30-nk54-bind-split/README.md diff --git a/docs/evidence/2026-10-30-nk54-bind-split/01-prepare.json b/docs/evidence/2026-10-30-nk54-bind-split/01-prepare.json new file mode 100644 index 0000000..50bc5fa --- /dev/null +++ b/docs/evidence/2026-10-30-nk54-bind-split/01-prepare.json @@ -0,0 +1,17 @@ +{ + "command": "prepare-custody", + "endpoint_sha256": null, + "observed_at": "2026-10-30T17:39:01.128334+00:00", + "result": { + "changed_keys": [ + "LLDAP_LDAP_USER_PASS", + "LLDAP_SVC_AUTHELIA_PASS", + "LLDAP_SVC_KEYCAPE_PASS", + "LLDAP_SVC_PRIVACYIDEA_PASS", + "LLDAP_SVC_PROVISIONER_PASS" + ] + }, + "schema": "netkingdom.lldap-bind-split.v1", + "task": "NK-WP-0054-T05", + "values_recorded": false +} diff --git a/docs/evidence/2026-10-30-nk54-bind-split/02-plan.json b/docs/evidence/2026-10-30-nk54-bind-split/02-plan.json new file mode 100644 index 0000000..09c3682 --- /dev/null +++ b/docs/evidence/2026-10-30-nk54-bind-split/02-plan.json @@ -0,0 +1,36 @@ +{ + "command": "plan", + "endpoint_sha256": "21b7c63da95eb01178305e49af9b3143295607b8ad68c522d1fa9545b7ef4d9f", + "observed_at": "2026-10-30T17:43:34.790101+00:00", + "result": { + "accounts": { + "admin": { + "exists": true, + "in_lldap_admin": true + }, + "ld-admin": { + "action": "create", + "exists": false + }, + "svc-authelia": { + "action": "create", + "exists": false + }, + "svc-keycape": { + "action": "create", + "exists": false + }, + "svc-privacyidea": { + "action": "create", + "exists": false + }, + "svc-provisioner": { + "action": "create", + "exists": false + } + } + }, + "schema": "netkingdom.lldap-bind-split.v1", + "task": "NK-WP-0054-T05", + "values_recorded": false +} diff --git a/docs/evidence/2026-10-30-nk54-bind-split/03a-accounts-preview.json b/docs/evidence/2026-10-30-nk54-bind-split/03a-accounts-preview.json new file mode 100644 index 0000000..3e551d8 --- /dev/null +++ b/docs/evidence/2026-10-30-nk54-bind-split/03a-accounts-preview.json @@ -0,0 +1,37 @@ +{ + "command": "accounts", + "endpoint_sha256": "21b7c63da95eb01178305e49af9b3143295607b8ad68c522d1fa9545b7ef4d9f", + "observed_at": "2026-10-30T17:45:04.549495+00:00", + "result": { + "accounts": { + "admin": { + "exists": true, + "in_lldap_admin": true + }, + "ld-admin": { + "action": "create", + "exists": false + }, + "svc-authelia": { + "action": "create", + "exists": false + }, + "svc-keycape": { + "action": "create", + "exists": false + }, + "svc-privacyidea": { + "action": "create", + "exists": false + }, + "svc-provisioner": { + "action": "create", + "exists": false + } + }, + "applied": false + }, + "schema": "netkingdom.lldap-bind-split.v1", + "task": "NK-WP-0054-T05", + "values_recorded": false +} diff --git a/docs/evidence/2026-10-30-nk54-bind-split/03b-accounts.json b/docs/evidence/2026-10-30-nk54-bind-split/03b-accounts.json new file mode 100644 index 0000000..b8a71e6 --- /dev/null +++ b/docs/evidence/2026-10-30-nk54-bind-split/03b-accounts.json @@ -0,0 +1,59 @@ +{ + "command": "accounts", + "endpoint_sha256": "21b7c63da95eb01178305e49af9b3143295607b8ad68c522d1fa9545b7ef4d9f", + "observed_at": "2026-10-30T17:45:53.893558+00:00", + "result": { + "accounts": { + "admin": { + "exists": true, + "in_lldap_admin": true + }, + "ld-admin": { + "action": "ok", + "exists": true, + "groups": [ + "lldap_admin" + ] + }, + "svc-authelia": { + "action": "ok", + "exists": true, + "groups": [ + "lldap_strict_readonly" + ] + }, + "svc-keycape": { + "action": "ok", + "exists": true, + "groups": [ + "lldap_strict_readonly" + ] + }, + "svc-privacyidea": { + "action": "ok", + "exists": true, + "groups": [ + "lldap_strict_readonly" + ] + }, + "svc-provisioner": { + "action": "ok", + "exists": true, + "groups": [ + "lldap_admin" + ] + } + }, + "applied": true, + "signs_in": { + "ld-admin": true, + "svc-authelia": true, + "svc-keycape": true, + "svc-privacyidea": true, + "svc-provisioner": true + } + }, + "schema": "netkingdom.lldap-bind-split.v1", + "task": "NK-WP-0054-T05", + "values_recorded": false +} diff --git a/docs/evidence/2026-10-30-nk54-bind-split/04-verify.json b/docs/evidence/2026-10-30-nk54-bind-split/04-verify.json new file mode 100644 index 0000000..91a1dbc --- /dev/null +++ b/docs/evidence/2026-10-30-nk54-bind-split/04-verify.json @@ -0,0 +1,59 @@ +{ + "command": "verify", + "endpoint_sha256": "21b7c63da95eb01178305e49af9b3143295607b8ad68c522d1fa9545b7ef4d9f", + "observed_at": "2026-10-30T17:46:48.728129+00:00", + "result": { + "accounts": { + "admin": { + "exists": true, + "in_lldap_admin": true + }, + "ld-admin": { + "action": "ok", + "exists": true, + "groups": [ + "lldap_admin" + ] + }, + "svc-authelia": { + "action": "ok", + "exists": true, + "groups": [ + "lldap_strict_readonly" + ] + }, + "svc-keycape": { + "action": "ok", + "exists": true, + "groups": [ + "lldap_strict_readonly" + ] + }, + "svc-privacyidea": { + "action": "ok", + "exists": true, + "groups": [ + "lldap_strict_readonly" + ] + }, + "svc-provisioner": { + "action": "ok", + "exists": true, + "groups": [ + "lldap_admin" + ] + } + }, + "signs_in": { + "ld-admin": true, + "svc-authelia": true, + "svc-keycape": true, + "svc-privacyidea": true, + "svc-provisioner": true + }, + "verdict": "pass" + }, + "schema": "netkingdom.lldap-bind-split.v1", + "task": "NK-WP-0054-T05", + "values_recorded": false +} diff --git a/docs/evidence/2026-10-30-nk54-bind-split/10-verify.json b/docs/evidence/2026-10-30-nk54-bind-split/10-verify.json new file mode 100644 index 0000000..7823a4c --- /dev/null +++ b/docs/evidence/2026-10-30-nk54-bind-split/10-verify.json @@ -0,0 +1,60 @@ +{ + "command": "verify", + "endpoint_sha256": "21b7c63da95eb01178305e49af9b3143295607b8ad68c522d1fa9545b7ef4d9f", + "observed_at": "2026-10-30T18:53:16.176849+00:00", + "result": { + "accounts": { + "admin": { + "exists": true, + "in_lldap_admin": true + }, + "ld-admin": { + "action": "ok", + "exists": true, + "groups": [ + "lldap_admin" + ] + }, + "svc-authelia": { + "action": "ok", + "exists": true, + "groups": [ + "lldap_strict_readonly" + ] + }, + "svc-keycape": { + "action": "ok", + "exists": true, + "groups": [ + "lldap_strict_readonly" + ] + }, + "svc-privacyidea": { + "action": "ok", + "exists": true, + "groups": [ + "lldap_strict_readonly" + ] + }, + "svc-provisioner": { + "action": "ok", + "exists": true, + "groups": [ + "lldap_admin" + ] + } + }, + "old_shared_password_refused": true, + "signs_in": { + "ld-admin": true, + "svc-authelia": true, + "svc-keycape": true, + "svc-privacyidea": true, + "svc-provisioner": true + }, + "verdict": "pass" + }, + "schema": "netkingdom.lldap-bind-split.v1", + "task": "NK-WP-0054-T05", + "values_recorded": false +} diff --git a/docs/evidence/2026-10-30-nk54-bind-split/README.md b/docs/evidence/2026-10-30-nk54-bind-split/README.md new file mode 100644 index 0000000..117c36e --- /dev/null +++ b/docs/evidence/2026-10-30-nk54-bind-split/README.md @@ -0,0 +1,40 @@ +# LLDAP bind split, live run 2026-10-30 (NK-WP-0054-T05) + +Founder-run in the founder's terminal following `sso-mfa/k8s/lldap/BIND-SPLIT-RUNBOOK.md`, 17:39-18:53 +UTC, agent (Claude Code) guiding. All receipts are value-free (`values_recorded: false`). + +| Step | Result | Receipt | +| --- | --- | --- | +| 1 prepare custody | five keys changed; custody copy re-sealed to `keys/age.pub` | `01-prepare.json` | +| 2 plan | five `create`, `admin` in `lldap_admin`, no collision | `02-plan.json` | +| 3 accounts | preview, then apply; all five sign in | `03a-accounts-preview.json`, `03b-accounts.json` | +| 4 verify | `pass` | `04-verify.json` | +| 5 identity-provisioner | Secret `identity-provisioner-ldap` created; image `sha256:6d1f3632…4199` ready (readiness includes directory login) | operator output | +| 6 Authelia | `ldap_password` rotated to `svc-authelia`; configmap applied; founder sign-in with password and OTP | operator output | +| 7 KeyCape | `directory-bind-rollout.py apply`, uid unchanged, `other_secret_data_unchanged: true`; sign-in ok | operator output | +| 8 privacyIDEA resolver | `realm_bootstrap.py --switch-bind --apply`; OTP sign-in ok | operator output | +| 9 LLDAP | `LLDAP_LDAP_USER_PASS` rotated; configured super-admin `ld-admin` | operator output | +| 10 retire shared password | `admin` password changed by the founder; old shared password refused; `pass` | `10-verify.json` | + +## Deviations and findings + +1. **Custody copy not at the runbook path.** Step 1 failed with `FileNotFoundError` (nothing written): + only the sealed `sso-mfa/bootstrap/secrets.enc/` exists. The founder decrypted it with + `make sops-custody-shell` + `decrypt-secrets.sh`, ran step 1, re-sealed with `--no-shred`, and sealed + and shredded after step 10. Runbook setup now says so. +2. **Wrong kubeconfig for the Secret scripts.** `apply-service-secrets.py` calls plain `kubectl`, which + used the limited default identity (`ops-mason-builder`, cannot read `sso` Secrets); step 5 was + refused before any write (presence check: `NotFound`). Fixed by + `export KUBECONFIG=$HOME/.kube/config-railiance01`; the runbook setup now exports it. +3. **Stale custody value: `AUTHELIA_KEYCAPE_CLIENT_SECRET`.** Step 6 was refused before any write: a + value-free comparison showed the four other Authelia values equal to live, `ldap_password` different + (intended), and the custody client secret **not** matching the live `keycape_client_secret_hash`. + The KeyCape-Authelia client secret was changed after the March custody seal without updating custody. + Step 6 used a one-off helper that calls the same `publish()` path for `ldap_password` only (uid, + resourceVersion and full-data test, readback) and skips only the client-secret check. Follow-up: + reconcile custody with the live client secret (founder-run). +4. Rollouts of identity-provisioner, KeyCape and LLDAP use `Recreate`: each had a short outage of + about 30 s. + +Closing condition still open: the restore drill on the first backup generation after this run +(NK-WP-0053 stages 2-5, T17/T19 under the new accounts). diff --git a/sso-mfa/k8s/lldap/BIND-SPLIT-RUNBOOK.md b/sso-mfa/k8s/lldap/BIND-SPLIT-RUNBOOK.md index fe79f99..c2aec0a 100644 --- a/sso-mfa/k8s/lldap/BIND-SPLIT-RUNBOOK.md +++ b/sso-mfa/k8s/lldap/BIND-SPLIT-RUNBOOK.md @@ -24,10 +24,22 @@ receipts under `~/nk54/`; they are value-free and go to `docs/evidence/` afterwa cd ~/net-kingdom && git pull --ff-only bridge up k3s-api-railiance01 K="kubectl --kubeconfig $HOME/.kube/config-hosteurope --server https://127.0.0.1:16444" +# The Secret scripts call plain kubectl; the default config is the limited ops-mason-builder. +export KUBECONFIG=$HOME/.kube/config-railiance01 +kubectl auth can-i create secrets -n sso # must print yes CUSTODY=sso-mfa/bootstrap/secrets # your private custody copy, as for make creds-apply mkdir -p -m 700 ~/nk54 ``` +If only the sealed `secrets.enc/` exists, decrypt it first and seal again after step 1 +(`--no-shred`, steps 5-10 still read it) and after step 10 (without `--no-shred`): +```bash +make sops-custody-shell # then, inside: bash sso-mfa/bootstrap/decrypt-secrets.sh sso-mfa/bootstrap/secrets "$SOPS_AGE_KEY_FILE"; exit +bash sso-mfa/bootstrap/encrypt-secrets.sh sso-mfa/bootstrap/secrets "" --no-shred +``` +2026-10-30: the custody `AUTHELIA_KEYCAPE_CLIENT_SECRET` no longer matched the live hash, so step 6's +`create-secrets.sh` refused; see `docs/evidence/2026-10-30-nk54-bind-split/README.md`. + ## 0. Preconditions 1. **Restore drill on the current backup** (NK-WP-0053-T02/T04) has passed. It proves today's backup diff --git a/workplans/NK-WP-0054-service-identity-policy.md b/workplans/NK-WP-0054-service-identity-policy.md index 51c2a12..57e0068 100644 --- a/workplans/NK-WP-0054-service-identity-policy.md +++ b/workplans/NK-WP-0054-service-identity-policy.md @@ -355,6 +355,16 @@ apply with uid and resourceVersion test, `--operation revert`; 8 tests) and `--switch-bind` runs. The founder's restore-drill rehearsal is the NK-WP-0053 run on the current backup (precondition), and the post-split drill closes this task. Agent preparation is complete. +**2026-10-30 live run: passed** ([evidence](../docs/evidence/2026-10-30-nk54-bind-split/README.md)). +The founder ran steps 1-10; `verify --old-admin-retired` passed: `ld-admin` is the configured +super-admin, Authelia, KeyCape, the privacyIDEA resolver and identity-provisioner bind with their own +accounts, `admin`'s password is human-only and the old shared password is refused. Deviations: the +custody copy had to be decrypted first and `KUBECONFIG` exported (runbook setup corrected); step 6 was +refused because the custody `AUTHELIA_KEYCAPE_CLIENT_SECRET` no longer matches the live hash, and +`ldap_password` was rotated alone through the same guarded `publish()` path. Open: the restore drill on +the next backup generation (closes this task, retires SI-X1) and reconciling the stale custody client +secret. + ## Disposition of per-service admin accounts ```task