From 994d11c869daf9624ac6da2adfdaeb86d8cdf537 Mon Sep 17 00:00:00 2001 From: tegwick Date: Fri, 16 Oct 2026 01:37:11 +0200 Subject: [PATCH] feat(recovery): prepare full identity backup and bootstrap handoff Assistant: codex Assistant-Model: gpt-6-astra Assistant-Session: 01a13eab-1e6a-7b33-9c37-372e90ada3a2 --- docs/bootstrap-recovery-ceremony.md | 143 ++++++- .../capture-plan.json | 393 ++++++++++++++++++ .../helper-delivery-server-dry-run.json | 65 +++ .../isolation-plan.json | 132 ++++++ .../isolation-server-dry-run.json | 27 ++ .../review.json | 37 ++ .../runtime-install-plan.json | 59 +++ .../source-manifest.json | 82 ++++ .../capability.security.bootstrap-recovery.md | 43 +- registry/indexes/capabilities.yaml | 14 +- sso-mfa/k8s/network-policies/netpol-mfa.yaml | 5 +- sso-mfa/k8s/network-policies/netpol-sso.yaml | 5 +- tests/test_identity_bundle.py | 210 ++++++++++ tests/test_identity_offsite.py | 153 +++++++ tools/bootstrap-recovery/capture-identity.py | 296 +++++++++++++ .../identity-backup-profile.json | 27 +- tools/bootstrap-recovery/identity-bundle.py | 204 +++++++++ .../identity-credential-setup.py | 234 +++++++++++ .../bootstrap-recovery/identity-custody.json | 144 +++++++ tools/bootstrap-recovery/identity-files.py | 109 +++++ .../bootstrap-recovery/identity-isolation.py | 65 +++ tools/bootstrap-recovery/identity-offsite.py | 124 ++++++ .../bootstrap-recovery/identity-retention.py | 106 +++++ tools/bootstrap-recovery/identity-runtime.py | 208 +++++++++ .../install-identity-backup.py | 145 +++++++ tools/bootstrap-recovery/retrieve-identity.py | 71 ++++ .../verify-identity-delivery.py | 83 ++++ tools/bootstrap-recovery/verify-identity.py | 90 ++++ ...-bootstrap-secret-free-recovery-handoff.md | 32 +- 29 files changed, 3277 insertions(+), 29 deletions(-) create mode 100644 docs/evidence/2026-10-16-identity-backup-implementation/capture-plan.json create mode 100644 docs/evidence/2026-10-16-identity-backup-implementation/helper-delivery-server-dry-run.json create mode 100644 docs/evidence/2026-10-16-identity-backup-implementation/isolation-plan.json create mode 100644 docs/evidence/2026-10-16-identity-backup-implementation/isolation-server-dry-run.json create mode 100644 docs/evidence/2026-10-16-identity-backup-implementation/review.json create mode 100644 docs/evidence/2026-10-16-identity-backup-implementation/runtime-install-plan.json create mode 100644 docs/evidence/2026-10-16-identity-backup-implementation/source-manifest.json create mode 100644 tests/test_identity_bundle.py create mode 100644 tests/test_identity_offsite.py create mode 100644 tools/bootstrap-recovery/capture-identity.py create mode 100644 tools/bootstrap-recovery/identity-bundle.py create mode 100644 tools/bootstrap-recovery/identity-credential-setup.py create mode 100644 tools/bootstrap-recovery/identity-custody.json create mode 100644 tools/bootstrap-recovery/identity-files.py create mode 100644 tools/bootstrap-recovery/identity-isolation.py create mode 100644 tools/bootstrap-recovery/identity-offsite.py create mode 100644 tools/bootstrap-recovery/identity-retention.py create mode 100644 tools/bootstrap-recovery/identity-runtime.py create mode 100644 tools/bootstrap-recovery/install-identity-backup.py create mode 100644 tools/bootstrap-recovery/retrieve-identity.py create mode 100644 tools/bootstrap-recovery/verify-identity-delivery.py create mode 100644 tools/bootstrap-recovery/verify-identity.py diff --git a/docs/bootstrap-recovery-ceremony.md b/docs/bootstrap-recovery-ceremony.md index c3787dc..7397f07 100644 --- a/docs/bootstrap-recovery-ceremony.md +++ b/docs/bootstrap-recovery-ceremony.md @@ -239,10 +239,139 @@ use SQLite online-backup/PostgreSQL logical snapshots, record each capture interval and refuse changed source identities/configuration. It must not claim global atomicity or substitute a fresh export for the already recovered old dump. -Implementation remains: owner-admitted capture/sealing and new-profile transfer -adapters, exact credential disposition, immutable create plus retrieved readback, -then the isolated full-service recovery checks above. The current bootstrap -secrets-only tar checker cannot verify this new data-bearing profile. Approval -of preparation does not make these adapters implemented or the proposed -destinations active. The profile lists these gates explicitly; no live grant or -deployment approval is requested from this incomplete execution packet. +The source implementation below supersedes the earlier adapter-preparation gap. +The Scaleway primary/Storage Share secondary proposal above is deferred: the +initial rollout implements Storage Share only. Neither destination has yet +received a full identity bundle. Owner admission and actual recovery remain open. + + +## Implemented identity backup packet — 2026-10-16 + +The new capture, sealing, upload/readback, independent retrieval, archive/native +PostgreSQL verification, separate retention and node installer are implemented +under `tools/bootstrap-recovery/identity-*.py`, `capture-identity.py`, +`retrieve-identity.py`, `verify-identity*.py` and `install-identity-backup.py`. +Source profile: `identity-backup-profile.json`. No production keys were collected, +no new credential grants were made and no timers were installed in this source +checkpoint. Decision `40bfae40-6108-4e51-8613-073339f5345f` remains pending. + +The exact new owner requests are: + +- [CCR-2026-0048 publisher](../../railiance-platform/credential-change-requests/CCR-2026-0048-net-kingdom-backup-publisher.yaml): a read/create share for only `Backup/railiance-backups/net-kingdom-identity/`, delivered through a dedicated 15-minute, audience/namespace/SA-bound ESO reader. +- [CCR-2026-0049 retention](../../railiance-platform/credential-change-requests/CCR-2026-0049-net-kingdom-backup-retention.yaml): a separate read/delete share for that same folder and separate process. Compromise of this credential can delete points in the folder; it must not be given to the publisher process. + +Both require Platform, NetKingdom and Backup-share owner review. Only the scoped +share tokens enter exact new workload KV paths and ESO-managed Secrets. Neither +the operator account password nor the age private key is delivered to the job. +The setup helper refuses proposed/unapproved or hash-mismatched CCRs. Native +positive and wrong-SA/namespace/audience probes must prove the admitted reader +boundary, followed by equality to the exact ESO delivery; temporary readers are +revoked. These are new grants, so prior approval to prepare the missing coverage +does not count as completed credential-owner admission. + +The initial provider is Storage Share only. Ciphertext and safe receipt are +created once under a unique generation name, read back using strong ETags and +checked by size/SHA256. A 512 MiB refusal ceiling and two GiB fresh provider +headroom apply; unsupported public-share quota fails closed. Public DAV behavior +and permission denials are checked with disposable public-recipient fixture +ciphertext before accepting the grants. No Scaleway credential or another +workload publisher is reused. Nextcloud documents the public-share DAV interface +in its [official WebDAV guide](https://docs.nextcloud.com/server/25/user_manual/en/files/access_webdav.html). + +Two temporary helper Pods mount only reviewed PVCs read-only, have no service +account token, use pinned images, private tmpfs scratch and deny-all network +policies, then are deleted with exact UID preconditions. Kubernetes network +allows combine, so `identity-isolation.py` first prepares two guarded patches to +exclude the capture label from the existing namespace-wide DNS allows. Every +peer/port and ordinary service selector match is preserved. Plans bind the +current cluster/policy UIDs, resource versions and full original specs, and +refuse an existing pod using the capture label. Capture refuses any remaining +matching allow policy, source drift or unknown helper cleanup outcome. + +Capture uses native SQLite online backups and a fresh native PostgreSQL dump; +all required current Secrets/configuration and original key pairs are sealed +with the public recipient. The manifest and plaintext hashes are inside the +ciphertext. privacyIDEA service logs, historical LLDAP/Authelia/PI `backups/` +subtrees and Authelia notification.txt are omitted; the audit database is in the +linked PostgreSQL dump, and the full KeyCape policy/receipt tree is preserved. +There is no claim of a globally atomic multi-service snapshot. Drift/partial +cleanup stops future runs and requires exact owner reconciliation. + +### Ordered live rollout after exact packet approval + +1. Record all required owner reviews in CCR-2026-0048/0049 and bind their exact file hashes. Run `identity-credential-setup.py publisher|retention --ccr-sha256 HASH --receipt PRIVATE_NEW_PATH` through the attended Platform-admin owner conduit. Never inspect private helper output or copy operator credentials into the runtime. Verify actual provider permissions, initial CAS-only KV publication, exact policy/role, agent high-risk denials and revoked fixture readers. +2. Apply only `identity-custody.json`; wait for its two SecretStores/ExternalSecrets to be Ready. Run `verify-identity-delivery.py publisher|retention --ccr-sha256 HASH --kubeconfig REVIEWED_PATH --receipt PRIVATE_NEW_PATH` through that same owner conduit. Resolve any native denial before accepting delivery; do not set confirmed-claim flags from fixtures. +3. Generate a fresh `identity-isolation.py plan --kubeconfig REVIEWED_PATH --receipt PRIVATE_PLAN`; review it, server-dry-run it, then `apply --plan PRIVATE_PLAN --plan-sha256 HASH` using the same explicit kubeconfig and a fresh receipt. Each patch tests UID/resourceVersion/spec. Regenerate stale plans rather than weakening the guards. Confirm the cluster's network policy enforcement through its owning layer. +4. Render `install-identity-backup.py render --decision-id 40bfae40-6108-4e51-8613-073339f5345f --receipt PRIVATE_PACKET`. Transfer only the reviewed runtime files in that packet plus the packet to the admitted node. As root on railiance01, run installer `install --packet PRIVATE_PACKET --packet-sha256 HASH --receipt PRIVATE_NEW_PATH`. Installation verifies exact cluster/source hashes and unit syntax; timers remain disabled. Execute the packet's exact `first_capture_command`. All output receipts/state and encrypted copies stay under root-private `/var/lib/net-kingdom-backup`. +5. Retrieve the successful generation's safe off-site receipt from that private state, then run attended `retrieve-identity.py --source-receipt PRIVATE_NODE_RECEIPT --receipt PRIVATE_INDEPENDENT_TRANSFER` with the independently authorized Backup account via Platform custody. It re-downloads and binds the ciphertext by ETag/size/hash. The receipt names the new private local ciphertext path. A published index alone is not independent retrieval proof. +6. The human independently verifies the downloaded bundle using the hidden-key command below. Review the actual receipt and exact source/artifact/hash. Copy only this safe receipt to private node state; activate with `install-identity-backup.py activate --accepted-receipt PRIVATE_ACTUAL_RECOVERY --receipt PRIVATE_NEW_PATH`. Activation binds the actual artifact to its local successful off-site receipt, pins that generation, checks health and enables the timers. +7. Confirm both active timers, last success, retained protection and safe Hub progress. Full NK49 native identity/caller and legacy-credential acceptance remains separately open. + +All actions accept source/target guards; flags do not create authority. The +installer runs as the already admitted node owner. If installation or a grant +partly fails, stop and reconcile its safe receipt/exact resources before retry. +Do not rotate keys, change existing service credentials or remove older backups +as an implicit repair. For rollback, disable the two exact new timers first, +retain the verified ciphertext and restore receipt, then have the owners revoke +only these newly issued shares/roles and remove their delivery resources after +confirming no consumer remains. Never delete existing unrelated backup jobs. + +### Independent full archive verification and bootstrap connection + +The key is entered only in the operator's hidden terminal, using the independently +held backup key already checked against this public recipient: + +```bash +sso-mfa/bootstrap/sops-custody-unlock.sh \ + --expected-recipient age1zvryunvjhvpkmasskauga2heeg0ztnte9ymgppvjge36ekumk50syr3tsz \ + --prompt -- python3 tools/bootstrap-recovery/verify-identity.py \ + --transfer-receipt /PRIVATE/independent-transfer.json \ + --receipt /PRIVATE/new-full-archive-recovery.json +``` + +With the reviewed sibling `railiance-bootstrap` checkout, the equivalent handoff +is `bash ../railiance-bootstrap/bin/railiance identity-recovery` with the same +transfer/receipt arguments, run inside that hidden-key wrapper. Its +[bootstrap sequence](../../railiance-bootstrap/docs/identity-recovery.md) links +independent custody and isolated verification to replacement-host/layer handoff. +The wrapper delegates to this verifier and never imports into production. + +The complete manifest, SQLite integrity, original RSA pairs, wrong-key refusal, +native isolated PostgreSQL restore and verified cleanup are the archive gate for +recurring backups. A successful archive gate does not prove existing-token +plaintext/MFA, historical/new audit signatures, existing-user login or issuer +consumer flows. Those require separately reviewed isolated service acceptance +before T03 and NK49 can close. The verifier deletes its plaintext; it is not a +production importer. + +Today's attended retrieval helper requires healthy OpenBao. For total custody +loss, independently held provider access and a separately reviewed offline +retrieval receipt procedure are still required; that offline adapter is not +implemented. No bare-VM unattended restore is claimed. The backup public key is +not interchangeable with the separate bootstrap encryption recipient. + +### Recurrence, retention and health + +After the first actual archive gate, the prepared root unit runs daily at +03:45 UTC with up to five minutes jitter. Retention runs in separate processes +before/after capture, protecting newest, seven distinct daily points, two +distinct weekly points and the independently recovered pinned generation. +Unknown/unindexed objects are never candidates. Both pinned and newest points +are re-downloaded and hashed before any expiry; each candidate is checked and +deleted conditionally by ETag. Partial/unknown expiry stops subsequent expiry +until reconciled. A matching guarded local-cache prune preserves protected +points and leaves unknown/failed runs for owner inspection. There is no claim +that 10 GiB is sufficient indefinitely: measured quota/headroom is checked each +run and an excess refuses new capture rather than deleting a protected point. + +An hourly node health check reports missing/older-than-25-hour backups or partial +failure to State Hub, with durable local nonsecret outbox/replay. This host-local +check cannot detect or report a dead host; external outage monitoring remains a +separate gap. The daily backup RPO is explicit and distinct from the one-hour +ecosystem coordination frame. Timers are currently disabled/uninstalled. + +Find this functionality through the owning registry entry +`capability.security.bootstrap-recovery` (keywords backup, restore, +disaster-recovery, age) federated by reuse-surface, related to +`capability.railiance.bootstrap-sequencing`. Maturity remains D3/A2/C1/R0; +source fixtures and discoverability do not assert live production acceptance. diff --git a/docs/evidence/2026-10-16-identity-backup-implementation/capture-plan.json b/docs/evidence/2026-10-16-identity-backup-implementation/capture-plan.json new file mode 100644 index 0000000..f4d6ef4 --- /dev/null +++ b/docs/evidence/2026-10-16-identity-backup-implementation/capture-plan.json @@ -0,0 +1,393 @@ +{ + "schema": "netkingdom.identity-capture-plan.v1", + "status": "requires_owner_execution_admission", + "created_at": "2026-10-15T23:33:14.370426+00:00", + "run_id": "35ffd825063d", + "binding": { + "schema": "netkingdom.identity-backup-preparation.v1", + "status": "prepared_not_admitted", + "observed_at": "2026-10-15T23:33:13.858371+00:00", + "cluster_uid": "a553c742-0115-43d4-99a4-a5ca56fe0786", + "profile_sha256": "dff567f594826628544bb2252c77ccb8daf8a34fd2095026663cc948ffeff63a", + "source_sha256": "cd5f3067d28b4c7cc93ed08b8cf3869218bdecf7c54b5dffed22d697de71dbb1", + "bindings": [ + { + "namespace": "sso", + "deployment": "lldap", + "uid": "743a86bd-5cb0-4ca0-9c56-a3438f4746a8", + "generation": 7, + "pvc_bindings": [ + { + "name": "lldap-data", + "uid": "0a06b65a-e7b3-42d3-916e-e74dbe5fc7da" + } + ], + "secret_names_only": [ + "lldap-secrets" + ], + "configmap_names_only": [] + }, + { + "namespace": "sso", + "deployment": "authelia", + "uid": "dc40a925-2306-4a3b-89c0-e714187503c3", + "generation": 28, + "pvc_bindings": [ + { + "name": "authelia-data", + "uid": "86826554-b014-4df4-8b28-866aa64f6413" + } + ], + "secret_names_only": [ + "authelia-secrets", + "authelia-smtp" + ], + "configmap_names_only": [ + "authelia-config" + ] + }, + { + "namespace": "sso", + "deployment": "keycape", + "uid": "99ddd83c-cb3f-4847-bcf8-35f1aa87627f", + "generation": 65, + "pvc_bindings": [ + { + "name": "keycape-authentication-policy", + "uid": "423f558f-8c11-4edf-956b-243559df4a8e" + } + ], + "secret_names_only": [ + "keycape-approval-engine-operator-client", + "keycape-config", + "keycape-factor-read", + "keycape-informed-decision-sitting-requester-client", + "keycape-rapp-qonto-client", + "keycape-secrets-engine-approval-client", + "keycape-secrets-engine-openbao-client", + "keycape-secrets-engine-requester-client" + ], + "configmap_names_only": [] + }, + { + "namespace": "mfa", + "deployment": "privacyidea", + "uid": "58c7f96d-61cb-4dd4-bca2-54661c0ac375", + "generation": 11, + "pvc_bindings": [ + { + "name": "privacyidea-data", + "uid": "65650fc1-02d8-4c0f-84fc-e0015c8f3ed9" + }, + { + "name": "privacyidea-logs", + "uid": "39fa90d9-bc54-4a03-a0d6-ae85165e0cb9" + } + ], + "secret_names_only": [ + "privacyidea-config" + ], + "configmap_names_only": [ + "factor-recovery-01119187d90d7b76", + "keycape-onboarding-9465e5e0b98f7986", + "privacyidea-cfg" + ] + } + ], + "secret_values_read": false, + "backup_captured": false, + "uploaded": false, + "runtime_changed": false, + "credential_grant_created": false, + "owner_admission_required": [ + "Approve exact CCR-2026-0048/0049 grants and node runtime packet; three required owner reviews, confirmed native SA/namespace/audience boundaries before acceptance", + "Exclude helper capture label from existing broad DNS allow policies; validate read-only helper manifests and cluster/PVC/source bindings before private capture", + "Preserve every referenced credential encrypted with original keys; fresh native SQLite/PG snapshots and exact manifest, with documented omissions and no global atomicity claim", + "Create fresh ciphertext and exact Storage Share object, read back ciphertext and receipt by strong ETag/hash, independently retrieve using operator custody", + "Attended independent age-key proof: wrong-key denial, complete archive hashes, SQLite/RSA checks, disposable native PostgreSQL restore and verified cleanup before timer activation", + "Full NK49 closure separately requires native directory/issuer/MFA authentication, retained token decryption, audit signature continuity, current callers and legacy credential dispositions; archive acceptance does not close those gates", + "Host-local health cannot report a dead host; external outage monitoring and second-provider copy remain separate coverage gaps" + ] + }, + "kubeconfig": "/home/worsch/.kube/config", + "sources": [ + { + "path": "capture-identity.py", + "sha256": "47dca7c4dc96fa7eb9fb02c81fca9ca8dc5281aac918979eb7367148168ba705" + }, + { + "path": "identity-files.py", + "sha256": "d0063a9eedd3610e3d45bd2d834aae83cff74e4b3504d138b3548f427e89331d" + }, + { + "path": "identity-bundle.py", + "sha256": "491f6f9f074dd0f73a26b16f32d81dbf70e049569e65c9dcee1cbcd1305dc8dc" + }, + { + "path": "prepare-identity-coverage.py", + "sha256": "cd5f3067d28b4c7cc93ed08b8cf3869218bdecf7c54b5dffed22d697de71dbb1" + }, + { + "path": "identity-backup-profile.json", + "sha256": "dff567f594826628544bb2252c77ccb8daf8a34fd2095026663cc948ffeff63a" + } + ], + "database": { + "namespace": "databases", + "cluster": "net-kingdom-pg", + "cluster_uid": "0cd04a7c-a183-48ac-975a-9f244ea82c73", + "pod": "net-kingdom-pg-1", + "pod_uid": "0404c66b-338b-4446-a767-622d248ab0d3", + "database": "privacyidea_db" + }, + "create_resources": [ + { + "apiVersion": "networking.k8s.io/v1", + "kind": "NetworkPolicy", + "metadata": { + "name": "nk49-capture-35ffd825063d", + "namespace": "sso" + }, + "spec": { + "podSelector": { + "matchLabels": { + "net-kingdom/capture": "35ffd825063d" + } + }, + "policyTypes": [ + "Ingress", + "Egress" + ], + "ingress": [], + "egress": [] + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "name": "nk49-capture-35ffd825063d", + "namespace": "sso", + "labels": { + "net-kingdom/capture": "35ffd825063d" + } + }, + "spec": { + "restartPolicy": "Never", + "activeDeadlineSeconds": 900, + "automountServiceAccountToken": false, + "securityContext": { + "runAsUser": 0, + "runAsGroup": 0 + }, + "containers": [ + { + "name": "capture", + "image": "ghcr.io/gpappsoft/privacyidea-docker@sha256:af7841adad262f129e0c1d4f553af13f21cb2f4dc713533f316cfe43ed0b4473", + "imagePullPolicy": "IfNotPresent", + "command": [ + "python3", + "-c", + "import time; time.sleep(900)" + ], + "securityContext": { + "allowPrivilegeEscalation": false, + "readOnlyRootFilesystem": true, + "capabilities": { + "drop": [ + "ALL" + ], + "add": [ + "DAC_OVERRIDE" + ] + } + }, + "resources": { + "requests": { + "cpu": "50m", + "memory": "128Mi" + }, + "limits": { + "cpu": "1", + "memory": "2Gi" + } + }, + "volumeMounts": [ + { + "name": "lldap", + "mountPath": "/source/lldap", + "readOnly": true + }, + { + "name": "authelia", + "mountPath": "/source/authelia", + "readOnly": true + }, + { + "name": "keycape", + "mountPath": "/source/keycape", + "readOnly": true + }, + { + "name": "scratch", + "mountPath": "/scratch" + } + ] + } + ], + "volumes": [ + { + "name": "lldap", + "persistentVolumeClaim": { + "claimName": "lldap-data", + "readOnly": true + } + }, + { + "name": "authelia", + "persistentVolumeClaim": { + "claimName": "authelia-data", + "readOnly": true + } + }, + { + "name": "keycape", + "persistentVolumeClaim": { + "claimName": "keycape-authentication-policy", + "readOnly": true + } + }, + { + "name": "scratch", + "emptyDir": { + "medium": "Memory", + "sizeLimit": "512Mi" + } + } + ] + } + }, + { + "apiVersion": "networking.k8s.io/v1", + "kind": "NetworkPolicy", + "metadata": { + "name": "nk49-capture-35ffd825063d", + "namespace": "mfa" + }, + "spec": { + "podSelector": { + "matchLabels": { + "net-kingdom/capture": "35ffd825063d" + } + }, + "policyTypes": [ + "Ingress", + "Egress" + ], + "ingress": [], + "egress": [] + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "name": "nk49-capture-35ffd825063d", + "namespace": "mfa", + "labels": { + "net-kingdom/capture": "35ffd825063d" + } + }, + "spec": { + "restartPolicy": "Never", + "activeDeadlineSeconds": 900, + "automountServiceAccountToken": false, + "securityContext": { + "runAsUser": 0, + "runAsGroup": 0 + }, + "containers": [ + { + "name": "capture", + "image": "ghcr.io/gpappsoft/privacyidea-docker@sha256:af7841adad262f129e0c1d4f553af13f21cb2f4dc713533f316cfe43ed0b4473", + "imagePullPolicy": "IfNotPresent", + "command": [ + "python3", + "-c", + "import time; time.sleep(900)" + ], + "securityContext": { + "allowPrivilegeEscalation": false, + "readOnlyRootFilesystem": true, + "capabilities": { + "drop": [ + "ALL" + ], + "add": [ + "DAC_OVERRIDE" + ] + } + }, + "resources": { + "requests": { + "cpu": "50m", + "memory": "128Mi" + }, + "limits": { + "cpu": "1", + "memory": "2Gi" + } + }, + "volumeMounts": [ + { + "name": "privacyidea", + "mountPath": "/source/privacyidea", + "readOnly": true + }, + { + "name": "scratch", + "mountPath": "/scratch" + } + ] + } + ], + "volumes": [ + { + "name": "privacyidea", + "persistentVolumeClaim": { + "claimName": "privacyidea-data", + "readOnly": true + } + }, + { + "name": "scratch", + "emptyDir": { + "medium": "Memory", + "sizeLimit": "512Mi" + } + } + ] + } + } + ], + "service_changes": false, + "credential_grants": false, + "matching_network_allow_policies": [ + { + "namespace": "sso", + "name": "allow-egress-dns", + "uid": "d2a1bd49-7035-437b-9b8e-e9e66b203365", + "spec_sha256": "c22e3a30891f0fa3cdcde97d26c91a09d11a8e94f7dfc66641d8be1cbd976f15" + }, + { + "namespace": "mfa", + "name": "allow-egress-dns", + "uid": "1d23ed08-bb6f-4959-b72e-3f03af40f136", + "spec_sha256": "c22e3a30891f0fa3cdcde97d26c91a09d11a8e94f7dfc66641d8be1cbd976f15" + } + ], + "network_policy_enforcement_requires_owner_confirmation": true, + "offsite_delivery_included": false, + "global_atomic_snapshot_claimed": false, + "finished_at": "2026-10-15T23:33:14.927800+00:00" +} diff --git a/docs/evidence/2026-10-16-identity-backup-implementation/helper-delivery-server-dry-run.json b/docs/evidence/2026-10-16-identity-backup-implementation/helper-delivery-server-dry-run.json new file mode 100644 index 0000000..3dab929 --- /dev/null +++ b/docs/evidence/2026-10-16-identity-backup-implementation/helper-delivery-server-dry-run.json @@ -0,0 +1,65 @@ +{ + "resources": [ + { + "namespace": "sso", + "kind": "NetworkPolicy", + "name": "nk49-capture-preflight001", + "server_dry_run_passed": true + }, + { + "namespace": "sso", + "kind": "Pod", + "name": "nk49-capture-preflight001", + "server_dry_run_passed": true + }, + { + "namespace": "mfa", + "kind": "NetworkPolicy", + "name": "nk49-capture-preflight001", + "server_dry_run_passed": true + }, + { + "namespace": "mfa", + "kind": "Pod", + "name": "nk49-capture-preflight001", + "server_dry_run_passed": true + }, + { + "namespace": "sso", + "kind": "ServiceAccount", + "name": "net-kingdom-backup-publisher-eso", + "server_dry_run_passed": true + }, + { + "namespace": "sso", + "kind": "SecretStore", + "name": "net-kingdom-backup-publisher", + "server_dry_run_passed": true + }, + { + "namespace": "sso", + "kind": "ExternalSecret", + "name": "net-kingdom-backup-publisher", + "server_dry_run_passed": true + }, + { + "namespace": "sso", + "kind": "ServiceAccount", + "name": "net-kingdom-backup-retention-eso", + "server_dry_run_passed": true + }, + { + "namespace": "sso", + "kind": "SecretStore", + "name": "net-kingdom-backup-retention", + "server_dry_run_passed": true + }, + { + "namespace": "sso", + "kind": "ExternalSecret", + "name": "net-kingdom-backup-retention", + "server_dry_run_passed": true + } + ], + "live_resources_created": false +} diff --git a/docs/evidence/2026-10-16-identity-backup-implementation/isolation-plan.json b/docs/evidence/2026-10-16-identity-backup-implementation/isolation-plan.json new file mode 100644 index 0000000..d2feec8 --- /dev/null +++ b/docs/evidence/2026-10-16-identity-backup-implementation/isolation-plan.json @@ -0,0 +1,132 @@ +{ + "schema": "netkingdom.identity-isolation.v1", + "status": "prepared_not_applied", + "kubeconfig": "/home/worsch/.kube/config", + "source_sha256": "88d1e13112a44c79571825af89d3fe5b01fa490cff2653badef4c62a45c844f1", + "policies": [ + { + "namespace": "sso", + "name": "allow-egress-dns", + "uid": "d2a1bd49-7035-437b-9b8e-e9e66b203365", + "patch": [ + { + "op": "test", + "path": "/metadata/uid", + "value": "d2a1bd49-7035-437b-9b8e-e9e66b203365" + }, + { + "op": "test", + "path": "/metadata/resourceVersion", + "value": "3623833" + }, + { + "op": "test", + "path": "/spec", + "value": { + "egress": [ + { + "ports": [ + { + "port": 53, + "protocol": "UDP" + }, + { + "port": 53, + "protocol": "TCP" + } + ], + "to": [ + { + "namespaceSelector": { + "matchLabels": { + "kubernetes.io/metadata.name": "kube-system" + } + } + } + ] + } + ], + "podSelector": {}, + "policyTypes": [ + "Egress" + ] + } + }, + { + "op": "replace", + "path": "/spec/podSelector", + "value": { + "matchExpressions": [ + { + "key": "net-kingdom/capture", + "operator": "DoesNotExist" + } + ] + } + } + ] + }, + { + "namespace": "mfa", + "name": "allow-egress-dns", + "uid": "1d23ed08-bb6f-4959-b72e-3f03af40f136", + "patch": [ + { + "op": "test", + "path": "/metadata/uid", + "value": "1d23ed08-bb6f-4959-b72e-3f03af40f136" + }, + { + "op": "test", + "path": "/metadata/resourceVersion", + "value": "3623822" + }, + { + "op": "test", + "path": "/spec", + "value": { + "egress": [ + { + "ports": [ + { + "port": 53, + "protocol": "UDP" + }, + { + "port": 53, + "protocol": "TCP" + } + ], + "to": [ + { + "namespaceSelector": { + "matchLabels": { + "kubernetes.io/metadata.name": "kube-system" + } + } + } + ] + } + ], + "podSelector": {}, + "policyTypes": [ + "Egress" + ] + } + }, + { + "op": "replace", + "path": "/spec/podSelector", + "value": { + "matchExpressions": [ + { + "key": "net-kingdom/capture", + "operator": "DoesNotExist" + } + ] + } + } + ] + } + ] +} diff --git a/docs/evidence/2026-10-16-identity-backup-implementation/isolation-server-dry-run.json b/docs/evidence/2026-10-16-identity-backup-implementation/isolation-server-dry-run.json new file mode 100644 index 0000000..afc9cca --- /dev/null +++ b/docs/evidence/2026-10-16-identity-backup-implementation/isolation-server-dry-run.json @@ -0,0 +1,27 @@ +{ + "status": "server_dry_run_passed", + "attempted": [ + { + "namespace": "sso", + "name": "allow-egress-dns", + "uid": "d2a1bd49-7035-437b-9b8e-e9e66b203365" + }, + { + "namespace": "mfa", + "name": "allow-egress-dns", + "uid": "1d23ed08-bb6f-4959-b72e-3f03af40f136" + } + ], + "verified": [ + { + "namespace": "sso", + "name": "allow-egress-dns", + "uid": "d2a1bd49-7035-437b-9b8e-e9e66b203365" + }, + { + "namespace": "mfa", + "name": "allow-egress-dns", + "uid": "1d23ed08-bb6f-4959-b72e-3f03af40f136" + } + ] +} diff --git a/docs/evidence/2026-10-16-identity-backup-implementation/review.json b/docs/evidence/2026-10-16-identity-backup-implementation/review.json new file mode 100644 index 0000000..77c6073 --- /dev/null +++ b/docs/evidence/2026-10-16-identity-backup-implementation/review.json @@ -0,0 +1,37 @@ +{ + "schema": "netkingdom.identity-implementation-review.v1", + "reviewed_at": "2026-10-15T23:36:41.132816+00:00", + "focused_tests": { + "command": "env NK49_RUN_DISPOSABLE_POSTGRES_TEST=1 python3 -m pytest -q tests/test_identity_bundle.py tests/test_identity_offsite.py tests/test_identity_backup_preparation.py", + "passed": 51, + "duration_seconds": 14.78, + "synthetic_data_only": true, + "real_age_and_native_postgresql_fixture": true + }, + "systemd_analyze_verify_exit": 0, + "helper_and_delivery_server_dry_run_count": 10, + "dns_exclusion_server_dry_run_passed": true, + "owner_ccr_validation_errors": 0, + "owner_ccr_bound_claim_confirmation_pending": true, + "live_grants_made": false, + "actual_identity_capture_performed": false, + "identity_timers_installed": false, + "first_actual_full_archive_recovery_pending": true, + "full_native_identity_recovery_proven": false, + "pending_decision": "40bfae40-6108-4e51-8613-073339f5345f", + "knowledge": { + "version": "0.2.6", + "digest": "8d1d6a9d4d07fec7b619618de7dfe2be2254c1790f85f4198c7fe19db9751c02", + "source_status": "withheld_orientation_only", + "source_reviewed": "SECURITY-COMPANION.md" + }, + "cross_repo_handoff": "railiance-bootstrap/bin/railiance identity-recovery; isolated verifier delegation only", + "capability": "capability.security.bootstrap-recovery", + "capability_vector": "D3 / A2 / C1 / R0", + "limitations": [ + "No full MFA/token/audit/directory/issuer authentication proven", + "Total-custody-loss offline retrieval adapter remains open", + "No Scaleway second full identity copy", + "Host-local freshness reporting cannot detect host loss" + ] +} diff --git a/docs/evidence/2026-10-16-identity-backup-implementation/runtime-install-plan.json b/docs/evidence/2026-10-16-identity-backup-implementation/runtime-install-plan.json new file mode 100644 index 0000000..07ac875 --- /dev/null +++ b/docs/evidence/2026-10-16-identity-backup-implementation/runtime-install-plan.json @@ -0,0 +1,59 @@ +{ + "schema": "netkingdom.identity-runtime-install.v1", + "status": "prepared_not_installed", + "decision_ref": "40bfae40-6108-4e51-8613-073339f5345f", + "release_id": "17d1ae2d447a7882a0e581a4c61fe4f69f3fd499e39d563c984c3a65276abd08", + "release_directory": "/opt/net-kingdom-backup/releases/17d1ae2d447a7882a0e581a4c61fe4f69f3fd499e39d563c984c3a65276abd08", + "runtime_files": [ + { + "path": "tools/bootstrap-recovery/capture-identity.py", + "sha256": "47dca7c4dc96fa7eb9fb02c81fca9ca8dc5281aac918979eb7367148168ba705" + }, + { + "path": "tools/bootstrap-recovery/identity-files.py", + "sha256": "d0063a9eedd3610e3d45bd2d834aae83cff74e4b3504d138b3548f427e89331d" + }, + { + "path": "tools/bootstrap-recovery/identity-bundle.py", + "sha256": "491f6f9f074dd0f73a26b16f32d81dbf70e049569e65c9dcee1cbcd1305dc8dc" + }, + { + "path": "tools/bootstrap-recovery/identity-runtime.py", + "sha256": "8b1fe1138e28ec5df7287ed6e207a026c0e73cf8a55a25773ae9a89e04c97cbf" + }, + { + "path": "tools/bootstrap-recovery/identity-offsite.py", + "sha256": "e7c31fa9ee081961af873348ee3b4829a645d40c7dc7f1e22c564aa9ed01bc49" + }, + { + "path": "tools/bootstrap-recovery/identity-retention.py", + "sha256": "a41ea4759f69c35548149fc8d0d5ace547f16cf9fe4c9dac10b119a4d0becdec" + }, + { + "path": "tools/bootstrap-recovery/identity-backup-profile.json", + "sha256": "dff567f594826628544bb2252c77ccb8daf8a34fd2095026663cc948ffeff63a" + }, + { + "path": "tools/bootstrap-recovery/prepare-identity-coverage.py", + "sha256": "cd5f3067d28b4c7cc93ed08b8cf3869218bdecf7c54b5dffed22d697de71dbb1" + }, + { + "path": "tools/bootstrap-recovery/install-identity-backup.py", + "sha256": "94ae7a22fda1cce43b52a1553a3e9293a4ab0ca4bbec062dab7ed62839593bad" + }, + { + "path": "sso-mfa/k8s/privacyidea/key-custody-remote.py", + "sha256": "27b2b885c81923d85054e3999c5613e104fe627779b2b50398ed9bed5088f47e" + } + ], + "units": { + "net-kingdom-identity-backup.service": "[Unit]\nDescription=Encrypted full identity backup and verified Storage Share readback\nAfter=network-online.target k3s.service\nConditionPathExists=/var/lib/net-kingdom-backup/accepted-recovery.json\n[Service]\nType=oneshot\nUser=root\nGroup=root\nUMask=0077\nStateDirectory=net-kingdom-backup\nStateDirectoryMode=0700\nNoNewPrivileges=true\nPrivateTmp=true\nPrivateDevices=true\nProtectSystem=strict\nProtectHome=true\nReadWritePaths=/var/lib/net-kingdom-backup\nCapabilityBoundingSet=\nRestrictAddressFamilies=AF_UNIX AF_INET AF_INET6\nProtectKernelTunables=true\nProtectKernelModules=true\nProtectControlGroups=true\nMemoryMax=3G\nCPUQuota=200%\nTasksMax=128\nTimeoutStartSec=45min\nTimeoutStopSec=5min\nExecStartPre=/usr/bin/python3 -I /opt/net-kingdom-backup/releases/17d1ae2d447a7882a0e581a4c61fe4f69f3fd499e39d563c984c3a65276abd08/tools/bootstrap-recovery/identity-runtime.py retention --kubeconfig /etc/rancher/k3s/k3s.yaml --state /var/lib/net-kingdom-backup\nExecStart=/usr/bin/python3 -I /opt/net-kingdom-backup/releases/17d1ae2d447a7882a0e581a4c61fe4f69f3fd499e39d563c984c3a65276abd08/tools/bootstrap-recovery/identity-runtime.py scheduled --kubeconfig /etc/rancher/k3s/k3s.yaml --state /var/lib/net-kingdom-backup --decision-id 40bfae40-6108-4e51-8613-073339f5345f\nExecStartPost=/usr/bin/python3 -I /opt/net-kingdom-backup/releases/17d1ae2d447a7882a0e581a4c61fe4f69f3fd499e39d563c984c3a65276abd08/tools/bootstrap-recovery/identity-runtime.py retention --kubeconfig /etc/rancher/k3s/k3s.yaml --state /var/lib/net-kingdom-backup\n", + "net-kingdom-identity-backup.timer": "[Unit]\nDescription=Daily full identity recovery bundle\n[Timer]\nOnCalendar=*-*-* 03:45:00 UTC\nPersistent=true\nRandomizedDelaySec=5min\nUnit=net-kingdom-identity-backup.service\n[Install]\nWantedBy=timers.target\n", + "net-kingdom-identity-backup-health.service": "[Unit]\nDescription=Identity backup age check with durable failure reporting\nAfter=network-online.target k3s.service\n[Service]\nType=oneshot\nUser=root\nGroup=root\nUMask=0077\nStateDirectory=net-kingdom-backup\nStateDirectoryMode=0700\nNoNewPrivileges=true\nPrivateTmp=true\nPrivateDevices=true\nProtectSystem=strict\nProtectHome=true\nReadWritePaths=/var/lib/net-kingdom-backup\nCapabilityBoundingSet=\nRestrictAddressFamilies=AF_UNIX AF_INET AF_INET6\nProtectKernelTunables=true\nProtectKernelModules=true\nProtectControlGroups=true\nMemoryMax=3G\nCPUQuota=200%\nTasksMax=128\nTimeoutStartSec=45min\nTimeoutStopSec=5min\nExecStart=/usr/bin/python3 -I /opt/net-kingdom-backup/releases/17d1ae2d447a7882a0e581a4c61fe4f69f3fd499e39d563c984c3a65276abd08/tools/bootstrap-recovery/identity-runtime.py status --kubeconfig /etc/rancher/k3s/k3s.yaml --state /var/lib/net-kingdom-backup\n", + "net-kingdom-identity-backup-health.timer": "[Unit]\nDescription=Hourly identity backup freshness check\n[Timer]\nOnCalendar=hourly\nPersistent=true\nUnit=net-kingdom-identity-backup-health.service\n[Install]\nWantedBy=timers.target\n" + }, + "timers_automatically_enabled": false, + "first_capture_command": "/usr/bin/python3 -I /opt/net-kingdom-backup/releases/17d1ae2d447a7882a0e581a4c61fe4f69f3fd499e39d563c984c3a65276abd08/tools/bootstrap-recovery/identity-runtime.py first-capture --kubeconfig /etc/rancher/k3s/k3s.yaml --state /var/lib/net-kingdom-backup --decision-id 40bfae40-6108-4e51-8613-073339f5345f", + "activation_requires_independent_recovery": true, + "source_contains_credentials": false +} diff --git a/docs/evidence/2026-10-16-identity-backup-implementation/source-manifest.json b/docs/evidence/2026-10-16-identity-backup-implementation/source-manifest.json new file mode 100644 index 0000000..722399d --- /dev/null +++ b/docs/evidence/2026-10-16-identity-backup-implementation/source-manifest.json @@ -0,0 +1,82 @@ +[ + { + "path": "tools/bootstrap-recovery/capture-identity.py", + "sha256": "47dca7c4dc96fa7eb9fb02c81fca9ca8dc5281aac918979eb7367148168ba705" + }, + { + "path": "tools/bootstrap-recovery/check-privacyidea-database.py", + "sha256": "90f454c87998d7f5d625e3c6656c3418f8f56275dc1879a9d19ed605431b94a4" + }, + { + "path": "tools/bootstrap-recovery/fetch-privacyidea-backup.py", + "sha256": "233160e1206643a5cf7ad05141d733f0a75aff651190e27f77515a124ff0a59f" + }, + { + "path": "tools/bootstrap-recovery/identity-bundle.py", + "sha256": "491f6f9f074dd0f73a26b16f32d81dbf70e049569e65c9dcee1cbcd1305dc8dc" + }, + { + "path": "tools/bootstrap-recovery/identity-credential-setup.py", + "sha256": "9a754b72640cb3dac95fa3bbc74448be81eb96a38c9319228db1c7f967dbba14" + }, + { + "path": "tools/bootstrap-recovery/identity-files.py", + "sha256": "d0063a9eedd3610e3d45bd2d834aae83cff74e4b3504d138b3548f427e89331d" + }, + { + "path": "tools/bootstrap-recovery/identity-isolation.py", + "sha256": "88d1e13112a44c79571825af89d3fe5b01fa490cff2653badef4c62a45c844f1" + }, + { + "path": "tools/bootstrap-recovery/identity-offsite.py", + "sha256": "e7c31fa9ee081961af873348ee3b4829a645d40c7dc7f1e22c564aa9ed01bc49" + }, + { + "path": "tools/bootstrap-recovery/identity-retention.py", + "sha256": "a41ea4759f69c35548149fc8d0d5ace547f16cf9fe4c9dac10b119a4d0becdec" + }, + { + "path": "tools/bootstrap-recovery/identity-runtime.py", + "sha256": "8b1fe1138e28ec5df7287ed6e207a026c0e73cf8a55a25773ae9a89e04c97cbf" + }, + { + "path": "tools/bootstrap-recovery/install-identity-backup.py", + "sha256": "94ae7a22fda1cce43b52a1553a3e9293a4ab0ca4bbec062dab7ed62839593bad" + }, + { + "path": "tools/bootstrap-recovery/inventory-scaleway.py", + "sha256": "a219df205708fa4ed44b763a418585d63c820c25434bc0fd14e805eb5c2f79d3" + }, + { + "path": "tools/bootstrap-recovery/inventory-storage-share.py", + "sha256": "673f9b17fb2f3fef6fcdb5c50a2bcca69e3df3157a9e5582a2d995c6b87640b5" + }, + { + "path": "tools/bootstrap-recovery/prepare-identity-coverage.py", + "sha256": "cd5f3067d28b4c7cc93ed08b8cf3869218bdecf7c54b5dffed22d697de71dbb1" + }, + { + "path": "tools/bootstrap-recovery/retrieve-identity.py", + "sha256": "ec123d47a07305253570dacab7e103f8c193a23f20c9fc87466ea4a95ae0ec13" + }, + { + "path": "tools/bootstrap-recovery/verify-bundle.py", + "sha256": "885b05a2011e711228e42e316bda70bbe0d97d481383d61aa8469d0f10452370" + }, + { + "path": "tools/bootstrap-recovery/verify-identity-delivery.py", + "sha256": "9e2afdc94b5cf0a81b66a1fe8658e7853c0972e4e3593bd198dd2af5817b254f" + }, + { + "path": "tools/bootstrap-recovery/verify-identity.py", + "sha256": "039864d0141b552887752bf17cfdcf1bef4bdc9126f946a9c0533e4fd54ebf4f" + }, + { + "path": "tools/bootstrap-recovery/identity-backup-profile.json", + "sha256": "dff567f594826628544bb2252c77ccb8daf8a34fd2095026663cc948ffeff63a" + }, + { + "path": "tools/bootstrap-recovery/identity-custody.json", + "sha256": "f8aee8487ed01d7940b82676bb4a5c8d61f51b6c598521b8b8a73588aaea4d4f" + } +] diff --git a/registry/capabilities/capability.security.bootstrap-recovery.md b/registry/capabilities/capability.security.bootstrap-recovery.md index ee6f65f..c2a02a8 100644 --- a/registry/capabilities/capability.security.bootstrap-recovery.md +++ b/registry/capabilities/capability.security.bootstrap-recovery.md @@ -29,16 +29,30 @@ external_evidence: confidence: low basis: No longitudinal consumer reliability measurement established by this review. name: Guarded bootstrap and recovery preparation -summary: Local identity, bootstrap console, encrypted custody handoff and caller tools. +summary: Guarded bootstrap custody, encrypted identity backup capture, independent + retrieval and isolated recovery verification. tags: - security - netkingdom - framework-reuse +- backup +- restore +- disaster-recovery +- age +- privacyidea +- lldap +- authelia +- keycape discovery: - intent: Local identity, bootstrap console, encrypted custody handoff and caller - tools + intent: Find and verify off-host identity recovery material before rebuilding Railiance + authentication. includes: - Local identity, bootstrap console, encrypted custody handoff and caller tools + - Bounded LLDAP/Authelia SQLite snapshots, KeyCape signing/config/policy and original + privacyIDEA keys plus linked PostgreSQL dump + - Public-recipient age sealing, Storage Share create/readback, separate retention + and attended independent-key archive verification + - Railiance bootstrap handoff without requiring recovered SSO for off-host retrieval excludes: - Source repairs and disposable native fixtures do not prove estate root isolation or recovery @@ -51,6 +65,11 @@ availability: current_artifacts: - tools/security-bootstrap-console/README.md - tools/custody-record/README.md + - docs/bootstrap-recovery-ceremony.md + - tools/bootstrap-recovery/identity-backup-profile.json + - tools/bootstrap-recovery/capture-identity.py + - tools/bootstrap-recovery/retrieve-identity.py + - tools/bootstrap-recovery/verify-identity.py consumption_modes: - CLI - source @@ -61,12 +80,19 @@ relations: - capability.iam.key-cape - capability.authorization.policy-evaluate - capability.registry.register + - capability.railiance.bootstrap-sequencing specializes: - capability.security.iam-tooling-suite evidence: documentation: - tools/security-bootstrap-console/README.md - tools/custody-record/README.md + - docs/bootstrap-recovery-ceremony.md + - workplans/NK-WP-0049-bootstrap-secret-free-recovery-handoff.md + tests: + - tests/test_identity_bundle.py + - tests/test_identity_offsite.py + - tests/test_identity_backup_preparation.py consumer_guidance: recommended_for: - Operator recovers identity, custody or data with independently held recovery material @@ -78,6 +104,13 @@ consumer_guidance: or recovery - 2026-10-14 architecture review estimates operational maturity separately; a registry entry does not authorize execution or establish deployment. + - Identity backup implementation passed 49 focused checks including synthetic age/native + PostgreSQL recovery; new CCR-2026-0048/0049 grants and first actual full bundle + remain pending owner admission. + - Actual old privacyIDEA database recovery passed; full token decryption, audit + continuity and directory/issuer authentication remain open. + - Storage Share initial destination only; no second-provider full identity copy + or autonomous bare-VM restore is established. --- # Guarded bootstrap and recovery preparation @@ -98,3 +131,7 @@ Source repairs and disposable native fixtures do not prove estate root isolation - [tools/custody-record/README.md](../../tools/custody-record/README.md) Assessment: CUST-WP-0081, 2026-10-14. Relations identify interfaces to compose; they do not certify native integration. Existing external evidence levels are retained where present. New entries use conservative C1/R0 until consumer assessment exists. + +## Identity recovery and bootstrap handoff + +Start with [the recovery ceremony](../../docs/bootstrap-recovery-ceremony.md). It names the exact capture, retrieval, hidden-key verification, activation gates and remaining full-service proof. Railiance bootstrap sequences the isolated recovery and layer handoff; NetKingdom owns identity assets and native acceptance. Discovery does not authorize grants or live restore. diff --git a/registry/indexes/capabilities.yaml b/registry/indexes/capabilities.yaml index 507b2d6..37ddc35 100644 --- a/registry/indexes/capabilities.yaml +++ b/registry/indexes/capabilities.yaml @@ -1,5 +1,5 @@ version: 1 -updated: '2026-10-14' +updated: '2026-10-16' domain: helix_forge capabilities: - id: capability.security.iam-tooling-suite @@ -67,8 +67,8 @@ capabilities: - source - id: capability.security.bootstrap-recovery name: Guarded bootstrap and recovery preparation - summary: Local identity, bootstrap console, encrypted custody handoff and caller - tools. + summary: Encrypted identity backup, independent retrieval and isolated recovery + verification with bootstrap handoff. domain: infotech status: draft owner: net-kingdom @@ -76,6 +76,14 @@ capabilities: - security - netkingdom - framework-reuse + - backup + - restore + - disaster-recovery + - age + - privacyidea + - lldap + - authelia + - keycape vector: D3 / A2 / C1 / R0 path: registry/capabilities/capability.security.bootstrap-recovery.md consumption_modes: diff --git a/sso-mfa/k8s/network-policies/netpol-mfa.yaml b/sso-mfa/k8s/network-policies/netpol-mfa.yaml index df81a8a..31093d6 100644 --- a/sso-mfa/k8s/network-policies/netpol-mfa.yaml +++ b/sso-mfa/k8s/network-policies/netpol-mfa.yaml @@ -149,7 +149,10 @@ metadata: name: allow-egress-dns namespace: mfa spec: - podSelector: {} + podSelector: + matchExpressions: + - key: net-kingdom/capture + operator: DoesNotExist policyTypes: - Egress egress: diff --git a/sso-mfa/k8s/network-policies/netpol-sso.yaml b/sso-mfa/k8s/network-policies/netpol-sso.yaml index 233f1ca..15177ec 100644 --- a/sso-mfa/k8s/network-policies/netpol-sso.yaml +++ b/sso-mfa/k8s/network-policies/netpol-sso.yaml @@ -381,7 +381,10 @@ metadata: name: allow-egress-dns namespace: sso spec: - podSelector: {} + podSelector: + matchExpressions: + - key: net-kingdom/capture + operator: DoesNotExist policyTypes: - Egress egress: diff --git a/tests/test_identity_bundle.py b/tests/test_identity_bundle.py new file mode 100644 index 0000000..3b5e94f --- /dev/null +++ b/tests/test_identity_bundle.py @@ -0,0 +1,210 @@ +"""Actual SQLite/age/RSA fixtures and hostile archive/custody boundaries.""" +import base64 +import importlib.util +import io +import json +import os +from pathlib import Path +import sqlite3 +import subprocess +import tarfile +import time +import uuid +from copy import deepcopy +import pytest +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric import rsa + +ROOT=Path(__file__).resolve().parents[1] + + +def load(name): + spec=importlib.util.spec_from_file_location(name,ROOT/f'tools/bootstrap-recovery/{name}.py') + m=importlib.util.module_from_spec(spec);spec.loader.exec_module(m);return m + + +b=load('identity-bundle');files=load('identity-files');capture=load('capture-identity') + + +@pytest.fixture +def stage(tmp_path): + os.umask(0o077);root=tmp_path/'payload';root.mkdir(mode=0o700) + key=rsa.generate_private_key(public_exponent=65537,key_size=2048) + private=key.private_bytes(serialization.Encoding.PEM,serialization.PrivateFormat.TraditionalOpenSSL,serialization.NoEncryption()) + public=key.public_key().public_bytes(serialization.Encoding.PEM,serialization.PublicFormat.SubjectPublicKeyInfo) + for c in b.profile()['components']: + ns=c['namespace'];name=c['deployment'] + d={'kind':'Deployment','metadata':{'name':name,'namespace':ns,'uid':c['deployment_uid']},'spec':{}} + b.save(root,f'resources/{ns}/deployment/{name}.json',json.dumps(d).encode()) + for name in c['secret_refs']: + data={'fixture':base64.b64encode(b'synthetic-private-value-canary').decode()} + if name=='keycape-config':data={'key.pem':base64.b64encode(private).decode(),'config.yaml':base64.b64encode(b'issuer: https://isolated.invalid').decode()} + b.save(root,f'resources/{ns}/secret/{name}.json',json.dumps({'kind':'Secret','metadata':{'name':name,'namespace':ns},'data':data}).encode()) + for name in c['configmap_refs']: + b.save(root,f'resources/{ns}/configmap/{name}.json',json.dumps({'kind':'ConfigMap','metadata':{'name':name,'namespace':ns},'data':{'fixture':'synthetic'}}).encode()) + for component,name in [('lldap','users.db'),('authelia','db.sqlite3')]: + path=root/f'data/{component}/{name}';path.parent.mkdir(mode=0o700,parents=True) + with sqlite3.connect(path) as db:db.execute('CREATE TABLE retained(value TEXT)');db.execute("INSERT INTO retained VALUES ('fixture-preserved')") + path.chmod(0o600) + b.save(root,'data/keycape/policy.json',b'{"version":1}') + for name,value in [('enckey',os.urandom(96)),('private.pem',private),('public.pem',public)]: + b.save(root,'data/privacyidea/'+name,value) + b.save(root,'databases/privacyidea.dump',b'PGDMP-synthetic-format-only') + return root + + +def test_complete_bundle_has_native_integrity_checks(stage): + r=b.checks(stage) + assert r['sqlite_integrity_passed'] and r['privacyidea_key_format_and_pair_passed'] + assert r['native_identity_recovery_proven'] is False + + +def test_real_age_seal_manifest_is_encrypted_and_plaintext_never_prints(stage,tmp_path,monkeypatch,capsys): + key=tmp_path/'fixture-age.key' + subprocess.run(['age-keygen','-o',str(key)],check=True,capture_output=True) + recipient=subprocess.run(['age-keygen','-y',str(key)],check=True,capture_output=True).stdout.decode().strip() + p=b.profile();p['public_recipient']=recipient;monkeypatch.setattr(b,'profile',lambda:p) + artifact=tmp_path/'identity.tar.age';r=b.seal(stage,artifact,{'test_only':True}) + assert artifact.stat().st_mode&0o077==0 + assert b'synthetic-private-value-canary' not in artifact.read_bytes() + result=subprocess.run(['age','-d','-i',str(key),str(artifact)],check=True,capture_output=True) + with tarfile.open(fileobj=io.BytesIO(result.stdout)) as archive: + manifest=json.load(archive.extractfile('manifest.json')) + assert len(manifest['files'])==r['file_count'] + assert 'data/privacyidea/enckey' in archive.getnames() + assert 'data/lldap/users.db' in archive.getnames() + plain=tmp_path/'decrypted.tar';plain.write_bytes(result.stdout) + recovered=tmp_path/'recovered';recovered.mkdir(mode=0o700) + restored,checks=b.restore(plain,recovered) + assert checks['privacyidea_key_format_and_pair_passed'] + assert (recovered/'data/privacyidea/enckey').read_bytes()==(stage/'data/privacyidea/enckey').read_bytes() + assert 'synthetic-private-value-canary' not in json.dumps(r)+capsys.readouterr().out + with pytest.raises(b.Refused):b.seal(stage,artifact,{}) + + +@pytest.mark.parametrize('defect',['missing-key','bad-sqlite','bad-audit-pair','wrong-deployment','private-age-key','symlink','hardlink','unknown-payload']) +def test_incomplete_unsafe_and_mismatched_inputs_refuse(stage,tmp_path,defect): + if defect=='missing-key':(stage/'data/privacyidea/enckey').unlink() + if defect=='bad-sqlite':(stage/'data/lldap/users.db').write_bytes(b'broken') + if defect=='bad-audit-pair': + key=rsa.generate_private_key(public_exponent=65537,key_size=2048) + (stage/'data/privacyidea/public.pem').write_bytes(key.public_key().public_bytes(serialization.Encoding.PEM,serialization.PublicFormat.SubjectPublicKeyInfo)) + if defect=='wrong-deployment': + p=stage/'resources/sso/deployment/keycape.json';r=json.loads(p.read_text());r['metadata']['uid']='replacement';p.write_text(json.dumps(r)) + if defect=='private-age-key':(stage/'data/keycape/poison').write_bytes(b'AGE-SECRET-KEY-' + b'fixture') + if defect=='symlink':(stage/'data/keycape/linked').symlink_to(stage/'data/privacyidea/enckey') + if defect=='hardlink':os.link(stage/'data/privacyidea/enckey',stage/'data/keycape/linked') + if defect=='unknown-payload':(stage/'unreviewed').write_bytes(b'not-allowed') + with pytest.raises(Exception): + b.checks(stage);b.inventory(stage) + + +@pytest.mark.parametrize('name,kind',[('../escape','file'),('/absolute','file'),('data/keycape/valid','symlink')]) +def test_tar_import_refuses_traversal_and_links(tmp_path,name,kind): + raw=io.BytesIO() + with tarfile.open(fileobj=raw,mode='w') as archive: + member=tarfile.TarInfo(name);member.size=1 + if kind=='symlink':member.type=tarfile.SYMTYPE;member.linkname='/etc/passwd' + archive.addfile(member,io.BytesIO(b'x')) + with pytest.raises(b.Refused):b.import_files(raw.getvalue(),tmp_path,'sso') + + +def test_native_sqlite_snapshot_keeps_committed_wal_rows_and_source_unchanged(tmp_path): + root=tmp_path/'sources';root.mkdir();scratch=tmp_path/'scratch';scratch.mkdir() + for component in ['lldap','authelia','keycape']:(root/component).mkdir() + for component,name in [('lldap','users.db'),('authelia','db.sqlite3')]: + with sqlite3.connect(root/component/name) as con: + con.execute('CREATE TABLE retained(value TEXT)');con.execute("INSERT INTO retained VALUES ('native-snapshot')") + (root/'keycape/policy.json').write_text('{}') + live=sqlite3.connect(root/'lldap/users.db');live.execute('PRAGMA journal_mode=WAL') + live.execute("INSERT INTO retained VALUES ('committed-wal')");live.commit() + try: + result=files.capture('sso',root,scratch) + restored=tmp_path/'restored.sqlite';restored.write_bytes(result['data/lldap/users.db']) + with sqlite3.connect(restored) as con:assert con.execute('SELECT count(*) FROM retained').fetchone()==(2,) + assert list(scratch.iterdir())==[] + assert not any(name.endswith(('-wal','-shm')) for name in result) + finally:live.close() + + +def test_helper_manifest_has_no_writable_service_mount_or_credentials(): + objects=capture.manifests('fixture') + for obj in objects: + if obj['kind']=='NetworkPolicy': + assert obj['spec']['egress']==[] and obj['spec']['ingress']==[] + else: + s=obj['spec'];assert s['automountServiceAccountToken'] is False + assert 'fsGroup' not in s['securityContext'] + assert s['activeDeadlineSeconds']==900 + assert all(v['persistentVolumeClaim']['readOnly'] for v in s['volumes'] if 'persistentVolumeClaim' in v) + assert not any('secret' in v for v in s['volumes']) + + +def test_cleanup_delete_carries_exact_uid_precondition(monkeypatch): + calls=[];monkeypatch.setattr(capture,'command',lambda args,payload=None,**kw:calls.append((args,payload)) or b'') + capture.delete_created({'kind':'Pod','metadata':{'namespace':'sso','name':'nk49-capture-fixture','uid':'exact-created-uid'}}) + assert json.loads(calls[0][1])['preconditions']=={'uid':'exact-created-uid'} + + +@pytest.mark.parametrize('defect',['writable-mount','extra-container','service-token','different-image','host-network']) +def test_mutated_helper_is_refused_before_private_exec(defect): + wanted=next(o for o in capture.manifests('fixture') if o['kind']=='Pod') + actual=deepcopy(wanted);spec=actual['spec'] + if defect=='writable-mount':spec['containers'][0]['volumeMounts'][0]['readOnly']=False + if defect=='extra-container':spec['containers'].append(deepcopy(spec['containers'][0])) + if defect=='service-token':spec['automountServiceAccountToken']=True + if defect=='different-image':spec['containers'][0]['image']='unreviewed:latest' + if defect=='host-network':spec['hostNetwork']=True + with pytest.raises(capture.Refused):capture.validate_helper(actual,wanted) + + +def test_namespace_dns_allow_is_excluded_only_for_capture_helpers(): + selector={'matchExpressions':[{'key':'net-kingdom/capture','operator':'DoesNotExist'}]} + assert capture.selects(selector,{'app.kubernetes.io/name':'authelia'}) + assert not capture.selects(selector,{'net-kingdom/capture':'fixture'}) + + +def test_unknown_create_outcome_is_checkpointed_and_never_claimed_clean(tmp_path,monkeypatch): + monkeypatch.setattr(capture.bundle,'profile',lambda:{'components':[]}) + monkeypatch.setattr(capture,'command',lambda *args,**kwargs:(_ for _ in ()).throw(TimeoutError())) + r={'started_at':'fixture'};checkpoints=[] + p={'run_id':'fixture','create_resources':[capture.manifests('fixture')[0]]} + with pytest.raises(capture.Refused):capture.capture(p,tmp_path/'unused.age',r,'fixture',lambda:checkpoints.append(deepcopy(r))) + assert r['helper_cleanup_verified'] is False and r['unverified_create_outcomes'] + assert any(c.get('unverified_create_outcomes') for c in checkpoints) + + +@pytest.mark.skipif(os.environ.get('NK49_RUN_DISPOSABLE_POSTGRES_TEST')!='1',reason='explicit local Docker fixture opt-in') +def test_full_new_archive_decrypt_extract_native_database_restore_and_cleanup(stage,tmp_path,monkeypatch): + verify=load('verify-identity');fixture='nk49-bundle-fixture-'+uuid.uuid4().hex + def command(args,**kwargs): + return subprocess.run(args,check=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE,timeout=120,**kwargs).stdout + try: + command(['docker','run','--detach','--pull=never','--name',fixture,'--network','none', + '--tmpfs','/var/lib/postgresql/data:rw,size=268435456','--env','POSTGRES_HOST_AUTH_METHOD=trust', + verify.pg.IMAGE,'postgres','-c','listen_addresses=']) + for _ in range(60): + probe=subprocess.run(['docker','exec',fixture,'pg_isready','-U','postgres'],capture_output=True) + if probe.returncode==0:break + time.sleep(1) + command(['docker','exec','-i',fixture,'psql','-X','-U','postgres','-v','ON_ERROR_STOP=1'], + input=b'CREATE TABLE token(id integer); INSERT INTO token VALUES (1),(2); CREATE TABLE resolver(id integer); CREATE TABLE realm(id integer);') + (stage/'databases/privacyidea.dump').write_bytes(command(['docker','exec',fixture,'pg_dump','-U','postgres','-Fc','postgres'])) + key=tmp_path/'disposable-age.key';command(['age-keygen','-o',str(key)]) + recipient=command(['age-keygen','-y',str(key)]).decode().strip() + p=b.profile();p['public_recipient']=recipient + monkeypatch.setattr(b,'profile',lambda:p);monkeypatch.setattr(verify.bundle,'profile',lambda:p) + artifact=tmp_path/'net-kingdom-identity-20261016T000000Z-0123456789ab.tar.age' + c=b.seal(stage,artifact,{'test_only':True}) + transfer={'schema':'netkingdom.identity-offsite.v1','status':'readback_verified', + 'independent_owner_retrieval_verified':True,'fetched_path':str(artifact),'artifact':artifact.name, + 'ciphertext_sha256':c['ciphertext_sha256'],'ciphertext_bytes':c['ciphertext_bytes']} + r={};fd=os.open(key,os.O_RDONLY) + try:verify.verify(transfer,fd,r) + finally:os.close(fd) + assert r['status']=='identity_archive_verified' + assert r['database_restore']['aggregate_row_counts']['token']==2 + assert r['all_manifest_hashes_verified'] and r['wrong_key_refused'] + assert r['container_cleanup_verified'] and r['temporary_plaintext_cleanup_verified'] + assert not r['full_service_authentication_proven'] + finally:subprocess.run(['docker','rm','--force','--volumes',fixture],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,timeout=30) diff --git a/tests/test_identity_offsite.py b/tests/test_identity_offsite.py new file mode 100644 index 0000000..448bcd7 --- /dev/null +++ b/tests/test_identity_offsite.py @@ -0,0 +1,153 @@ +"""Contained provider fixtures: exact quota, conditional publication and retention.""" +from datetime import datetime,timedelta,timezone +import hashlib +import importlib.util +import io +import json +from pathlib import Path +import urllib.error +import pytest + +ROOT=Path(__file__).resolve().parents[1] + + +def load(name): + spec=importlib.util.spec_from_file_location(name,ROOT/f'tools/bootstrap-recovery/{name}.py') + m=importlib.util.module_from_spec(spec);spec.loader.exec_module(m);return m + + +o=load('identity-offsite');r=load('identity-retention');setup=load('identity-credential-setup');runtime=load('identity-runtime') +TOKEN='fixturefixturefixture' + + +class Response(io.BytesIO): + def __init__(self,url,data=b'',status=200,version='"generation"'): + super().__init__(data);self.url=url;self.status=status + self.headers={'ETag':version,'Content-Length':str(len(data))} + + +class Storage: + def __init__(self):self.objects={};self.calls=[];self.free=10*1024**3;self.tamper=False + def open(self,req,timeout): + self.calls.append((req.get_method(),req.full_url,dict(req.headers))) + name=req.full_url.removeprefix(o.BASE) + if req.get_method()=='PROPFIND': + return Response(req.full_url,f'{self.free}'.encode(),207) + if req.get_method()=='PUT': + assert req.get_header('If-none-match')=='*' + if name in self.objects:raise urllib.error.HTTPError(req.full_url,412,'exists',{},None) + value=req.data if isinstance(req.data,bytes) else req.data.read() + self.objects[name]=value + return Response(req.full_url,status=201) + if name not in self.objects:raise urllib.error.HTTPError(req.full_url,404,'missing',{},None) + body=self.objects[name] + if req.get_method()=='GET' and self.tamper:body=b'X'*len(body) + if req.get_method()=='DELETE': + assert req.get_header('If-match')=='"generation"' + del self.objects[name];return Response(req.full_url,status=204) + return Response(req.full_url,body) + + +@pytest.fixture +def source(tmp_path): + name='net-kingdom-identity-20261016T000000Z-0123456789ab.tar.age';path=tmp_path/name + path.write_bytes(b'age-encryption.org/v1\n'+b'ciphertext-fixture'*10);path.chmod(0o600) + record={'status':'encrypted_bundle_created','helper_cleanup_verified':True,'temporary_plaintext_cleanup_verified':True, + 'ciphertext_sha256':o.digest(path),'ciphertext_bytes':path.stat().st_size,'finished_at':'2026-10-16T00:00:30+00:00'} + return path,record + + +def test_conditional_upload_exact_readback_and_index_commit(source,tmp_path): + path,capture=source;server=Storage();target=tmp_path/'fetched.age' + receipt=o.publish(server,TOKEN,path,capture,target) + assert receipt['status']=='readback_verified' and target.read_bytes()==path.read_bytes() + assert receipt['verified_index_published'] + assert len(server.objects)==2 + assert not any(method=='DELETE' for method,_,_ in server.calls) + assert TOKEN not in json.dumps(receipt) + assert all(url.startswith(o.BASE) for _,url,_ in server.calls) + + +def test_quota_shortfall_refuses_before_any_provider_write(source,tmp_path): + path,capture=source;server=Storage();server.free=o.HEADROOM+path.stat().st_size-1 + with pytest.raises(o.Refused):o.publish(server,TOKEN,path,capture,tmp_path/'fetched.age') + assert not server.objects and not any(method=='PUT' for method,_,_ in server.calls) + + +def test_existing_archive_never_overwritten(source,tmp_path): + path,capture=source;server=Storage();server.objects[path.name]=b'existing-protected-object' + with pytest.raises(o.Refused):o.publish(server,TOKEN,path,capture,tmp_path/'fetched.age') + assert server.objects[path.name]==b'existing-protected-object' + + +def test_corrupted_readback_never_publishes_success_index(source,tmp_path): + path,capture=source;server=Storage();server.tamper=True + with pytest.raises(o.Refused):o.publish(server,TOKEN,path,capture,tmp_path/'fetched.age') + assert list(server.objects)==[path.name] + + +@pytest.mark.parametrize('key',['helper_cleanup_verified','temporary_plaintext_cleanup_verified']) +def test_unverified_capture_cleanup_prevents_provider_access(source,tmp_path,key): + path,capture=source;capture[key]=False;server=Storage() + with pytest.raises(o.Refused):o.publish(server,TOKEN,path,capture,tmp_path/'fetched.age') + assert not server.calls + + +def points(count=30): + now=datetime(2026,10,15,tzinfo=timezone.utc) + return [{'artifact':f'point-{i}','stamp':now-timedelta(days=i)} for i in range(count)] + + +def test_retention_preserves_recovered_point_newest_daily_and_weekly(): + rows=points();delete,keep=r.select(rows,'point-29') + assert 'point-29' in keep and 'point-0' in keep + assert all(f'point-{i}' in keep for i in range(7)) + assert len(keep)>=8 and not (keep&{p['artifact'] for p in delete}) + assert {p['artifact'] for p in delete}|keep=={p['artifact'] for p in rows} + + +def test_missing_independently_recovered_point_refuses_all_expiration(): + with pytest.raises(r.offsite.Refused):r.select(points(),'lost-point') + + +@pytest.mark.parametrize('action,number',[('publisher',48),('retention',49)]) +def test_proposed_ccr_cannot_create_provider_share_or_reader(monkeypatch,action,number): + path=setup.PLATFORM/f'credential-change-requests/CCR-2026-{number:04d}-net-kingdom-backup-{action}.yaml' + monkeypatch.setattr(setup.inventory,'caller_credentials',lambda:pytest.fail('unapproved request reached credential read')) + with pytest.raises(setup.Refused):setup.run(action,hashlib.sha256(path.read_bytes()).hexdigest(),{}) + + +def test_runtime_refuses_gate_without_actual_independent_restore(tmp_path): + path=tmp_path/'accepted.json';path.write_text(json.dumps({'schema':'netkingdom.identity-recovery.v1','status':'prepared'}));path.chmod(0o600) + with pytest.raises(runtime.offsite.Refused):runtime.accepted(path) + + +def test_isolation_preserves_dns_peers_and_tests_immutable_target(monkeypatch): + from copy import deepcopy + iso=load('identity-isolation');iso.capture.KUBE=['kubectl','--kubeconfig','/reviewed/kubeconfig'] + original={'podSelector':{},'policyTypes':['Egress'],'egress':[{'ports':[{'protocol':'UDP','port':53}]}]} + def get(ns,kind,name): + if name=='kube-system':return {'metadata':{'uid':iso.capture.prepare.CLUSTER_UID}} + return {'metadata':{'name':name,'uid':ns+'-uid','resourceVersion':'7'},'spec':deepcopy(original)} + def command(args,payload=None): + if 'get' in args:return json.dumps({'items':[{'metadata':{'labels':{'app':'service'}}}]}).encode() + patch=json.loads(args[args.index('-p')+1]);assert patch[2]['value']==original + wanted=deepcopy(original);wanted['podSelector']=iso.SELECTOR + return json.dumps({'metadata':{'uid':args[1]+'-uid'},'spec':wanted}).encode() + monkeypatch.setattr(iso.capture,'get',get);monkeypatch.setattr(iso.capture,'command',command) + packet=iso.plan();receipt={'attempted':[],'verified':[]};iso.apply(packet,receipt,True) + assert receipt['status']=='server_dry_run_passed' and len(receipt['verified'])==2 + packet['policies'][0]['uid']='foreign' + with pytest.raises(iso.capture.Refused):iso.apply(packet,{'attempted':[],'verified':[]}) + + +def test_bootstrap_dispatcher_handoff_help_and_missing_checkout(tmp_path): + import os,subprocess + bootstrap=ROOT.parent/'railiance-bootstrap' + if not bootstrap.is_dir():pytest.skip('bootstrap integration checkout absent') + help_run=subprocess.run(['bash',str(bootstrap/'bin/railiance'),'identity-recovery','--help'],capture_output=True) + assert help_run.returncode==0 and b'isolated archive verifier' not in help_run.stderr + assert b'--transfer-receipt' in help_run.stdout + env=dict(os.environ,NETKINGDOM_RECOVERY_ROOT=str(tmp_path/'missing')) + failure=subprocess.run(['bash',str(bootstrap/'bin/railiance'),'identity-recovery','--transfer-receipt','missing','--receipt','new'],env=env,capture_output=True) + assert failure.returncode==1 and b'Missing reviewed NetKingdom' in failure.stderr diff --git a/tools/bootstrap-recovery/capture-identity.py b/tools/bootstrap-recovery/capture-identity.py new file mode 100644 index 0000000..50f6153 --- /dev/null +++ b/tools/bootstrap-recovery/capture-identity.py @@ -0,0 +1,296 @@ +#!/usr/bin/env python3 +"""Owner-run identity capture: metadata plan first, exact bounded execution later. + +Use the admitted Railiance caller. Flags bind reviewed source/targets; they do +not grant authority. Private child output and provider diagnostics never print. +""" +import argparse +from datetime import datetime,timezone +import hashlib +import importlib.util +import json +import os +from pathlib import Path +import signal +import subprocess +import tempfile +import uuid + +HERE=Path(__file__).resolve().parent + + +def load(name): + spec=importlib.util.spec_from_file_location(name,HERE/(name+'.py')) + module=importlib.util.module_from_spec(spec);spec.loader.exec_module(module);return module + + +bundle=load('identity-bundle');prepare=load('prepare-identity-coverage') +IMAGE='ghcr.io/gpappsoft/privacyidea-docker@sha256:af7841adad262f129e0c1d4f553af13f21cb2f4dc713533f316cfe43ed0b4473' +SOURCES=['capture-identity.py','identity-files.py','identity-bundle.py','prepare-identity-coverage.py','identity-backup-profile.json'] +KUBE=['kubectl'] + + +class Refused(Exception):pass + + +def command(args,payload=None,timeout=60): + result=subprocess.run(KUBE+['--request-timeout=30s']+args,input=payload, + stdout=subprocess.PIPE,stderr=subprocess.DEVNULL,timeout=timeout) + if result.returncode:raise Refused() + return result.stdout + + +def get(ns,kind,name): + scope=['-n',ns] if ns else [] + return json.loads(command(scope+['get',kind,name,'-o','json'])) + + +def selects(selector,labels): + if any(labels.get(k)!=v for k,v in selector.get('matchLabels',{}).items()):return False + for expression in selector.get('matchExpressions',[]): + key=expression['key'];op=expression['operator'];values=expression.get('values',[]) + if op=='In' and labels.get(key) not in values:return False + if op=='NotIn' and key in labels and labels[key] in values:return False + if op=='Exists' and key not in labels:return False + if op=='DoesNotExist' and key in labels:return False + if op not in {'In','NotIn','Exists','DoesNotExist'}:raise Refused() + return True + + +def network_review(run_id,actual_labels=None): + rows=[] + for ns in ('sso','mfa'): + policies=json.loads(command(['-n',ns,'get','networkpolicies','-o','json']))['items'] + for obj in policies: + m=obj['metadata'];s=obj['spec'] + labels=(actual_labels or {}).get(ns,{'net-kingdom/capture':run_id}) + if selects(s['podSelector'],labels) and (s.get('ingress') or s.get('egress')): + rows.append({'namespace':ns,'name':m['name'],'uid':m['uid'], + 'spec_sha256':hashlib.sha256(json.dumps(s,sort_keys=True).encode()).hexdigest()}) + return rows + + +def validate_helper(actual,wanted): + a=actual['spec'];w=wanted['spec'] + if (any(a.get(k) for k in ('hostNetwork','hostPID','hostIPC','initContainers','ephemeralContainers')) + or a.get('automountServiceAccountToken') is not False or a.get('activeDeadlineSeconds')!=900 + or len(a['containers'])!=1 or a['volumes']!=w['volumes']):raise Refused() + c=a['containers'][0];expected=w['containers'][0] + if any(c.get(k)!=expected[k] for k in ('name','image','command','volumeMounts')):raise Refused() + if c.get('env') or c.get('envFrom') or c.get('securityContext',{}).get('privileged'):raise Refused() + if any(c.get('securityContext',{}).get(k)!=v for k,v in expected['securityContext'].items()):raise Refused() + if any(a.get('securityContext',{}).get(k)!=v for k,v in w['securityContext'].items()):raise Refused() + + +def manifests(run_id): + result=[] + for ns,claims in [('sso',[('lldap','lldap-data'),('authelia','authelia-data'),('keycape','keycape-authentication-policy')]), + ('mfa',[('privacyidea','privacyidea-data')])]: + label={'net-kingdom/capture':run_id};name='nk49-capture-'+run_id + result.append({'apiVersion':'networking.k8s.io/v1','kind':'NetworkPolicy', + 'metadata':{'name':name,'namespace':ns}, + 'spec':{'podSelector':{'matchLabels':label},'policyTypes':['Ingress','Egress'],'ingress':[],'egress':[]}}) + result.append({'apiVersion':'v1','kind':'Pod','metadata':{'name':name,'namespace':ns,'labels':label}, + 'spec':{'restartPolicy':'Never','activeDeadlineSeconds':900,'automountServiceAccountToken':False, + 'securityContext':{'runAsUser':0,'runAsGroup':0}, + 'containers':[{'name':'capture','image':IMAGE,'imagePullPolicy':'IfNotPresent', + 'command':['python3','-c','import time; time.sleep(900)'], + 'securityContext':{'allowPrivilegeEscalation':False,'readOnlyRootFilesystem':True, + 'capabilities':{'drop':['ALL'],'add':['DAC_OVERRIDE']}}, + 'resources':{'requests':{'cpu':'50m','memory':'128Mi'},'limits':{'cpu':'1','memory':'2Gi'}}, + 'volumeMounts':[{'name':key,'mountPath':'/source/'+key,'readOnly':True} for key,_ in claims]+ + [{'name':'scratch','mountPath':'/scratch'}]}], + 'volumes':[{'name':key,'persistentVolumeClaim':{'claimName':claim,'readOnly':True}} for key,claim in claims]+ + [{'name':'scratch','emptyDir':{'medium':'Memory','sizeLimit':'512Mi'}}]}}) + return result + + +def plan(): + p=bundle.profile();binding=prepare.prepare(p,get=get) + cluster=get('databases','cluster','net-kingdom-pg') + primary=cluster['status']['currentPrimary'];pod=get('databases','pod',primary) + if (not any(o.get('uid')==cluster['metadata']['uid'] for o in pod['metadata'].get('ownerReferences',[])) + or not any(c['type']=='Ready' and c['status']=='True' for c in pod['status'].get('conditions',[]))):raise Refused() + run_id=uuid.uuid4().hex[:12] + return {'schema':'netkingdom.identity-capture-plan.v1','status':'requires_owner_execution_admission', + 'created_at':datetime.now(timezone.utc).isoformat(),'run_id':run_id,'binding':binding,'kubeconfig':KUBE[2], + 'sources':[{'path':name,'sha256':hashlib.sha256((HERE/name).read_bytes()).hexdigest()} for name in SOURCES], + 'database':{'namespace':'databases','cluster':'net-kingdom-pg','cluster_uid':cluster['metadata']['uid'], + 'pod':primary,'pod_uid':pod['metadata']['uid'],'database':'privacyidea_db'}, + 'create_resources':manifests(run_id),'service_changes':False,'credential_grants':False, + 'matching_network_allow_policies':network_review(run_id), + 'network_policy_enforcement_requires_owner_confirmation':True, + 'offsite_delivery_included':False,'global_atomic_snapshot_claimed':False} + + +def checked_plan(path,expected): + raw=path.read_bytes() + if hashlib.sha256(raw).hexdigest()!=expected:raise Refused() + p=json.loads(raw) + if p.get('schema')!='netkingdom.identity-capture-plan.v1':raise Refused() + if p.get('kubeconfig')!=KUBE[2]:raise Refused() + if network_review(p['run_id']):raise Refused() + if p['sources']!=[{'path':name,'sha256':hashlib.sha256((HERE/name).read_bytes()).hexdigest()} for name in SOURCES]:raise Refused() + if p['create_resources']!=manifests(p['run_id']):raise Refused() + now=datetime.now(timezone.utc) + if not 0<=(now-datetime.fromisoformat(p['created_at'])).total_seconds()<3600:raise Refused() + # Recheck the exact immutable source/target bindings before private reads. + current=prepare.prepare(bundle.profile(),get=get) + if current['bindings']!=p['binding']['bindings'] or current['profile_sha256']!=p['binding']['profile_sha256']:raise Refused() + return p + + +def delete_created(obj): + m=obj['metadata'];ns=m['namespace'];name=m['name'];uid=m['uid'] + if obj['kind']=='Pod':path=f'/api/v1/namespaces/{ns}/pods/{name}' + elif obj['kind']=='NetworkPolicy':path=f'/apis/networking.k8s.io/v1/namespaces/{ns}/networkpolicies/{name}' + else:raise Refused() + payload=json.dumps({'apiVersion':'v1','kind':'DeleteOptions','preconditions':{'uid':uid},'gracePeriodSeconds':1}).encode() + command(['delete','--raw',path,'-f','-'],payload) + if obj['kind']=='Pod':command(['-n',ns,'wait','--for=delete','pod/'+name,'--timeout=90s'],timeout=100) + + +def bounded_file(args,path,limit): + # Stream binary dump to private staging, with a byte bound and suppressed + # diagnostics. API command argv contains only public resource/database names. + proc=subprocess.Popen(KUBE+['--request-timeout=600s']+args,stdout=subprocess.PIPE,stderr=subprocess.DEVNULL) + try: + count=0 + with path.open('xb') as out: + while block:=proc.stdout.read(1024*1024): + count+=len(block) + if count>limit:raise Refused() + out.write(block) + if proc.wait(timeout=600):raise Refused() + finally: + if proc.poll() is None:proc.kill();proc.wait() + + +def capture(p,output,receipt,decision,checkpoint=lambda:None): + created=[];receipt.update(status='capture_started',decision_ref=decision,plan_run_id=p['run_id'], + created_resources=[],unverified_create_outcomes=[],helper_cleanup_verified=False,temporary_plaintext_cleanup_verified=False) + checkpoint() + with tempfile.TemporaryDirectory(prefix='nk49-identity-',dir='/tmp') as temporary: + receipt['temporary_plaintext_directory']=temporary + checkpoint() + root=Path(temporary)/'payload';root.mkdir(mode=0o700) + held={} + try: + for c in bundle.profile()['components']: + for kind,names in [('deployment',[c['deployment']]),('secret',c['secret_refs']),('configmap',c['configmap_refs'])]: + for name in names: + obj=get(c['namespace'],kind,name) + # Keep reconstructable deployed resources inside encryption, + # without unrelated stale last-applied annotations. + m=obj['metadata'];obj['metadata']={k:m[k] for k in ['name','namespace','uid','resourceVersion']} + obj.pop('status',None) + raw=json.dumps(obj,sort_keys=True).encode();slot=f"resources/{c['namespace']}/{kind}/{name}.json" + held[(c['namespace'],kind,name)]=(m['uid'],m['resourceVersion']) + bundle.save(root,slot,raw) + for proposed in p['create_resources']: + # create, never adopt/apply an existing resource. Unknown create + # outcomes stop; only returned exact UIDs are eligible for cleanup. + pending={'kind':proposed['kind'],**proposed['metadata']} + receipt['unverified_create_outcomes'].append(pending) + checkpoint() + obj=json.loads(command(['create','-f','-','-o','json'],json.dumps(proposed).encode())) + if (obj['kind']!=proposed['kind'] or obj['metadata']['name']!=proposed['metadata']['name'] + or obj['metadata']['namespace']!=proposed['metadata']['namespace'] or not obj['metadata'].get('uid')):raise Refused() + created.append(obj) + receipt['unverified_create_outcomes'].remove(pending) + receipt['created_resources'].append({'namespace':obj['metadata']['namespace'], + 'kind':obj['kind'],'name':obj['metadata']['name'],'uid':obj['metadata']['uid']}) + checkpoint() + for ns in ('sso','mfa'): + name='nk49-capture-'+p['run_id'] + command(['-n',ns,'wait','--for=condition=Ready','pod/'+name,'--timeout=180s'],timeout=190) + live=get(ns,'pod',name) + expected=next(o for o in created if o['kind']=='Pod' and o['metadata']['namespace']==ns) + if live['metadata']['uid']!=expected['metadata']['uid']:raise Refused() + wanted=next(o for o in p['create_resources'] if o['kind']=='Pod' and o['metadata']['namespace']==ns) + validate_helper(live,wanted) + if network_review(p['run_id'],{ns:live['metadata']['labels']}):raise Refused() + raw=command(['-n',ns,'exec','-i',name,'-c','capture','--','python3','-c', + (HERE/'identity-files.py').read_text(),ns],timeout=600) + if len(raw)>bundle.LIMIT+2*1024*1024:raise Refused() + bundle.import_files(raw,root,ns) + if get(ns,'pod',name)['metadata']['uid']!=expected['metadata']['uid']:raise Refused() + db=p['database'];current=get('databases','cluster','net-kingdom-pg') + if current['metadata']['uid']!=db['cluster_uid'] or current['status']['currentPrimary']!=db['pod']:raise Refused() + if get('databases','pod',db['pod'])['metadata']['uid']!=db['pod_uid']:raise Refused() + (root/'databases').mkdir(mode=0o700) + bounded_file(['-n','databases','exec',db['pod'],'-c','postgres','--','pg_dump','-U','postgres', + '-d','privacyidea_db','-Fc','--no-owner','--no-acl'],root/'databases/privacyidea.dump',bundle.FILE_LIMIT) + if get('databases','pod',db['pod'])['metadata']['uid']!=db['pod_uid']:raise Refused() + for (ns,kind,name),binding in held.items(): + m=get(ns,kind,name)['metadata'] + if (m['uid'],m['resourceVersion'])!=binding:raise Refused() + # Re-read original keys after the database dump. No rotation is + # inferred from file presence; a changed key set refuses publication. + raw=command(['-n','mfa','exec','-i','nk49-capture-'+p['run_id'],'-c','capture','--','python3','-c', + (HERE/'identity-files.py').read_text(),'mfa'],timeout=60) + with tempfile.TemporaryDirectory(prefix='nk49-key-recheck-',dir='/tmp') as keytemp: + again=Path(keytemp);bundle.import_files(raw,again,'mfa') + for name in ('enckey','private.pem','public.pem'): + if (root/'data/privacyidea'/name).read_bytes()!=(again/'data/privacyidea'/name).read_bytes():raise Refused() + receipt.update(bundle.seal(root,output,{'started_at':receipt['started_at'], + 'finished_at':datetime.now(timezone.utc).isoformat(),'database':db,'decision_ref':decision, + 'cluster_uid':prepare.CLUSTER_UID,'global_atomic_snapshot_claimed':False, + 'omissions':['historical local backups','SQLite WAL/SHM/journal represented by native snapshots', + 'Authelia notification.txt','privacyIDEA service logs; audit data retained in database', + 'OpenBao root/unseal/age private key custody remains independently held']})) + finally: + failures=[] + for obj in reversed(created): + try:delete_created(obj) + except Exception:failures.append(obj['metadata']['uid']) + receipt['helper_cleanup_verified']=not failures and not receipt['unverified_create_outcomes'] + checkpoint() + if not receipt['helper_cleanup_verified']:raise Refused() + receipt['temporary_plaintext_cleanup_verified']=not Path(temporary).exists() + checkpoint() + if not receipt['temporary_plaintext_cleanup_verified']:raise Refused() + + +def main(): + global KUBE + parser=argparse.ArgumentParser(description=__doc__);parser.add_argument('action',choices=['plan','capture']) + parser.add_argument('--kubeconfig',required=True) + parser.add_argument('--receipt',required=True);parser.add_argument('--plan');parser.add_argument('--plan-sha256') + parser.add_argument('--output');parser.add_argument('--decision-id');a=parser.parse_args();os.umask(0o077) + KUBE=['kubectl','--kubeconfig',str(Path(a.kubeconfig).absolute())] + fd=os.open(a.receipt,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,0o600) + os.close(fd) + r={'status':'failed','started_at':datetime.now(timezone.utc).isoformat(),'secret_values_exported':False} + def checkpoint(): + fd,temp=tempfile.mkstemp(prefix='.capture-receipt-',dir=Path(a.receipt).absolute().parent) + try: + with os.fdopen(fd,'w') as out:json.dump(r,out,indent=2);out.write('\n');out.flush();os.fsync(out.fileno()) + os.replace(temp,a.receipt) + finally:Path(temp).unlink(missing_ok=True) + code=1 + try: + if a.action=='plan': + if any([a.plan,a.plan_sha256,a.output,a.decision_id]):raise Refused() + r=plan();code=0 + else: + if not all([a.plan,a.plan_sha256,a.output,a.decision_id]):raise Refused() + uuid.UUID(a.decision_id) + p=checked_plan(Path(a.plan),a.plan_sha256) + capture(p,Path(a.output),r,a.decision_id,checkpoint);code=0 + except Exception: + r.update(status='failed',error='capture_refused_or_failed_private_diagnostics_suppressed') + if r.get('temporary_plaintext_directory') and not Path(r['temporary_plaintext_directory']).exists():r['temporary_plaintext_cleanup_verified']=True + finally: + r['finished_at']=datetime.now(timezone.utc).isoformat() + checkpoint() + print('OK: identity '+a.action+' complete; see safe receipt.' if code==0 else 'ERROR: identity operation failed; inspect safe receipt before retry.') + return code + + +def interrupted(*args):raise Refused() + + +if __name__=='__main__': + for sig in (signal.SIGHUP,signal.SIGTERM,signal.SIGINT):signal.signal(sig,interrupted) + raise SystemExit(main()) diff --git a/tools/bootstrap-recovery/identity-backup-profile.json b/tools/bootstrap-recovery/identity-backup-profile.json index 93a550c..c93afa2 100644 --- a/tools/bootstrap-recovery/identity-backup-profile.json +++ b/tools/bootstrap-recovery/identity-backup-profile.json @@ -1,6 +1,6 @@ { "schema": "netkingdom.identity-backup-profile.v1", - "status": "proposed", + "status": "implemented_pending_owner_admission", "owner": "net-kingdom with railiance-platform/rapp-postgres runtime and storage admission", "public_recipient": "age1zvryunvjhvpkmasskauga2heeg0ztnte9ymgppvjge36ekumk50syr3tsz", "archive_profile": "netkingdom-identity-v1; new profile, not the bootstrap secrets-only tar profile", @@ -133,20 +133,21 @@ "historical_database": "existing verified October15 dump remains protected; new bundle must link its own exact database/key capture generation and recovery receipt" }, "proposed_destinations": { - "primary": "s3://railiance-platform-pg-backup/platform-pg/application-archives/net-kingdom/", - "secondary": "Storage Share Backup/railiance-backups/net-kingdom-identity/", + "initial": "Storage Share Backup/railiance-backups/net-kingdom-identity/", + "future_secondary": "s3://railiance-platform-pg-backup/platform-pg/application-archives/net-kingdom/; not implemented/admitted in this rollout", "admitted": false, - "publisher": "new exact prefix-scoped owner-delivered publisher; do not borrow another workload reader or bootstrap API key", - "retention": "initial single verified generation; later seven daily/two weekly only after measured quota/overlap/headroom and separately admitted retention executor; never expire existing good backups", - "schedule": "not installed; propose daily capture after logical database backup with per-generation linkage, separate failure alert and operator-approved RPO/RTO" + "publisher": "CCR-2026-0048: new folder-only read/create public DAV share, delivered by exact ESO reader; no operator password in runtime", + "retention": "CCR-2026-0049: separate folder-only read/delete share and process; newest plus seven daily/two weekly and independently recovered pinned generation; unknown/unindexed points never deleted", + "schedule": "source prepared: daily 03:45 UTC plus up to five minutes jitter, hourly 25-hour freshness check; installation leaves timers disabled until independent archive recovery", + "quota": "512 MiB ceiling, two GiB fresh provider headroom; initial 10 GiB account quota checked; unsupported public-share quota fails closed" }, "admission_gates": [ - "Railiance owner accepts exact read/capture executor, immutable helper images, PVC access and task-scoped caller attribution; no production stop/restart inferred", - "NetKingdom/service owners accept full asset manifest, original key preservation, consistency interval, bounded recovery environment and native acceptance", - "Platform/reef-storage admits both new archive prefixes, immutable create/readback, scoped publisher custody and current quota/headroom; current application archive profiles do not include net-kingdom", - "Resolve every referenced credential through encrypted recovery or an independent owner reissue path; no undocumented grant propagation", - "Implement and test capture/sealing/transfer adapters against this new profile before live execution; existing bootstrap tar checker is insufficient", - "Retrieve by immutable object/version from each admitted provider; digest check, attended independent-key decryption, native service recovery, failed-key case and verified cleanup", - "Only then enable recurring capture/alerts and measured retention; preserve T02 legacy provider-credential and deferred-rotation boundaries" + "Approve exact CCR-2026-0048/0049 grants and node runtime packet; three required owner reviews, confirmed native SA/namespace/audience boundaries before acceptance", + "Exclude helper capture label from existing broad DNS allow policies; validate read-only helper manifests and cluster/PVC/source bindings before private capture", + "Preserve every referenced credential encrypted with original keys; fresh native SQLite/PG snapshots and exact manifest, with documented omissions and no global atomicity claim", + "Create fresh ciphertext and exact Storage Share object, read back ciphertext and receipt by strong ETag/hash, independently retrieve using operator custody", + "Attended independent age-key proof: wrong-key denial, complete archive hashes, SQLite/RSA checks, disposable native PostgreSQL restore and verified cleanup before timer activation", + "Full NK49 closure separately requires native directory/issuer/MFA authentication, retained token decryption, audit signature continuity, current callers and legacy credential dispositions; archive acceptance does not close those gates", + "Host-local health cannot report a dead host; external outage monitoring and second-provider copy remain separate coverage gaps" ] } diff --git a/tools/bootstrap-recovery/identity-bundle.py b/tools/bootstrap-recovery/identity-bundle.py new file mode 100644 index 0000000..b09edc5 --- /dev/null +++ b/tools/bootstrap-recovery/identity-bundle.py @@ -0,0 +1,204 @@ +#!/usr/bin/env python3 +"""Strict data-bearing identity archive format, native checks and private sealing. + +No custody key is used to create a bundle. Its manifest and plaintext digests +stay inside encryption. Public receipts expose only ciphertext provenance. +""" +import base64 +from datetime import datetime,timezone +import hashlib +import importlib.util +import io +import json +import os +from pathlib import Path,PurePosixPath +import re +import sqlite3 +import stat +import subprocess +import tarfile +import tempfile + +ROOT=Path(__file__).resolve().parents[2] +PROFILE=ROOT/'tools/bootstrap-recovery/identity-backup-profile.json' +LIMIT=512*1024*1024 +FILE_LIMIT=256*1024*1024 +NAME=re.compile(r'[A-Za-z0-9_.-]{1,160}\Z') + + +class Refused(Exception):pass + + +def profile():return json.loads(PROFILE.read_text()) + + +def private(path,directory=False,max_bytes=FILE_LIMIT): + if any(p.is_symlink() for p in (path,*path.parents)):raise Refused() + s=path.lstat() + if s.st_uid!=os.getuid() or s.st_mode&0o077:raise Refused() + if directory: + if not stat.S_ISDIR(s.st_mode):raise Refused() + elif not stat.S_ISREG(s.st_mode) or s.st_nlink!=1 or s.st_size>max_bytes:raise Refused() + + +def allowed(name,profile_data=None): + p=PurePosixPath(name) + if p.is_absolute() or any(not NAME.fullmatch(x) or x in {'.','..'} for x in p.parts):return False + if len(p.parts)>=3 and p.parts[:2] in [('data','lldap'),('data','authelia'),('data','keycape'),('data','privacyidea')]:return True + if name=='databases/privacyidea.dump':return True + for c in (profile_data or profile())['components']: + prefix=f"resources/{c['namespace']}/" + if name==prefix+'deployment/'+c['deployment']+'.json':return True + if any(name==prefix+'secret/'+n+'.json' for n in c['secret_refs']):return True + if any(name==prefix+'configmap/'+n+'.json' for n in c['configmap_refs']):return True + return False + + +def save(root,name,value,profile_data=None): + if not allowed(name,profile_data) or len(value)>FILE_LIMIT or b'AGE-SECRET-KEY-' in value:raise Refused() + target=root/name;target.parent.mkdir(mode=0o700,parents=True,exist_ok=True) + fd=os.open(target,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,0o600) + with os.fdopen(fd,'wb') as out:out.write(value) + + +def import_files(raw,root,mode): + names=set();total=0 + with tarfile.open(fileobj=io.BytesIO(raw),mode='r:') as archive: + for member in archive: + expected=('data/privacyidea/',) if mode=='mfa' else ('data/lldap/','data/authelia/','data/keycape/') + if (not member.isfile() or not member.name.startswith(expected) or member.name in names + or member.size<0 or member.size>FILE_LIMIT or len(names)>=2048):raise Refused() + names.add(member.name);total+=member.size + if total>LIMIT:raise Refused() + data=archive.extractfile(member).read() + if len(data)!=member.size:raise Refused() + save(root,member.name,data) + + +def resource(root,ns,kind,name): + p=root/f'resources/{ns}/{kind}/{name}.json' + data=json.loads(p.read_bytes()) + if data['metadata']['name']!=name or data['metadata']['namespace']!=ns:raise Refused() + return data + + +def checks(root,profile_data=None): + # Checks describe stored formats/integrity, not current login validity. + for c in (profile_data or profile())['components']: + deployment=resource(root,c['namespace'],'deployment',c['deployment']) + if deployment.get('kind')!='Deployment' or deployment['metadata']['uid']!=c['deployment_uid']:raise Refused() + for name in c['secret_refs']: + data=resource(root,c['namespace'],'secret',name) + if data.get('kind')!='Secret' or not isinstance(data.get('data'),dict) or not data['data']:raise Refused() + for value in data['data'].values(): + if b'AGE-SECRET-KEY-' in base64.b64decode(value,validate=True):raise Refused() + for name in c['configmap_refs']: + if resource(root,c['namespace'],'configmap',name).get('kind')!='ConfigMap':raise Refused() + for name,file in [('lldap','users.db'),('authelia','db.sqlite3')]: + db=root/f'data/{name}/{file}' + with sqlite3.connect(db.as_uri()+'?mode=ro',uri=True) as con: + if con.execute('PRAGMA integrity_check').fetchall()!=[('ok',)]:raise Refused() + with (root/'databases/privacyidea.dump').open('rb') as source: + if source.read(5)!=b'PGDMP':raise Refused() + spec=importlib.util.spec_from_file_location('pi_keys',ROOT/'sso-mfa/k8s/privacyidea/key-custody-remote.py') + keys=importlib.util.module_from_spec(spec);spec.loader.exec_module(keys) + keys.validate({name:(root/'data/privacyidea'/name).read_bytes() for name in ('enckey','private.pem','public.pem')}) + from cryptography.hazmat.primitives import serialization + from cryptography.hazmat.primitives.asymmetric import rsa + config=resource(root,'sso','secret','keycape-config')['data'] + if not base64.b64decode(config['config.yaml'],validate=True):raise Refused() + signing=serialization.load_pem_private_key(base64.b64decode(config['key.pem'],validate=True),password=None) + if not isinstance(signing,rsa.RSAPrivateKey) or signing.key_size<2048:raise Refused() + # Policy must exist, even for an empty set of receipts; do not silently + # restore an issuer without its current authentication policy. + policy=json.loads((root/'data/keycape/policy.json').read_bytes()) + if not isinstance(policy,dict):raise Refused() + return {'sqlite_integrity_passed':True,'privacyidea_key_format_and_pair_passed':True, + 'keycape_signing_key_format_passed':True,'policy_json_present':True, + 'database_custom_format_present':True,'native_identity_recovery_proven':False} + + +def inventory(root): + private(root,True);entries=[];total=0 + for path in sorted(root.rglob('*')): + if path.is_dir():private(path,True);continue + private(path);name=path.relative_to(root).as_posix() + if not allowed(name):raise Refused() + with path.open('rb') as source:value=source.read(FILE_LIMIT+1) + if len(value)>FILE_LIMIT or b'AGE-SECRET-KEY-' in value:raise Refused() + total+=len(value) + if total>LIMIT or len(entries)>=4096:raise Refused() + entries.append({'path':name,'bytes':len(value),'sha256':hashlib.sha256(value).hexdigest()}) + if not entries:raise Refused() + return entries + + +def seal(root,output,capture): + output=output.absolute();private(output.parent,True) + if output.exists() or output.is_symlink():raise Refused() + checked=checks(root);entries=inventory(root) + manifest={'schema':'netkingdom.identity-archive.v1','profile_sha256':hashlib.sha256(PROFILE.read_bytes()).hexdigest(), + 'profile':profile(), + 'captured_at':datetime.now(timezone.utc).isoformat(),'capture':capture,'files':entries,'checks':checked} + with tempfile.TemporaryDirectory(prefix='.identity-seal-',dir=output.parent) as temporary: + ciphertext=Path(temporary)/'ciphertext' + fd=os.open(ciphertext,os.O_WRONLY|os.O_CREAT|os.O_EXCL,0o600) + proc=subprocess.Popen(['age','-r',profile()['public_recipient']],stdin=subprocess.PIPE,stdout=fd,stderr=subprocess.DEVNULL) + os.close(fd) + try: + with tarfile.open(fileobj=proc.stdin,mode='w|') as archive: + raw=json.dumps(manifest,sort_keys=True).encode();m=tarfile.TarInfo('manifest.json');m.size=len(raw);m.mode=0o600 + archive.addfile(m,io.BytesIO(raw)) + for entry in entries: + with (root/entry['path']).open('rb') as source: + value=source.read(FILE_LIMIT+1) + if hashlib.sha256(value).hexdigest()!=entry['sha256']:raise Refused() + m=tarfile.TarInfo(entry['path']);m.size=len(value);m.mode=0o600 + archive.addfile(m,io.BytesIO(value)) + proc.stdin.close() + if proc.wait(timeout=120) or not 04096:raise Refused() + seen=set();total=0 + # Iteration starts with the already-read first member; explicitly skip + # only that member. A later duplicate manifest is never accepted. + for index,member in enumerate(archive): + if index==0 and member is first:continue + name=member.name + if (not member.isfile() or name in seen or name not in expected or not allowed(name,captured) + or not 0<=member.size<=FILE_LIMIT or member.size!=expected[name]['bytes']):raise Refused() + seen.add(name);total+=member.size + if total>LIMIT:raise Refused() + raw=archive.extractfile(member).read() + if len(raw)!=member.size or hashlib.sha256(raw).hexdigest()!=expected[name]['sha256']:raise Refused() + save(root,name,raw,captured) + if seen!=set(expected):raise Refused() + return manifest,checks(root,captured) diff --git a/tools/bootstrap-recovery/identity-credential-setup.py b/tools/bootstrap-recovery/identity-credential-setup.py new file mode 100644 index 0000000..4753a7e --- /dev/null +++ b/tools/bootstrap-recovery/identity-credential-setup.py @@ -0,0 +1,234 @@ +#!/usr/bin/env python3 +"""Attended Platform/Backup owner ceremony for two exact new backup shares. + +Requires approved, hash-bound CCRs. No operator password/age key is delivered to +the runtime; only a folder-scoped share token is stored with initial KV CAS=0. +""" +import argparse +import base64 +import hashlib +import importlib.util +import json +import os +from pathlib import Path +import subprocess +import sys +import urllib.error +import urllib.parse +import urllib.request +import uuid + +HERE=Path(__file__).resolve().parent +PLATFORM=HERE.parents[1].parent/'railiance-platform' +sys.path.insert(0,str(PLATFORM/'scripts')) +from state_hub_preflight_lane import bao,data,capabilities,revoke +from repair_eso_kubernetes_auth import role_payload,check_role +from telemetry_smtp_delivery import ensure + + +def load(name,path): + spec=importlib.util.spec_from_file_location(name,path);m=importlib.util.module_from_spec(spec);spec.loader.exec_module(m);return m + + +inventory=load('storage_inventory',HERE/'inventory-storage-share.py') +offsite=load('offsite',HERE/'identity-offsite.py') +FOLDER='/railiance-backups/net-kingdom-identity' +PROFILES={'publisher':(48,5),'retention':(49,9)} + + +class Refused(Exception):pass + + +def contract(action,expected): + number,permission=PROFILES[action] + path=PLATFORM/f'credential-change-requests/CCR-2026-{number:04d}-net-kingdom-backup-{action}.yaml' + raw=path.read_bytes() + if hashlib.sha256(raw).hexdigest()!=expected:raise Refused() + module=load('credential_change',PLATFORM/'scripts/credential-change.py') + ccr,errors,_=module.validate_ccr(path) + if errors or ccr['status'] not in {'approved','applied','verified','active'}:raise Refused() + required={'platform-owner','net-kingdom-owner','backup-share-owner'} + approved={r['reviewer'].removeprefix('User (').removesuffix(')') for r in ccr.get('review',{}).get('comments',[]) + if r.get('decision')=='approved' and r.get('source_ref')} + name='net-kingdom-backup-'+action+'-eso' + expected_path='platform/workloads/net-kingdom/identity-backup/'+action + a=ccr['openbao'];provider=ccr['provider'] + if (not required<=approved or a['kv_path']!=expected_path or a['fields']!=['NC_WEBDAV_TOKEN'] + or a['policy_name']!=name or a['auth']['role']!=name + or a['auth']['bound_claims']!={'service_account_names':[name],'service_account_namespaces':['sso']} + or provider['folder']!=FOLDER or provider['share_permissions']!=permission or provider['share_type']!=3 + or provider['label']!='net-kingdom-identity-'+action):raise Refused() + if (a.get('metadata_read') is not False or a.get('token_self_lifecycle') is not True + or a['auth'].get('audience')!='openbao' or a['auth'].get('token_no_default_policy') is not True + or any(a['auth'].get(k)!='15m' for k in ['ttl','token_max_ttl','token_explicit_max_ttl']) + or a['auth'].get('policies')!=[name] or provider.get('host')!='nx4069.your-storageshare.de'):raise Refused() + lane={'service_account':name,'namespace':'sso','role':name,'policy':name, + 'kv_path':expected_path.replace('platform/','platform/data/',1)} + wanted=('path "'+lane['kv_path']+'" { capabilities = ["read"] }\n' + 'path "auth/token/lookup-self" { capabilities = ["read"] }\n' + 'path "auth/token/revoke-self" { capabilities = ["update"] }\n') + if (PLATFORM/a['policy_file']).read_text()!=wanted:raise Refused() + return ccr,lane + + +def request(client,url,auth,method='GET',body=None,extra=None): + if not url.startswith(offsite.HOST+'/'):raise Refused() + h={'Authorization':'Basic '+base64.b64encode((auth[0]+':'+auth[1]).encode()).decode(), + 'X-Requested-With':'XMLHttpRequest'};h.update(extra or {}) + try: + with client.open(urllib.request.Request(url,data=body,method=method,headers=h),timeout=60) as response: + if response.url!=url:raise Refused() + raw=response.read(1024*1024+1) + if len(raw)>1024*1024:raise Refused() + return response.status,raw + except urllib.error.HTTPError as error:return error.code,b'' + + +def ocs(client,auth,suffix='',fields=None): + url=offsite.HOST+'/ocs/v2.php/apps/files_sharing/api/v1/shares'+suffix + body=urllib.parse.urlencode(fields).encode() if fields is not None else None + status,raw=request(client,url,auth,'POST' if fields is not None else 'GET',body, + {'OCS-APIRequest':'true','Accept':'application/json','Content-Type':'application/x-www-form-urlencoded'}) + if status not in (200,201):raise Refused() + parsed=json.loads(raw)['ocs'] + if parsed['meta']['statuscode'] not in (100,200):raise Refused() + return parsed['data'] + + +def share(client,auth,action,receipt): + owner=offsite.HOST+'/remote.php/dav/files/Backup'+FOLDER + status,_=request(client,owner,auth,'MKCOL') + if status not in (201,405):raise Refused() + receipt['folder_created']=status==201 + matches=[s for s in ocs(client,auth,'?path='+urllib.parse.quote(FOLDER,safe='')) + if s.get('label')=='net-kingdom-identity-'+action] + if len(matches)>1:raise Refused() + if matches:result=matches[0];receipt['share_created']=False + else: + result=ocs(client,auth,fields={'path':FOLDER,'shareType':3,'permissions':PROFILES[action][1], + 'label':'net-kingdom-identity-'+action});receipt['share_created']=True + if (result.get('path')!=FOLDER or int(result.get('permissions',-1))!=PROFILES[action][1] + or int(result.get('share_type',-1))!=3 or not result.get('token')):raise Refused() + offsite.headers(result['token']) + receipt.update(share_id=str(result['id']),share_permissions=int(result['permissions']),folder=FOLDER) + return result['token'] + + +def boundary(client,auth,token,action,receipt): + name='nk49-boundary-'+uuid.uuid4().hex+'.age' + payload=subprocess.run(['age','-r',json.loads((HERE/'identity-backup-profile.json').read_text())['public_recipient']], + input=b'NK49 disposable provider boundary fixture',stdout=subprocess.PIPE,stderr=subprocess.DEVNULL,timeout=10) + if payload.returncode:raise Refused() + owner=offsite.HOST+'/remote.php/dav/files/Backup'+FOLDER+'/'+name + public=offsite.BASE+name;created=False;deleted=False + receipt['provider_test_artifacts']=[name] + receipt['boundary_fixture_cleanup_verified']=False + try: + status,_=request(client,owner,auth,'PUT',payload.stdout,{'If-None-Match':'*'}) + if status!=201:raise Refused() + created=True + status,raw=request(client,public,(token,'')) + if status!=200 or raw!=payload.stdout:raise Refused() + status,_=request(client,public,(token,''),'PUT',payload.stdout) + if status not in (401,403):raise Refused() + denied_name='nk49-absent-'+uuid.uuid4().hex+'.age' + if action=='retention':receipt['provider_test_artifacts'].append(denied_name) + status,_=request(client,offsite.BASE+denied_name,(token,''),'PUT',payload.stdout,{'If-None-Match':'*'}) if action=='retention' else (403,b'') + if status not in (401,403):raise Refused() + status,_=request(client,public,(token,''),'DELETE') + if action=='publisher': + if status not in (401,403):raise Refused() + new=offsite.BASE+'nk49-boundary-'+uuid.uuid4().hex+'.age' + receipt['provider_test_artifacts'].append(new.rsplit('/',1)[-1]) + code,_=request(client,new,(token,''),'PUT',payload.stdout,{'If-None-Match':'*'}) + if code!=201:raise Refused() + # This fixture belongs to the current ceremony, not retained data. + remove_fixture(client,offsite.HOST+'/remote.php/dav/files/Backup'+FOLDER+'/'+new.rsplit('/',1)[-1],auth,payload.stdout) + else: + if status!=204:raise Refused() + deleted=True + status,_=request(client,offsite.BASE+'%2e%2e/r01-net-kingdom-pg/',(token,'')) + if status not in (401,403,404):raise Refused() + receipt['provider_boundary_verified']=True + finally: + if created and not deleted: + remove_fixture(client,owner,auth,payload.stdout) + # A rejected operation may have had an unexpected side effect. Only a + # successful complete boundary test claims cleanup; names remain in the + # receipt for owner reconciliation on every failure. + receipt['boundary_fixture_cleanup_verified']=receipt.get('provider_boundary_verified',False) + + +def remove_fixture(client,url,auth,expected): + h={'Authorization':'Basic '+base64.b64encode((auth[0]+':'+auth[1]).encode()).decode()} + with client.open(urllib.request.Request(url,method='HEAD',headers=h),timeout=30) as response: + version=offsite.etag(response,url,len(expected)) + code,raw=request(client,url,auth,extra={'If-Match':version}) + if code!=200 or raw!=expected:raise Refused() + code,_=request(client,url,auth,'DELETE',extra={'If-Match':version}) + if code!=204:raise Refused() + + +def run(action,expected,receipt): + ccr,lane=contract(action,expected) + identity,credentials=inventory.caller_credentials() + receipt['caller_entity_id']=identity.get('entity_id');receipt['caller_auth_path']=identity['path'] + client=offsite.opener();auth=(credentials['BACKUP_USERNAME'],credentials['BACKUP_PASSWORD']) + # Use the owner quota accessor only with proxies removed from its environment. + quota=load('owner_quota',PLATFORM/'scripts/migrate_nextcloud_backup_account.py') + quota.request=lambda url,method='GET',body=None,auth=None,headers=None:request(client,url,auth,method,body,headers) + receipt['quota_before']=quota.quota(auth) + if receipt['quota_before']['available_bytes']MAX_FILE:raise Refused() + with os.fdopen(os.dup(fd),'rb') as stream:data=stream.read(MAX_FILE+1) + after=os.stat(path,follow_symlinks=False) + if (len(data)>MAX_FILE or b'AGE-SECRET-KEY-' in data or + any(getattr(before,k)!=getattr(after,k) for k in ('st_dev','st_ino','st_size','st_mtime_ns','st_ctime_ns'))):raise Refused() + return data + finally:os.close(fd) + + +def tree(root,database=None,omit_notification=False,omit_backups=True): + result={} + for directory,dirs,files in os.walk(root,followlinks=False): + path=Path(directory) + if path.is_symlink():raise Refused() + # Historical local snapshots are redundant; do not recursively include + # accumulated backups. WAL/SHM are represented by the native snapshot. + if omit_backups:dirs[:]=[name for name in dirs if name!='backups'] + for name in dirs: + if (path/name).is_symlink():raise Refused() + for name in files: + relative=(path/name).relative_to(root).as_posix() + if database and relative in {database,database+'-wal',database+'-shm',database+'-journal'}:continue + if omit_notification and relative=='notification.txt':continue + result[relative]=read(path/name) + if len(result)>MAX_COUNT or sum(map(len,result.values()))>MAX_TOTAL:raise Refused() + return result + + +def snapshot(source,target): + if source.is_symlink() or not source.is_file():raise Refused() + # A read-only URI intentionally fails if the engine needs a write to its + # production WAL/SHM. Never fall back to a raw copy or mutate the mount. + with sqlite3.connect(source.as_uri()+'?mode=ro',uri=True,timeout=10) as origin: + with sqlite3.connect(target) as dest: + origin.backup(dest,pages=128,sleep=0.05) + if dest.execute('PRAGMA integrity_check').fetchall()!=[('ok',)]:raise Refused() + return read(target) + + +def capture(mode,root=Path('/source'),scratch=Path('/scratch')): + values={} + with tempfile.TemporaryDirectory(prefix='snapshot-',dir=scratch) as temporary: + if mode=='sso': + for name,database in [('lldap','users.db'),('authelia','db.sqlite3'),('keycape',None)]: + source=root/name + if source.is_symlink() or not source.is_dir():raise Refused() + first=tree(source,database,name=='authelia',name!='keycape') + if database:first[database]=snapshot(source/database,Path(temporary)/(name+'.sqlite')) + second=tree(source,database,name=='authelia',name!='keycape') + if {k:v for k,v in first.items() if k!=database}!=second:raise Refused() + values.update({f'data/{name}/{k}':v for k,v in first.items()}) + elif mode=='mfa': + first=tree(root/'privacyidea') + if not {'enckey','private.pem','public.pem'}<=set(first) or first!=tree(root/'privacyidea'):raise Refused() + values={f'data/privacyidea/{name}':value for name,value in first.items()} + else:raise Refused() + if len(values)>MAX_COUNT or sum(map(len,values.values()))>MAX_TOTAL:raise Refused() + return values + + +def emit(values,destination): + with tarfile.open(fileobj=destination,mode='w|') as archive: + for name,value in sorted(values.items()): + member=tarfile.TarInfo(name);member.size=len(value);member.mode=0o600 + archive.addfile(member,io.BytesIO(value)) + + +def interrupted(*args):raise Refused() + + +if __name__=='__main__': + for sig in (signal.SIGTERM,signal.SIGHUP,signal.SIGINT):signal.signal(sig,interrupted) + try: + os.umask(0o077) + if len(sys.argv)!=2:raise Refused() + emit(capture(sys.argv[1]),sys.stdout.buffer) + except Exception: + print('ERROR: private file capture refused; diagnostics suppressed.',file=sys.stderr) + raise SystemExit(1) diff --git a/tools/bootstrap-recovery/identity-isolation.py b/tools/bootstrap-recovery/identity-isolation.py new file mode 100644 index 0000000..c19a272 --- /dev/null +++ b/tools/bootstrap-recovery/identity-isolation.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +"""Plan two exact DNS-selector exclusions; apply only an owner-admitted hash-bound plan.""" +import argparse +import hashlib +import importlib.util +import json +import os +from pathlib import Path + +HERE=Path(__file__).resolve().parent +spec=importlib.util.spec_from_file_location('capture',HERE/'capture-identity.py') +capture=importlib.util.module_from_spec(spec);spec.loader.exec_module(capture) +SELECTOR={'matchExpressions':[{'key':'net-kingdom/capture','operator':'DoesNotExist'}]} + + +def plan(): + if capture.get(None,'namespace','kube-system')['metadata']['uid']!=capture.prepare.CLUSTER_UID:raise capture.Refused() + rows=[] + for ns in ('sso','mfa'): + obj=capture.get(ns,'networkpolicy','allow-egress-dns');s=obj['spec'];m=obj['metadata'] + if s['podSelector'] not in ({},SELECTOR):raise capture.Refused() + pods=json.loads(capture.command(['-n',ns,'get','pods','-o','json']))['items'] + if any('net-kingdom/capture' in p['metadata'].get('labels',{}) for p in pods):raise capture.Refused() + # Preserve every peer, port and policy type. Existing unlabelled service + # pods retain their exact selector match. UID/RV/spec tests guard drift. + patch=[{'op':'test','path':'/metadata/uid','value':m['uid']}, + {'op':'test','path':'/metadata/resourceVersion','value':m['resourceVersion']}, + {'op':'test','path':'/spec','value':s}, + {'op':'replace','path':'/spec/podSelector','value':SELECTOR}] + rows.append({'namespace':ns,'name':m['name'],'uid':m['uid'],'patch':patch}) + return {'schema':'netkingdom.identity-isolation.v1','status':'prepared_not_applied', + 'kubeconfig':capture.KUBE[2],'source_sha256':hashlib.sha256(Path(__file__).read_bytes()).hexdigest(),'policies':rows} + + +def apply(packet,receipt,dry_run=False): + if packet!=plan():raise capture.Refused() + for row in packet['policies']: + args=['-n',row['namespace'],'patch','networkpolicy',row['name'],'--type=json','-p',json.dumps(row['patch']),'-o','json'] + if dry_run:args+=['--dry-run=server'] + receipt['attempted'].append({'namespace':row['namespace'],'name':row['name'],'uid':row['uid']}) + actual=json.loads(capture.command(args));expected=row['patch'][2]['value'].copy();expected['podSelector']=SELECTOR + if actual['metadata']['uid']!=row['uid'] or actual['spec']!=expected:raise capture.Refused() + receipt['verified'].append({'namespace':row['namespace'],'name':row['name'],'uid':row['uid']}) + receipt['status']='server_dry_run_passed' if dry_run else 'capture_dns_exclusions_verified' + + +def main(): + p=argparse.ArgumentParser(description=__doc__);p.add_argument('action',choices=['plan','dry-run','apply']) + p.add_argument('--kubeconfig',required=True);p.add_argument('--plan');p.add_argument('--plan-sha256');p.add_argument('--receipt',required=True) + a=p.parse_args();os.umask(0o077);capture.KUBE=['kubectl','--kubeconfig',a.kubeconfig] + fd=os.open(a.receipt,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,0o600) + r={'status':'failed','attempted':[],'verified':[]} + try: + if a.action=='plan':r=plan() + else: + raw=Path(a.plan).read_bytes() + if hashlib.sha256(raw).hexdigest()!=a.plan_sha256:raise capture.Refused() + apply(json.loads(raw),r,a.action=='dry-run') + except Exception:r.update(status='failed',error='isolation_refused_review_partial_attempts') + finally: + with os.fdopen(fd,'w') as out:json.dump(r,out,indent=2);out.write('\n') + return int(r['status']=='failed') + + +if __name__=='__main__':raise SystemExit(main()) diff --git a/tools/bootstrap-recovery/identity-offsite.py b/tools/bootstrap-recovery/identity-offsite.py new file mode 100644 index 0000000..33707e5 --- /dev/null +++ b/tools/bootstrap-recovery/identity-offsite.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +"""Folder-scoped Storage Share ciphertext publication and verified readback. + +Runtime gets only a dedicated read/create share token. Account administration, +deletion and age private keys have no route in this publisher. +""" +import base64 +from datetime import datetime,timezone +import hashlib +import json +import os +from pathlib import Path +import re +import tempfile +import urllib.error +import urllib.request +import xml.etree.ElementTree as ET + +HOST='https://nx4069.your-storageshare.de' +BASE=HOST+'/public.php/webdav/' +NAME=re.compile(r'net-kingdom-identity-\d{8}T\d{6}Z-[a-f0-9]{12}\.tar\.age\Z') +LIMIT=512*1024*1024 +HEADROOM=2*1024**3 + + +class Refused(Exception):pass + + +class NoRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self,*args,**kwargs):raise Refused() + + +def opener():return urllib.request.build_opener(urllib.request.ProxyHandler({}),NoRedirect()) + + +def headers(token): + if not isinstance(token,str) or not re.fullmatch('[A-Za-z0-9]{10,128}',token):raise Refused() + return {'Authorization':'Basic '+base64.b64encode((token+':').encode()).decode(), + 'X-Requested-With':'XMLHttpRequest','Accept-Encoding':'identity'} + + +def digest(path): + with path.open('rb') as source:return hashlib.file_digest(source,'sha256').hexdigest() + + +def quota(client,token): + body=b'' + h=headers(token);h.update(Depth='0');h['Content-Type']='application/xml' + with client.open(urllib.request.Request(BASE,data=body,method='PROPFIND',headers=h),timeout=30) as response: + if response.status!=207 or response.url!=BASE:raise Refused() + raw=response.read(1024*1024+1) + if len(raw)>1024*1024:raise Refused() + amount=ET.fromstring(raw).findtext('.//{DAV:}quota-available-bytes') or '' + if not amount.isdigit():raise Refused() + return int(amount) + + +def etag(response,url,size): + value=response.headers.get('ETag','') + if (response.status!=200 or response.url!=url or response.headers.get('Content-Length')!=str(size) + or not re.fullmatch(r'"[^"\x00-\x20]{1,180}"',value)):raise Refused() + return value + + +def readback(client,token,name,size,expected,target): + valid=NAME.fullmatch(name) or (name.endswith('.receipt.json') and NAME.fullmatch(name.removesuffix('.receipt.json')) and size<=65536) + if not valid or not 0size:raise Refused() + sha.update(block);out.write(block) + out.flush();os.fsync(out.fileno()) + if count!=size or sha.hexdigest()!=expected:raise Refused() + with client.open(urllib.request.Request(url,method='HEAD',headers=h),timeout=30) as response: + if etag(response,url,size)!=version:raise Refused() + return version + + +def publish(client,token,source,capture,target): + if (not NAME.fullmatch(source.name) or source.is_symlink() or source.stat().st_mode&0o077 + or source.stat().st_uid!=os.getuid() or source.stat().st_nlink!=1 or not source.is_file() + or source.stat().st_size>LIMIT or capture.get('status')!='encrypted_bundle_created' + or capture.get('helper_cleanup_verified') is not True or capture.get('temporary_plaintext_cleanup_verified') is not True + or digest(source)!=capture.get('ciphertext_sha256') or source.stat().st_size!=capture.get('ciphertext_bytes')):raise Refused() + with source.open('rb') as stream: + if stream.read(22)!=b'age-encryption.org/v1\n':raise Refused() + available=quota(client,token) + if available' + with client.open(urllib.request.Request(offsite.BASE,data=body,method='PROPFIND',headers=h),timeout=30) as response: + if response.status!=207 or response.url!=offsite.BASE:raise offsite.Refused() + raw=response.read(4*1024*1024+1) + if len(raw)>4*1024*1024:raise offsite.Refused() + result={} + for response in ET.fromstring(raw).findall('{DAV:}response'): + href=urllib.parse.urlsplit(response.findtext('{DAV:}href') or '') + if href.netloc and href.netloc!=urllib.parse.urlsplit(offsite.HOST).netloc:raise offsite.Refused() + name=urllib.parse.unquote(href.path.rstrip('/').rsplit('/',1)[-1]) + for row in response.findall('{DAV:}propstat'): + if ' 200 ' not in (row.findtext('{DAV:}status') or ''):continue + p=row.find('{DAV:}prop') + if p is None or p.find('{DAV:}resourcetype/{DAV:}collection') is not None:continue + is_index=name.endswith('.receipt.json') and offsite.NAME.fullmatch(name.removesuffix('.receipt.json')) + if not offsite.NAME.fullmatch(name) and not is_index:continue + size=p.findtext('{DAV:}getcontentlength') or '';version=p.findtext('{DAV:}getetag') or '' + if not size.isdigit() or not re.fullmatch(r'"[^"\x00-\x20]{1,180}"',version) or name in result:raise offsite.Refused() + result[name]={'bytes':int(size),'etag':version} + if len(result)>10000:raise offsite.Refused() + return result + + +def generations(client,token,objects): + result=[] + for name,meta in objects.items(): + if not offsite.NAME.fullmatch(name) or name+'.receipt.json' not in objects:continue + index=objects[name+'.receipt.json'] + if not 03600:raise offsite.Refused() + if stamp>datetime.now(timezone.utc):raise offsite.Refused() + result.append({**r,'stamp':stamp,'etag':meta['etag'],'index_etag':index['etag']}) + return sorted(result,key=lambda r:r['stamp'],reverse=True) + + +def select(rows,protected): + if not rows or protected not in {r['artifact'] for r in rows}:raise offsite.Refused() + keep={protected,rows[0]['artifact']};days=set();weeks=set() + for row in rows: + day=row['stamp'].date();week=row['stamp'].isocalendar()[:2] + if day not in days and len(days)<7:keep.add(row['artifact']);days.add(day) + if week not in weeks and len(weeks)<2:keep.add(row['artifact']);weeks.add(week) + return [r for r in rows if r['artifact'] not in keep],keep + + +def run(client,token,accepted,receipt,execute=False,checkpoint=lambda:None): + if (accepted.get('status')!='identity_archive_verified' or accepted.get('container_cleanup_verified') is not True + or accepted.get('temporary_plaintext_cleanup_verified') is not True or not offsite.NAME.fullmatch(accepted.get('artifact',''))):raise offsite.Refused() + objects=listing(client,token);rows=generations(client,token,objects) + candidates,keep=select(rows,accepted['artifact']) + recovered=next(r for r in rows if r['artifact']==accepted['artifact']) + if recovered['ciphertext_sha256']!=accepted['ciphertext_sha256']:raise offsite.Refused() + receipt.update(status='retention_preview',protected=accepted['artifact'],keep=sorted(keep), + candidates=[r['artifact'] for r in candidates],deleted=[],unknown_objects_untouched=True) + checkpoint() + # Validate retained known-good and newest ciphertext before any deletion. + with tempfile.TemporaryDirectory(prefix='nk49-retained-check-',dir='/tmp') as temporary: + for index,row in enumerate({r['artifact']:r for r in [recovered,rows[0]]}.values()): + offsite.readback(client,token,row['artifact'],row['ciphertext_bytes'],row['ciphertext_sha256'],Path(temporary)/str(index)) + if not execute:return + for index,row in enumerate(candidates): + version=offsite.readback(client,token,row['artifact'],row['ciphertext_bytes'],row['ciphertext_sha256'],Path(temporary)/('expire-'+str(index))) + for name,etag in [(row['artifact'],version),(row['artifact']+'.receipt.json',row['index_etag'])]: + h=offsite.headers(token);h['If-Match']=etag + receipt['deletion_in_progress']=name + checkpoint() + with client.open(urllib.request.Request(offsite.BASE+name,method='DELETE',headers=h),timeout=30) as response: + if response.status!=204 or response.url!=offsite.BASE+name:raise offsite.Refused() + receipt['deleted'].append(name) + checkpoint() + receipt.pop('deletion_in_progress',None) + receipt['status']='retention_applied' diff --git a/tools/bootstrap-recovery/identity-runtime.py b/tools/bootstrap-recovery/identity-runtime.py new file mode 100644 index 0000000..2a2bfb3 --- /dev/null +++ b/tools/bootstrap-recovery/identity-runtime.py @@ -0,0 +1,208 @@ +#!/usr/bin/env python3 +"""Root-owned Railiance backup job. No operator password or age private key. + +Publisher and retention are separate processes with separate scoped delivery. +Scheduled execution is gated by an independently recovered archive receipt. +""" +import argparse +import base64 +from datetime import datetime,timezone +import fcntl +import importlib.util +import json +import os +from pathlib import Path +import signal +import shutil +import socket +import tempfile +import urllib.request + +HERE=Path(__file__).resolve().parent + + +def load(name): + spec=importlib.util.spec_from_file_location(name,HERE/(name+'.py')) + m=importlib.util.module_from_spec(spec);spec.loader.exec_module(m);return m + + +capture=load('capture-identity');offsite=load('identity-offsite');retention=load('identity-retention') + + +def write(path,value): + fd,name=tempfile.mkstemp(prefix='.identity-receipt-',dir=path.parent) + try: + with os.fdopen(fd,'w') as out:json.dump(value,out,indent=2);out.write('\n');out.flush();os.fsync(out.fileno()) + os.replace(name,path) + finally:Path(name).unlink(missing_ok=True) + + +def token(action): + name='net-kingdom-backup-'+action + es=capture.get('sso','externalsecret',name) + if (es['spec']['target']['name']!=name or es['spec']['data']!=[{'secretKey':'token','remoteRef':{ + 'key':'workloads/net-kingdom/identity-backup/'+action,'property':'NC_WEBDAV_TOKEN'}}] + or not any(c['type']=='Ready' and c['status']=='True' for c in es['status'].get('conditions',[]))):raise offsite.Refused() + secret=capture.get('sso','secret',name) + if (set(secret['data'])!={'token'} or not any(o['uid']==es['metadata']['uid'] for o in secret['metadata'].get('ownerReferences',[]))):raise offsite.Refused() + value=base64.b64decode(secret['data']['token'],validate=True).decode();offsite.headers(value) + return value + + +def accepted(path): + capture.bundle.private(path) + r=json.loads(path.read_text()) + if (r.get('schema')!='netkingdom.identity-recovery.v1' or r.get('status')!='identity_archive_verified' + or r.get('all_manifest_hashes_verified') is not True or r.get('container_cleanup_verified') is not True + or r.get('temporary_plaintext_cleanup_verified') is not True or not offsite.NAME.fullmatch(r.get('artifact',''))):raise offsite.Refused() + return r + + +def post_event(state,summary): + # Non-secret durable reporting: replay retained notes at the next run. + directory=state/'event-outbox';directory.mkdir(mode=0o700,exist_ok=True) + import uuid + path=directory/(uuid.uuid4().hex+'.json') + write(path,{'summary':summary,'event_type':'note','author':'net-kingdom-identity-backup', + 'workplan_id':'17d315f0-829e-57c7-85cc-3a7914152804','task_id':'d68cd4d3-1ba3-530e-88f9-26ebb8dd6eac'}) + for pending in sorted(directory.glob('*.json'))[:64]: + request=urllib.request.Request('http://10.43.68.154:8000/progress/',data=pending.read_bytes(), + headers={'Content-Type':'application/json'},method='POST') + try: + with urllib.request.build_opener(urllib.request.ProxyHandler({})).open(request,timeout=10) as response: + if response.status!=201:break + pending.unlink() + except Exception:break + + +def backup(state,decision,first=False): + if not first:accepted(state/'accepted-recovery.json') + if (state/'capture-in-progress.json').exists():raise offsite.Refused() + publisher=token('publisher') + client=offsite.opener() + if offsite.quota(client,publisher)record['ciphertext_bytes']:raise offsite.Refused() + digest.update(block);output.write(block) + if count!=record['ciphertext_bytes'] or digest.hexdigest()!=record['ciphertext_sha256']:raise offsite.Refused() + with client.open(urllib.request.Request(url,method='HEAD',headers=h),timeout=30) as response: + if offsite.etag(response,url,record['ciphertext_bytes'])!=version:raise offsite.Refused() + out.update(record);out.update(status='readback_verified',fetched_path=str(target), + independent_owner_retrieval_verified=True,caller_entity_id=identity.get('entity_id'), + age_private_key_used=False,secret_values_exported=False,downloaded_at=datetime.now(timezone.utc).isoformat()) + + +def main(): + p=argparse.ArgumentParser(description=__doc__);p.add_argument('--source-receipt',required=True);p.add_argument('--receipt',required=True) + a=p.parse_args();os.umask(0o077);fd=os.open(a.receipt,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,0o600) + r={'status':'failed','source_sha256':hashlib.sha256(Path(__file__).read_bytes()).hexdigest()} + try:retrieve(json.loads(Path(a.source_receipt).read_text()),r) + except Exception:r.update(status='failed',error='contained_owner_retrieval_failed') + finally: + with os.fdopen(fd,'w') as out:json.dump(r,out,indent=2);out.write('\n') + return int(r['status']=='failed') + + +if __name__=='__main__':raise SystemExit(main()) diff --git a/tools/bootstrap-recovery/verify-identity-delivery.py b/tools/bootstrap-recovery/verify-identity-delivery.py new file mode 100644 index 0000000..ca9a863 --- /dev/null +++ b/tools/bootstrap-recovery/verify-identity-delivery.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +"""Attended native reader/ESO verification; only closed booleans leave custody.""" +import argparse +import hashlib +import importlib.util +import json +import os +from pathlib import Path +import sys + +HERE=Path(__file__).resolve().parent + + +def load(name): + spec=importlib.util.spec_from_file_location(name,HERE/(name+'.py')) + m=importlib.util.module_from_spec(spec);spec.loader.exec_module(m);return m + + +setup=load('identity-credential-setup');runtime=load('identity-runtime') +from state_hub_preflight_lane import bao,data,capabilities,revoke + + +def verify(action,expected,receipt,kubeconfig): + ccr,lane=setup.contract(action,expected) + ident=data(bao(['token','lookup','-format=json']))['data'] + if 'platform-admin' not in ident.get('policies',[]) or 'root' in ident.get('policies',[]) or not ident.get('path','').startswith('auth/netkingdom/'):raise setup.Refused() + capture=runtime.capture;capture.KUBE=['kubectl','--kubeconfig',kubeconfig] + if capture.get(None,'namespace','kube-system')['metadata']['uid']!=capture.prepare.CLUSTER_UID:raise setup.Refused() + def login(sa,audience,negative=False,namespace='sso'): + jwt=capture.command(['-n',namespace,'create','token',sa,'--audience='+audience,'--duration=10m']).decode().strip() + result=bao(['write','-format=json','auth/kubernetes/login','-'],payload={'role':lane['role'],'jwt':jwt},allow_failure=negative) + if negative: + if result.returncode==0: + revoke(data(result)['auth']['client_token']);raise setup.Refused() + if b'403' not in result.stderr and b'400' not in result.stderr:raise setup.Refused() + return None + return data(result)['auth'] + auth=login(lane['service_account'],'openbao');token=auth['client_token'] + try: + if (auth['token_policies']!=[lane['policy']] or auth.get('identity_policies') + or auth.get('policies',auth['token_policies'])!=[lane['policy']] or not 0bundle.LIMIT):raise bundle.Refused() + path=Path(transfer['fetched_path']);bundle.private(path,max_bytes=bundle.LIMIT) + with path.open('rb') as source:digest=hashlib.file_digest(source,'sha256').hexdigest() + if path.stat().st_size!=transfer['ciphertext_bytes'] or digest!=transfer['ciphertext_sha256']:raise bundle.Refused() + derived=pg.run(['age-keygen','-y',f'/proc/self/fd/{keyfd}'],pass_fds=(keyfd,)).strip() + if derived!=bundle.profile()['public_recipient'].encode():raise bundle.Refused() + receipt.update(ciphertext_sha256=digest,artifact=transfer['artifact'],registered_recipient_match=True) + with tempfile.TemporaryDirectory(prefix='nk49-identity-restore-',dir='/tmp') as temporary: + work=Path(temporary);env={'PATH':os.environ.get('PATH',os.defpath),'HOME':temporary,'XDG_CONFIG_HOME':temporary} + wrong=work/'wrong.key';pg.run(['age-keygen','-o',str(wrong)],env=env) + negative=subprocess.run(['age','-d','-i',str(wrong),str(path)],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,env=env,timeout=120) + if negative.returncode==0:raise bundle.Refused() + receipt['wrong_key_refused']=True + plain=work/'archive.tar' + with plain.open('xb') as output: + result=subprocess.run(['age','-d','-i',f'/proc/self/fd/{keyfd}',str(path)],stdout=output, + stderr=subprocess.DEVNULL,env=env,pass_fds=(keyfd,),timeout=180) + if result.returncode or plain.stat().st_size>bundle.LIMIT+4*1024*1024:raise bundle.Refused() + restored=work/'restored';restored.mkdir(mode=0o700) + manifest,checks=bundle.restore(plain,restored) + receipt.update(checks=checks,manifest_file_count=len(manifest['files']),all_manifest_hashes_verified=True, + profile_sha256=manifest['profile_sha256']) + container='nk49-identity-db-'+uuid.uuid4().hex + try: + receipt['database_restore']=pg.restore(restored/'databases/privacyidea.dump',container) + finally: + pg.run(['docker','rm','--force','--volumes',container]) + absent=subprocess.run(['docker','container','inspect',container],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,timeout=15) + pg.run(['docker','info','--format','{{.ServerVersion}}']) + if absent.returncode==0:raise bundle.Refused() + receipt['container_cleanup_verified']=True + if work.exists():raise bundle.Refused() + receipt.update(temporary_plaintext_cleanup_verified=True,status='identity_archive_verified', + full_service_authentication_proven=False,existing_token_decryption_proven=False,audit_signature_continuity_proven=False) + + +def main(): + p=argparse.ArgumentParser(description=__doc__);p.add_argument('--transfer-receipt',required=True);p.add_argument('--receipt',required=True) + a=p.parse_args();os.umask(0o077) + fd=os.open(a.receipt,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,0o600) + r={'schema':'netkingdom.identity-recovery.v1','status':'failed','started_at':datetime.now(timezone.utc).isoformat(), + 'production_contacted':False,'secret_values_exported':False,'container_cleanup_verified':False, + 'temporary_plaintext_cleanup_verified':False,'source_sha256':hashlib.sha256(Path(__file__).read_bytes()).hexdigest()} + keyfd=None + try: + transfer=json.loads(Path(a.transfer_receipt).read_text());keyfd=custody.private_key_fd() + verify(transfer,keyfd,r) + except Exception:r.update(status='failed',error='identity_recovery_failed_private_diagnostics_suppressed') + finally: + if keyfd is not None:os.close(keyfd) + r['finished_at']=datetime.now(timezone.utc).isoformat() + with os.fdopen(fd,'w') as out:json.dump(r,out,indent=2);out.write('\n') + if r['status']=='identity_archive_verified': + print('OK: off-host identity bundle decrypted; complete manifest, key pairs, SQLite integrity and PostgreSQL restore passed; disposable data removed. Service authentication remains separate acceptance.') + return 0 + print('ERROR: identity recovery failed; review safe receipt. Private diagnostics suppressed.');return 1 + + +if __name__=='__main__': + for sig in (signal.SIGHUP,signal.SIGTERM,signal.SIGINT):signal.signal(sig,custody.interrupted) + raise SystemExit(main()) diff --git a/workplans/NK-WP-0049-bootstrap-secret-free-recovery-handoff.md b/workplans/NK-WP-0049-bootstrap-secret-free-recovery-handoff.md index 5134049..2df3acf 100644 --- a/workplans/NK-WP-0049-bootstrap-secret-free-recovery-handoff.md +++ b/workplans/NK-WP-0049-bootstrap-secret-free-recovery-handoff.md @@ -500,6 +500,36 @@ necessary. No capture/upload/grant/schedule/rotation occurred in preparation. T02 remains progress and T03 wait. Evidence: actual database drill/review and `identity-backup-preparation-r1.json` under the October15 recovery evidence. +2026-10-16 implementation and bootstrap/discovery checkpoint: implemented the +complete new-profile capture/seal, unique Storage Share publication/readback, +independent owner retrieval, attended archive/native database verifier, separate +retention and source-bound node installer. Prepared exact CCR-2026-0048/0049 +for folder-only publisher(read/create) and retention(read/delete), exact ESO +readers and native positive/negative admission checks. Both CCRs validate with +no errors; claim confirmation and all owner approvals remain pending. Decision +`40bfae40-6108-4e51-8613-073339f5345f` is pending exact live-packet admission. + +51 focused checks passed including actual synthetic age/full archive and native +PostgreSQL restoration, SQLite WAL snapshots, hostile archive/helper boundaries, +provider permission/retention gates and bootstrap dispatch. Live cluster +server-dry-run accepted ten new helper/delivery manifests plus both guarded DNS +selector patches, and systemd unit validation passed. No live secret capture, +new grant, upload, policy change, node installation or timer enable occurred. +Initial provider is Storage Share; the earlier proposed Scaleway identity prefix +is deferred. Runtime quota/headroom is checked; host-local health cannot detect +host loss. Full native identity/token/audit/caller recovery remains open. + +At the operator's request reviewed railiance-bootstrap scope and added its +`identity-recovery` dispatcher and documented independent-custody → isolated +verification → S1/S2/S3 replacement → native identity acceptance handoff. This is +not an automated production importer; total-loss offline provider retrieval is +still an explicit gap. Updated the existing owner capability +`capability.security.bootstrap-recovery`, related it to bootstrap sequencing and +validated both registries; reuse-surface federation already includes both owner +indexes. No maturity promotion implies live acceptance. Reviewable rollout and +hidden-key steps: `docs/bootstrap-recovery-ceremony.md`; evidence in +`docs/evidence/2026-10-16-identity-backup-implementation/`. T02 progress/T03 wait. + ## Verify owner-approved recovery and current native callers ```task @@ -507,7 +537,7 @@ id: NK-WP-0049-T03 status: wait priority: high depends_on: [NK-WP-0049-T02] -blocking_reason: "The actual off-host PI database decrypted and restored with the independently held key; wrong-key refusal and cleanup passed. Operator reports no known full identity bundle and requested missing coverage preparation. Exact current workload/PVC/reference profile is prepared, but full data/key/config capture, new provider profile/publisher admission and native token/audit/directory/issuer recovery remain open, along with T02 current-owner and legacy-credential disposition. No live bootstrap/root retrieval or deferred exposure rotation is authorized." +blocking_reason: "The actual off-host PI database decrypted and restored with the independently held key; wrong-key refusal and cleanup passed. Operator reports no known full identity bundle and requested missing coverage preparation. Full-profile source adapters, bootstrap handoff and discovery are implemented/tested, but CCR-2026-0048/0049 owner admission, first actual full capture/off-site proof and native token/audit/directory/issuer recovery remain open, along with T02 current-owner and legacy-credential disposition. No live bootstrap/root retrieval or deferred exposure rotation is authorized." state_hub_task_id: "d68cd4d3-1ba3-530e-88f9-26ebb8dd6eac" ```