feat(recovery): verify scoped delivery and create first full offsite bundle
All checks were successful
CI Smoke / host-smoke (push) Successful in 0s
CI Smoke / container-smoke (push) Successful in 1s

Assistant: codex
Assistant-Model: gpt-6-astra
Assistant-Session: 01a13eab-1e6a-7b33-9c37-372e90ada3a2
This commit is contained in:
tegwick 2026-10-16 09:21:55 +02:00
parent 8857d9ff6f
commit 17a8fa1495
15 changed files with 435 additions and 12 deletions

View file

@ -420,3 +420,40 @@ checks pass after this correction. A fresh metadata capture plan confirms zero
allow policies matching helper Pods. The next owner login timed out before
handoff; no corrected provider probe or KV/ESO publication ran in that attempt.
The installed runtime's timers remain disabled while attended admission resumes.
## First full bundle created — 2026-10-16
Both exact roles passed native login, effective single-policy/900-second lifetime,
metadata/sibling/old-escrow/token-create denials, wrong-SA/namespace/audience
rejection, ESO value equality and temporary-reader revocation. Both ExternalSecrets
are Ready and KV versions are1. The operator session self-revoked. CCR-0048/0049
are applied with actual bound-claim evidence; the provider-only request5/9
assumption has been superseded by separately approved21/25.
ESO adds known no-transformation defaults (`conversionStrategy: Default`,
`decodingStrategy: None`, `metadataPolicy: None`) to its spec. The delivery guard
now accepts only those defaults while rejecting extra sources or altered
conversion/property/version/reader bindings. The disabled node units were
replaced with exact previous-packet/content guards, timers verified disabled and
inactive, and preserved rollback bytes; source release
`758ba4d170ba5ed4b6c563d3a379e496c5f81e8bff28c45d142a6bacbfaa21bf`
is installed.56 focused checks passed, including native synthetic full-archive
PostgreSQL recovery. The task-only disabled-upgrade source and packet are in the
live evidence folder; this is not permission for an unchecked active-job upgrade.
Actual first capture completed under the approved node owner, with all required
source identity guards and native data snapshots. Both temporary helpers and
private plaintext were cleaned. The exact new encrypted artifact is:
- Name: `net-kingdom-identity-20261016T063159Z-5882771aa2ce.tar.age`
- Folder: Storage Share `Backup/railiance-backups/net-kingdom-identity/`
- Bytes:109738344
- Ciphertext SHA256:`7117b3fe1e4840c86b301c94c85e6b16709e5626bb0884bd0580bde630e30e30`
- Status: unique publication and fresh downloaded readback verified; safe index published last.
Private node receipts live under `/var/lib/net-kingdom-backup/runs/` for that
artifact. No age private key was used in capture/upload. The first independent
owner retrieval and human hidden-key archive proof follow; both timers remain
disabled until the actual proof is reviewed. Full directory/issuer/MFA/token and
audit-continuity acceptance remains distinct from archive recovery.

View file

@ -0,0 +1,53 @@
{
"started_at": "2026-10-16T06:32:01.718185+00:00",
"status": "encrypted_bundle_created",
"decision_ref": "40bfae40-6108-4e51-8613-073339f5345f",
"plan_run_id": "5882771aa2ce",
"created_resources": [
{
"namespace": "sso",
"kind": "NetworkPolicy",
"name": "nk49-capture-5882771aa2ce",
"uid": "39853a14-4875-4320-b0f5-fda26d4d04eb"
},
{
"namespace": "sso",
"kind": "Pod",
"name": "nk49-capture-5882771aa2ce",
"uid": "6cb00afb-05b5-4196-929b-fe5c142a9d66"
},
{
"namespace": "mfa",
"kind": "NetworkPolicy",
"name": "nk49-capture-5882771aa2ce",
"uid": "d9f9e2fe-56ee-4c59-950d-09d33d9c95da"
},
{
"namespace": "mfa",
"kind": "Pod",
"name": "nk49-capture-5882771aa2ce",
"uid": "f3328766-1a74-4272-b5b1-d9d07a4aff7a"
}
],
"unverified_create_outcomes": [],
"helper_cleanup_verified": true,
"temporary_plaintext_cleanup_verified": true,
"temporary_plaintext_directory": "/tmp/nk49-identity-qcsy2iyi",
"schema": "netkingdom.identity-capture.v1",
"ciphertext_sha256": "7117b3fe1e4840c86b301c94c85e6b16709e5626bb0884bd0580bde630e30e30",
"ciphertext_bytes": 109738344,
"ciphertext_path": "/var/lib/net-kingdom-backup/runs/net-kingdom-identity-20261016T063159Z-5882771aa2ce.tar.age/net-kingdom-identity-20261016T063159Z-5882771aa2ce.tar.age",
"profile_sha256": "40f9e5b5bb957e5b23e03b31701702b850502d16a92a0fa0ebf92618cc3f18a1",
"file_count": 36,
"checks": {
"sqlite_integrity_passed": true,
"privacyidea_key_format_and_pair_passed": true,
"keycape_signing_key_format_passed": true,
"policy_json_present": true,
"database_custom_format_present": true,
"native_identity_recovery_proven": false
},
"offsite_recovery_proven": false,
"secret_values_exported": false,
"finished_at": "2026-10-16T06:32:35.934734+00:00"
}

View file

@ -0,0 +1,21 @@
{
"schema": "netkingdom.identity-offsite.v1",
"status": "readback_verified",
"observed_at": "2026-10-16T06:32:39.402297+00:00",
"provider": "Storage Share",
"folder": "Backup/railiance-backups/net-kingdom-identity",
"artifact": "net-kingdom-identity-20261016T063159Z-5882771aa2ce.tar.age",
"ciphertext_sha256": "7117b3fe1e4840c86b301c94c85e6b16709e5626bb0884bd0580bde630e30e30",
"ciphertext_bytes": 109738344,
"fetched_path": "/var/lib/net-kingdom-backup/runs/net-kingdom-identity-20261016T063159Z-5882771aa2ce.tar.age/fetched.tar.age",
"etag": "\"28ec1f7014e492de4c40e1ae75502f08\"",
"quota_available_before": 3643309771,
"new_object_created": true,
"downloaded_digest_verified": true,
"verified_index_published": true,
"index_name": "net-kingdom-identity-20261016T063159Z-5882771aa2ce.tar.age.receipt.json",
"age_private_key_used": false,
"secret_values_exported": false,
"retained_objects_deleted": false,
"native_identity_recovery_proven": false
}

View file

@ -0,0 +1,59 @@
{
"schema": "netkingdom.identity-runtime-install.v1",
"status": "prepared_not_installed",
"decision_ref": "40bfae40-6108-4e51-8613-073339f5345f",
"release_id": "758ba4d170ba5ed4b6c563d3a379e496c5f81e8bff28c45d142a6bacbfaa21bf",
"release_directory": "/opt/net-kingdom-backup/releases/758ba4d170ba5ed4b6c563d3a379e496c5f81e8bff28c45d142a6bacbfaa21bf",
"runtime_files": [
{
"path": "tools/bootstrap-recovery/capture-identity.py",
"sha256": "47dca7c4dc96fa7eb9fb02c81fca9ca8dc5281aac918979eb7367148168ba705"
},
{
"path": "tools/bootstrap-recovery/identity-files.py",
"sha256": "d0063a9eedd3610e3d45bd2d834aae83cff74e4b3504d138b3548f427e89331d"
},
{
"path": "tools/bootstrap-recovery/identity-bundle.py",
"sha256": "524a03c065db4307fcde83d33af39503f3de19cb6ef3a8ca72c991d4a8c08ee6"
},
{
"path": "tools/bootstrap-recovery/identity-runtime.py",
"sha256": "a2c0acf567c204705b90be64d561d0fda2285e0fc71a962d6e1b149f571e1940"
},
{
"path": "tools/bootstrap-recovery/identity-offsite.py",
"sha256": "e7c31fa9ee081961af873348ee3b4829a645d40c7dc7f1e22c564aa9ed01bc49"
},
{
"path": "tools/bootstrap-recovery/identity-retention.py",
"sha256": "a41ea4759f69c35548149fc8d0d5ace547f16cf9fe4c9dac10b119a4d0becdec"
},
{
"path": "tools/bootstrap-recovery/identity-backup-profile.json",
"sha256": "40f9e5b5bb957e5b23e03b31701702b850502d16a92a0fa0ebf92618cc3f18a1"
},
{
"path": "tools/bootstrap-recovery/prepare-identity-coverage.py",
"sha256": "cd5f3067d28b4c7cc93ed08b8cf3869218bdecf7c54b5dffed22d697de71dbb1"
},
{
"path": "tools/bootstrap-recovery/install-identity-backup.py",
"sha256": "5c612a2a51c9b59aad8cc01d418c2f563cbd92c62dc1fa8be994e5fb59df90f1"
},
{
"path": "sso-mfa/k8s/privacyidea/key-custody-remote.py",
"sha256": "27b2b885c81923d85054e3999c5613e104fe627779b2b50398ed9bed5088f47e"
}
],
"units": {
"net-kingdom-identity-backup.service": "[Unit]\nDescription=Encrypted full identity backup and verified Storage Share readback\nAfter=network-online.target k3s.service\nConditionPathExists=/var/lib/net-kingdom-backup/accepted-recovery.json\n[Service]\nType=oneshot\nUser=root\nGroup=root\nUMask=0077\nStateDirectory=net-kingdom-backup\nStateDirectoryMode=0700\nNoNewPrivileges=true\nPrivateTmp=true\nPrivateDevices=true\nProtectSystem=strict\nProtectHome=true\nReadWritePaths=/var/lib/net-kingdom-backup\nCapabilityBoundingSet=\nRestrictAddressFamilies=AF_UNIX AF_INET AF_INET6\nProtectKernelTunables=true\nProtectKernelModules=true\nProtectControlGroups=true\nMemoryMax=3G\nCPUQuota=200%\nTasksMax=128\nTimeoutStartSec=45min\nTimeoutStopSec=5min\nExecStartPre=/usr/bin/python3 -I /opt/net-kingdom-backup/releases/758ba4d170ba5ed4b6c563d3a379e496c5f81e8bff28c45d142a6bacbfaa21bf/tools/bootstrap-recovery/identity-runtime.py retention --kubeconfig /etc/rancher/k3s/k3s.yaml --state /var/lib/net-kingdom-backup\nExecStart=/usr/bin/python3 -I /opt/net-kingdom-backup/releases/758ba4d170ba5ed4b6c563d3a379e496c5f81e8bff28c45d142a6bacbfaa21bf/tools/bootstrap-recovery/identity-runtime.py scheduled --kubeconfig /etc/rancher/k3s/k3s.yaml --state /var/lib/net-kingdom-backup --decision-id 40bfae40-6108-4e51-8613-073339f5345f\nExecStartPost=/usr/bin/python3 -I /opt/net-kingdom-backup/releases/758ba4d170ba5ed4b6c563d3a379e496c5f81e8bff28c45d142a6bacbfaa21bf/tools/bootstrap-recovery/identity-runtime.py retention --kubeconfig /etc/rancher/k3s/k3s.yaml --state /var/lib/net-kingdom-backup\n",
"net-kingdom-identity-backup.timer": "[Unit]\nDescription=Daily full identity recovery bundle\n[Timer]\nOnCalendar=*-*-* 03:45:00 UTC\nPersistent=true\nRandomizedDelaySec=5min\nUnit=net-kingdom-identity-backup.service\n[Install]\nWantedBy=timers.target\n",
"net-kingdom-identity-backup-health.service": "[Unit]\nDescription=Identity backup age check with durable failure reporting\nAfter=network-online.target k3s.service\n[Service]\nType=oneshot\nUser=root\nGroup=root\nUMask=0077\nStateDirectory=net-kingdom-backup\nStateDirectoryMode=0700\nNoNewPrivileges=true\nPrivateTmp=true\nPrivateDevices=true\nProtectSystem=strict\nProtectHome=true\nReadWritePaths=/var/lib/net-kingdom-backup\nCapabilityBoundingSet=\nRestrictAddressFamilies=AF_UNIX AF_INET AF_INET6\nProtectKernelTunables=true\nProtectKernelModules=true\nProtectControlGroups=true\nMemoryMax=3G\nCPUQuota=200%\nTasksMax=128\nTimeoutStartSec=45min\nTimeoutStopSec=5min\nExecStart=/usr/bin/python3 -I /opt/net-kingdom-backup/releases/758ba4d170ba5ed4b6c563d3a379e496c5f81e8bff28c45d142a6bacbfaa21bf/tools/bootstrap-recovery/identity-runtime.py status --kubeconfig /etc/rancher/k3s/k3s.yaml --state /var/lib/net-kingdom-backup\n",
"net-kingdom-identity-backup-health.timer": "[Unit]\nDescription=Hourly identity backup freshness check\n[Timer]\nOnCalendar=hourly\nPersistent=true\nUnit=net-kingdom-identity-backup-health.service\n[Install]\nWantedBy=timers.target\n"
},
"timers_automatically_enabled": false,
"first_capture_command": "/usr/bin/python3 -I /opt/net-kingdom-backup/releases/758ba4d170ba5ed4b6c563d3a379e496c5f81e8bff28c45d142a6bacbfaa21bf/tools/bootstrap-recovery/identity-runtime.py first-capture --kubeconfig /etc/rancher/k3s/k3s.yaml --state /var/lib/net-kingdom-backup --decision-id 40bfae40-6108-4e51-8613-073339f5345f",
"activation_requires_independent_recovery": true,
"source_contains_credentials": false
}

View file

@ -0,0 +1,49 @@
{
"status": "reader_and_delivery_verified",
"phase": "negative_reader_logins",
"native_reader_metadata": {
"token_policies": [
"net-kingdom-backup-publisher-eso"
],
"identity_policies": null,
"policies": [
"net-kingdom-backup-publisher-eso"
],
"lease_duration": 900
},
"capabilities_checked": {
"auth/token/create": [
"deny"
],
"platform/data/workloads/net-kingdom/identity-backup/publisher": [
"read"
],
"platform/data/workloads/net-kingdom/identity-backup/publisher-sibling": [
"deny"
],
"platform/data/workloads/railiance/backup/offsite-lane": [
"deny"
],
"platform/metadata/workloads/net-kingdom/identity-backup/publisher": [
"deny"
]
},
"kv_version": 1,
"wrong_namespace_probe_attempted": {
"name": "net-kingdom-backup-publisher-eso",
"namespace": "default"
},
"wrong_namespace_probe_cleanup_verified": true,
"exact_namespace_binding_verified": true,
"exact_policy_and_ttl_verified": true,
"wrong_service_account_denied": true,
"wrong_audience_denied": true,
"wrong_namespace_denied": true,
"metadata_sibling_escrow_and_token_create_denied": true,
"kv_write_denied": true,
"eso_delivery_equality_verified": true,
"temporary_reader_revoked": true,
"secret_values_exported": false,
"source_ccr": "CCR-2026-0048",
"ccr_sha256": "a7d5c356ce0c4df9de439a58ad3d36e1578f27cda327c9387e97cfc2e9dc7cdc"
}

View file

@ -0,0 +1,37 @@
{
"status": "provider_and_reader_prepared",
"action": "publisher",
"caller_entity_id": "4a1f663c-0621-0483-edd0-f852274f4b6b",
"caller_auth_path": "auth/netkingdom/oidc/callback",
"quota_before": {
"quota_bytes": 10737418240,
"used_bytes": 7094108469,
"available_bytes": 3643309771
},
"folder_created": false,
"share_created": false,
"share_id": "26",
"share_permissions": 21,
"folder": "/railiance-backups/net-kingdom-identity",
"provider_test_artifacts": [
"nk49-boundary-2d01d6795bb8422282c9af18c965df49.age",
"nk49-boundary-7d6dd29f0d5a495cab2ee6e51bed1843.age"
],
"boundary_fixture_cleanup_verified": true,
"boundary_phase": "deny_other_account_folder",
"parent_probe_status": 400,
"parent_path_rejected": true,
"other_account_folder_status": 401,
"provider_boundary_verified": true,
"owner_fixture_cleanup_verified": true,
"publisher_quota_available": 3643309771,
"coding_agent_fence_verified_before_publication": true,
"kv_version": 1,
"coding_agents_denied": true,
"escrow_read": false,
"operator_password_delivered": false,
"retained_backups_deleted": false,
"source_ccr": "CCR-2026-0048",
"ccr_sha256": "a7d5c356ce0c4df9de439a58ad3d36e1578f27cda327c9387e97cfc2e9dc7cdc",
"eso_delivery_and_native_login_verification_pending": true
}

View file

@ -0,0 +1,49 @@
{
"status": "reader_and_delivery_verified",
"phase": "negative_reader_logins",
"native_reader_metadata": {
"token_policies": [
"net-kingdom-backup-retention-eso"
],
"identity_policies": null,
"policies": [
"net-kingdom-backup-retention-eso"
],
"lease_duration": 900
},
"capabilities_checked": {
"auth/token/create": [
"deny"
],
"platform/data/workloads/net-kingdom/identity-backup/retention": [
"read"
],
"platform/data/workloads/net-kingdom/identity-backup/retention-sibling": [
"deny"
],
"platform/data/workloads/railiance/backup/offsite-lane": [
"deny"
],
"platform/metadata/workloads/net-kingdom/identity-backup/retention": [
"deny"
]
},
"kv_version": 1,
"wrong_namespace_probe_attempted": {
"name": "net-kingdom-backup-retention-eso",
"namespace": "default"
},
"wrong_namespace_probe_cleanup_verified": true,
"exact_namespace_binding_verified": true,
"exact_policy_and_ttl_verified": true,
"wrong_service_account_denied": true,
"wrong_audience_denied": true,
"wrong_namespace_denied": true,
"metadata_sibling_escrow_and_token_create_denied": true,
"kv_write_denied": true,
"eso_delivery_equality_verified": true,
"temporary_reader_revoked": true,
"secret_values_exported": false,
"source_ccr": "CCR-2026-0049",
"ccr_sha256": "d391018abbda0fbc1acec4b5a22c0f99ce65525de940431df376a7552b3746cf"
}

View file

@ -0,0 +1,35 @@
{
"status": "provider_and_reader_prepared",
"action": "retention",
"caller_entity_id": "4a1f663c-0621-0483-edd0-f852274f4b6b",
"caller_auth_path": "auth/netkingdom/oidc/callback",
"quota_before": {
"quota_bytes": 10737418240,
"used_bytes": 7094108469,
"available_bytes": 3643309771
},
"folder_created": false,
"share_created": true,
"share_id": "29",
"share_permissions": 25,
"folder": "/railiance-backups/net-kingdom-identity",
"provider_test_artifacts": [
"nk49-boundary-6f5413c177894fdabdfb7828ef76f823.age",
"nk49-absent-abb34d9dafdc46e2babb5922b2d341d7.age"
],
"boundary_fixture_cleanup_verified": true,
"boundary_phase": "deny_other_account_folder",
"parent_probe_status": 400,
"parent_path_rejected": true,
"other_account_folder_status": 401,
"provider_boundary_verified": true,
"coding_agent_fence_verified_before_publication": true,
"kv_version": 1,
"coding_agents_denied": true,
"escrow_read": false,
"operator_password_delivered": false,
"retained_backups_deleted": false,
"source_ccr": "CCR-2026-0049",
"ccr_sha256": "d391018abbda0fbc1acec4b5a22c0f99ce65525de940431df376a7552b3746cf",
"eso_delivery_and_native_login_verification_pending": true
}

View file

@ -1,13 +1,13 @@
{
"status": "runtime_installed_provider_admission_waiting_attended_login",
"status": "actual_full_bundle_readback_verified_independent_recovery_pending",
"observed_at": "2026-10-15T23:56:00.293983+00:00",
"approval_decision": "40bfae40-6108-4e51-8613-073339f5345f",
"approved_platform_source_commit": "69a792c",
"first_owner_session": "child_failed_session_revoked",
"second_owner_session": "metadata_review_succeeded_session_revoked",
"new_identity_bundle_created": false,
"new_identity_bundle_created": true,
"identity_job_installed": true,
"new_credentials_published_to_openbao": false,
"new_credentials_published_to_openbao": true,
"existing_backup_changes": false,
"observed_permissions": 21,
"requested_permissions": 5,
@ -35,10 +35,10 @@
"publisher": 21,
"retention": 25
},
"focused_tests_passed": 52,
"focused_tests_passed": 56,
"temporary_dns_selector_exclusions_applied": true,
"network_policy_rule_lines_present": 649,
"installed_runtime_release": "bf8b38f185037515cfad9eebcddefc9f60ace724c32e10d89ae8307dc95a4eb7",
"installed_runtime_release": "758ba4d170ba5ed4b6c563d3a379e496c5f81e8bff28c45d142a6bacbfaa21bf",
"current_login_session_active": false,
"latest_login_failure": "callback_timeout_before_handoff",
"latest_timeout_possible_session_revocation_confirmed": false,
@ -46,6 +46,10 @@
"publisher_fixture_cleanup_verified": true,
"parent_path_rejection_status": 400,
"corrected_boundary_tests_passed": 16,
"corrected_native_boundary_and_delivery_not_yet_run": true,
"helper_network_allow_count": 0
"corrected_native_boundary_and_delivery_not_yet_run": false,
"helper_network_allow_count": 0,
"actual_provider_boundary_and_native_delivery_verified": true,
"actual_ciphertext_bytes": 109738344,
"actual_ciphertext_sha256": "7117b3fe1e4840c86b301c94c85e6b16709e5626bb0884bd0580bde630e30e30",
"actual_independent_key_recovery_pending": true
}

View file

@ -0,0 +1,29 @@
"""Exact task-only replacement of disabled NK49 units; preserve rollback bytes."""
import hashlib,importlib.util,json,os,subprocess,sys,tempfile
from pathlib import Path
os.umask(0o077)
root=Path(sys.argv[1]);old=Path(sys.argv[2]);out=Path(sys.argv[3])
expected_old='bb5579ce72967f0ce78e6e0afb361cfd5e3793768d3eb5b422718033505a0014'
assert hashlib.sha256(old.read_bytes()).hexdigest()==expected_old
previous=json.loads(old.read_text());packet=json.loads((root/'install-packet.json').read_text())
s=importlib.util.spec_from_file_location('installer',root/'tools/bootstrap-recovery/install-identity-backup.py');m=importlib.util.module_from_spec(s);s.loader.exec_module(m)
m.owner();assert packet==m.render(packet['decision_ref'])
assert packet['decision_ref']==previous['decision_ref']
for unit in ['net-kingdom-identity-backup.timer','net-kingdom-identity-backup-health.timer']:
for action,expected in [('is-enabled',b'disabled'),('is-active',b'inactive')]:
r=subprocess.run(['systemctl',action,unit],stdout=subprocess.PIPE,stderr=subprocess.DEVNULL,timeout=20);assert r.stdout.strip()==expected
paths={name:Path('/etc/systemd/system')/name for name in previous['units']}
assert set(paths)==set(packet['units'])
for name,path in paths.items():assert not path.is_symlink() and path.read_text()==previous['units'][name]
backup=Path(tempfile.mkdtemp(prefix='nk49-disabled-unit-rollback-',dir='/tmp'))
try:
for name,path in paths.items():path.rename(backup/name)
m.install(packet)
except BaseException:
for name,path in paths.items():
if (backup/name).exists():
assert not path.exists() or path.read_text()==packet['units'][name]
os.replace(backup/name,path)
subprocess.run(['systemctl','daemon-reload'],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,timeout=30)
raise
out.write_text(json.dumps({'status':'installed_timers_disabled','release_id':packet['release_id'],'previous_release':previous['release_id'],'rollback_units':str(backup)},indent=2)+'\n')