feat(recovery): verify scoped delivery and create first full offsite bundle
All checks were successful
CI Smoke / host-smoke (push) Successful in 0s
CI Smoke / container-smoke (push) Successful in 1s

Assistant: codex
Assistant-Model: gpt-6-astra
Assistant-Session: 01a13eab-1e6a-7b33-9c37-372e90ada3a2
This commit is contained in:
tegwick 2026-10-16 09:21:55 +02:00
parent 8857d9ff6f
commit 17a8fa1495
15 changed files with 435 additions and 12 deletions

View file

@ -0,0 +1,53 @@
{
"started_at": "2026-10-16T06:32:01.718185+00:00",
"status": "encrypted_bundle_created",
"decision_ref": "40bfae40-6108-4e51-8613-073339f5345f",
"plan_run_id": "5882771aa2ce",
"created_resources": [
{
"namespace": "sso",
"kind": "NetworkPolicy",
"name": "nk49-capture-5882771aa2ce",
"uid": "39853a14-4875-4320-b0f5-fda26d4d04eb"
},
{
"namespace": "sso",
"kind": "Pod",
"name": "nk49-capture-5882771aa2ce",
"uid": "6cb00afb-05b5-4196-929b-fe5c142a9d66"
},
{
"namespace": "mfa",
"kind": "NetworkPolicy",
"name": "nk49-capture-5882771aa2ce",
"uid": "d9f9e2fe-56ee-4c59-950d-09d33d9c95da"
},
{
"namespace": "mfa",
"kind": "Pod",
"name": "nk49-capture-5882771aa2ce",
"uid": "f3328766-1a74-4272-b5b1-d9d07a4aff7a"
}
],
"unverified_create_outcomes": [],
"helper_cleanup_verified": true,
"temporary_plaintext_cleanup_verified": true,
"temporary_plaintext_directory": "/tmp/nk49-identity-qcsy2iyi",
"schema": "netkingdom.identity-capture.v1",
"ciphertext_sha256": "7117b3fe1e4840c86b301c94c85e6b16709e5626bb0884bd0580bde630e30e30",
"ciphertext_bytes": 109738344,
"ciphertext_path": "/var/lib/net-kingdom-backup/runs/net-kingdom-identity-20261016T063159Z-5882771aa2ce.tar.age/net-kingdom-identity-20261016T063159Z-5882771aa2ce.tar.age",
"profile_sha256": "40f9e5b5bb957e5b23e03b31701702b850502d16a92a0fa0ebf92618cc3f18a1",
"file_count": 36,
"checks": {
"sqlite_integrity_passed": true,
"privacyidea_key_format_and_pair_passed": true,
"keycape_signing_key_format_passed": true,
"policy_json_present": true,
"database_custom_format_present": true,
"native_identity_recovery_proven": false
},
"offsite_recovery_proven": false,
"secret_values_exported": false,
"finished_at": "2026-10-16T06:32:35.934734+00:00"
}

View file

@ -0,0 +1,21 @@
{
"schema": "netkingdom.identity-offsite.v1",
"status": "readback_verified",
"observed_at": "2026-10-16T06:32:39.402297+00:00",
"provider": "Storage Share",
"folder": "Backup/railiance-backups/net-kingdom-identity",
"artifact": "net-kingdom-identity-20261016T063159Z-5882771aa2ce.tar.age",
"ciphertext_sha256": "7117b3fe1e4840c86b301c94c85e6b16709e5626bb0884bd0580bde630e30e30",
"ciphertext_bytes": 109738344,
"fetched_path": "/var/lib/net-kingdom-backup/runs/net-kingdom-identity-20261016T063159Z-5882771aa2ce.tar.age/fetched.tar.age",
"etag": "\"28ec1f7014e492de4c40e1ae75502f08\"",
"quota_available_before": 3643309771,
"new_object_created": true,
"downloaded_digest_verified": true,
"verified_index_published": true,
"index_name": "net-kingdom-identity-20261016T063159Z-5882771aa2ce.tar.age.receipt.json",
"age_private_key_used": false,
"secret_values_exported": false,
"retained_objects_deleted": false,
"native_identity_recovery_proven": false
}

View file

@ -0,0 +1,59 @@
{
"schema": "netkingdom.identity-runtime-install.v1",
"status": "prepared_not_installed",
"decision_ref": "40bfae40-6108-4e51-8613-073339f5345f",
"release_id": "758ba4d170ba5ed4b6c563d3a379e496c5f81e8bff28c45d142a6bacbfaa21bf",
"release_directory": "/opt/net-kingdom-backup/releases/758ba4d170ba5ed4b6c563d3a379e496c5f81e8bff28c45d142a6bacbfaa21bf",
"runtime_files": [
{
"path": "tools/bootstrap-recovery/capture-identity.py",
"sha256": "47dca7c4dc96fa7eb9fb02c81fca9ca8dc5281aac918979eb7367148168ba705"
},
{
"path": "tools/bootstrap-recovery/identity-files.py",
"sha256": "d0063a9eedd3610e3d45bd2d834aae83cff74e4b3504d138b3548f427e89331d"
},
{
"path": "tools/bootstrap-recovery/identity-bundle.py",
"sha256": "524a03c065db4307fcde83d33af39503f3de19cb6ef3a8ca72c991d4a8c08ee6"
},
{
"path": "tools/bootstrap-recovery/identity-runtime.py",
"sha256": "a2c0acf567c204705b90be64d561d0fda2285e0fc71a962d6e1b149f571e1940"
},
{
"path": "tools/bootstrap-recovery/identity-offsite.py",
"sha256": "e7c31fa9ee081961af873348ee3b4829a645d40c7dc7f1e22c564aa9ed01bc49"
},
{
"path": "tools/bootstrap-recovery/identity-retention.py",
"sha256": "a41ea4759f69c35548149fc8d0d5ace547f16cf9fe4c9dac10b119a4d0becdec"
},
{
"path": "tools/bootstrap-recovery/identity-backup-profile.json",
"sha256": "40f9e5b5bb957e5b23e03b31701702b850502d16a92a0fa0ebf92618cc3f18a1"
},
{
"path": "tools/bootstrap-recovery/prepare-identity-coverage.py",
"sha256": "cd5f3067d28b4c7cc93ed08b8cf3869218bdecf7c54b5dffed22d697de71dbb1"
},
{
"path": "tools/bootstrap-recovery/install-identity-backup.py",
"sha256": "5c612a2a51c9b59aad8cc01d418c2f563cbd92c62dc1fa8be994e5fb59df90f1"
},
{
"path": "sso-mfa/k8s/privacyidea/key-custody-remote.py",
"sha256": "27b2b885c81923d85054e3999c5613e104fe627779b2b50398ed9bed5088f47e"
}
],
"units": {
"net-kingdom-identity-backup.service": "[Unit]\nDescription=Encrypted full identity backup and verified Storage Share readback\nAfter=network-online.target k3s.service\nConditionPathExists=/var/lib/net-kingdom-backup/accepted-recovery.json\n[Service]\nType=oneshot\nUser=root\nGroup=root\nUMask=0077\nStateDirectory=net-kingdom-backup\nStateDirectoryMode=0700\nNoNewPrivileges=true\nPrivateTmp=true\nPrivateDevices=true\nProtectSystem=strict\nProtectHome=true\nReadWritePaths=/var/lib/net-kingdom-backup\nCapabilityBoundingSet=\nRestrictAddressFamilies=AF_UNIX AF_INET AF_INET6\nProtectKernelTunables=true\nProtectKernelModules=true\nProtectControlGroups=true\nMemoryMax=3G\nCPUQuota=200%\nTasksMax=128\nTimeoutStartSec=45min\nTimeoutStopSec=5min\nExecStartPre=/usr/bin/python3 -I /opt/net-kingdom-backup/releases/758ba4d170ba5ed4b6c563d3a379e496c5f81e8bff28c45d142a6bacbfaa21bf/tools/bootstrap-recovery/identity-runtime.py retention --kubeconfig /etc/rancher/k3s/k3s.yaml --state /var/lib/net-kingdom-backup\nExecStart=/usr/bin/python3 -I /opt/net-kingdom-backup/releases/758ba4d170ba5ed4b6c563d3a379e496c5f81e8bff28c45d142a6bacbfaa21bf/tools/bootstrap-recovery/identity-runtime.py scheduled --kubeconfig /etc/rancher/k3s/k3s.yaml --state /var/lib/net-kingdom-backup --decision-id 40bfae40-6108-4e51-8613-073339f5345f\nExecStartPost=/usr/bin/python3 -I /opt/net-kingdom-backup/releases/758ba4d170ba5ed4b6c563d3a379e496c5f81e8bff28c45d142a6bacbfaa21bf/tools/bootstrap-recovery/identity-runtime.py retention --kubeconfig /etc/rancher/k3s/k3s.yaml --state /var/lib/net-kingdom-backup\n",
"net-kingdom-identity-backup.timer": "[Unit]\nDescription=Daily full identity recovery bundle\n[Timer]\nOnCalendar=*-*-* 03:45:00 UTC\nPersistent=true\nRandomizedDelaySec=5min\nUnit=net-kingdom-identity-backup.service\n[Install]\nWantedBy=timers.target\n",
"net-kingdom-identity-backup-health.service": "[Unit]\nDescription=Identity backup age check with durable failure reporting\nAfter=network-online.target k3s.service\n[Service]\nType=oneshot\nUser=root\nGroup=root\nUMask=0077\nStateDirectory=net-kingdom-backup\nStateDirectoryMode=0700\nNoNewPrivileges=true\nPrivateTmp=true\nPrivateDevices=true\nProtectSystem=strict\nProtectHome=true\nReadWritePaths=/var/lib/net-kingdom-backup\nCapabilityBoundingSet=\nRestrictAddressFamilies=AF_UNIX AF_INET AF_INET6\nProtectKernelTunables=true\nProtectKernelModules=true\nProtectControlGroups=true\nMemoryMax=3G\nCPUQuota=200%\nTasksMax=128\nTimeoutStartSec=45min\nTimeoutStopSec=5min\nExecStart=/usr/bin/python3 -I /opt/net-kingdom-backup/releases/758ba4d170ba5ed4b6c563d3a379e496c5f81e8bff28c45d142a6bacbfaa21bf/tools/bootstrap-recovery/identity-runtime.py status --kubeconfig /etc/rancher/k3s/k3s.yaml --state /var/lib/net-kingdom-backup\n",
"net-kingdom-identity-backup-health.timer": "[Unit]\nDescription=Hourly identity backup freshness check\n[Timer]\nOnCalendar=hourly\nPersistent=true\nUnit=net-kingdom-identity-backup-health.service\n[Install]\nWantedBy=timers.target\n"
},
"timers_automatically_enabled": false,
"first_capture_command": "/usr/bin/python3 -I /opt/net-kingdom-backup/releases/758ba4d170ba5ed4b6c563d3a379e496c5f81e8bff28c45d142a6bacbfaa21bf/tools/bootstrap-recovery/identity-runtime.py first-capture --kubeconfig /etc/rancher/k3s/k3s.yaml --state /var/lib/net-kingdom-backup --decision-id 40bfae40-6108-4e51-8613-073339f5345f",
"activation_requires_independent_recovery": true,
"source_contains_credentials": false
}

View file

@ -0,0 +1,49 @@
{
"status": "reader_and_delivery_verified",
"phase": "negative_reader_logins",
"native_reader_metadata": {
"token_policies": [
"net-kingdom-backup-publisher-eso"
],
"identity_policies": null,
"policies": [
"net-kingdom-backup-publisher-eso"
],
"lease_duration": 900
},
"capabilities_checked": {
"auth/token/create": [
"deny"
],
"platform/data/workloads/net-kingdom/identity-backup/publisher": [
"read"
],
"platform/data/workloads/net-kingdom/identity-backup/publisher-sibling": [
"deny"
],
"platform/data/workloads/railiance/backup/offsite-lane": [
"deny"
],
"platform/metadata/workloads/net-kingdom/identity-backup/publisher": [
"deny"
]
},
"kv_version": 1,
"wrong_namespace_probe_attempted": {
"name": "net-kingdom-backup-publisher-eso",
"namespace": "default"
},
"wrong_namespace_probe_cleanup_verified": true,
"exact_namespace_binding_verified": true,
"exact_policy_and_ttl_verified": true,
"wrong_service_account_denied": true,
"wrong_audience_denied": true,
"wrong_namespace_denied": true,
"metadata_sibling_escrow_and_token_create_denied": true,
"kv_write_denied": true,
"eso_delivery_equality_verified": true,
"temporary_reader_revoked": true,
"secret_values_exported": false,
"source_ccr": "CCR-2026-0048",
"ccr_sha256": "a7d5c356ce0c4df9de439a58ad3d36e1578f27cda327c9387e97cfc2e9dc7cdc"
}

View file

@ -0,0 +1,37 @@
{
"status": "provider_and_reader_prepared",
"action": "publisher",
"caller_entity_id": "4a1f663c-0621-0483-edd0-f852274f4b6b",
"caller_auth_path": "auth/netkingdom/oidc/callback",
"quota_before": {
"quota_bytes": 10737418240,
"used_bytes": 7094108469,
"available_bytes": 3643309771
},
"folder_created": false,
"share_created": false,
"share_id": "26",
"share_permissions": 21,
"folder": "/railiance-backups/net-kingdom-identity",
"provider_test_artifacts": [
"nk49-boundary-2d01d6795bb8422282c9af18c965df49.age",
"nk49-boundary-7d6dd29f0d5a495cab2ee6e51bed1843.age"
],
"boundary_fixture_cleanup_verified": true,
"boundary_phase": "deny_other_account_folder",
"parent_probe_status": 400,
"parent_path_rejected": true,
"other_account_folder_status": 401,
"provider_boundary_verified": true,
"owner_fixture_cleanup_verified": true,
"publisher_quota_available": 3643309771,
"coding_agent_fence_verified_before_publication": true,
"kv_version": 1,
"coding_agents_denied": true,
"escrow_read": false,
"operator_password_delivered": false,
"retained_backups_deleted": false,
"source_ccr": "CCR-2026-0048",
"ccr_sha256": "a7d5c356ce0c4df9de439a58ad3d36e1578f27cda327c9387e97cfc2e9dc7cdc",
"eso_delivery_and_native_login_verification_pending": true
}

View file

@ -0,0 +1,49 @@
{
"status": "reader_and_delivery_verified",
"phase": "negative_reader_logins",
"native_reader_metadata": {
"token_policies": [
"net-kingdom-backup-retention-eso"
],
"identity_policies": null,
"policies": [
"net-kingdom-backup-retention-eso"
],
"lease_duration": 900
},
"capabilities_checked": {
"auth/token/create": [
"deny"
],
"platform/data/workloads/net-kingdom/identity-backup/retention": [
"read"
],
"platform/data/workloads/net-kingdom/identity-backup/retention-sibling": [
"deny"
],
"platform/data/workloads/railiance/backup/offsite-lane": [
"deny"
],
"platform/metadata/workloads/net-kingdom/identity-backup/retention": [
"deny"
]
},
"kv_version": 1,
"wrong_namespace_probe_attempted": {
"name": "net-kingdom-backup-retention-eso",
"namespace": "default"
},
"wrong_namespace_probe_cleanup_verified": true,
"exact_namespace_binding_verified": true,
"exact_policy_and_ttl_verified": true,
"wrong_service_account_denied": true,
"wrong_audience_denied": true,
"wrong_namespace_denied": true,
"metadata_sibling_escrow_and_token_create_denied": true,
"kv_write_denied": true,
"eso_delivery_equality_verified": true,
"temporary_reader_revoked": true,
"secret_values_exported": false,
"source_ccr": "CCR-2026-0049",
"ccr_sha256": "d391018abbda0fbc1acec4b5a22c0f99ce65525de940431df376a7552b3746cf"
}

View file

@ -0,0 +1,35 @@
{
"status": "provider_and_reader_prepared",
"action": "retention",
"caller_entity_id": "4a1f663c-0621-0483-edd0-f852274f4b6b",
"caller_auth_path": "auth/netkingdom/oidc/callback",
"quota_before": {
"quota_bytes": 10737418240,
"used_bytes": 7094108469,
"available_bytes": 3643309771
},
"folder_created": false,
"share_created": true,
"share_id": "29",
"share_permissions": 25,
"folder": "/railiance-backups/net-kingdom-identity",
"provider_test_artifacts": [
"nk49-boundary-6f5413c177894fdabdfb7828ef76f823.age",
"nk49-absent-abb34d9dafdc46e2babb5922b2d341d7.age"
],
"boundary_fixture_cleanup_verified": true,
"boundary_phase": "deny_other_account_folder",
"parent_probe_status": 400,
"parent_path_rejected": true,
"other_account_folder_status": 401,
"provider_boundary_verified": true,
"coding_agent_fence_verified_before_publication": true,
"kv_version": 1,
"coding_agents_denied": true,
"escrow_read": false,
"operator_password_delivered": false,
"retained_backups_deleted": false,
"source_ccr": "CCR-2026-0049",
"ccr_sha256": "d391018abbda0fbc1acec4b5a22c0f99ce65525de940431df376a7552b3746cf",
"eso_delivery_and_native_login_verification_pending": true
}

View file

@ -1,13 +1,13 @@
{
"status": "runtime_installed_provider_admission_waiting_attended_login",
"status": "actual_full_bundle_readback_verified_independent_recovery_pending",
"observed_at": "2026-10-15T23:56:00.293983+00:00",
"approval_decision": "40bfae40-6108-4e51-8613-073339f5345f",
"approved_platform_source_commit": "69a792c",
"first_owner_session": "child_failed_session_revoked",
"second_owner_session": "metadata_review_succeeded_session_revoked",
"new_identity_bundle_created": false,
"new_identity_bundle_created": true,
"identity_job_installed": true,
"new_credentials_published_to_openbao": false,
"new_credentials_published_to_openbao": true,
"existing_backup_changes": false,
"observed_permissions": 21,
"requested_permissions": 5,
@ -35,10 +35,10 @@
"publisher": 21,
"retention": 25
},
"focused_tests_passed": 52,
"focused_tests_passed": 56,
"temporary_dns_selector_exclusions_applied": true,
"network_policy_rule_lines_present": 649,
"installed_runtime_release": "bf8b38f185037515cfad9eebcddefc9f60ace724c32e10d89ae8307dc95a4eb7",
"installed_runtime_release": "758ba4d170ba5ed4b6c563d3a379e496c5f81e8bff28c45d142a6bacbfaa21bf",
"current_login_session_active": false,
"latest_login_failure": "callback_timeout_before_handoff",
"latest_timeout_possible_session_revocation_confirmed": false,
@ -46,6 +46,10 @@
"publisher_fixture_cleanup_verified": true,
"parent_path_rejection_status": 400,
"corrected_boundary_tests_passed": 16,
"corrected_native_boundary_and_delivery_not_yet_run": true,
"helper_network_allow_count": 0
"corrected_native_boundary_and_delivery_not_yet_run": false,
"helper_network_allow_count": 0,
"actual_provider_boundary_and_native_delivery_verified": true,
"actual_ciphertext_bytes": 109738344,
"actual_ciphertext_sha256": "7117b3fe1e4840c86b301c94c85e6b16709e5626bb0884bd0580bde630e30e30",
"actual_independent_key_recovery_pending": true
}

View file

@ -0,0 +1,29 @@
"""Exact task-only replacement of disabled NK49 units; preserve rollback bytes."""
import hashlib,importlib.util,json,os,subprocess,sys,tempfile
from pathlib import Path
os.umask(0o077)
root=Path(sys.argv[1]);old=Path(sys.argv[2]);out=Path(sys.argv[3])
expected_old='bb5579ce72967f0ce78e6e0afb361cfd5e3793768d3eb5b422718033505a0014'
assert hashlib.sha256(old.read_bytes()).hexdigest()==expected_old
previous=json.loads(old.read_text());packet=json.loads((root/'install-packet.json').read_text())
s=importlib.util.spec_from_file_location('installer',root/'tools/bootstrap-recovery/install-identity-backup.py');m=importlib.util.module_from_spec(s);s.loader.exec_module(m)
m.owner();assert packet==m.render(packet['decision_ref'])
assert packet['decision_ref']==previous['decision_ref']
for unit in ['net-kingdom-identity-backup.timer','net-kingdom-identity-backup-health.timer']:
for action,expected in [('is-enabled',b'disabled'),('is-active',b'inactive')]:
r=subprocess.run(['systemctl',action,unit],stdout=subprocess.PIPE,stderr=subprocess.DEVNULL,timeout=20);assert r.stdout.strip()==expected
paths={name:Path('/etc/systemd/system')/name for name in previous['units']}
assert set(paths)==set(packet['units'])
for name,path in paths.items():assert not path.is_symlink() and path.read_text()==previous['units'][name]
backup=Path(tempfile.mkdtemp(prefix='nk49-disabled-unit-rollback-',dir='/tmp'))
try:
for name,path in paths.items():path.rename(backup/name)
m.install(packet)
except BaseException:
for name,path in paths.items():
if (backup/name).exists():
assert not path.exists() or path.read_text()==packet['units'][name]
os.replace(backup/name,path)
subprocess.run(['systemctl','daemon-reload'],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,timeout=30)
raise
out.write_text(json.dumps({'status':'installed_timers_disabled','release_id':packet['release_id'],'previous_release':previous['release_id'],'rollback_units':str(backup)},indent=2)+'\n')