From bf77e378a8498051c5dade23403e1e9f4d3974f2 Mon Sep 17 00:00:00 2001 From: tegwick Date: Fri, 6 Nov 2026 10:55:12 +0100 Subject: [PATCH] Record native recovery success and prepare attended pilot backups Assistant: codex Assistant-Model: gpt-6.1-sol Assistant-Session: 01a1abf1-7e95-7141-87c9-ee78332bf790 --- ...6-11-06-vergabe-native-restore-return.json | 60 ++++ ...rgabe-native-restore-transport-return.json | 13 + .../2026-11-06-vergabe-operating-choice.json | 21 ++ .../README.md | 151 ++++++++++ .../plan.json | 52 ++++ .../validation.json | 15 + .../README.md | 22 +- .../README.md | 17 +- docs/vergabe-demo-company-acceptance.md | 48 ++- docs/vergabe-demo-company-binding.md | 47 +-- ...o-company-operating-acceptance.proposed.md | 58 ++++ tests/test_vergabe_pilot_attended.py | 283 ++++++++++++++++++ tools/vergabe_pilot_backup.py | 268 +++++++++++++++++ tools/vergabe_pilot_backup_node.py | 76 +++++ tools/vergabe_pilot_operating_restore.py | 120 ++++++++ tools/vergabe_pilot_point.py | 117 ++++++++ tools/vergabe_pilot_retention.py | 97 ++++++ .../RAPPS-WP-0014-vergabe-invited-pilot.md | 47 ++- 18 files changed, 1467 insertions(+), 45 deletions(-) create mode 100644 docs/evidence/2026-11-06-vergabe-native-restore-return.json create mode 100644 docs/evidence/2026-11-06-vergabe-native-restore-transport-return.json create mode 100644 docs/evidence/2026-11-06-vergabe-operating-choice.json create mode 100644 docs/proposals/2026-11-06-vergabe-attended-backups/README.md create mode 100644 docs/proposals/2026-11-06-vergabe-attended-backups/plan.json create mode 100644 docs/proposals/2026-11-06-vergabe-attended-backups/validation.json create mode 100644 docs/vergabe-demo-company-operating-acceptance.proposed.md create mode 100644 tests/test_vergabe_pilot_attended.py create mode 100644 tools/vergabe_pilot_backup.py create mode 100644 tools/vergabe_pilot_backup_node.py create mode 100644 tools/vergabe_pilot_operating_restore.py create mode 100644 tools/vergabe_pilot_point.py create mode 100644 tools/vergabe_pilot_retention.py diff --git a/docs/evidence/2026-11-06-vergabe-native-restore-return.json b/docs/evidence/2026-11-06-vergabe-native-restore-return.json new file mode 100644 index 0000000..3246536 --- /dev/null +++ b/docs/evidence/2026-11-06-vergabe-native-restore-return.json @@ -0,0 +1,60 @@ +{ + "schema": "railiance.vergabe-pilot-isolated-restore.v1", + "run_id": "20261105T233543Z-89bfbe04dece", + "nonce": "2d9a74da048c", + "namespace": "vergabe-restore-2d9a74da048c", + "status": "restored_and_verified", + "restore_verified": true, + "application_changed": false, + "production_application_changed": false, + "provider_retrieved_source": true, + "ciphertext_sha256": "4915288384e3ba8a87f1427b8e10106ee7241c04184adb97f439e57846bef76f", + "plaintext_packet_sha256": "791a2cf0af25d9e54ea37203c584e6aa5cb1b8a0d4d467a281947160a7bb32e0", + "application_image": "forgejo.coulomb.social/coulomb/vergabe-teilnahme@sha256:a26444f59c259698159c69ccb96f73dc648a261ece4c86bb2037a9d977870d91", + "postgres_image": "postgres@sha256:ff23cdce56cac62ada6f66013e1a50864c0abbe79d132d40b6e05bd80f378a70", + "native_sso_login_verified": false, + "helm_rollback_verified": false, + "stage": "complete", + "database_restored": true, + "all_postgres_table_digests_match": true, + "database_tables": 39, + "database_rows": 247, + "database_logical_sha256": "33191c43c437f1b05da410a3f22989c86ce50f4fca8fd32d40fbb327b8b73765", + "all_media_hashes_match": true, + "sqlite_integrity_and_rows_match": true, + "local_issue_link_verified": true, + "restored_files_uid_gid_verified": true, + "media_files": 1, + "sqlite_tables": 9, + "sqlite_rows": 7, + "two_restored_members_verified": true, + "restored_identity_mappings": 2, + "member_document_checksums_match": true, + "cross_member_task_edit_verified": true, + "anonymous_pages_and_binary_denied": true, + "inactive_existing_session_denied": true, + "csrf_write_denied": true, + "workflow_test_transaction_rolled_back": true, + "captured_tender_status": 10, + "captured_tender_archived": false, + "captured_status_in_dashboard_range": false, + "outgoing_integrations_enabled": false, + "restore_elapsed_seconds": 55.41, + "configuration_rollback_verified": true, + "committed_clone_write_preserved": true, + "pod_replacements": 2, + "rollback_same_image_and_schema": true, + "older_image_or_schema_downgrade_verified": false, + "configuration_inverse_elapsed_seconds": 55.24, + "separate_fresh_claims": true, + "network_policy_default_deny": true, + "application_image_cached_no_pull": true, + "namespace_and_volumes_removed": true, + "production_spec_and_readiness_unchanged": true, + "elapsed_seconds": 215.0, + "evidence_kind": "user_reported_terminal_output", + "reported_on": "2026-11-06", + "workplan": "RAPPS-WP-0014", + "task": "RAPPS-WP-0014-T03", + "source_revision": "1a35ee81132537c3461f46cc495e56a15dc1eae5" +} diff --git a/docs/evidence/2026-11-06-vergabe-native-restore-transport-return.json b/docs/evidence/2026-11-06-vergabe-native-restore-transport-return.json new file mode 100644 index 0000000..dee8a63 --- /dev/null +++ b/docs/evidence/2026-11-06-vergabe-native-restore-transport-return.json @@ -0,0 +1,13 @@ +{ + "schema": "railiance.vergabe-pilot-native-restore-transport.v1", + "nonce": "2d9a74da048c", + "run_id": "20261105T233543Z-89bfbe04dece", + "worker_exit_code": 0, + "node_stage_removed": true, + "status": "complete", + "evidence_kind": "user_reported_terminal_output", + "reported_on": "2026-11-06", + "workplan": "RAPPS-WP-0014", + "task": "RAPPS-WP-0014-T03", + "source_revision": "1a35ee81132537c3461f46cc495e56a15dc1eae5" +} diff --git a/docs/evidence/2026-11-06-vergabe-operating-choice.json b/docs/evidence/2026-11-06-vergabe-operating-choice.json new file mode 100644 index 0000000..4e4a9a4 --- /dev/null +++ b/docs/evidence/2026-11-06-vergabe-operating-choice.json @@ -0,0 +1,21 @@ +{ + "schema": "railiance.vergabe-pilot-operating-choice.v1", + "reported_on": "2026-11-06", + "evidence_kind": "user_question_reply", + "workplan": "RAPPS-WP-0014", + "task": "RAPPS-WP-0014-T03", + "answer": "Attended pilot backups (Recommended)", + "selected_model": "attended", + "backup_execution_and_incident_contact": "Bernd Worsch", + "cadence": "after each active pilot day and before changes", + "provider_retention": { + "daily": 7, + "weekly": 2, + "baseline_and_newest_protected": true + }, + "data_loss_target": "at most one active pilot day", + "recovery_target": "attended within one working day", + "scheduled_service": false, + "operating_path_native_verified": false, + "note": "Targets presented in the question; selected without amendments. Original baseline recovery already passes; fresh repeatable operating run remains pending." +} diff --git a/docs/proposals/2026-11-06-vergabe-attended-backups/README.md b/docs/proposals/2026-11-06-vergabe-attended-backups/README.md new file mode 100644 index 0000000..9aa8279 --- /dev/null +++ b/docs/proposals/2026-11-06-vergabe-attended-backups/README.md @@ -0,0 +1,151 @@ +# Attended demo-company backups + +The founder selected attended end-of-active-day and pre-change backups, with +himself as backup execution and first incident/support contact. The accepted +[operating targets](../../vergabe-demo-company-operating-acceptance.proposed.md) +are seven daily/two weekly verified provider points, at most one active pilot +day's data loss and attended recovery within one working day. No scheduled +service, notification worker or SLA is introduced. + +The [baseline restore](../../evidence/2026-11-06-vergabe-native-restore-return.json) +passes. This packet makes fresh points repeatable without changing the frozen +acceptance tools. [Exact scope and code hashes](plan.json). + +## First operating run — founder terminal + +```bash +git pull --ff-only +python3 tools/vergabe_pilot_backup.py --check +python3 tools/vergabe_pilot_backup.py --execute --verify-restore +``` + +The check is read-only/server dry-run. Execute briefly pauses only the existing +demo-company application for a coherent new PostgreSQL/media/SQLite capture, +then resumes and verifies it before sealing or transfer. It creates a unique +UTC run ID and refuses to overwrite an earlier point. The existing root-private +capture journal, operation lock, deployment/claim identity checks and guarded +resume are reused. An unresolved older capture refuses a new pause. + +The second command also uploads one new encrypted object and creates/removes an +isolated restore namespace with three fresh claims. Founder execution is the go +for these exact runtime changes under `the-custodian/docs/agent-environment-orientation.md` +section 0. No separate approval machinery or credential grant is added. + +Encryption occurs on the source node for the existing platform age recipient. +Only ciphertext and its safe transfer metadata are fetched to the private +workstation kit. The existing contained OpenBao owner login then performs the +same create-only Nextcloud upload, independent Backup-account readback, escrow +decryption and full packet/media/SQLite verification that passed for baseline. +Token aliases are cleared, owner helper pins/policies are checked, and Warden +revokes/removes the session. No credential or backup content is printed. + +With `--verify-restore`, that newly retrieved/decrypted point is then restored +through the already-proved isolated worker: all PostgreSQL row digests and file/ +SQLite fingerprints, both restored ordinary member sessions, document/task +workflow and access denials, same-image configuration inverse preserving a +committed clone write, UID-guarded namespace/PV deletion and production readback. +The private node stage is removed on success. Native SSO/callback/ingress recovery +and older-image/schema/Helm rollback remain untested. This is a rehearsal on the +healthy current cluster with the admitted app image cached, not a node-rebuild +or unattended disaster-recovery demonstration. + +Allow several minutes plus interactive SSH/sudo and OpenBao login. Keep the run +ID and any restore nonce. Success ends with `railiance.vergabe-pilot-attended-backup.v1`, +`status: complete`, publication/readback/decryption/session cleanup true and +`restore_verified: true` for this first run. Return the sanitized receipts. +The isolated worker's time is not end-to-end RTO or a live outage measurement. + +## Each active pilot day and before changes + +```bash +python3 tools/vergabe_pilot_backup.py --check +python3 tools/vergabe_pilot_backup.py --execute +python3 tools/vergabe_pilot_backup.py --retention +``` + +The routine backup omits a fresh restore rehearsal and reports +`restore_verified: false`; it still requires provider readback/decryption and +owner session cleanup. Repeat `--verify-restore` after a change to the operating +path or as an attended recovery drill. Reconcile the new point's success receipt +before ending the working session. If it fails, treat the prior successful point +as the recovery boundary and report its age; do not assume that capture or +workstation transfer alone satisfied the day's backup. + +This path is intentionally bound to the admitted image, Helm revision 4, chart +0.2.1, cluster, database and claims. A release/placement change requires review +and updated binding first. Fixture tender/task/document 1 and its local issue +must remain available. Safety limits remain 1 MiB per captured packet member, +4 MiB encrypted/decrypted packet, 128 MiB inflated stores and 100,000 database +rows. Growth beyond those limits refuses; review sizing before using larger data. + +## Interrupted steps and recovery + +Use the same printed run ID; do not recapture simply because publication failed: + +```bash +python3 tools/vergabe_pilot_backup.py --resume +python3 tools/vergabe_pilot_backup.py --fetch +python3 tools/vergabe_pilot_backup.py --publish --verify-restore +python3 tools/vergabe_pilot_backup.py --restore +python3 tools/vergabe_pilot_backup.py --restore-cleanup +``` + +Resume is only for an interrupted application pause: it requires the active +journal's matching run and unchanged deployment/claims, then restores readiness +without accepting an incomplete capture. Fetch reseals only a completed capture +and fetches its encrypted pair. Publish retries the same deterministic object; +identical existing ciphertext is read back, different bytes refuse. Restore uses +a successful fully cleaned-up provider receipt and checks its fetched ciphertext, +decrypted ZIP and every recovered file before staging; no extra key retrieval or +provider login is needed. Restore cleanup only removes the journal-owned clone +and stage and never claims data acceptance. + +Restoring into or cutting over the production instance is a separate attended +incident change: retain current data, inspect image/schema compatibility and +prepare the exact database/claim/route change before executing. The rehearsal +never overwrites production or repoints its SSO callback. Production passwords, +runtime Secrets and the recovery identity are not copied into the clone. + +## Provider retention + +`--retention` previews points from private workstation kits with matching +ciphertext and complete independent-readback/decryption/session-cleanup receipts. +It selects the latest point for each of seven days and two ISO weeks; daily and +weekly representatives can overlap. Baseline and newest remain protected. +Failed, missing, unknown or unrelated objects are never deletion candidates. + +Review the printed candidate list, then execute separately when candidates exist: + +```bash +python3 tools/vergabe_pilot_backup.py --prune +``` + +This is a separate founder go for provider deletion and opens a contained owner +session. Before any delete it independently retrieves every retained and candidate +ciphertext and compares exact size/hash. The proven baseline's fixed ciphertext +must match. Deletes require a strong ETag with `If-Match`, followed by 404 readback. +Each deletion gets a private local tombstone so later inventories exclude removed +objects. The batch is capped at 50 deletes; no provider-wide listing or unknown +object adoption is performed. Backup execution itself never deletes. + +Interruption between provider deletion and the durable local tombstone refuses +the next batch on the missing object; reconcile from owner receipts before any +further deletion. A changed/missing retained point also stops pruning. The +operator checks quota/last verified point and retains the baseline until an +explicit later review; no automatic unpin is introduced. Local encrypted kits, +private recovered data and root node capture/journal copies remain private and +are not removed by provider retention. Their removal requires owner review of +the verified inventory; they are not an independent offsite destination. + +## Validation and closure + +The original capture, crypto/provider and native restore implementations are +unchanged and source-pinned. Synthetic tests cover distinct fresh points through +real age encryption/decryption and mocked provider round-trips, file/metadata +tampering, incomplete capture and failed cleanup exclusion, selected-point +restore verification, embedded source compilation and founder boundary, failure +sequencing, and retention protection/hash/conditional-delete races. + +No agent reads the founder's real private kit or executes this native credential +path. RAPPS-WP-0014-T03 remains waiting for the first operating run above; the +existing baseline's technical proof and accepted targets are already recorded. diff --git a/docs/proposals/2026-11-06-vergabe-attended-backups/plan.json b/docs/proposals/2026-11-06-vergabe-attended-backups/plan.json new file mode 100644 index 0000000..79e7936 --- /dev/null +++ b/docs/proposals/2026-11-06-vergabe-attended-backups/plan.json @@ -0,0 +1,52 @@ +{ + "schema": "railiance.vergabe-pilot-attended-backup-plan.v1", + "prepared_on": "2026-11-06", + "workplan": "RAPPS-WP-0014", + "target": "railiance01", + "cluster_uid": "a553c742-0115-43d4-99a4-a5ca56fe0786", + "helm_revision": 4, + "image": "forgejo.coulomb.social/coulomb/vergabe-teilnahme@sha256:a26444f59c259698159c69ccb96f73dc648a261ece4c86bb2037a9d977870d91", + "sources": { + "vergabe_pilot_backup": "ba9a33d50ed63cbf9f10b92c7b1eae815c3beb82753c05a1cbfb3b6a7a321354", + "vergabe_pilot_backup_node": "b12474806cab5c440c60ef9f1bfc90344a02f2d34ddc16e83da746ab2ee2b39f", + "vergabe_pilot_export": "b45733fa10ae59b56563045c43c53b5f824a74e1114f68fbed28404f14b43e3a", + "vergabe_pilot_export_remote": "11c32dc2011dc96f604cebd7681cb4aa27302d90b1d80f765c94e4dddc31c9d5", + "vergabe_pilot_operating_restore": "14072a12873d27827b815295fb45b2250d18414740df852c07fc4ca700049eb8", + "vergabe_pilot_point": "676191b8ade0e4dd0e6607341a0398d52ce909d2e27d1fb8d43ac8f4f0e003a3", + "vergabe_pilot_publish": "4340f386e448984b2b34d13d59f318ca4dda35b218a2b28cd034dcd00881244e", + "vergabe_pilot_recovery_store": "724a2dd9db8a0f57e1efa709ec989d8a47c681a86b1fca923bfdfea1698ae23c", + "vergabe_pilot_restore": "407df786ce8994744df7079771ddadf56019f2d384d4b6cbda8dc8d051270dd4", + "vergabe_pilot_restore_remote": "cda578e169199f7e319423b41786f2baf9cd7e7a992f1dda3f19bb134f69ad91", + "vergabe_pilot_retention": "3945b5c565b42e74db901a8d2aa240fd940e384076ace61f4c47fb5f16e6f880", + "vergabe_pilot_snapshot": "35c3aa9906a17c82867541d27f41456b75b86f6fda0c87f6dfda5a63985ff796" + }, + "capture_mutations": [ + "briefly pause only vergabe-demo-company/vergabe-teilnahme", + "temporary read-only capture Pod on two existing claims", + "resume exact deployment and verify fixture/readiness" + ], + "provider_actions": [ + "create-only unique encrypted point", + "independent readback and escrow decryption in contained founder session" + ], + "prune_actions": [ + "preview only by default with --retention", + "--prune separately opens contained owner session", + "verify retained and candidate ciphertext before any deletion", + "If-Match ETag guarded delete and 404 readback", + "protect baseline and newest; ignore unknown objects" + ], + "scheduled_service": false, + "automatic_deletion_during_backup": false, + "limits": { + "capture_files_bytes_each": 1048576, + "encrypted_packet_bytes": 4194304, + "store_inflation_bytes": 134217728, + "database_rows": 100000, + "provider_delete_batch": 50 + }, + "execution": "Founder terminal via existing SSH/sudo and contained OpenBao login; no agent execution", + "isolated_restore": "--restore RUN reuses the proven same-image fresh-claim restore from that successful provider-retrieved point; --restore-cleanup RUN NONCE is guarded cleanup only", + "first_execution": "python3 tools/vergabe_pilot_backup.py --execute --verify-restore", + "routine_execution": "python3 tools/vergabe_pilot_backup.py --execute" +} diff --git a/docs/proposals/2026-11-06-vergabe-attended-backups/validation.json b/docs/proposals/2026-11-06-vergabe-attended-backups/validation.json new file mode 100644 index 0000000..c3cb347 --- /dev/null +++ b/docs/proposals/2026-11-06-vergabe-attended-backups/validation.json @@ -0,0 +1,15 @@ +{ + "schema": "railiance.vergabe-pilot-attended-backup-validation.v1", + "unit_tests": 76, + "new_attended_cases": 17, + "unit_status": "passed", + "lint": "passed", + "native_new_operating_run_executed": false, + "original_capture_export_publish_restore_sources_unchanged": true, + "synthetic_real_age_two_distinct_points": true, + "provider_mocked": true, + "retention_baseline_newest_protection": true, + "retained_bytes_prechecked_before_delete": true, + "conditional_delete_race_refused": true, + "native_backup_or_credential_read": false +} diff --git a/docs/proposals/2026-11-06-vergabe-native-restore/README.md b/docs/proposals/2026-11-06-vergabe-native-restore/README.md index c1e729b..0bcb232 100644 --- a/docs/proposals/2026-11-06-vergabe-native-restore/README.md +++ b/docs/proposals/2026-11-06-vergabe-native-restore/README.md @@ -6,6 +6,21 @@ escrow decryption, packet/media/SQLite verification and successful Warden cleanu The point is `20261105T233543Z-89bfbe04dece`, ciphertext `49152883…6bef76f`. [Exact scope and source pins](plan.json) · [synthetic validation](validation.json). +Execution succeeded under nonce `2d9a74da048c`: the +[worker receipt](../../evidence/2026-11-06-vergabe-native-restore-return.json) +reports `restored_and_verified`, all 39 tables / 247 rows and both file stores +matching, two restored member sessions/workflow/denials passing, committed clone +writes preserved through the configuration inverse, and namespace/PV cleanup +with production specification/readiness unchanged. Total worker time was +215 seconds; restore/checks 55.41 seconds, inverse/recheck 55.24 seconds. +The [transport receipt](../../evidence/2026-11-06-vergabe-native-restore-transport-return.json) +reports `complete`, exit 0 and private staging removed. Captured tender status 10 +(Gewonnen), not archived, explains its exclusion from the overview's status range. +The command below is retained for reproducibility; no rerun is needed for this +point. [Operating acceptance](../../vergabe-demo-company-operating-acceptance.proposed.md) +now records the selected attended model and prepared repeatable operating path. +Native SSO/ingress and older-image/Helm rollback remain untested. + ## Founder workstation command ```bash @@ -131,9 +146,10 @@ pass a local synthetic three-store restore (39 tables / 202 rows): full table an file/SQLite fingerprints, both restored member sessions, edit/download and denial checks, transaction preservation, two database restarts and the bad-configuration inverse retaining a committed write. The PostgreSQL query format matches capture. -Those tests use no native backup, credential or Kubernetes mutation. Native -scheduling/admission/claims still need the founder execution receipt. +Those synthetic tests use no native backup, credential or Kubernetes mutation. +The founder's execution receipt above now also proves native scheduling, +admission, fresh claims, complete data recovery and guarded cleanup for this run. -RAPPS-WP-0014-T03 remains open for that receipt and operating acceptance: named +RAPPS-WP-0014-T03 remains open for operating acceptance: named backup/support owners, cadence, retention, incident handling and accepted RPO/RTO; any broader restored SSO/ingress or version-rollback requirement must be explicit. diff --git a/docs/proposals/2026-11-06-vergabe-provider-publication/README.md b/docs/proposals/2026-11-06-vergabe-provider-publication/README.md index cedd354..4ac7503 100644 --- a/docs/proposals/2026-11-06-vergabe-provider-publication/README.md +++ b/docs/proposals/2026-11-06-vergabe-provider-publication/README.md @@ -7,8 +7,10 @@ kit is already on the founder's workstation. [Exact scope and source pins](plan. Execution succeeded: [the native provider receipt](../../evidence/2026-11-06-vergabe-provider-recovery-return.json) confirms upload 201, independent readback 200, decryption and packet/media/SQLite -verification, Warden exit 0 and session cleanup. Next is the -[isolated native restore/inverse rehearsal](../2026-11-06-vergabe-native-restore/README.md). +verification, Warden exit 0 and session cleanup. The subsequent +[isolated native restore/inverse rehearsal](../2026-11-06-vergabe-native-restore/README.md) +also passes, including all stores, restored member workflow/access denials, +current-write-preserving configuration inverse and cleanup. The publication command below is retained for reproducibility. ## Earlier attempts @@ -132,9 +134,10 @@ isolated restore. The directory is 0700 and files 0600, outside Git; do not open or paste backup contents. The decryption identity is not stored with them. The operation's safe `provider-receipt-*.json` is persisted in that kit. -The receipt keeps `restore_verified: false`: an isolated PostgreSQL restore and -restored application workflow, access refusal, compatible rollback and agreed -backup/support ownership, cadence, retention and RPO/RTO remain T03 requirements. +This publication-only receipt keeps `restore_verified: false`; the separate +native restore receipt now supplies database/workflow/access/configuration-inverse +proof. Agreed backup/support ownership, repeatable cadence, retention and RPO/RTO +remain T03 requirements. No second application pause or recapture is needed for this point. ## Validation @@ -148,5 +151,5 @@ The additional cases test all token aliases, API/effective-policy handling, separate session outcomes and the installed Warden envelope using an offline fake Bao client. They issue no native session and inspect no real helper or credential. Both existing URL forms pass the synthetic publication/readback/ -decryption flow. All 47 pilot helper checks pass. Native provider publication and -decryption remain unverified until a successful founder receipt returns. +decryption flow. All 47 pilot helper checks passed at publication preparation; +the successful founder receipt above supplies native publication/decryption proof. diff --git a/docs/vergabe-demo-company-acceptance.md b/docs/vergabe-demo-company-acceptance.md index e992f26..9c4c27e 100644 --- a/docs/vergabe-demo-company-acceptance.md +++ b/docs/vergabe-demo-company-acceptance.md @@ -9,8 +9,9 @@ The user confirmed the uploaded file is available from the other account; see the [document return](evidence/2026-11-05-vergabe-document-return.json). The document UI still needs user review. The November 6 native check verifies the stored file matches the original PDF checksum and its anonymous binary URL -redirects to login. A second-account downloaded-file hash comparison remains -unreported. The user also confirms the **LOKAL** issue badge is +redirects to login. The later isolated restore verifies both restored members' +downloaded bytes; a live browser-downloaded hash return remains unreported. +The user also confirms the **LOKAL** issue badge is visible in both accounts; the [issue return](evidence/2026-11-05-vergabe-local-issue-return.json) binds Ausschreibung 1, task 1 and document 1 from their supplied URLs. The user now confirms cross-account task editing and the flow through documented @@ -34,10 +35,11 @@ the helper was removed. Total operation time was 98.61 seconds, including resume/postflight, rather than a separately measured outage duration. The [human restart return](evidence/2026-11-06-vergabe-post-restart-user-return.json) confirms both accounts work and everything else appears as before, except the -tender is absent from Übersicht. The deployed dashboard lists only status 1–9; -final statuses 10–13 would explain exclusion. The actual current status is not -yet reported, so this remains a UI follow-up. At that stage, off-host publication -and isolated restore were unverified. +tender is absent from Übersicht. The restored captured tender is status 10 +(Gewonnen), not archived; the deployed dashboard lists only status 1–9. This +explains its exclusion in the captured state. It is a discoverability follow-up, +not evidence of lost tender data. No later live status read is inferred. +At that stage, off-host publication and isolated restore were unverified. The [encrypted workstation copy](evidence/2026-11-06-vergabe-encrypted-workstation-copy-return.json) now passes: 182521 bytes, hash matching the node handoff. The subsequent @@ -46,11 +48,27 @@ executed without another app pause. That [provider execution now passes](evidence/2026-11-06-vergabe-provider-recovery-return.json): create-only publication, independent readback, escrow decryption, file/SQLite -verification and Warden cleanup all succeed. PostgreSQL has not yet been restored. -The [native isolated restore packet](proposals/2026-11-06-vergabe-native-restore/README.md) -prepares fresh claims and a same-image configuration inverse, with explicit -restored-session versus native-SSO limits. Native execution and operating -acceptance remain open. +verification and Warden cleanup all succeed. + +The [native isolated restore](evidence/2026-11-06-vergabe-native-restore-return.json) +now also passes from that provider-retrieved point: all 39 PostgreSQL tables / +247 rows, the PDF and SQLite's nine tables / seven rows match capture. Two +restored ordinary member sessions can read/download/edit across accounts; +anonymous, inactive-session and missing-CSRF requests are denied. Two Pod +replacements demonstrate the same-image configuration inverse retains a +committed clone write. Fresh namespace/PVs and private node staging are removed; +production specification and readiness are unchanged. The +[transport receipt](evidence/2026-11-06-vergabe-native-restore-transport-return.json) +confirms exit 0 and complete cleanup. + +Restore/checks took 55.41 seconds; the configuration inverse/recheck took 55.24 +seconds; total worker time was 215 seconds. Workstation authentication/transfer +are outside that timing. Restored sessions use Django test-client login, so +native SSO/MFA/ingress recovery remains untested. No older-image/schema downgrade +or Helm rollback is claimed. The [review packet](proposals/2026-11-06-vergabe-native-restore/README.md) +retains the exact scope. [Operating acceptance](vergabe-demo-company-operating-acceptance.proposed.md) +records the founder-selected attended model and targets. The repeatable fresh +backup/retention path is prepared; its first operating run remains open. The other session does not update automatically, and a fresh login was needed after a short interval. The user wants a full working day without renewed login @@ -149,6 +167,8 @@ and omit both file stores. They cannot establish this three-store recovery claim The two-account functional walkthrough is user-confirmed with UI follow-up. The attended capture/restart action has executed successfully, with automated and human restart checks (overview follow-up above). Off-host publication, -isolated restore and rollback remain outstanding. -Logged-in account deactivation remains a separate owner-run access-control check -from the pilot contract. No account is disabled as part of these walkthrough steps. +isolated restore and the same-image configuration inverse now pass. Deactivation +of an already-authenticated restored member was verified transactionally in the +clone; live accounts were unchanged. Native SSO/ingress recovery and any broader +version-rollback claim remain outside that proof. Named operating responsibilities, +repeatable cadence/retention and accepted recovery targets still need completion. diff --git a/docs/vergabe-demo-company-binding.md b/docs/vergabe-demo-company-binding.md index 638fd84..99cc28f 100644 --- a/docs/vergabe-demo-company-binding.md +++ b/docs/vergabe-demo-company-binding.md @@ -3,24 +3,30 @@ Updated 2026-11-06 under RAPPS-WP-0014 and VERGABE-WP-0019. Release binding is complete (RAPPS-WP-0014-T02). Recovery and invited-recipient acceptance remain open. -November 6 recovery update: the founder's matched PostgreSQL/media/SQLite capture -and app resume pass at revision 4; the full operation took 98.61 seconds. Both -accounts work after restart, with an overview-list UI follow-up. See the -[capture receipt](evidence/2026-11-06-vergabe-three-store-capture-return.json). -Provider publication/readback, isolated restore, rollback/access and operating -ownership remain open. The accepted point can be transferred without another -pause using the [encrypted export packet](proposals/2026-11-06-vergabe-recovery-export/README.md). -That export now passes: [the founder's workstation receipt](evidence/2026-11-06-vergabe-encrypted-workstation-copy-return.json) -verifies 182521 encrypted bytes, SHA256 `49152883…6bef76f`, outside the production -node. Provider publication, escrow decryption and restore remain open; the -[contained owner handoff](proposals/2026-11-06-vergabe-provider-publication/README.md) -is prepared. +November 6 recovery update: matched PostgreSQL/media/SQLite capture and app +resume pass at revision 4 (98.61 seconds total). Both accounts work after +restart. The captured tender is status 10 (Gewonnen), outside the overview's +1–9 range; it remains present and not archived. See the +[capture](evidence/2026-11-06-vergabe-three-store-capture-return.json) and +[restore](evidence/2026-11-06-vergabe-native-restore-return.json) receipts. +The [encrypted workstation copy](evidence/2026-11-06-vergabe-encrypted-workstation-copy-return.json) +verifies 182521 bytes, SHA256 `49152883…6bef76f`, outside the production node. Provider publication/readback/decryption is now verified by the [founder receipt](evidence/2026-11-06-vergabe-provider-recovery-return.json): 201/200, correct ciphertext and packet/store hashes, Warden cleanup confirmed. -The [isolated native restore/inverse packet](proposals/2026-11-06-vergabe-native-restore/README.md) -is prepared; its runtime evidence and operating acceptance remain open. +Native isolated recovery now passes for all 39 PostgreSQL tables / 247 rows, +media and issue-state fingerprints, two restored member sessions/workflow and +access denials. The same-image configuration inverse preserves committed clone +writes across two Pod replacements. Worker time was 215 seconds; restore/checks +55.41 seconds, inverse/recheck 55.24 seconds. Namespace/PVs and private staging +were removed, with production specification/readiness unchanged. See the +[restore scope](proposals/2026-11-06-vergabe-native-restore/README.md) and +[transport receipt](evidence/2026-11-06-vergabe-native-restore-transport-return.json). +Restored native SSO/ingress and older-image/schema/Helm rollback are untested. +[Operating acceptance](vergabe-demo-company-operating-acceptance.proposed.md) +now selects attended backups and named targets. The repeatable fresh-point +backup/retention path is prepared and awaits its first operating run. | Item | Current binding | | --- | --- | @@ -125,8 +131,10 @@ bootstrap helpers or substitute apps_admin credentials. Revision 4 is the current binding; revisions 1 through 3 are historical deployment receipts, not demonstrated safe rollback targets. On a failed next upgrade, stop public traffic and reconcile schema/image compatibility before reverting. Retain the -claims and database; namespace deletion is not rollback. Pod replacement and -isolated restore must still be proven with matched PostgreSQL/media/SQLite data. +claims and database; namespace deletion is not rollback. Populated restart, +isolated three-store recovery and a same-image configuration inverse are now +proved by the November 6 receipts. Those receipts do not admit an older image +or incompatible schema downgrade. Current recovery rollout: docs/evidence/2026-09-12-account-recovery-live.md. @@ -135,9 +143,10 @@ Current recovery rollout: docs/evidence/2026-09-12-account-recovery-live.md. Read-only deployment, Helm, claim, certificate and HTTPS checks match this binding; see [the verification receipt](evidence/2026-09-27-vergabe-binding-review.md). -The September 25 database restore and pod restart passed, but populated media +At that review, the September 25 database restore and pod restart passed, but populated media and issue-state recovery, an off-host copy, restored two-user workflow and rollback/support acceptance remain RAPPS-WP-0014-T03. That task is waiting and -the workplan is blocked. Earlier dated sections describe their original -snapshots; the current binding table and this review supersede their pending +the workplan was blocked. The November 6 update supersedes that recovery wait; +operating acceptance remains open. Earlier dated sections describe their original +snapshots; the current binding table and latest update supersede their pending DNS, deployment and first-login statements. diff --git a/docs/vergabe-demo-company-operating-acceptance.proposed.md b/docs/vergabe-demo-company-operating-acceptance.proposed.md new file mode 100644 index 0000000..64864cc --- /dev/null +++ b/docs/vergabe-demo-company-operating-acceptance.proposed.md @@ -0,0 +1,58 @@ +# Demo-company operating acceptance — attended model selected + +Prepared 2026-11-06 for RAPPS-WP-0014-T03. The founder selected the attended model +with the contacts, cadence, retention and recovery targets below. The repeatable +operating path is prepared and tested; its first native run remains open. +No scheduled service or SLA is inferred. + +The populated three-store recovery drill now passes from the independently +retrieved Nextcloud object. The [native receipt](evidence/2026-11-06-vergabe-native-restore-return.json) +records 55.41 seconds for restore/checks, 55.24 seconds for the configuration +inverse/recheck and 215 seconds including temporary resources and cleanup. +Workstation authentication/transfer and native SSO/ingress recovery are outside +those timings. The clone used the admitted image and schema; an earlier image +or schema downgrade is not an admitted recovery procedure. + +## Accepted attended pilot arrangement + +| Item | Accepted target | +| --- | --- | +| Scope | The current small invited demo pilot, with the existing single replica and planned maintenance interruptions | +| Backup execution and first incident/support contact | Bernd Worsch, using his own attended operator lane | +| Provider/storage custody route | Existing railiance-platform Nextcloud Backup/OpenBao recovery lane; no new credentials or grants | +| Cadence | A fresh coherent off-host point at the end of every active pilot day and before deployment/data changes | +| Retention | Keep seven daily and two weekly verified provider points (representatives may overlap); protect newest and the successful baseline, with later baseline replacement requiring explicit review | +| Data-loss objective | At most one active pilot day's work, conditional on that day's verified backup completing | +| Recovery objective | Attended recovery within one working day after the operator receives the incident; an accepted target, not an SLA inferred from this drill | +| Failure handling | A missing/failed backup or provider verification is an incident for the named operator; report the last verified point's age before further changes | +| Upgrade handling | Inspect migrations and image/schema compatibility before each upgrade, make a fresh point first, and retain current writes; historical Helm revision alone is insufficient | + +The [attended operating packet](proposals/2026-11-06-vergabe-attended-backups/README.md) +prepares `tools/vergabe_pilot_backup.py --execute` for each active pilot day and +before changes. It reuses the frozen capture/encryption/provider algorithms, +selects a new run with validated metadata/hashes, and requires complete provider +readback/decryption/session cleanup. `--verify-restore` adds the already-proved +isolated worker for that new point. Seventeen new synthetic cases pass; all +76 pilot helper tests pass. The first attended operating run must still pass +before the cadence is recorded active. The baseline's exact-point helpers remain +unchanged for reproducibility. + +Provider retention has a bounded private-inventory preview and a separate +contained owner executor: protect baseline/newest, independently verify all +retained/candidate bytes, ETag-guard deletes and require absence readback. Unknown +objects are excluded. No retention deletion, schedule or notification worker has +been executed/installed. Local private recovery copies and node journals remain; +provider retention does not remove them. Use only the reviewed founder command +as the go for a deletion batch, as specified in the packet. + +## Work still to finish + +Execute the first fresh attended backup with `--verify-restore` and return the +sanitized backup/worker/transport receipts. Review provider retention previews +and execute only when eligible excess points exist; no artificial deletion is +needed to accept an empty retention plan. +Restored interactive SSO/callback/ingress remains untested; the isolated drill +proved restored ordinary member sessions and access denials. Any required native +SSO recovery demonstration must use an admitted isolated callback rather than +changing the live product callback. Broader recipient acceptance remains with +VERGABE-WP-0019. diff --git a/tests/test_vergabe_pilot_attended.py b/tests/test_vergabe_pilot_attended.py new file mode 100644 index 0000000..6e1e39e --- /dev/null +++ b/tests/test_vergabe_pilot_attended.py @@ -0,0 +1,283 @@ +"""Synthetic fresh-point/retention boundaries; no native backup or credentials.""" + +import datetime as dt +import hashlib +import io +import json +import shlex +import shutil +import subprocess +import sys +import unittest +from contextlib import redirect_stdout +from pathlib import Path +from unittest.mock import patch + +import test_vergabe_pilot_publish as fixtures + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tools")) +import vergabe_pilot_backup as backup +import vergabe_pilot_export as export +import vergabe_pilot_operating_restore as recovery +import vergabe_pilot_point as point +import vergabe_pilot_publish as publisher +import vergabe_pilot_retention as retention +from vergabe_pilot_recovery_store import Refused + + +class ScopeTests(unittest.TestCase): + def test_dates_and_shell_injection_refuse_before_transport(self): + for value in ["../../other", "20261105T000000Z-$(cat /etc/passwd)", + "20269905T233543Z-0123456789ab", "20991105T233543Z-0123456789ab"]: + with self.subTest(value=value), patch.object(backup, "source_pins") as source, self.assertRaises(Refused): + backup.remote_command(value, "capture") + source.assert_not_called() + + def test_embedded_node_sources_compile_without_native_access(self): + command = backup.remote_command("20261105T233543Z-0123456789ab", "check") + operation = shlex.split(command[-1]) + self.assertEqual(operation[:5], ["sudo", "/usr/bin/env", "KUBECONFIG=/etc/rancher/k3s/k3s.yaml", "/usr/bin/python3", "-c"]) + result = subprocess.run([sys.executable, "-c", operation[5], "--help"], capture_output=True, + timeout=10, check=False, cwd="/tmp") + self.assertEqual(result.returncode, 0) + self.assertIn(b"capture", result.stdout) + + def test_changed_source_pin_refuses_before_ssh(self): + body = json.loads(backup.PLAN.read_text()) + body["sources"]["vergabe_pilot_point"] = "0" * 64 + with patch.object(backup.Path, "read_text", return_value=json.dumps(body)), self.assertRaises(Refused): + backup.source_pins() + + def test_selected_restore_bootstrap_compiles_and_refuses_nonfounder_without_native_access(self): + command = recovery.remote_command(backup.source_pins(), "20261105T234001Z-0123456789ab", "0123456789ab") + operation = shlex.split(command[-1]) + result = subprocess.run([sys.executable, "-c", operation[5], *operation[6:]], + capture_output=True, timeout=10, check=False, cwd="/tmp", env={"PATH": "/usr/bin:/bin"}) + self.assertEqual(result.returncode, 1) + self.assertEqual(json.loads(result.stdout)["failure"], "founder_node_sudo_required") + self.assertEqual(result.stderr, b"") + + def test_no_restore_transport_after_failed_owner_publication(self): + with patch.object(backup.sys, "argv", ["backup", "--publish", "20261105T234001Z-0123456789ab", "--verify-restore"]), \ + patch.object(backup.sys.stdin, "isatty", return_value=True), patch.object(backup, "source_pins"), \ + patch.object(backup, "configure"), patch.object(backup, "owner_action", side_effect=Refused("fixture")), \ + patch.object(recovery, "execute") as restore, redirect_stdout(io.StringIO()), self.assertRaises(Refused): + backup.main() + restore.assert_not_called() + + def test_capture_failure_never_fetches_or_opens_owner_login(self): + with patch.object(backup.sys, "argv", ["backup", "--execute"]), \ + patch.object(backup.sys.stdin, "isatty", return_value=True), \ + patch.object(backup, "source_pins"), patch.object(backup, "remote_command", return_value=["fake-ssh"]), \ + patch.object(backup.subprocess, "run", return_value=subprocess.CompletedProcess([], 1)), \ + patch.object(backup, "fetch") as fetch, patch.object(backup, "owner_action") as owner, \ + redirect_stdout(io.StringIO()), self.assertRaises(Refused): + backup.main() + fetch.assert_not_called() + owner.assert_not_called() + + +@unittest.skipUnless(shutil.which("age") and shutil.which("age-keygen"), "requires synthetic age tools") +class FreshPointTests(unittest.TestCase): + def setUp(self): + self.fixture = fixtures.PublisherTests() + self.fixture.setUp() + self.addCleanup(self.fixture.doCleanups) + for target, names in [(export, ("RUN_ID", "NAME", "EXPECTED", "BASE")), + (publisher, ("OBJECT", "CIPHER_SHA256", "CIPHER_BYTES")), + (recovery.restore, ("CIPHER", "PLAIN"))]: + for name in names: + patcher = patch.object(target, name, getattr(target, name)) + patcher.start() + self.addCleanup(patcher.stop) + self.node = self.fixture.root / "node" + self.node.mkdir(mode=0o700) + export.BASE = self.node + self.kits = self.fixture.root / "kits" + self.kits.mkdir(mode=0o700) + + def make_point(self, run, content): + directory = self.node / run + shutil.copytree(self.fixture.point, directory) + directory.chmod(0o700) + for name in ("seal.json", "recovery-point.zip.age"): + (directory / name).unlink() + dump = directory / "database.dump" + dump.write_bytes(content) + receipt = json.loads((directory / "receipt.json").read_text()) + receipt.update(run_id=run, database_bytes=len(content), database_sha256=hashlib.sha256(content).hexdigest()) + (directory / "receipt.json").write_text(json.dumps(receipt)) + point.select_node(run) + cipher, sealed = export.seal(directory, "age") + kit = self.kits / run + kit.mkdir(mode=0o700) + shutil.copy2(cipher, kit / (export.NAME + ".zip.age")) + shutil.copy2(directory / "seal.json", kit / (export.NAME + ".transfer.json")) + return kit, sealed + + def test_two_different_fresh_points_encrypt_publish_readback_decrypt_and_inventory(self): + runs = ("20261105T234001Z-0123456789ab", "20261105T234002Z-0123456789ac") + hashes = [] + for number, run in enumerate(runs): + kit, sealed = self.make_point(run, b"PGDMP changed synthetic data " + str(number).encode()) + backup.configure(kit, run) + provider = self.fixture.provider() + body = {"run_id": run} + publisher.publish(kit, self.fixture.credentials, provider, body) + body.update(warden_exit_code=0, session_cleanup_confirmed=True) + self.assertTrue(point.successful_provider(body, run, sealed)) + self.assertTrue(body["packet_file_hashes_verified"]) + self.assertTrue(body["sqlite_link_verified"]) + self.assertFalse(body["restore_verified"]) + backup.write_receipt(kit / "provider-receipt-fixture.json", body) + self.assertTrue(provider.put_calls[0].endswith("/vergabe-pilot-" + run + ".zip.age")) + hashes.append(body["ciphertext_sha256"]) + self.assertNotEqual(*hashes) + self.assertEqual({p["run_id"] for p in backup.inventory(self.kits)}, set(runs)) + self.assertNotIn("synthetic private row", json.dumps(body)) + + def test_selected_new_point_reuses_full_recovered_file_checks_before_restore(self): + run = "20261105T234001Z-0123456789ab" + kit, sealed = self.make_point(run, b"PGDMP changed") + backup.configure(kit, run) + body = {"run_id": run} + publisher.publish(kit, self.fixture.credentials, self.fixture.provider(), body) + body.update(warden_exit_code=0, session_cleanup_confirmed=True) + backup.write_receipt(kit / "provider-receipt-fixture.json", body) + _, recovered, files, _ = recovery.selected_receipt(kit, run, sealed) + self.assertEqual((recovered / "database.dump").read_bytes(), b"PGDMP changed") + self.assertEqual(files, sealed["packet"]["files"]) + (recovered / "database.dump").write_bytes(b"tampered") + with self.assertRaises(Refused): + recovery.selected_receipt(kit, run, sealed) + + def test_tampered_or_public_transfer_refuses_before_owner_login(self): + run = "20261105T234001Z-0123456789ab" + kit, _ = self.make_point(run, b"PGDMP changed") + path = kit / ("vergabe-pilot-" + run + ".transfer.json") + metadata = json.loads(path.read_text()) + metadata["packet"]["image"] = "unreviewed:image" + path.write_text(json.dumps(metadata)) + with self.assertRaises(Refused): + backup.configure(kit, run) + self.assertEqual(backup.inventory(self.kits), []) + path.chmod(0o644) + with self.assertRaises(Refused): + point.select_transfer(kit, run) + + def test_node_rejects_incomplete_capture_before_sealing(self): + run = "20261105T234001Z-0123456789ab" + self.make_point(run, b"PGDMP changed") + path = self.node / run / "receipt.json" + body = json.loads(path.read_text()) + body["app_resumed"] = False + path.write_text(json.dumps(body)) + with self.assertRaises(Refused): + point.select_node(run) + + def test_provider_failure_and_removed_points_never_count_as_current_backups(self): + run = "20261105T234001Z-0123456789ab" + kit, sealed = self.make_point(run, b"PGDMP changed") + body = {"run_id": run} + backup.configure(kit, run) + publisher.publish(kit, self.fixture.credentials, self.fixture.provider(), body) + body.update(warden_exit_code=14, session_cleanup_confirmed=False) + path = kit / "provider-receipt-fixture.json" + backup.write_receipt(path, body) + self.assertEqual(backup.inventory(self.kits), []) + body.update(warden_exit_code=0, session_cleanup_confirmed=True) + path.write_text(json.dumps(body)) + self.assertEqual(len(backup.inventory(self.kits)), 1) + backup.write_receipt(kit / "provider-removed.json", {"run_id": run, + "ciphertext_sha256": sealed["ciphertext_sha256"], + "status": "provider_removed_and_readback_absent"}) + self.assertEqual(backup.inventory(self.kits), []) + + +class RetentionTests(unittest.TestCase): + def item(self, run): + content = ("synthetic ciphertext " + run).encode() + return {"run_id": run, "ciphertext_sha256": hashlib.sha256(content).hexdigest(), + "ciphertext_bytes": len(content)} + + def fixture(self): + # Multiple captures on one day exercise candidate deletion without + # inventing future native evidence; baseline/newest remain protected. + runs = [point.BASELINE, "20261105T235001Z-0123456789ab", "20261105T235002Z-0123456789ac", + "20261105T235003Z-0123456789ad"] + plan = point.retention_plan([self.item(r) for r in runs]) + return plan + + def objects(self, plan, bad_run=None, changed_on_delete=False): + class Fake: + def __init__(self): + self.deleted = [] + def request(self, item, delete=False, etag=None): + run = item["run_id"] + if delete: + if changed_on_delete: + return 412, b"", None + assert etag == '"synthetic-' + run + '"' + self.deleted.append(run) + return 204, b"", None + if run in self.deleted: + return 404, b"", None + content = b"changed provider bytes" if run == bad_run else ("synthetic ciphertext " + run).encode() + return 200, content, '"synthetic-' + run + '"' + return Fake() + + def anchor(self, plan): + baseline = next(p for p in plan["keep"] if p["run_id"] == point.BASELINE) + return patch.multiple(retention, BASELINE_SHA=baseline["ciphertext_sha256"], BASELINE_BYTES=baseline["ciphertext_bytes"]) + + def test_baseline_newest_and_daily_weekly_representatives_are_protected(self): + plan = self.fixture() + self.assertIn(point.BASELINE, {p["run_id"] for p in plan["keep"]}) + self.assertIn("20261105T235003Z-0123456789ad", {p["run_id"] for p in plan["keep"]}) + self.assertEqual(len(plan["candidates"]), 2) + self.assertFalse(plan["unknown_objects_eligible"]) + + def test_seven_daily_two_weekly_representatives_across_three_weeks(self): + class FutureDateTime(dt.datetime): + @classmethod + def now(cls, tz=None): + return cls(2026, 11, 30, tzinfo=dt.UTC) + runs = [f"202611{day:02d}T120000Z-0123456789ab" for day in range(5, 26)] + with patch.object(point.dt, "datetime", FutureDateTime): + plan = point.retention_plan([self.item(r) for r in [point.BASELINE, *runs]]) + retained = {p["run_id"] for p in plan["keep"]} + self.assertTrue({f"202611{day:02d}T120000Z-0123456789ab" for day in range(19, 26)} <= retained) + self.assertIn(point.BASELINE, retained) + self.assertGreater(len(plan["candidates"]), 0) + + def test_all_retained_bytes_are_checked_before_any_conditional_delete(self): + plan = self.fixture() + objects = self.objects(plan, bad_run=point.BASELINE) + with self.anchor(plan), self.assertRaises(Refused): + retention.execute(plan, objects, lambda item: self.fail("unexpected tombstone")) + self.assertEqual(objects.deleted, []) + + def test_conditional_delete_and_absence_readback_persist_each_tombstone(self): + plan = self.fixture() + objects = self.objects(plan) + markers = [] + with self.anchor(plan): + deleted = retention.execute(plan, objects, lambda item: markers.append(item["run_id"])) + self.assertEqual(deleted, markers) + self.assertEqual(set(deleted), {p["run_id"] for p in plan["candidates"]}) + self.assertNotIn(point.BASELINE, deleted) + + def test_provider_race_refuses_without_tombstone_or_delete(self): + plan = self.fixture() + objects = self.objects(plan, changed_on_delete=True) + with self.anchor(plan), self.assertRaises(Refused): + retention.execute(plan, objects, lambda item: self.fail("unexpected tombstone")) + self.assertEqual(objects.deleted, []) + + def test_unsafe_runs_in_inventory_refuse_instead_of_becoming_delete_urls(self): + with self.assertRaises(Refused): + point.retention_plan([self.item("../../foreign")]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/vergabe_pilot_backup.py b/tools/vergabe_pilot_backup.py new file mode 100644 index 0000000..01bc934 --- /dev/null +++ b/tools/vergabe_pilot_backup.py @@ -0,0 +1,268 @@ +#!/usr/bin/env python3 +"""Founder workstation attended backup for the admitted demo-company release. + +Default reviews only. --execute captures a fresh point (brief app pause), seals +on-node, fetches ciphertext, then opens contained owner login to publish/readback/ +decrypt. --fetch RUN or --publish RUN retries without another capture/pause. +--retention previews owner-verified provider retention; --prune executes it. +""" + +import argparse +import datetime as dt +import hashlib +import json +import os +import shlex +import shutil +import subprocess +import sys +import tempfile +import uuid +from pathlib import Path + +import vergabe_pilot_export as export +import vergabe_pilot_export_remote as transfer +import vergabe_pilot_point as point +import vergabe_pilot_publish as publisher +from vergabe_pilot_recovery_store import Refused, private_directory, require + +PLAN = Path(__file__).resolve().parents[1] / "docs/proposals/2026-11-06-vergabe-attended-backups/plan.json" +NODE_MODULES = ("vergabe_pilot_recovery_store", "vergabe_pilot_export", "vergabe_pilot_point", + "vergabe_pilot_snapshot", "vergabe_pilot_backup_node") + + +def source_pins(): + plan = json.loads(PLAN.read_text()) + expected = {*NODE_MODULES, "vergabe_pilot_export_remote", "vergabe_pilot_publish", + "vergabe_pilot_backup", "vergabe_pilot_retention", "vergabe_pilot_operating_restore", + "vergabe_pilot_restore", "vergabe_pilot_restore_remote"} + require(set(plan["sources"]) == expected, "attended_source_set_changed") + sources = {} + for name, checksum in plan["sources"].items(): + raw = Path(__file__).with_name(name + ".py").read_bytes() + require(hashlib.sha256(raw).hexdigest() == checksum, "attended_source_pin_changed") + sources[name] = raw.decode() + publisher.pins() + return sources + + +def remote_command(run, mode): + point.run_time(run) + require(mode in {"check", "capture", "seal", "resume"}, "invalid_backup_mode") + sources = source_pins() + bootstrap = "import sys,types;" + for name in NODE_MODULES[:-1]: + bootstrap += ("m=types.ModuleType(" + repr(name) + ");sys.modules[m.__name__]=m;" + "exec(compile(" + repr(sources[name]) + "," + repr(name + ".py") + ",'exec'),m.__dict__);") + bootstrap += ("sys.argv=['vergabe_pilot_backup_node.py',*sys.argv[1:]];exec(compile(" + + repr(sources[NODE_MODULES[-1]]) + ",'vergabe_pilot_backup_node.py','exec'),{'__name__':'__main__'})") + command = shlex.join(["sudo", "/usr/bin/env", "KUBECONFIG=/etc/rancher/k3s/k3s.yaml", + "/usr/bin/python3", "-c", bootstrap, "--run", run, mode]) + require(len(command.encode()) < 120000, "attended_transport_size_bound") + return ["ssh", "-t", "railiance01", command] + + +def kit_root(): + return Path.home() / ".local/state/vergabe-pilot-recovery" + + +def configure(directory, run): + require(transfer.outside_git(directory.resolve()), "private_kit_outside_git_required") + sealed = point.select_transfer(directory, run) + verified = transfer.verify(directory) + publisher.OBJECT = export.NAME + ".zip.age" + publisher.CIPHER_SHA256 = sealed["ciphertext_sha256"] + publisher.CIPHER_BYTES = sealed["ciphertext_bytes"] + require(verified["ciphertext_sha256"] == sealed["ciphertext_sha256"], "sealed_point_changed") + return sealed + + +def fetch(run, already_sealed=False): + point.run_time(run) + root = kit_root() + require(transfer.outside_git(root.resolve()), "private_kit_outside_git_required") + root.mkdir(parents=True, exist_ok=True, mode=0o700) + private_directory(root) + directory = root / run + if directory.exists() or directory.is_symlink(): + configure(directory, run) + return directory + if not already_sealed: + result = subprocess.run(remote_command(run, "seal"), check=False) + require(result.returncode == 0, "node_seal_failed_use_same_run_for_retry") + with tempfile.TemporaryDirectory(prefix="fetch-", dir=root) as temporary: + temporary = Path(temporary) + for suffix in (".transfer.json", ".zip.age"): + name = "vergabe-pilot-" + run + suffix + copied = subprocess.run(["scp", "-p", "railiance01:" + name, str(temporary / name)], + capture_output=True, timeout=120, check=False) + require(copied.returncode == 0, "encrypted_fetch_failed_use_same_run_for_retry") + configure(temporary, run) + temporary.rename(directory) + return directory + + +def inventory(root): + """Only private local kits with ciphertext and complete owner receipts qualify.""" + private_directory(root) + children = list(root.iterdir()) + require(len(children) <= 2000, "local_inventory_bound") + points = [] + for directory in sorted(children): + if not directory.is_dir() or directory.is_symlink(): + continue + try: + point.run_time(directory.name) + sealed = configure(directory, directory.name) + tombstone = directory / "provider-removed.json" + if tombstone.exists() or tombstone.is_symlink(): + removed = point.read_private(tombstone) + require(removed["run_id"] == directory.name and removed["ciphertext_sha256"] == sealed["ciphertext_sha256"] + and removed["status"] == "provider_removed_and_readback_absent", "retention_tombstone_invalid") + continue + receipts = list(directory.glob("provider-receipt-*.json")) + require(len(receipts) <= 100, "provider_receipt_count_bound") + if any(point.successful_provider(point.read_private(p), directory.name, sealed) for p in receipts): + points.append({"run_id": directory.name, "ciphertext_sha256": sealed["ciphertext_sha256"], + "ciphertext_bytes": sealed["ciphertext_bytes"]}) + except (OSError, ValueError, KeyError, TypeError, Refused): + continue # Incomplete/unknown kits never qualify for provider deletion. + return points + + +def write_receipt(path, body): + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) + with os.fdopen(fd, "w") as stream: + json.dump(body, stream, indent=2) + stream.flush() + os.fsync(stream.fileno()) + + +def owner_child(args): + # Child stays silent under Warden, including setup and parser failures. + try: + source_pins() + require(args.directory and args.receipt and args.directory.is_absolute() + and args.receipt.parent == args.directory, "private_owner_paths_required") + configure(args.directory, args.run) + if args.action == "publish": + return publisher.child(args.directory, args.receipt) + require(args.action == "prune", "invalid_owner_action") + import vergabe_pilot_retention as retention + return retention.child(args.directory.parent, args.receipt, inventory) + except BaseException: # noqa: BLE001 — contained child must never print private output + return 1 + + +def owner_action(directory, run, action="publish"): + configure(directory, run) + require(all(shutil.which(t) for t in ("warden", "bao", "age", "age-keygen")), "owner_tools_required") + output = directory / (("provider-receipt-" if action == "publish" else "retention-receipt-") + uuid.uuid4().hex[:12] + ".json") + command = [sys.executable, str(publisher.PLATFORM / "scripts/openbao-attended-exec.py"), "--", + sys.executable, str(Path(__file__).resolve()), "--contained", "--run", run, + "--directory", str(directory), "--receipt", str(output), "--action", action] + result = subprocess.run(command, check=False, env=publisher.owner_login_environment()) + if output.exists() or output.is_symlink(): + body = point.read_private(output) + else: + body = {"status": "owner_login_or_child_not_completed", "provider_outcome": "unconfirmed"} + body.update(warden_exit_code=result.returncode, receipt_file=str(output), **publisher.session_outcome(result.returncode)) + if output.exists(): + export.regular(output, os.geteuid()) + fd = os.open(output, os.O_WRONLY | os.O_TRUNC | os.O_NOFOLLOW) + with os.fdopen(fd, "w") as stream: + json.dump(body, stream, indent=2) + stream.flush() + os.fsync(stream.fileno()) + else: + write_receipt(output, body) + print(json.dumps(body, indent=2)) + expected = "published_read_back_and_decrypted" if action == "publish" else "provider_retention_complete" + require(result.returncode == 0 and body["status"] == expected, "owner_operation_incomplete") + return body + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + modes = parser.add_mutually_exclusive_group() + modes.add_argument("--check", action="store_true") + modes.add_argument("--execute", action="store_true") + modes.add_argument("--fetch") + modes.add_argument("--publish") + modes.add_argument("--resume") + modes.add_argument("--retention", action="store_true") + modes.add_argument("--prune", action="store_true") + modes.add_argument("--restore") + modes.add_argument("--restore-cleanup", nargs=2, metavar=("RUN", "NONCE")) + modes.add_argument("--contained", action="store_true", help=argparse.SUPPRESS) + parser.add_argument("--run", help=argparse.SUPPRESS) + parser.add_argument("--directory", type=Path, help=argparse.SUPPRESS) + parser.add_argument("--receipt", type=Path, help=argparse.SUPPRESS) + parser.add_argument("--action", choices=("publish", "prune"), default="publish", help=argparse.SUPPRESS) + parser.add_argument("--verify-restore", action="store_true", help="also rehearse this point in fresh isolated claims") + args = parser.parse_args() + if args.contained: + return owner_child(args) + require(not args.verify_restore or args.execute or args.publish, "verify_restore_requires_capture_or_publish") + sources = source_pins() + if not any((args.check, args.execute, args.fetch, args.publish, args.resume, args.retention, args.prune, + args.restore, args.restore_cleanup)): + print(json.dumps({"status": "prepared", "target": "railiance01", "helm_revision": 4, + "next": "--check then --execute", "runtime_mutations": False, + "capture_briefly_pauses_pilot": True, "scheduled_service": False})) + return 0 + require(sys.stdin.isatty(), "founder_interactive_terminal_required") + if args.restore or args.restore_cleanup: + import vergabe_pilot_operating_restore as recovery + if args.restore_cleanup: + return recovery.cleanup(sources, *args.restore_cleanup) + directory = kit_root() / args.restore + sealed = configure(directory, args.restore) + return recovery.execute(directory, args.restore, sealed, sources) + if args.retention or args.prune: + plan = point.retention_plan(inventory(kit_root())) + print(json.dumps({"status": "retention_preview", **plan}, indent=2), flush=True) + if args.prune and plan["candidates"]: + require(any(p["run_id"] == point.BASELINE for p in plan["keep"]), "baseline_receipt_required") + newest = plan["keep"][0]["run_id"] + owner_action(kit_root() / newest, newest, "prune") + return 0 + run = args.fetch or args.publish or args.resume or dt.datetime.now(dt.UTC).strftime("%Y%m%dT%H%M%SZ-") + uuid.uuid4().hex[:12] + point.run_time(run) + if args.check or args.resume: + return subprocess.run(remote_command(run, "resume" if args.resume else "check"), check=False).returncode + print(json.dumps({"status": "attended_backup_started", "run_id": run, + "capture_requested": args.execute}), flush=True) + if args.execute: + result = subprocess.run(remote_command(run, "capture"), check=False) + require(result.returncode == 0, "capture_or_seal_failed_keep_run_id_and_review_resume") + directory = kit_root() / run if args.publish else fetch(run, already_sealed=args.execute) + configure(directory, run) + if args.fetch: + print(json.dumps({"status": "encrypted_workstation_copy_verified", "run_id": run, + "provider_publication_verified": False, "local_directory": str(directory)})) + return 0 + body = owner_action(directory, run) + if args.verify_restore: + import vergabe_pilot_operating_restore as recovery + sealed = configure(directory, run) + require(recovery.execute(directory, run, sealed, sources) == 0, "backup_published_but_restore_requires_review") + summary = {"schema": "railiance.vergabe-pilot-attended-backup.v1", "status": "complete", "run_id": run, + "provider_publication_verified": True, "independent_readback_verified": True, + "decryption_verified": True, "session_cleanup_confirmed": True, "restore_verified": args.verify_restore, + "provider_receipt": body["receipt_file"], "capture_requested": args.execute, + "retention_execution_pending": bool(point.retention_plan(inventory(kit_root()))["candidates"])} + write_receipt(directory / ("attended-backup-" + uuid.uuid4().hex[:12] + ".json"), summary) + print(json.dumps(summary, indent=2)) + return 0 + + +if __name__ == "__main__": + os.umask(0o077) + try: + raise SystemExit(main()) + except Exception as error: # noqa: BLE001 — keep private transport/parser output suppressed + if "--contained" not in sys.argv: + print(json.dumps({"status": "attended_backup_failed", "failure": str(error) if isinstance(error, Refused) + else "attended_backup_error", "details_suppressed": True})) + raise SystemExit(1) diff --git a/tools/vergabe_pilot_backup_node.py b/tools/vergabe_pilot_backup_node.py new file mode 100644 index 0000000..2bef2a6 --- /dev/null +++ b/tools/vergabe_pilot_backup_node.py @@ -0,0 +1,76 @@ +"""Attended capture/seal entry point; frozen capture and export implementations.""" + +import argparse +import fcntl +import json +import os +import pwd +import signal + +import vergabe_pilot_export as export +import vergabe_pilot_point as point +import vergabe_pilot_snapshot as snapshot +from vergabe_pilot_recovery_store import Refused, private_directory, require + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--run", required=True) + parser.add_argument("mode", choices=("check", "capture", "seal", "resume")) + args = parser.parse_args() + point.run_time(args.run) + owner = pwd.getpwnam("tegwick") + require(os.geteuid() == 0 and os.environ.get("SUDO_USER") == owner.pw_name + and os.environ.get("SUDO_UID") == str(owner.pw_uid), "founder_node_sudo_required") + if args.mode == "check": + _, fixture, _ = snapshot.preflight(args.run) + print(json.dumps({"status": "ready_for_attended_capture", "runtime_mutations": False, + "fixture_present": fixture["fixture_present"], "helm_revision": snapshot.REVISION})) + return 0 + if not snapshot.BASE.exists(): + snapshot.BASE.mkdir(mode=0o700) + private_directory(snapshot.BASE) + fd = os.open(snapshot.BASE / "operation.lock", os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, 0o600) + with os.fdopen(fd, "w") as lock: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + if args.mode == "resume": + private_directory(snapshot.BASE / args.run) + require(point.read_private(snapshot.BASE / "active.json")["run_id"] == args.run, + "resume_must_match_active_run") + journal = point.read_private(snapshot.BASE / args.run / "journal.json") + require(journal["run_id"] == args.run, "journal_run_mismatch") + result = snapshot.resume(journal) + if result["helper_cleanup_confirmed"]: + snapshot.checkpoint(snapshot.BASE / "active.json", {"run_id": args.run, "status": "resumed_after_failure"}) + print(json.dumps({"status": "application_resumed", "run_id": args.run, + "helper_cleanup_confirmed": result["helper_cleanup_confirmed"], + "backup_acceptance": False})) + return 0 if result["helper_cleanup_confirmed"] else 1 + if args.mode == "capture": + require(args.run != point.BASELINE and not (snapshot.BASE / args.run).exists(), "fresh_run_required") + # Refuse a forgotten older interrupted pause before creating any new one. + active = snapshot.BASE / "active.json" + if active.exists() or active.is_symlink(): + old = point.read_private(active) + require(old["status"] in {"captured_and_resumed", "resumed_after_failure"}, "previous_capture_requires_review_or_resume") + baseline, fixture, status = snapshot.preflight(args.run) + code = snapshot.execute(args.run, baseline, fixture, status) + if code: + return code + point.select_node(args.run) + export.main() + return 0 + + +if __name__ == "__main__": + os.umask(0o077) + def interrupted(signum, frame): + raise Refused("attended_backup_interrupted") + signal.signal(signal.SIGHUP, interrupted) + signal.signal(signal.SIGTERM, interrupted) + try: + raise SystemExit(main()) + except Exception as error: # noqa: BLE001 — never expose private subprocess/parser output + print(json.dumps({"status": "attended_backup_failed", "failure": str(error) if isinstance(error, Refused) + else "attended_backup_node_error", "details_suppressed": True})) + raise SystemExit(1) diff --git a/tools/vergabe_pilot_operating_restore.py b/tools/vergabe_pilot_operating_restore.py new file mode 100644 index 0000000..453707f --- /dev/null +++ b/tools/vergabe_pilot_operating_restore.py @@ -0,0 +1,120 @@ +"""Select a successful attended point for the already-proved isolated restore.""" + +import json +import os +import re +import shlex +import shutil +import subprocess +import tempfile +import uuid +from pathlib import Path + +import vergabe_pilot_point as point +import vergabe_pilot_restore as restore +import vergabe_pilot_restore_remote as transport +from vergabe_pilot_recovery_store import require + + +def remote_command(sources, run, nonce, cleanup=False): + point.run_time(run) + require(re.fullmatch(r"[0-9a-f]{12}", nonce), "invalid_restore_nonce") + bootstrap = "import sys,types;" + for name in ("vergabe_pilot_recovery_store", "vergabe_pilot_export", "vergabe_pilot_point", "vergabe_pilot_restore"): + bootstrap += ("m=types.ModuleType(" + repr(name) + ");sys.modules[m.__name__]=m;" + "exec(compile(" + repr(sources[name]) + "," + repr(name + ".py") + ",'exec'),m.__dict__);") + selection = ''' +import json,os,re,signal +from pathlib import Path +import vergabe_pilot_export as export +import vergabe_pilot_point as point +import vergabe_pilot_restore as restore +from vergabe_pilot_recovery_store import Refused,require +try: + run,nonce,mode=sys.argv[1:] + point.run_time(run) + require(os.geteuid()==0 and os.environ.get('SUDO_USER')=='tegwick','founder_node_sudo_required') + export.RUN_ID=run;export.NAME='vergabe-pilot-'+run + if mode!='cleanup': + stage=Path('/home/tegwick')/('vergabe-pilot-restore-'+nonce) + info=stage.lstat() + owner=int(os.environ['SUDO_UID']) + require(stage.is_dir() and not stage.is_symlink() and info.st_uid==owner and not info.st_mode&0o077,'private_founder_stage_required') + def read(name): + path=stage/name + info=export.regular(path,owner) + require(0= dt.datetime(2026, 11, 5, tzinfo=dt.UTC) + and value <= dt.datetime.now(dt.UTC) + dt.timedelta(minutes=5), "backup_date_outside_scope") + return value + + +def read_private(path, maximum=16384): + info = export.regular(path, os.geteuid()) + require(0 < info.st_size <= maximum, "point_metadata_bound") + return json.loads(path.read_text()) + + +def select(run, files): + run_time(run) + require(set(files) == FILES, "point_file_set_changed") + for value in files.values(): + require(set(value) == {"bytes", "sha256"} and type(value["bytes"]) is int + and 0 < value["bytes"] <= 1024**2 and isinstance(value["sha256"], str) + and re.fullmatch(r"[0-9a-f]{64}", value["sha256"]), "point_file_metadata_invalid") + export.RUN_ID = run + export.NAME = "vergabe-pilot-" + run + export.EXPECTED = {n: (files[n]["bytes"], files[n]["sha256"]) for n in ("database.dump", "stores.tar.gz")} + + +def select_node(run): + run_time(run) + directory = export.BASE / run + private_directory(export.BASE) + private_directory(directory) + receipt = read_private(directory / "receipt.json") + require(receipt["run_id"] == run and receipt["status"] == "captured_and_resumed", "capture_not_complete") + files = {} + for name in FILES: + path = directory / name + info = export.regular(path, os.geteuid()) + require(0 < info.st_size <= 1024**2, "point_file_size_bound") + files[name] = {"bytes": info.st_size, "sha256": digest(path)} + select(run, files) + # Reuses completed-writer/resume, revision, image and receipt-hash guards. + require(export.capture_files(directory) == files, "node_capture_changed") + return files + + +def select_transfer(directory, run): + run_time(run) + private_directory(directory) + metadata = read_private(directory / ("vergabe-pilot-" + run + ".transfer.json")) + packet = metadata["packet"] + require(metadata["schema"] == "railiance.vergabe-pilot-sealed-export.v1" + and metadata["run_id"] == packet["run_id"] == run + and metadata["recipient"] == export.RECIPIENT and packet["image"] == export.IMAGE + and packet["database"] == "vergabe_demo_company", "point_binding_changed") + require(type(metadata["ciphertext_bytes"]) is int and 0 < metadata["ciphertext_bytes"] <= 4 * 1024**2 + and re.fullmatch(r"[0-9a-f]{64}", metadata["ciphertext_sha256"]), "ciphertext_metadata_invalid") + select(run, packet["files"]) + return metadata + + +def successful_provider(body, run, sealed): + return (body.get("run_id") == run and body.get("status") == "published_read_back_and_decrypted" + and body.get("warden_exit_code") == 0 + and all(body.get(k) is True for k in ("provider_publication_verified", "independent_readback_verified", + "decryption_verified", "session_cleanup_confirmed")) + and body.get("ciphertext_sha256") == sealed["ciphertext_sha256"] + and body.get("ciphertext_bytes") == sealed["ciphertext_bytes"] + and body.get("object_path") == "r01-platform-pg/vergabe-pilot-" + run + ".zip.age") + + +def retention_plan(points): + """Seven daily plus two weekly representatives; always protect baseline/newest. + + Input is the owner's locally verified provider inventory. Unknown remote + objects are never inferred eligible. Multiple points on a day don't consume + extra daily slots; weekly representatives may overlap the daily set. + """ + require(len(points) <= 1000 and len({p["run_id"] for p in points}) == len(points), "inventory_bound_or_duplicate") + ordered = sorted(points, key=lambda p: run_time(p["run_id"]), reverse=True) + keep = {BASELINE} + if ordered: + keep.add(ordered[0]["run_id"]) + days, weeks = set(), set() + for point in ordered: + date = run_time(point["run_id"]).date() + day, week = date.isoformat(), tuple(date.isocalendar()[:2]) + if day not in days and len(days) < 7: + keep.add(point["run_id"]) + days.add(day) + if week not in weeks and len(weeks) < 2: + keep.add(point["run_id"]) + weeks.add(week) + return {"keep": [p for p in ordered if p["run_id"] in keep], + "candidates": [p for p in ordered if p["run_id"] not in keep], + "baseline_protected": BASELINE, "unknown_objects_eligible": False} diff --git a/tools/vergabe_pilot_retention.py b/tools/vergabe_pilot_retention.py new file mode 100644 index 0000000..3a29500 --- /dev/null +++ b/tools/vergabe_pilot_retention.py @@ -0,0 +1,97 @@ +"""Contained owner retention for locally verified Vergabe provider points only.""" + +import base64 +import json +import os +import urllib.error +import urllib.request + +import vergabe_pilot_point as point +import vergabe_pilot_publish as publisher +from vergabe_pilot_recovery_store import Refused, require + +BASELINE_SHA = "4915288384e3ba8a87f1427b8e10106ee7241c04184adb97f439e57846bef76f" +BASELINE_BYTES = 182521 + + +class Objects: + def __init__(self, helper, auth): + self.root = publisher.HOST + "/remote.php/dav/files/Backup/railiance-backups/" + publisher.PREFIX + "/" + self.headers = {"Authorization": "Basic " + base64.b64encode((auth[0] + ":" + auth[1]).encode()).decode()} + self.opener = urllib.request.build_opener(urllib.request.ProxyHandler({}), helper.NoRedirect()) + + def request(self, item, delete=False, etag=None): + point.run_time(item["run_id"]) + url = self.root + "vergabe-pilot-" + item["run_id"] + ".zip.age" + headers = dict(self.headers) + if delete: + require(isinstance(etag, str) and etag.startswith('"') and etag.endswith('"') + and len(etag) < 1024 and not any(c in etag for c in "\r\n"), "strong_provider_etag_required") + headers["If-Match"] = etag + try: + with self.opener.open(urllib.request.Request(url, method="DELETE" if delete else "GET", headers=headers), timeout=60) as response: + content = response.read(1024 if delete else item["ciphertext_bytes"] + 1) + return response.status, content, response.headers.get("ETag") + except urllib.error.HTTPError as error: + return error.code, b"", None + + +def execute(plan, objects, removed): + import hashlib + require(0 < len(plan["keep"]) <= 10 and len(plan["candidates"]) <= 50, "retention_batch_bound") + baseline = [p for p in plan["keep"] if p["run_id"] == point.BASELINE] + require(len(baseline) == 1 and baseline[0]["ciphertext_sha256"] == BASELINE_SHA + and baseline[0]["ciphertext_bytes"] == BASELINE_BYTES, "verified_baseline_required") + require(not ({p["run_id"] for p in plan["keep"]} & {p["run_id"] for p in plan["candidates"]}), + "retention_sets_overlap") + etags = {} + # Independently recheck every retained point before allowing any deletion. + for item in [*plan["keep"], *plan["candidates"]]: + code, content, etag = objects.request(item) + require(code == 200 and len(content) == item["ciphertext_bytes"] + and hashlib.sha256(content).hexdigest() == item["ciphertext_sha256"], "retention_provider_bytes_changed") + require(isinstance(etag, str) and etag.startswith('"') and etag.endswith('"') + and not etag.startswith('W/') and not any(c in etag for c in "\r\n"), "provider_etag_required") + etags[item["run_id"]] = etag + deleted = [] + for item in plan["candidates"]: + require(item["run_id"] != point.BASELINE, "baseline_delete_refused") + code, _, _ = objects.request(item, delete=True, etag=etags[item["run_id"]]) + require(code == 204, "conditional_retention_delete_failed") + require(objects.request(item)[0] == 404, "retention_delete_readback_failed") + # Durable per-point tombstones make partial batches safely resumable. + removed(item) + deleted.append(item["run_id"]) + return deleted + + +def child(root, output, inventory): + fd = os.open(output, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) + receipt = {"schema": "railiance.vergabe-pilot-provider-retention.v1", "status": "failed", + "deleted": [], "application_changed": False, "local_copies_deleted": False} + try: + helper = publisher.owner_helpers() + publisher.require_owner_handoff() + credentials = publisher.custody(helper) + plan = point.retention_plan(inventory(root)) + receipt["retained"] = [p["run_id"] for p in plan["keep"]] + receipt["planned_candidates"] = [p["run_id"] for p in plan["candidates"]] + receipt["quota_before"] = publisher.Provider(helper).quota(credentials["owner_auth"]) + def removed(item): + path = root / item["run_id"] / "provider-removed.json" + marker = {**item, "status": "provider_removed_and_readback_absent"} + with os.fdopen(os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600), "w") as stream: + json.dump(marker, stream) + stream.flush() + os.fsync(stream.fileno()) + receipt["deleted"].append(item["run_id"]) + execute(plan, Objects(helper, credentials["owner_auth"]), removed) + receipt["status"] = "provider_retention_complete" + except BaseException as error: # noqa: BLE001 — preserve partial progress without private diagnostics + receipt["failure"] = str(error) if isinstance(error, Refused) else "provider_retention_failed" + finally: + with os.fdopen(fd, "w") as stream: + json.dump(receipt, stream, indent=2) + stream.flush() + os.fsync(stream.fileno()) + return 0 if receipt["status"] == "provider_retention_complete" else 1 diff --git a/workplans/RAPPS-WP-0014-vergabe-invited-pilot.md b/workplans/RAPPS-WP-0014-vergabe-invited-pilot.md index 1a0deda..43d79d8 100644 --- a/workplans/RAPPS-WP-0014-vergabe-invited-pilot.md +++ b/workplans/RAPPS-WP-0014-vergabe-invited-pilot.md @@ -196,10 +196,10 @@ id: RAPPS-WP-0014-T03 status: wait needs_human: true decision_id: d06307e5-2737-461a-a2f4-a12c3b492a9a -intervention_note: "Provider publication, independent readback and escrow decryption pass; Warden cleanup confirmed. Prepared founder-run isolated native restore/configuration-inverse rehearsal from the provider-retrieved point with new namespace/claims, no live app pause. Native restore/cleanup evidence and operating acceptance remain; SSO/ingress and older-image/Helm rollback are not inferred." +intervention_note: "Native three-store restore/configuration inverse and cleanup pass (215 seconds). Founder selected attended backups with himself as backup/incident contact, daily active-day/pre-change cadence, seven daily/two weekly points, one active day data-loss and one working day recovery targets. Repeatable fresh-point backup/retention/isolated-restore path prepared; 76 tests pass. Next founder command: tools/vergabe_pilot_backup.py --execute --verify-restore after --check. Native SSO/ingress and older-image/Helm rollback remain outside the proof." priority: high assignee: the-custodian -blocking_reason: "Capture/restart, provider publication/readback and decryption pass. Await native isolated restore/workflow/access/configuration-inverse receipt and operating ownership/cadence/retention/RPO/RTO acceptance; broader restored SSO or version-rollback scope remains explicit." +blocking_reason: "Recovery and clone configuration-inverse proof pass; attended operating model selected. Finish and verify repeatable fresh-point capture/publication and bounded retention procedure before claiming the accepted cadence active. Restored native SSO/ingress and broader version rollback remain explicit limits." state_hub_task_id: "dd069c6d-fcc1-5bac-b233-976f2f0d5cd1" ``` @@ -515,6 +515,49 @@ not yet evidenced. Session tests use force_login; no restored native SSO/MFA, ingress or older-image/schema/Helm rollback is claimed. T03 remains waiting for the reviewed execution and operating acceptance. +2026-11-06 native restore return: nonce `2d9a74da048c`, worker +`restored_and_verified`, transport `complete` / exit 0. The provider-retrieved +point matches all 39 PostgreSQL tables / 247 rows (logical SHA256 +`33191c43…3765`), one PDF, SQLite nine tables / seven rows and linked local issue. +Both restored ordinary member sessions pass document hashes and cross-member +task editing; anonymous, already-authenticated inactive-member and missing-CSRF +requests are denied. Test transactions roll back. A clone-only committed marker +survives two Pod replacements through bad configuration and its exact inverse. +All new namespace/PVs and private staging are removed; production specification +and readiness are unchanged. Restore/checks 55.41 seconds, inverse/recheck 55.24 +seconds, total worker 215 seconds; workstation authentication/transfer excluded. +Receipts: `docs/evidence/2026-11-06-vergabe-native-restore-return.json` and +`docs/evidence/2026-11-06-vergabe-native-restore-transport-return.json`. +Captured tender status 10 is Gewonnen and not archived, explaining its exclusion +from the deployed dashboard's status 1–9 range; no later live status is inferred. +Restored sessions use test-client login; native SSO/ingress recovery and an older +image/schema or Helm rollback remain untested. + +The founder selected attended pilot backups: Bernd Worsch as backup execution +and first incident/support contact, existing railiance-platform storage/custody +lane, a fresh point after every active pilot day and before changes, seven daily +and two weekly verified points, at most one active day's data loss and attended +recovery within one working day. These are accepted operating targets, not an +SLA inferred from the drill. `docs/vergabe-demo-company-operating-acceptance.proposed.md` +records the arrangement and implementation gap: exact-point acceptance helpers +are not yet a repeatable fresh-backup procedure. T03 remains wait until that path +and bounded retention handling are ready and a fresh attended run passes. + +Prepared `docs/proposals/2026-11-06-vergabe-attended-backups/README.md` and +`tools/vergabe_pilot_backup.py`: the frozen capture, encryption, provider and +native restore modules remain unchanged. Validated per-run selection supports +fresh capture/seal/fetch, contained publication/readback/decryption, retries, +guarded resume and optional full isolated restore for the new point. Provider +retention previews only complete owner-verified kits, protects baseline/newest, +and independently verifies retained/candidate ciphertext before separate +ETag-guarded deletion/absence readback. Unknown provider objects are ineligible; +local private copies remain. Seventeen new synthetic cases and 76 total pilot +helper tests pass, including real age round-trips for two distinct fresh points, +selected recovered-file checks and conditional retention races. No new native +capture, provider object or retention deletion has been performed by the agent. +T03 waits for the first attended operating run: +`python3 tools/vergabe_pilot_backup.py --execute --verify-restore`, after `--check`. + ## Validate synthetic sizing against the exact release and shared contention