Record native recovery success and prepare attended pilot backups
Assistant: codex Assistant-Model: gpt-6.1-sol Assistant-Session: 01a1abf1-7e95-7141-87c9-ee78332bf790
This commit is contained in:
parent
1a35ee8113
commit
bf77e378a8
18 changed files with 1467 additions and 45 deletions
60
docs/evidence/2026-11-06-vergabe-native-restore-return.json
Normal file
60
docs/evidence/2026-11-06-vergabe-native-restore-return.json
Normal file
|
|
@ -0,0 +1,60 @@
|
|||
{
|
||||
"schema": "railiance.vergabe-pilot-isolated-restore.v1",
|
||||
"run_id": "20261105T233543Z-89bfbe04dece",
|
||||
"nonce": "2d9a74da048c",
|
||||
"namespace": "vergabe-restore-2d9a74da048c",
|
||||
"status": "restored_and_verified",
|
||||
"restore_verified": true,
|
||||
"application_changed": false,
|
||||
"production_application_changed": false,
|
||||
"provider_retrieved_source": true,
|
||||
"ciphertext_sha256": "4915288384e3ba8a87f1427b8e10106ee7241c04184adb97f439e57846bef76f",
|
||||
"plaintext_packet_sha256": "791a2cf0af25d9e54ea37203c584e6aa5cb1b8a0d4d467a281947160a7bb32e0",
|
||||
"application_image": "forgejo.coulomb.social/coulomb/vergabe-teilnahme@sha256:a26444f59c259698159c69ccb96f73dc648a261ece4c86bb2037a9d977870d91",
|
||||
"postgres_image": "postgres@sha256:ff23cdce56cac62ada6f66013e1a50864c0abbe79d132d40b6e05bd80f378a70",
|
||||
"native_sso_login_verified": false,
|
||||
"helm_rollback_verified": false,
|
||||
"stage": "complete",
|
||||
"database_restored": true,
|
||||
"all_postgres_table_digests_match": true,
|
||||
"database_tables": 39,
|
||||
"database_rows": 247,
|
||||
"database_logical_sha256": "33191c43c437f1b05da410a3f22989c86ce50f4fca8fd32d40fbb327b8b73765",
|
||||
"all_media_hashes_match": true,
|
||||
"sqlite_integrity_and_rows_match": true,
|
||||
"local_issue_link_verified": true,
|
||||
"restored_files_uid_gid_verified": true,
|
||||
"media_files": 1,
|
||||
"sqlite_tables": 9,
|
||||
"sqlite_rows": 7,
|
||||
"two_restored_members_verified": true,
|
||||
"restored_identity_mappings": 2,
|
||||
"member_document_checksums_match": true,
|
||||
"cross_member_task_edit_verified": true,
|
||||
"anonymous_pages_and_binary_denied": true,
|
||||
"inactive_existing_session_denied": true,
|
||||
"csrf_write_denied": true,
|
||||
"workflow_test_transaction_rolled_back": true,
|
||||
"captured_tender_status": 10,
|
||||
"captured_tender_archived": false,
|
||||
"captured_status_in_dashboard_range": false,
|
||||
"outgoing_integrations_enabled": false,
|
||||
"restore_elapsed_seconds": 55.41,
|
||||
"configuration_rollback_verified": true,
|
||||
"committed_clone_write_preserved": true,
|
||||
"pod_replacements": 2,
|
||||
"rollback_same_image_and_schema": true,
|
||||
"older_image_or_schema_downgrade_verified": false,
|
||||
"configuration_inverse_elapsed_seconds": 55.24,
|
||||
"separate_fresh_claims": true,
|
||||
"network_policy_default_deny": true,
|
||||
"application_image_cached_no_pull": true,
|
||||
"namespace_and_volumes_removed": true,
|
||||
"production_spec_and_readiness_unchanged": true,
|
||||
"elapsed_seconds": 215.0,
|
||||
"evidence_kind": "user_reported_terminal_output",
|
||||
"reported_on": "2026-11-06",
|
||||
"workplan": "RAPPS-WP-0014",
|
||||
"task": "RAPPS-WP-0014-T03",
|
||||
"source_revision": "1a35ee81132537c3461f46cc495e56a15dc1eae5"
|
||||
}
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
{
|
||||
"schema": "railiance.vergabe-pilot-native-restore-transport.v1",
|
||||
"nonce": "2d9a74da048c",
|
||||
"run_id": "20261105T233543Z-89bfbe04dece",
|
||||
"worker_exit_code": 0,
|
||||
"node_stage_removed": true,
|
||||
"status": "complete",
|
||||
"evidence_kind": "user_reported_terminal_output",
|
||||
"reported_on": "2026-11-06",
|
||||
"workplan": "RAPPS-WP-0014",
|
||||
"task": "RAPPS-WP-0014-T03",
|
||||
"source_revision": "1a35ee81132537c3461f46cc495e56a15dc1eae5"
|
||||
}
|
||||
21
docs/evidence/2026-11-06-vergabe-operating-choice.json
Normal file
21
docs/evidence/2026-11-06-vergabe-operating-choice.json
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
{
|
||||
"schema": "railiance.vergabe-pilot-operating-choice.v1",
|
||||
"reported_on": "2026-11-06",
|
||||
"evidence_kind": "user_question_reply",
|
||||
"workplan": "RAPPS-WP-0014",
|
||||
"task": "RAPPS-WP-0014-T03",
|
||||
"answer": "Attended pilot backups (Recommended)",
|
||||
"selected_model": "attended",
|
||||
"backup_execution_and_incident_contact": "Bernd Worsch",
|
||||
"cadence": "after each active pilot day and before changes",
|
||||
"provider_retention": {
|
||||
"daily": 7,
|
||||
"weekly": 2,
|
||||
"baseline_and_newest_protected": true
|
||||
},
|
||||
"data_loss_target": "at most one active pilot day",
|
||||
"recovery_target": "attended within one working day",
|
||||
"scheduled_service": false,
|
||||
"operating_path_native_verified": false,
|
||||
"note": "Targets presented in the question; selected without amendments. Original baseline recovery already passes; fresh repeatable operating run remains pending."
|
||||
}
|
||||
151
docs/proposals/2026-11-06-vergabe-attended-backups/README.md
Normal file
151
docs/proposals/2026-11-06-vergabe-attended-backups/README.md
Normal file
|
|
@ -0,0 +1,151 @@
|
|||
# Attended demo-company backups
|
||||
|
||||
The founder selected attended end-of-active-day and pre-change backups, with
|
||||
himself as backup execution and first incident/support contact. The accepted
|
||||
[operating targets](../../vergabe-demo-company-operating-acceptance.proposed.md)
|
||||
are seven daily/two weekly verified provider points, at most one active pilot
|
||||
day's data loss and attended recovery within one working day. No scheduled
|
||||
service, notification worker or SLA is introduced.
|
||||
|
||||
The [baseline restore](../../evidence/2026-11-06-vergabe-native-restore-return.json)
|
||||
passes. This packet makes fresh points repeatable without changing the frozen
|
||||
acceptance tools. [Exact scope and code hashes](plan.json).
|
||||
|
||||
## First operating run — founder terminal
|
||||
|
||||
```bash
|
||||
git pull --ff-only
|
||||
python3 tools/vergabe_pilot_backup.py --check
|
||||
python3 tools/vergabe_pilot_backup.py --execute --verify-restore
|
||||
```
|
||||
|
||||
The check is read-only/server dry-run. Execute briefly pauses only the existing
|
||||
demo-company application for a coherent new PostgreSQL/media/SQLite capture,
|
||||
then resumes and verifies it before sealing or transfer. It creates a unique
|
||||
UTC run ID and refuses to overwrite an earlier point. The existing root-private
|
||||
capture journal, operation lock, deployment/claim identity checks and guarded
|
||||
resume are reused. An unresolved older capture refuses a new pause.
|
||||
|
||||
The second command also uploads one new encrypted object and creates/removes an
|
||||
isolated restore namespace with three fresh claims. Founder execution is the go
|
||||
for these exact runtime changes under `the-custodian/docs/agent-environment-orientation.md`
|
||||
section 0. No separate approval machinery or credential grant is added.
|
||||
|
||||
Encryption occurs on the source node for the existing platform age recipient.
|
||||
Only ciphertext and its safe transfer metadata are fetched to the private
|
||||
workstation kit. The existing contained OpenBao owner login then performs the
|
||||
same create-only Nextcloud upload, independent Backup-account readback, escrow
|
||||
decryption and full packet/media/SQLite verification that passed for baseline.
|
||||
Token aliases are cleared, owner helper pins/policies are checked, and Warden
|
||||
revokes/removes the session. No credential or backup content is printed.
|
||||
|
||||
With `--verify-restore`, that newly retrieved/decrypted point is then restored
|
||||
through the already-proved isolated worker: all PostgreSQL row digests and file/
|
||||
SQLite fingerprints, both restored ordinary member sessions, document/task
|
||||
workflow and access denials, same-image configuration inverse preserving a
|
||||
committed clone write, UID-guarded namespace/PV deletion and production readback.
|
||||
The private node stage is removed on success. Native SSO/callback/ingress recovery
|
||||
and older-image/schema/Helm rollback remain untested. This is a rehearsal on the
|
||||
healthy current cluster with the admitted app image cached, not a node-rebuild
|
||||
or unattended disaster-recovery demonstration.
|
||||
|
||||
Allow several minutes plus interactive SSH/sudo and OpenBao login. Keep the run
|
||||
ID and any restore nonce. Success ends with `railiance.vergabe-pilot-attended-backup.v1`,
|
||||
`status: complete`, publication/readback/decryption/session cleanup true and
|
||||
`restore_verified: true` for this first run. Return the sanitized receipts.
|
||||
The isolated worker's time is not end-to-end RTO or a live outage measurement.
|
||||
|
||||
## Each active pilot day and before changes
|
||||
|
||||
```bash
|
||||
python3 tools/vergabe_pilot_backup.py --check
|
||||
python3 tools/vergabe_pilot_backup.py --execute
|
||||
python3 tools/vergabe_pilot_backup.py --retention
|
||||
```
|
||||
|
||||
The routine backup omits a fresh restore rehearsal and reports
|
||||
`restore_verified: false`; it still requires provider readback/decryption and
|
||||
owner session cleanup. Repeat `--verify-restore` after a change to the operating
|
||||
path or as an attended recovery drill. Reconcile the new point's success receipt
|
||||
before ending the working session. If it fails, treat the prior successful point
|
||||
as the recovery boundary and report its age; do not assume that capture or
|
||||
workstation transfer alone satisfied the day's backup.
|
||||
|
||||
This path is intentionally bound to the admitted image, Helm revision 4, chart
|
||||
0.2.1, cluster, database and claims. A release/placement change requires review
|
||||
and updated binding first. Fixture tender/task/document 1 and its local issue
|
||||
must remain available. Safety limits remain 1 MiB per captured packet member,
|
||||
4 MiB encrypted/decrypted packet, 128 MiB inflated stores and 100,000 database
|
||||
rows. Growth beyond those limits refuses; review sizing before using larger data.
|
||||
|
||||
## Interrupted steps and recovery
|
||||
|
||||
Use the same printed run ID; do not recapture simply because publication failed:
|
||||
|
||||
```bash
|
||||
python3 tools/vergabe_pilot_backup.py --resume <run-id>
|
||||
python3 tools/vergabe_pilot_backup.py --fetch <run-id>
|
||||
python3 tools/vergabe_pilot_backup.py --publish <run-id> --verify-restore
|
||||
python3 tools/vergabe_pilot_backup.py --restore <run-id>
|
||||
python3 tools/vergabe_pilot_backup.py --restore-cleanup <run-id> <nonce>
|
||||
```
|
||||
|
||||
Resume is only for an interrupted application pause: it requires the active
|
||||
journal's matching run and unchanged deployment/claims, then restores readiness
|
||||
without accepting an incomplete capture. Fetch reseals only a completed capture
|
||||
and fetches its encrypted pair. Publish retries the same deterministic object;
|
||||
identical existing ciphertext is read back, different bytes refuse. Restore uses
|
||||
a successful fully cleaned-up provider receipt and checks its fetched ciphertext,
|
||||
decrypted ZIP and every recovered file before staging; no extra key retrieval or
|
||||
provider login is needed. Restore cleanup only removes the journal-owned clone
|
||||
and stage and never claims data acceptance.
|
||||
|
||||
Restoring into or cutting over the production instance is a separate attended
|
||||
incident change: retain current data, inspect image/schema compatibility and
|
||||
prepare the exact database/claim/route change before executing. The rehearsal
|
||||
never overwrites production or repoints its SSO callback. Production passwords,
|
||||
runtime Secrets and the recovery identity are not copied into the clone.
|
||||
|
||||
## Provider retention
|
||||
|
||||
`--retention` previews points from private workstation kits with matching
|
||||
ciphertext and complete independent-readback/decryption/session-cleanup receipts.
|
||||
It selects the latest point for each of seven days and two ISO weeks; daily and
|
||||
weekly representatives can overlap. Baseline and newest remain protected.
|
||||
Failed, missing, unknown or unrelated objects are never deletion candidates.
|
||||
|
||||
Review the printed candidate list, then execute separately when candidates exist:
|
||||
|
||||
```bash
|
||||
python3 tools/vergabe_pilot_backup.py --prune
|
||||
```
|
||||
|
||||
This is a separate founder go for provider deletion and opens a contained owner
|
||||
session. Before any delete it independently retrieves every retained and candidate
|
||||
ciphertext and compares exact size/hash. The proven baseline's fixed ciphertext
|
||||
must match. Deletes require a strong ETag with `If-Match`, followed by 404 readback.
|
||||
Each deletion gets a private local tombstone so later inventories exclude removed
|
||||
objects. The batch is capped at 50 deletes; no provider-wide listing or unknown
|
||||
object adoption is performed. Backup execution itself never deletes.
|
||||
|
||||
Interruption between provider deletion and the durable local tombstone refuses
|
||||
the next batch on the missing object; reconcile from owner receipts before any
|
||||
further deletion. A changed/missing retained point also stops pruning. The
|
||||
operator checks quota/last verified point and retains the baseline until an
|
||||
explicit later review; no automatic unpin is introduced. Local encrypted kits,
|
||||
private recovered data and root node capture/journal copies remain private and
|
||||
are not removed by provider retention. Their removal requires owner review of
|
||||
the verified inventory; they are not an independent offsite destination.
|
||||
|
||||
## Validation and closure
|
||||
|
||||
The original capture, crypto/provider and native restore implementations are
|
||||
unchanged and source-pinned. Synthetic tests cover distinct fresh points through
|
||||
real age encryption/decryption and mocked provider round-trips, file/metadata
|
||||
tampering, incomplete capture and failed cleanup exclusion, selected-point
|
||||
restore verification, embedded source compilation and founder boundary, failure
|
||||
sequencing, and retention protection/hash/conditional-delete races.
|
||||
|
||||
No agent reads the founder's real private kit or executes this native credential
|
||||
path. RAPPS-WP-0014-T03 remains waiting for the first operating run above; the
|
||||
existing baseline's technical proof and accepted targets are already recorded.
|
||||
52
docs/proposals/2026-11-06-vergabe-attended-backups/plan.json
Normal file
52
docs/proposals/2026-11-06-vergabe-attended-backups/plan.json
Normal file
|
|
@ -0,0 +1,52 @@
|
|||
{
|
||||
"schema": "railiance.vergabe-pilot-attended-backup-plan.v1",
|
||||
"prepared_on": "2026-11-06",
|
||||
"workplan": "RAPPS-WP-0014",
|
||||
"target": "railiance01",
|
||||
"cluster_uid": "a553c742-0115-43d4-99a4-a5ca56fe0786",
|
||||
"helm_revision": 4,
|
||||
"image": "forgejo.coulomb.social/coulomb/vergabe-teilnahme@sha256:a26444f59c259698159c69ccb96f73dc648a261ece4c86bb2037a9d977870d91",
|
||||
"sources": {
|
||||
"vergabe_pilot_backup": "ba9a33d50ed63cbf9f10b92c7b1eae815c3beb82753c05a1cbfb3b6a7a321354",
|
||||
"vergabe_pilot_backup_node": "b12474806cab5c440c60ef9f1bfc90344a02f2d34ddc16e83da746ab2ee2b39f",
|
||||
"vergabe_pilot_export": "b45733fa10ae59b56563045c43c53b5f824a74e1114f68fbed28404f14b43e3a",
|
||||
"vergabe_pilot_export_remote": "11c32dc2011dc96f604cebd7681cb4aa27302d90b1d80f765c94e4dddc31c9d5",
|
||||
"vergabe_pilot_operating_restore": "14072a12873d27827b815295fb45b2250d18414740df852c07fc4ca700049eb8",
|
||||
"vergabe_pilot_point": "676191b8ade0e4dd0e6607341a0398d52ce909d2e27d1fb8d43ac8f4f0e003a3",
|
||||
"vergabe_pilot_publish": "4340f386e448984b2b34d13d59f318ca4dda35b218a2b28cd034dcd00881244e",
|
||||
"vergabe_pilot_recovery_store": "724a2dd9db8a0f57e1efa709ec989d8a47c681a86b1fca923bfdfea1698ae23c",
|
||||
"vergabe_pilot_restore": "407df786ce8994744df7079771ddadf56019f2d384d4b6cbda8dc8d051270dd4",
|
||||
"vergabe_pilot_restore_remote": "cda578e169199f7e319423b41786f2baf9cd7e7a992f1dda3f19bb134f69ad91",
|
||||
"vergabe_pilot_retention": "3945b5c565b42e74db901a8d2aa240fd940e384076ace61f4c47fb5f16e6f880",
|
||||
"vergabe_pilot_snapshot": "35c3aa9906a17c82867541d27f41456b75b86f6fda0c87f6dfda5a63985ff796"
|
||||
},
|
||||
"capture_mutations": [
|
||||
"briefly pause only vergabe-demo-company/vergabe-teilnahme",
|
||||
"temporary read-only capture Pod on two existing claims",
|
||||
"resume exact deployment and verify fixture/readiness"
|
||||
],
|
||||
"provider_actions": [
|
||||
"create-only unique encrypted point",
|
||||
"independent readback and escrow decryption in contained founder session"
|
||||
],
|
||||
"prune_actions": [
|
||||
"preview only by default with --retention",
|
||||
"--prune separately opens contained owner session",
|
||||
"verify retained and candidate ciphertext before any deletion",
|
||||
"If-Match ETag guarded delete and 404 readback",
|
||||
"protect baseline and newest; ignore unknown objects"
|
||||
],
|
||||
"scheduled_service": false,
|
||||
"automatic_deletion_during_backup": false,
|
||||
"limits": {
|
||||
"capture_files_bytes_each": 1048576,
|
||||
"encrypted_packet_bytes": 4194304,
|
||||
"store_inflation_bytes": 134217728,
|
||||
"database_rows": 100000,
|
||||
"provider_delete_batch": 50
|
||||
},
|
||||
"execution": "Founder terminal via existing SSH/sudo and contained OpenBao login; no agent execution",
|
||||
"isolated_restore": "--restore RUN reuses the proven same-image fresh-claim restore from that successful provider-retrieved point; --restore-cleanup RUN NONCE is guarded cleanup only",
|
||||
"first_execution": "python3 tools/vergabe_pilot_backup.py --execute --verify-restore",
|
||||
"routine_execution": "python3 tools/vergabe_pilot_backup.py --execute"
|
||||
}
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
{
|
||||
"schema": "railiance.vergabe-pilot-attended-backup-validation.v1",
|
||||
"unit_tests": 76,
|
||||
"new_attended_cases": 17,
|
||||
"unit_status": "passed",
|
||||
"lint": "passed",
|
||||
"native_new_operating_run_executed": false,
|
||||
"original_capture_export_publish_restore_sources_unchanged": true,
|
||||
"synthetic_real_age_two_distinct_points": true,
|
||||
"provider_mocked": true,
|
||||
"retention_baseline_newest_protection": true,
|
||||
"retained_bytes_prechecked_before_delete": true,
|
||||
"conditional_delete_race_refused": true,
|
||||
"native_backup_or_credential_read": false
|
||||
}
|
||||
|
|
@ -6,6 +6,21 @@ escrow decryption, packet/media/SQLite verification and successful Warden cleanu
|
|||
The point is `20261105T233543Z-89bfbe04dece`, ciphertext `49152883…6bef76f`.
|
||||
[Exact scope and source pins](plan.json) · [synthetic validation](validation.json).
|
||||
|
||||
Execution succeeded under nonce `2d9a74da048c`: the
|
||||
[worker receipt](../../evidence/2026-11-06-vergabe-native-restore-return.json)
|
||||
reports `restored_and_verified`, all 39 tables / 247 rows and both file stores
|
||||
matching, two restored member sessions/workflow/denials passing, committed clone
|
||||
writes preserved through the configuration inverse, and namespace/PV cleanup
|
||||
with production specification/readiness unchanged. Total worker time was
|
||||
215 seconds; restore/checks 55.41 seconds, inverse/recheck 55.24 seconds.
|
||||
The [transport receipt](../../evidence/2026-11-06-vergabe-native-restore-transport-return.json)
|
||||
reports `complete`, exit 0 and private staging removed. Captured tender status 10
|
||||
(Gewonnen), not archived, explains its exclusion from the overview's status range.
|
||||
The command below is retained for reproducibility; no rerun is needed for this
|
||||
point. [Operating acceptance](../../vergabe-demo-company-operating-acceptance.proposed.md)
|
||||
now records the selected attended model and prepared repeatable operating path.
|
||||
Native SSO/ingress and older-image/Helm rollback remain untested.
|
||||
|
||||
## Founder workstation command
|
||||
|
||||
```bash
|
||||
|
|
@ -131,9 +146,10 @@ pass a local synthetic three-store restore (39 tables / 202 rows): full table an
|
|||
file/SQLite fingerprints, both restored member sessions, edit/download and denial
|
||||
checks, transaction preservation, two database restarts and the bad-configuration
|
||||
inverse retaining a committed write. The PostgreSQL query format matches capture.
|
||||
Those tests use no native backup, credential or Kubernetes mutation. Native
|
||||
scheduling/admission/claims still need the founder execution receipt.
|
||||
Those synthetic tests use no native backup, credential or Kubernetes mutation.
|
||||
The founder's execution receipt above now also proves native scheduling,
|
||||
admission, fresh claims, complete data recovery and guarded cleanup for this run.
|
||||
|
||||
RAPPS-WP-0014-T03 remains open for that receipt and operating acceptance: named
|
||||
RAPPS-WP-0014-T03 remains open for operating acceptance: named
|
||||
backup/support owners, cadence, retention, incident handling and accepted RPO/RTO;
|
||||
any broader restored SSO/ingress or version-rollback requirement must be explicit.
|
||||
|
|
|
|||
|
|
@ -7,8 +7,10 @@ kit is already on the founder's workstation. [Exact scope and source pins](plan.
|
|||
|
||||
Execution succeeded: [the native provider receipt](../../evidence/2026-11-06-vergabe-provider-recovery-return.json)
|
||||
confirms upload 201, independent readback 200, decryption and packet/media/SQLite
|
||||
verification, Warden exit 0 and session cleanup. Next is the
|
||||
[isolated native restore/inverse rehearsal](../2026-11-06-vergabe-native-restore/README.md).
|
||||
verification, Warden exit 0 and session cleanup. The subsequent
|
||||
[isolated native restore/inverse rehearsal](../2026-11-06-vergabe-native-restore/README.md)
|
||||
also passes, including all stores, restored member workflow/access denials,
|
||||
current-write-preserving configuration inverse and cleanup.
|
||||
The publication command below is retained for reproducibility.
|
||||
|
||||
## Earlier attempts
|
||||
|
|
@ -132,9 +134,10 @@ isolated restore. The directory is 0700 and files 0600, outside Git; do not open
|
|||
or paste backup contents. The decryption identity is not stored with them.
|
||||
The operation's safe `provider-receipt-*.json` is persisted in that kit.
|
||||
|
||||
The receipt keeps `restore_verified: false`: an isolated PostgreSQL restore and
|
||||
restored application workflow, access refusal, compatible rollback and agreed
|
||||
backup/support ownership, cadence, retention and RPO/RTO remain T03 requirements.
|
||||
This publication-only receipt keeps `restore_verified: false`; the separate
|
||||
native restore receipt now supplies database/workflow/access/configuration-inverse
|
||||
proof. Agreed backup/support ownership, repeatable cadence, retention and RPO/RTO
|
||||
remain T03 requirements.
|
||||
No second application pause or recapture is needed for this point.
|
||||
|
||||
## Validation
|
||||
|
|
@ -148,5 +151,5 @@ The additional cases test all token aliases, API/effective-policy handling,
|
|||
separate session outcomes and the installed Warden envelope using an offline
|
||||
fake Bao client. They issue no native session and inspect no real helper or
|
||||
credential. Both existing URL forms pass the synthetic publication/readback/
|
||||
decryption flow. All 47 pilot helper checks pass. Native provider publication and
|
||||
decryption remain unverified until a successful founder receipt returns.
|
||||
decryption flow. All 47 pilot helper checks passed at publication preparation;
|
||||
the successful founder receipt above supplies native publication/decryption proof.
|
||||
|
|
|
|||
|
|
@ -9,8 +9,9 @@ The user confirmed the uploaded file is available from the other account; see
|
|||
the [document return](evidence/2026-11-05-vergabe-document-return.json). The
|
||||
document UI still needs user review. The November 6 native check verifies the
|
||||
stored file matches the original PDF checksum and its anonymous binary URL
|
||||
redirects to login. A second-account downloaded-file hash comparison remains
|
||||
unreported. The user also confirms the **LOKAL** issue badge is
|
||||
redirects to login. The later isolated restore verifies both restored members'
|
||||
downloaded bytes; a live browser-downloaded hash return remains unreported.
|
||||
The user also confirms the **LOKAL** issue badge is
|
||||
visible in both accounts; the [issue return](evidence/2026-11-05-vergabe-local-issue-return.json)
|
||||
binds Ausschreibung 1, task 1 and document 1 from their supplied URLs. The user
|
||||
now confirms cross-account task editing and the flow through documented
|
||||
|
|
@ -34,10 +35,11 @@ the helper was removed. Total operation time was 98.61 seconds, including
|
|||
resume/postflight, rather than a separately measured outage duration.
|
||||
The [human restart return](evidence/2026-11-06-vergabe-post-restart-user-return.json)
|
||||
confirms both accounts work and everything else appears as before, except the
|
||||
tender is absent from Übersicht. The deployed dashboard lists only status 1–9;
|
||||
final statuses 10–13 would explain exclusion. The actual current status is not
|
||||
yet reported, so this remains a UI follow-up. At that stage, off-host publication
|
||||
and isolated restore were unverified.
|
||||
tender is absent from Übersicht. The restored captured tender is status 10
|
||||
(Gewonnen), not archived; the deployed dashboard lists only status 1–9. This
|
||||
explains its exclusion in the captured state. It is a discoverability follow-up,
|
||||
not evidence of lost tender data. No later live status read is inferred.
|
||||
At that stage, off-host publication and isolated restore were unverified.
|
||||
|
||||
The [encrypted workstation copy](evidence/2026-11-06-vergabe-encrypted-workstation-copy-return.json)
|
||||
now passes: 182521 bytes, hash matching the node handoff. The subsequent
|
||||
|
|
@ -46,11 +48,27 @@ executed without another app pause.
|
|||
|
||||
That [provider execution now passes](evidence/2026-11-06-vergabe-provider-recovery-return.json):
|
||||
create-only publication, independent readback, escrow decryption, file/SQLite
|
||||
verification and Warden cleanup all succeed. PostgreSQL has not yet been restored.
|
||||
The [native isolated restore packet](proposals/2026-11-06-vergabe-native-restore/README.md)
|
||||
prepares fresh claims and a same-image configuration inverse, with explicit
|
||||
restored-session versus native-SSO limits. Native execution and operating
|
||||
acceptance remain open.
|
||||
verification and Warden cleanup all succeed.
|
||||
|
||||
The [native isolated restore](evidence/2026-11-06-vergabe-native-restore-return.json)
|
||||
now also passes from that provider-retrieved point: all 39 PostgreSQL tables /
|
||||
247 rows, the PDF and SQLite's nine tables / seven rows match capture. Two
|
||||
restored ordinary member sessions can read/download/edit across accounts;
|
||||
anonymous, inactive-session and missing-CSRF requests are denied. Two Pod
|
||||
replacements demonstrate the same-image configuration inverse retains a
|
||||
committed clone write. Fresh namespace/PVs and private node staging are removed;
|
||||
production specification and readiness are unchanged. The
|
||||
[transport receipt](evidence/2026-11-06-vergabe-native-restore-transport-return.json)
|
||||
confirms exit 0 and complete cleanup.
|
||||
|
||||
Restore/checks took 55.41 seconds; the configuration inverse/recheck took 55.24
|
||||
seconds; total worker time was 215 seconds. Workstation authentication/transfer
|
||||
are outside that timing. Restored sessions use Django test-client login, so
|
||||
native SSO/MFA/ingress recovery remains untested. No older-image/schema downgrade
|
||||
or Helm rollback is claimed. The [review packet](proposals/2026-11-06-vergabe-native-restore/README.md)
|
||||
retains the exact scope. [Operating acceptance](vergabe-demo-company-operating-acceptance.proposed.md)
|
||||
records the founder-selected attended model and targets. The repeatable fresh
|
||||
backup/retention path is prepared; its first operating run remains open.
|
||||
|
||||
The other session does not update automatically, and a fresh login was needed
|
||||
after a short interval. The user wants a full working day without renewed login
|
||||
|
|
@ -149,6 +167,8 @@ and omit both file stores. They cannot establish this three-store recovery claim
|
|||
The two-account functional walkthrough is user-confirmed with UI follow-up.
|
||||
The attended capture/restart action has executed successfully, with automated
|
||||
and human restart checks (overview follow-up above). Off-host publication,
|
||||
isolated restore and rollback remain outstanding.
|
||||
Logged-in account deactivation remains a separate owner-run access-control check
|
||||
from the pilot contract. No account is disabled as part of these walkthrough steps.
|
||||
isolated restore and the same-image configuration inverse now pass. Deactivation
|
||||
of an already-authenticated restored member was verified transactionally in the
|
||||
clone; live accounts were unchanged. Native SSO/ingress recovery and any broader
|
||||
version-rollback claim remain outside that proof. Named operating responsibilities,
|
||||
repeatable cadence/retention and accepted recovery targets still need completion.
|
||||
|
|
|
|||
|
|
@ -3,24 +3,30 @@
|
|||
Updated 2026-11-06 under RAPPS-WP-0014 and VERGABE-WP-0019.
|
||||
Release binding is complete (RAPPS-WP-0014-T02). Recovery and invited-recipient acceptance remain open.
|
||||
|
||||
November 6 recovery update: the founder's matched PostgreSQL/media/SQLite capture
|
||||
and app resume pass at revision 4; the full operation took 98.61 seconds. Both
|
||||
accounts work after restart, with an overview-list UI follow-up. See the
|
||||
[capture receipt](evidence/2026-11-06-vergabe-three-store-capture-return.json).
|
||||
Provider publication/readback, isolated restore, rollback/access and operating
|
||||
ownership remain open. The accepted point can be transferred without another
|
||||
pause using the [encrypted export packet](proposals/2026-11-06-vergabe-recovery-export/README.md).
|
||||
That export now passes: [the founder's workstation receipt](evidence/2026-11-06-vergabe-encrypted-workstation-copy-return.json)
|
||||
verifies 182521 encrypted bytes, SHA256 `49152883…6bef76f`, outside the production
|
||||
node. Provider publication, escrow decryption and restore remain open; the
|
||||
[contained owner handoff](proposals/2026-11-06-vergabe-provider-publication/README.md)
|
||||
is prepared.
|
||||
November 6 recovery update: matched PostgreSQL/media/SQLite capture and app
|
||||
resume pass at revision 4 (98.61 seconds total). Both accounts work after
|
||||
restart. The captured tender is status 10 (Gewonnen), outside the overview's
|
||||
1–9 range; it remains present and not archived. See the
|
||||
[capture](evidence/2026-11-06-vergabe-three-store-capture-return.json) and
|
||||
[restore](evidence/2026-11-06-vergabe-native-restore-return.json) receipts.
|
||||
The [encrypted workstation copy](evidence/2026-11-06-vergabe-encrypted-workstation-copy-return.json)
|
||||
verifies 182521 bytes, SHA256 `49152883…6bef76f`, outside the production node.
|
||||
|
||||
Provider publication/readback/decryption is now verified by the
|
||||
[founder receipt](evidence/2026-11-06-vergabe-provider-recovery-return.json):
|
||||
201/200, correct ciphertext and packet/store hashes, Warden cleanup confirmed.
|
||||
The [isolated native restore/inverse packet](proposals/2026-11-06-vergabe-native-restore/README.md)
|
||||
is prepared; its runtime evidence and operating acceptance remain open.
|
||||
Native isolated recovery now passes for all 39 PostgreSQL tables / 247 rows,
|
||||
media and issue-state fingerprints, two restored member sessions/workflow and
|
||||
access denials. The same-image configuration inverse preserves committed clone
|
||||
writes across two Pod replacements. Worker time was 215 seconds; restore/checks
|
||||
55.41 seconds, inverse/recheck 55.24 seconds. Namespace/PVs and private staging
|
||||
were removed, with production specification/readiness unchanged. See the
|
||||
[restore scope](proposals/2026-11-06-vergabe-native-restore/README.md) and
|
||||
[transport receipt](evidence/2026-11-06-vergabe-native-restore-transport-return.json).
|
||||
Restored native SSO/ingress and older-image/schema/Helm rollback are untested.
|
||||
[Operating acceptance](vergabe-demo-company-operating-acceptance.proposed.md)
|
||||
now selects attended backups and named targets. The repeatable fresh-point
|
||||
backup/retention path is prepared and awaits its first operating run.
|
||||
|
||||
| Item | Current binding |
|
||||
| --- | --- |
|
||||
|
|
@ -125,8 +131,10 @@ bootstrap helpers or substitute apps_admin credentials.
|
|||
Revision 4 is the current binding; revisions 1 through 3 are historical deployment
|
||||
receipts, not demonstrated safe rollback targets. On a failed next upgrade,
|
||||
stop public traffic and reconcile schema/image compatibility before reverting. Retain the
|
||||
claims and database; namespace deletion is not rollback. Pod replacement and
|
||||
isolated restore must still be proven with matched PostgreSQL/media/SQLite data.
|
||||
claims and database; namespace deletion is not rollback. Populated restart,
|
||||
isolated three-store recovery and a same-image configuration inverse are now
|
||||
proved by the November 6 receipts. Those receipts do not admit an older image
|
||||
or incompatible schema downgrade.
|
||||
|
||||
Current recovery rollout: docs/evidence/2026-09-12-account-recovery-live.md.
|
||||
|
||||
|
|
@ -135,9 +143,10 @@ Current recovery rollout: docs/evidence/2026-09-12-account-recovery-live.md.
|
|||
|
||||
Read-only deployment, Helm, claim, certificate and HTTPS checks match this
|
||||
binding; see [the verification receipt](evidence/2026-09-27-vergabe-binding-review.md).
|
||||
The September 25 database restore and pod restart passed, but populated media
|
||||
At that review, the September 25 database restore and pod restart passed, but populated media
|
||||
and issue-state recovery, an off-host copy, restored two-user workflow and
|
||||
rollback/support acceptance remain RAPPS-WP-0014-T03. That task is waiting and
|
||||
the workplan is blocked. Earlier dated sections describe their original
|
||||
snapshots; the current binding table and this review supersede their pending
|
||||
the workplan was blocked. The November 6 update supersedes that recovery wait;
|
||||
operating acceptance remains open. Earlier dated sections describe their original
|
||||
snapshots; the current binding table and latest update supersede their pending
|
||||
DNS, deployment and first-login statements.
|
||||
|
|
|
|||
58
docs/vergabe-demo-company-operating-acceptance.proposed.md
Normal file
58
docs/vergabe-demo-company-operating-acceptance.proposed.md
Normal file
|
|
@ -0,0 +1,58 @@
|
|||
# Demo-company operating acceptance — attended model selected
|
||||
|
||||
Prepared 2026-11-06 for RAPPS-WP-0014-T03. The founder selected the attended model
|
||||
with the contacts, cadence, retention and recovery targets below. The repeatable
|
||||
operating path is prepared and tested; its first native run remains open.
|
||||
No scheduled service or SLA is inferred.
|
||||
|
||||
The populated three-store recovery drill now passes from the independently
|
||||
retrieved Nextcloud object. The [native receipt](evidence/2026-11-06-vergabe-native-restore-return.json)
|
||||
records 55.41 seconds for restore/checks, 55.24 seconds for the configuration
|
||||
inverse/recheck and 215 seconds including temporary resources and cleanup.
|
||||
Workstation authentication/transfer and native SSO/ingress recovery are outside
|
||||
those timings. The clone used the admitted image and schema; an earlier image
|
||||
or schema downgrade is not an admitted recovery procedure.
|
||||
|
||||
## Accepted attended pilot arrangement
|
||||
|
||||
| Item | Accepted target |
|
||||
| --- | --- |
|
||||
| Scope | The current small invited demo pilot, with the existing single replica and planned maintenance interruptions |
|
||||
| Backup execution and first incident/support contact | Bernd Worsch, using his own attended operator lane |
|
||||
| Provider/storage custody route | Existing railiance-platform Nextcloud Backup/OpenBao recovery lane; no new credentials or grants |
|
||||
| Cadence | A fresh coherent off-host point at the end of every active pilot day and before deployment/data changes |
|
||||
| Retention | Keep seven daily and two weekly verified provider points (representatives may overlap); protect newest and the successful baseline, with later baseline replacement requiring explicit review |
|
||||
| Data-loss objective | At most one active pilot day's work, conditional on that day's verified backup completing |
|
||||
| Recovery objective | Attended recovery within one working day after the operator receives the incident; an accepted target, not an SLA inferred from this drill |
|
||||
| Failure handling | A missing/failed backup or provider verification is an incident for the named operator; report the last verified point's age before further changes |
|
||||
| Upgrade handling | Inspect migrations and image/schema compatibility before each upgrade, make a fresh point first, and retain current writes; historical Helm revision alone is insufficient |
|
||||
|
||||
The [attended operating packet](proposals/2026-11-06-vergabe-attended-backups/README.md)
|
||||
prepares `tools/vergabe_pilot_backup.py --execute` for each active pilot day and
|
||||
before changes. It reuses the frozen capture/encryption/provider algorithms,
|
||||
selects a new run with validated metadata/hashes, and requires complete provider
|
||||
readback/decryption/session cleanup. `--verify-restore` adds the already-proved
|
||||
isolated worker for that new point. Seventeen new synthetic cases pass; all
|
||||
76 pilot helper tests pass. The first attended operating run must still pass
|
||||
before the cadence is recorded active. The baseline's exact-point helpers remain
|
||||
unchanged for reproducibility.
|
||||
|
||||
Provider retention has a bounded private-inventory preview and a separate
|
||||
contained owner executor: protect baseline/newest, independently verify all
|
||||
retained/candidate bytes, ETag-guard deletes and require absence readback. Unknown
|
||||
objects are excluded. No retention deletion, schedule or notification worker has
|
||||
been executed/installed. Local private recovery copies and node journals remain;
|
||||
provider retention does not remove them. Use only the reviewed founder command
|
||||
as the go for a deletion batch, as specified in the packet.
|
||||
|
||||
## Work still to finish
|
||||
|
||||
Execute the first fresh attended backup with `--verify-restore` and return the
|
||||
sanitized backup/worker/transport receipts. Review provider retention previews
|
||||
and execute only when eligible excess points exist; no artificial deletion is
|
||||
needed to accept an empty retention plan.
|
||||
Restored interactive SSO/callback/ingress remains untested; the isolated drill
|
||||
proved restored ordinary member sessions and access denials. Any required native
|
||||
SSO recovery demonstration must use an admitted isolated callback rather than
|
||||
changing the live product callback. Broader recipient acceptance remains with
|
||||
VERGABE-WP-0019.
|
||||
283
tests/test_vergabe_pilot_attended.py
Normal file
283
tests/test_vergabe_pilot_attended.py
Normal file
|
|
@ -0,0 +1,283 @@
|
|||
"""Synthetic fresh-point/retention boundaries; no native backup or credentials."""
|
||||
|
||||
import datetime as dt
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import shlex
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import unittest
|
||||
from contextlib import redirect_stdout
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
import test_vergabe_pilot_publish as fixtures
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tools"))
|
||||
import vergabe_pilot_backup as backup
|
||||
import vergabe_pilot_export as export
|
||||
import vergabe_pilot_operating_restore as recovery
|
||||
import vergabe_pilot_point as point
|
||||
import vergabe_pilot_publish as publisher
|
||||
import vergabe_pilot_retention as retention
|
||||
from vergabe_pilot_recovery_store import Refused
|
||||
|
||||
|
||||
class ScopeTests(unittest.TestCase):
|
||||
def test_dates_and_shell_injection_refuse_before_transport(self):
|
||||
for value in ["../../other", "20261105T000000Z-$(cat /etc/passwd)",
|
||||
"20269905T233543Z-0123456789ab", "20991105T233543Z-0123456789ab"]:
|
||||
with self.subTest(value=value), patch.object(backup, "source_pins") as source, self.assertRaises(Refused):
|
||||
backup.remote_command(value, "capture")
|
||||
source.assert_not_called()
|
||||
|
||||
def test_embedded_node_sources_compile_without_native_access(self):
|
||||
command = backup.remote_command("20261105T233543Z-0123456789ab", "check")
|
||||
operation = shlex.split(command[-1])
|
||||
self.assertEqual(operation[:5], ["sudo", "/usr/bin/env", "KUBECONFIG=/etc/rancher/k3s/k3s.yaml", "/usr/bin/python3", "-c"])
|
||||
result = subprocess.run([sys.executable, "-c", operation[5], "--help"], capture_output=True,
|
||||
timeout=10, check=False, cwd="/tmp")
|
||||
self.assertEqual(result.returncode, 0)
|
||||
self.assertIn(b"capture", result.stdout)
|
||||
|
||||
def test_changed_source_pin_refuses_before_ssh(self):
|
||||
body = json.loads(backup.PLAN.read_text())
|
||||
body["sources"]["vergabe_pilot_point"] = "0" * 64
|
||||
with patch.object(backup.Path, "read_text", return_value=json.dumps(body)), self.assertRaises(Refused):
|
||||
backup.source_pins()
|
||||
|
||||
def test_selected_restore_bootstrap_compiles_and_refuses_nonfounder_without_native_access(self):
|
||||
command = recovery.remote_command(backup.source_pins(), "20261105T234001Z-0123456789ab", "0123456789ab")
|
||||
operation = shlex.split(command[-1])
|
||||
result = subprocess.run([sys.executable, "-c", operation[5], *operation[6:]],
|
||||
capture_output=True, timeout=10, check=False, cwd="/tmp", env={"PATH": "/usr/bin:/bin"})
|
||||
self.assertEqual(result.returncode, 1)
|
||||
self.assertEqual(json.loads(result.stdout)["failure"], "founder_node_sudo_required")
|
||||
self.assertEqual(result.stderr, b"")
|
||||
|
||||
def test_no_restore_transport_after_failed_owner_publication(self):
|
||||
with patch.object(backup.sys, "argv", ["backup", "--publish", "20261105T234001Z-0123456789ab", "--verify-restore"]), \
|
||||
patch.object(backup.sys.stdin, "isatty", return_value=True), patch.object(backup, "source_pins"), \
|
||||
patch.object(backup, "configure"), patch.object(backup, "owner_action", side_effect=Refused("fixture")), \
|
||||
patch.object(recovery, "execute") as restore, redirect_stdout(io.StringIO()), self.assertRaises(Refused):
|
||||
backup.main()
|
||||
restore.assert_not_called()
|
||||
|
||||
def test_capture_failure_never_fetches_or_opens_owner_login(self):
|
||||
with patch.object(backup.sys, "argv", ["backup", "--execute"]), \
|
||||
patch.object(backup.sys.stdin, "isatty", return_value=True), \
|
||||
patch.object(backup, "source_pins"), patch.object(backup, "remote_command", return_value=["fake-ssh"]), \
|
||||
patch.object(backup.subprocess, "run", return_value=subprocess.CompletedProcess([], 1)), \
|
||||
patch.object(backup, "fetch") as fetch, patch.object(backup, "owner_action") as owner, \
|
||||
redirect_stdout(io.StringIO()), self.assertRaises(Refused):
|
||||
backup.main()
|
||||
fetch.assert_not_called()
|
||||
owner.assert_not_called()
|
||||
|
||||
|
||||
@unittest.skipUnless(shutil.which("age") and shutil.which("age-keygen"), "requires synthetic age tools")
|
||||
class FreshPointTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.fixture = fixtures.PublisherTests()
|
||||
self.fixture.setUp()
|
||||
self.addCleanup(self.fixture.doCleanups)
|
||||
for target, names in [(export, ("RUN_ID", "NAME", "EXPECTED", "BASE")),
|
||||
(publisher, ("OBJECT", "CIPHER_SHA256", "CIPHER_BYTES")),
|
||||
(recovery.restore, ("CIPHER", "PLAIN"))]:
|
||||
for name in names:
|
||||
patcher = patch.object(target, name, getattr(target, name))
|
||||
patcher.start()
|
||||
self.addCleanup(patcher.stop)
|
||||
self.node = self.fixture.root / "node"
|
||||
self.node.mkdir(mode=0o700)
|
||||
export.BASE = self.node
|
||||
self.kits = self.fixture.root / "kits"
|
||||
self.kits.mkdir(mode=0o700)
|
||||
|
||||
def make_point(self, run, content):
|
||||
directory = self.node / run
|
||||
shutil.copytree(self.fixture.point, directory)
|
||||
directory.chmod(0o700)
|
||||
for name in ("seal.json", "recovery-point.zip.age"):
|
||||
(directory / name).unlink()
|
||||
dump = directory / "database.dump"
|
||||
dump.write_bytes(content)
|
||||
receipt = json.loads((directory / "receipt.json").read_text())
|
||||
receipt.update(run_id=run, database_bytes=len(content), database_sha256=hashlib.sha256(content).hexdigest())
|
||||
(directory / "receipt.json").write_text(json.dumps(receipt))
|
||||
point.select_node(run)
|
||||
cipher, sealed = export.seal(directory, "age")
|
||||
kit = self.kits / run
|
||||
kit.mkdir(mode=0o700)
|
||||
shutil.copy2(cipher, kit / (export.NAME + ".zip.age"))
|
||||
shutil.copy2(directory / "seal.json", kit / (export.NAME + ".transfer.json"))
|
||||
return kit, sealed
|
||||
|
||||
def test_two_different_fresh_points_encrypt_publish_readback_decrypt_and_inventory(self):
|
||||
runs = ("20261105T234001Z-0123456789ab", "20261105T234002Z-0123456789ac")
|
||||
hashes = []
|
||||
for number, run in enumerate(runs):
|
||||
kit, sealed = self.make_point(run, b"PGDMP changed synthetic data " + str(number).encode())
|
||||
backup.configure(kit, run)
|
||||
provider = self.fixture.provider()
|
||||
body = {"run_id": run}
|
||||
publisher.publish(kit, self.fixture.credentials, provider, body)
|
||||
body.update(warden_exit_code=0, session_cleanup_confirmed=True)
|
||||
self.assertTrue(point.successful_provider(body, run, sealed))
|
||||
self.assertTrue(body["packet_file_hashes_verified"])
|
||||
self.assertTrue(body["sqlite_link_verified"])
|
||||
self.assertFalse(body["restore_verified"])
|
||||
backup.write_receipt(kit / "provider-receipt-fixture.json", body)
|
||||
self.assertTrue(provider.put_calls[0].endswith("/vergabe-pilot-" + run + ".zip.age"))
|
||||
hashes.append(body["ciphertext_sha256"])
|
||||
self.assertNotEqual(*hashes)
|
||||
self.assertEqual({p["run_id"] for p in backup.inventory(self.kits)}, set(runs))
|
||||
self.assertNotIn("synthetic private row", json.dumps(body))
|
||||
|
||||
def test_selected_new_point_reuses_full_recovered_file_checks_before_restore(self):
|
||||
run = "20261105T234001Z-0123456789ab"
|
||||
kit, sealed = self.make_point(run, b"PGDMP changed")
|
||||
backup.configure(kit, run)
|
||||
body = {"run_id": run}
|
||||
publisher.publish(kit, self.fixture.credentials, self.fixture.provider(), body)
|
||||
body.update(warden_exit_code=0, session_cleanup_confirmed=True)
|
||||
backup.write_receipt(kit / "provider-receipt-fixture.json", body)
|
||||
_, recovered, files, _ = recovery.selected_receipt(kit, run, sealed)
|
||||
self.assertEqual((recovered / "database.dump").read_bytes(), b"PGDMP changed")
|
||||
self.assertEqual(files, sealed["packet"]["files"])
|
||||
(recovered / "database.dump").write_bytes(b"tampered")
|
||||
with self.assertRaises(Refused):
|
||||
recovery.selected_receipt(kit, run, sealed)
|
||||
|
||||
def test_tampered_or_public_transfer_refuses_before_owner_login(self):
|
||||
run = "20261105T234001Z-0123456789ab"
|
||||
kit, _ = self.make_point(run, b"PGDMP changed")
|
||||
path = kit / ("vergabe-pilot-" + run + ".transfer.json")
|
||||
metadata = json.loads(path.read_text())
|
||||
metadata["packet"]["image"] = "unreviewed:image"
|
||||
path.write_text(json.dumps(metadata))
|
||||
with self.assertRaises(Refused):
|
||||
backup.configure(kit, run)
|
||||
self.assertEqual(backup.inventory(self.kits), [])
|
||||
path.chmod(0o644)
|
||||
with self.assertRaises(Refused):
|
||||
point.select_transfer(kit, run)
|
||||
|
||||
def test_node_rejects_incomplete_capture_before_sealing(self):
|
||||
run = "20261105T234001Z-0123456789ab"
|
||||
self.make_point(run, b"PGDMP changed")
|
||||
path = self.node / run / "receipt.json"
|
||||
body = json.loads(path.read_text())
|
||||
body["app_resumed"] = False
|
||||
path.write_text(json.dumps(body))
|
||||
with self.assertRaises(Refused):
|
||||
point.select_node(run)
|
||||
|
||||
def test_provider_failure_and_removed_points_never_count_as_current_backups(self):
|
||||
run = "20261105T234001Z-0123456789ab"
|
||||
kit, sealed = self.make_point(run, b"PGDMP changed")
|
||||
body = {"run_id": run}
|
||||
backup.configure(kit, run)
|
||||
publisher.publish(kit, self.fixture.credentials, self.fixture.provider(), body)
|
||||
body.update(warden_exit_code=14, session_cleanup_confirmed=False)
|
||||
path = kit / "provider-receipt-fixture.json"
|
||||
backup.write_receipt(path, body)
|
||||
self.assertEqual(backup.inventory(self.kits), [])
|
||||
body.update(warden_exit_code=0, session_cleanup_confirmed=True)
|
||||
path.write_text(json.dumps(body))
|
||||
self.assertEqual(len(backup.inventory(self.kits)), 1)
|
||||
backup.write_receipt(kit / "provider-removed.json", {"run_id": run,
|
||||
"ciphertext_sha256": sealed["ciphertext_sha256"],
|
||||
"status": "provider_removed_and_readback_absent"})
|
||||
self.assertEqual(backup.inventory(self.kits), [])
|
||||
|
||||
|
||||
class RetentionTests(unittest.TestCase):
|
||||
def item(self, run):
|
||||
content = ("synthetic ciphertext " + run).encode()
|
||||
return {"run_id": run, "ciphertext_sha256": hashlib.sha256(content).hexdigest(),
|
||||
"ciphertext_bytes": len(content)}
|
||||
|
||||
def fixture(self):
|
||||
# Multiple captures on one day exercise candidate deletion without
|
||||
# inventing future native evidence; baseline/newest remain protected.
|
||||
runs = [point.BASELINE, "20261105T235001Z-0123456789ab", "20261105T235002Z-0123456789ac",
|
||||
"20261105T235003Z-0123456789ad"]
|
||||
plan = point.retention_plan([self.item(r) for r in runs])
|
||||
return plan
|
||||
|
||||
def objects(self, plan, bad_run=None, changed_on_delete=False):
|
||||
class Fake:
|
||||
def __init__(self):
|
||||
self.deleted = []
|
||||
def request(self, item, delete=False, etag=None):
|
||||
run = item["run_id"]
|
||||
if delete:
|
||||
if changed_on_delete:
|
||||
return 412, b"", None
|
||||
assert etag == '"synthetic-' + run + '"'
|
||||
self.deleted.append(run)
|
||||
return 204, b"", None
|
||||
if run in self.deleted:
|
||||
return 404, b"", None
|
||||
content = b"changed provider bytes" if run == bad_run else ("synthetic ciphertext " + run).encode()
|
||||
return 200, content, '"synthetic-' + run + '"'
|
||||
return Fake()
|
||||
|
||||
def anchor(self, plan):
|
||||
baseline = next(p for p in plan["keep"] if p["run_id"] == point.BASELINE)
|
||||
return patch.multiple(retention, BASELINE_SHA=baseline["ciphertext_sha256"], BASELINE_BYTES=baseline["ciphertext_bytes"])
|
||||
|
||||
def test_baseline_newest_and_daily_weekly_representatives_are_protected(self):
|
||||
plan = self.fixture()
|
||||
self.assertIn(point.BASELINE, {p["run_id"] for p in plan["keep"]})
|
||||
self.assertIn("20261105T235003Z-0123456789ad", {p["run_id"] for p in plan["keep"]})
|
||||
self.assertEqual(len(plan["candidates"]), 2)
|
||||
self.assertFalse(plan["unknown_objects_eligible"])
|
||||
|
||||
def test_seven_daily_two_weekly_representatives_across_three_weeks(self):
|
||||
class FutureDateTime(dt.datetime):
|
||||
@classmethod
|
||||
def now(cls, tz=None):
|
||||
return cls(2026, 11, 30, tzinfo=dt.UTC)
|
||||
runs = [f"202611{day:02d}T120000Z-0123456789ab" for day in range(5, 26)]
|
||||
with patch.object(point.dt, "datetime", FutureDateTime):
|
||||
plan = point.retention_plan([self.item(r) for r in [point.BASELINE, *runs]])
|
||||
retained = {p["run_id"] for p in plan["keep"]}
|
||||
self.assertTrue({f"202611{day:02d}T120000Z-0123456789ab" for day in range(19, 26)} <= retained)
|
||||
self.assertIn(point.BASELINE, retained)
|
||||
self.assertGreater(len(plan["candidates"]), 0)
|
||||
|
||||
def test_all_retained_bytes_are_checked_before_any_conditional_delete(self):
|
||||
plan = self.fixture()
|
||||
objects = self.objects(plan, bad_run=point.BASELINE)
|
||||
with self.anchor(plan), self.assertRaises(Refused):
|
||||
retention.execute(plan, objects, lambda item: self.fail("unexpected tombstone"))
|
||||
self.assertEqual(objects.deleted, [])
|
||||
|
||||
def test_conditional_delete_and_absence_readback_persist_each_tombstone(self):
|
||||
plan = self.fixture()
|
||||
objects = self.objects(plan)
|
||||
markers = []
|
||||
with self.anchor(plan):
|
||||
deleted = retention.execute(plan, objects, lambda item: markers.append(item["run_id"]))
|
||||
self.assertEqual(deleted, markers)
|
||||
self.assertEqual(set(deleted), {p["run_id"] for p in plan["candidates"]})
|
||||
self.assertNotIn(point.BASELINE, deleted)
|
||||
|
||||
def test_provider_race_refuses_without_tombstone_or_delete(self):
|
||||
plan = self.fixture()
|
||||
objects = self.objects(plan, changed_on_delete=True)
|
||||
with self.anchor(plan), self.assertRaises(Refused):
|
||||
retention.execute(plan, objects, lambda item: self.fail("unexpected tombstone"))
|
||||
self.assertEqual(objects.deleted, [])
|
||||
|
||||
def test_unsafe_runs_in_inventory_refuse_instead_of_becoming_delete_urls(self):
|
||||
with self.assertRaises(Refused):
|
||||
point.retention_plan([self.item("../../foreign")])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
268
tools/vergabe_pilot_backup.py
Normal file
268
tools/vergabe_pilot_backup.py
Normal file
|
|
@ -0,0 +1,268 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Founder workstation attended backup for the admitted demo-company release.
|
||||
|
||||
Default reviews only. --execute captures a fresh point (brief app pause), seals
|
||||
on-node, fetches ciphertext, then opens contained owner login to publish/readback/
|
||||
decrypt. --fetch RUN or --publish RUN retries without another capture/pause.
|
||||
--retention previews owner-verified provider retention; --prune executes it.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shlex
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
import vergabe_pilot_export as export
|
||||
import vergabe_pilot_export_remote as transfer
|
||||
import vergabe_pilot_point as point
|
||||
import vergabe_pilot_publish as publisher
|
||||
from vergabe_pilot_recovery_store import Refused, private_directory, require
|
||||
|
||||
PLAN = Path(__file__).resolve().parents[1] / "docs/proposals/2026-11-06-vergabe-attended-backups/plan.json"
|
||||
NODE_MODULES = ("vergabe_pilot_recovery_store", "vergabe_pilot_export", "vergabe_pilot_point",
|
||||
"vergabe_pilot_snapshot", "vergabe_pilot_backup_node")
|
||||
|
||||
|
||||
def source_pins():
|
||||
plan = json.loads(PLAN.read_text())
|
||||
expected = {*NODE_MODULES, "vergabe_pilot_export_remote", "vergabe_pilot_publish",
|
||||
"vergabe_pilot_backup", "vergabe_pilot_retention", "vergabe_pilot_operating_restore",
|
||||
"vergabe_pilot_restore", "vergabe_pilot_restore_remote"}
|
||||
require(set(plan["sources"]) == expected, "attended_source_set_changed")
|
||||
sources = {}
|
||||
for name, checksum in plan["sources"].items():
|
||||
raw = Path(__file__).with_name(name + ".py").read_bytes()
|
||||
require(hashlib.sha256(raw).hexdigest() == checksum, "attended_source_pin_changed")
|
||||
sources[name] = raw.decode()
|
||||
publisher.pins()
|
||||
return sources
|
||||
|
||||
|
||||
def remote_command(run, mode):
|
||||
point.run_time(run)
|
||||
require(mode in {"check", "capture", "seal", "resume"}, "invalid_backup_mode")
|
||||
sources = source_pins()
|
||||
bootstrap = "import sys,types;"
|
||||
for name in NODE_MODULES[:-1]:
|
||||
bootstrap += ("m=types.ModuleType(" + repr(name) + ");sys.modules[m.__name__]=m;"
|
||||
"exec(compile(" + repr(sources[name]) + "," + repr(name + ".py") + ",'exec'),m.__dict__);")
|
||||
bootstrap += ("sys.argv=['vergabe_pilot_backup_node.py',*sys.argv[1:]];exec(compile(" +
|
||||
repr(sources[NODE_MODULES[-1]]) + ",'vergabe_pilot_backup_node.py','exec'),{'__name__':'__main__'})")
|
||||
command = shlex.join(["sudo", "/usr/bin/env", "KUBECONFIG=/etc/rancher/k3s/k3s.yaml",
|
||||
"/usr/bin/python3", "-c", bootstrap, "--run", run, mode])
|
||||
require(len(command.encode()) < 120000, "attended_transport_size_bound")
|
||||
return ["ssh", "-t", "railiance01", command]
|
||||
|
||||
|
||||
def kit_root():
|
||||
return Path.home() / ".local/state/vergabe-pilot-recovery"
|
||||
|
||||
|
||||
def configure(directory, run):
|
||||
require(transfer.outside_git(directory.resolve()), "private_kit_outside_git_required")
|
||||
sealed = point.select_transfer(directory, run)
|
||||
verified = transfer.verify(directory)
|
||||
publisher.OBJECT = export.NAME + ".zip.age"
|
||||
publisher.CIPHER_SHA256 = sealed["ciphertext_sha256"]
|
||||
publisher.CIPHER_BYTES = sealed["ciphertext_bytes"]
|
||||
require(verified["ciphertext_sha256"] == sealed["ciphertext_sha256"], "sealed_point_changed")
|
||||
return sealed
|
||||
|
||||
|
||||
def fetch(run, already_sealed=False):
|
||||
point.run_time(run)
|
||||
root = kit_root()
|
||||
require(transfer.outside_git(root.resolve()), "private_kit_outside_git_required")
|
||||
root.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
private_directory(root)
|
||||
directory = root / run
|
||||
if directory.exists() or directory.is_symlink():
|
||||
configure(directory, run)
|
||||
return directory
|
||||
if not already_sealed:
|
||||
result = subprocess.run(remote_command(run, "seal"), check=False)
|
||||
require(result.returncode == 0, "node_seal_failed_use_same_run_for_retry")
|
||||
with tempfile.TemporaryDirectory(prefix="fetch-", dir=root) as temporary:
|
||||
temporary = Path(temporary)
|
||||
for suffix in (".transfer.json", ".zip.age"):
|
||||
name = "vergabe-pilot-" + run + suffix
|
||||
copied = subprocess.run(["scp", "-p", "railiance01:" + name, str(temporary / name)],
|
||||
capture_output=True, timeout=120, check=False)
|
||||
require(copied.returncode == 0, "encrypted_fetch_failed_use_same_run_for_retry")
|
||||
configure(temporary, run)
|
||||
temporary.rename(directory)
|
||||
return directory
|
||||
|
||||
|
||||
def inventory(root):
|
||||
"""Only private local kits with ciphertext and complete owner receipts qualify."""
|
||||
private_directory(root)
|
||||
children = list(root.iterdir())
|
||||
require(len(children) <= 2000, "local_inventory_bound")
|
||||
points = []
|
||||
for directory in sorted(children):
|
||||
if not directory.is_dir() or directory.is_symlink():
|
||||
continue
|
||||
try:
|
||||
point.run_time(directory.name)
|
||||
sealed = configure(directory, directory.name)
|
||||
tombstone = directory / "provider-removed.json"
|
||||
if tombstone.exists() or tombstone.is_symlink():
|
||||
removed = point.read_private(tombstone)
|
||||
require(removed["run_id"] == directory.name and removed["ciphertext_sha256"] == sealed["ciphertext_sha256"]
|
||||
and removed["status"] == "provider_removed_and_readback_absent", "retention_tombstone_invalid")
|
||||
continue
|
||||
receipts = list(directory.glob("provider-receipt-*.json"))
|
||||
require(len(receipts) <= 100, "provider_receipt_count_bound")
|
||||
if any(point.successful_provider(point.read_private(p), directory.name, sealed) for p in receipts):
|
||||
points.append({"run_id": directory.name, "ciphertext_sha256": sealed["ciphertext_sha256"],
|
||||
"ciphertext_bytes": sealed["ciphertext_bytes"]})
|
||||
except (OSError, ValueError, KeyError, TypeError, Refused):
|
||||
continue # Incomplete/unknown kits never qualify for provider deletion.
|
||||
return points
|
||||
|
||||
|
||||
def write_receipt(path, body):
|
||||
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600)
|
||||
with os.fdopen(fd, "w") as stream:
|
||||
json.dump(body, stream, indent=2)
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
|
||||
|
||||
def owner_child(args):
|
||||
# Child stays silent under Warden, including setup and parser failures.
|
||||
try:
|
||||
source_pins()
|
||||
require(args.directory and args.receipt and args.directory.is_absolute()
|
||||
and args.receipt.parent == args.directory, "private_owner_paths_required")
|
||||
configure(args.directory, args.run)
|
||||
if args.action == "publish":
|
||||
return publisher.child(args.directory, args.receipt)
|
||||
require(args.action == "prune", "invalid_owner_action")
|
||||
import vergabe_pilot_retention as retention
|
||||
return retention.child(args.directory.parent, args.receipt, inventory)
|
||||
except BaseException: # noqa: BLE001 — contained child must never print private output
|
||||
return 1
|
||||
|
||||
|
||||
def owner_action(directory, run, action="publish"):
|
||||
configure(directory, run)
|
||||
require(all(shutil.which(t) for t in ("warden", "bao", "age", "age-keygen")), "owner_tools_required")
|
||||
output = directory / (("provider-receipt-" if action == "publish" else "retention-receipt-") + uuid.uuid4().hex[:12] + ".json")
|
||||
command = [sys.executable, str(publisher.PLATFORM / "scripts/openbao-attended-exec.py"), "--",
|
||||
sys.executable, str(Path(__file__).resolve()), "--contained", "--run", run,
|
||||
"--directory", str(directory), "--receipt", str(output), "--action", action]
|
||||
result = subprocess.run(command, check=False, env=publisher.owner_login_environment())
|
||||
if output.exists() or output.is_symlink():
|
||||
body = point.read_private(output)
|
||||
else:
|
||||
body = {"status": "owner_login_or_child_not_completed", "provider_outcome": "unconfirmed"}
|
||||
body.update(warden_exit_code=result.returncode, receipt_file=str(output), **publisher.session_outcome(result.returncode))
|
||||
if output.exists():
|
||||
export.regular(output, os.geteuid())
|
||||
fd = os.open(output, os.O_WRONLY | os.O_TRUNC | os.O_NOFOLLOW)
|
||||
with os.fdopen(fd, "w") as stream:
|
||||
json.dump(body, stream, indent=2)
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
else:
|
||||
write_receipt(output, body)
|
||||
print(json.dumps(body, indent=2))
|
||||
expected = "published_read_back_and_decrypted" if action == "publish" else "provider_retention_complete"
|
||||
require(result.returncode == 0 and body["status"] == expected, "owner_operation_incomplete")
|
||||
return body
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
modes = parser.add_mutually_exclusive_group()
|
||||
modes.add_argument("--check", action="store_true")
|
||||
modes.add_argument("--execute", action="store_true")
|
||||
modes.add_argument("--fetch")
|
||||
modes.add_argument("--publish")
|
||||
modes.add_argument("--resume")
|
||||
modes.add_argument("--retention", action="store_true")
|
||||
modes.add_argument("--prune", action="store_true")
|
||||
modes.add_argument("--restore")
|
||||
modes.add_argument("--restore-cleanup", nargs=2, metavar=("RUN", "NONCE"))
|
||||
modes.add_argument("--contained", action="store_true", help=argparse.SUPPRESS)
|
||||
parser.add_argument("--run", help=argparse.SUPPRESS)
|
||||
parser.add_argument("--directory", type=Path, help=argparse.SUPPRESS)
|
||||
parser.add_argument("--receipt", type=Path, help=argparse.SUPPRESS)
|
||||
parser.add_argument("--action", choices=("publish", "prune"), default="publish", help=argparse.SUPPRESS)
|
||||
parser.add_argument("--verify-restore", action="store_true", help="also rehearse this point in fresh isolated claims")
|
||||
args = parser.parse_args()
|
||||
if args.contained:
|
||||
return owner_child(args)
|
||||
require(not args.verify_restore or args.execute or args.publish, "verify_restore_requires_capture_or_publish")
|
||||
sources = source_pins()
|
||||
if not any((args.check, args.execute, args.fetch, args.publish, args.resume, args.retention, args.prune,
|
||||
args.restore, args.restore_cleanup)):
|
||||
print(json.dumps({"status": "prepared", "target": "railiance01", "helm_revision": 4,
|
||||
"next": "--check then --execute", "runtime_mutations": False,
|
||||
"capture_briefly_pauses_pilot": True, "scheduled_service": False}))
|
||||
return 0
|
||||
require(sys.stdin.isatty(), "founder_interactive_terminal_required")
|
||||
if args.restore or args.restore_cleanup:
|
||||
import vergabe_pilot_operating_restore as recovery
|
||||
if args.restore_cleanup:
|
||||
return recovery.cleanup(sources, *args.restore_cleanup)
|
||||
directory = kit_root() / args.restore
|
||||
sealed = configure(directory, args.restore)
|
||||
return recovery.execute(directory, args.restore, sealed, sources)
|
||||
if args.retention or args.prune:
|
||||
plan = point.retention_plan(inventory(kit_root()))
|
||||
print(json.dumps({"status": "retention_preview", **plan}, indent=2), flush=True)
|
||||
if args.prune and plan["candidates"]:
|
||||
require(any(p["run_id"] == point.BASELINE for p in plan["keep"]), "baseline_receipt_required")
|
||||
newest = plan["keep"][0]["run_id"]
|
||||
owner_action(kit_root() / newest, newest, "prune")
|
||||
return 0
|
||||
run = args.fetch or args.publish or args.resume or dt.datetime.now(dt.UTC).strftime("%Y%m%dT%H%M%SZ-") + uuid.uuid4().hex[:12]
|
||||
point.run_time(run)
|
||||
if args.check or args.resume:
|
||||
return subprocess.run(remote_command(run, "resume" if args.resume else "check"), check=False).returncode
|
||||
print(json.dumps({"status": "attended_backup_started", "run_id": run,
|
||||
"capture_requested": args.execute}), flush=True)
|
||||
if args.execute:
|
||||
result = subprocess.run(remote_command(run, "capture"), check=False)
|
||||
require(result.returncode == 0, "capture_or_seal_failed_keep_run_id_and_review_resume")
|
||||
directory = kit_root() / run if args.publish else fetch(run, already_sealed=args.execute)
|
||||
configure(directory, run)
|
||||
if args.fetch:
|
||||
print(json.dumps({"status": "encrypted_workstation_copy_verified", "run_id": run,
|
||||
"provider_publication_verified": False, "local_directory": str(directory)}))
|
||||
return 0
|
||||
body = owner_action(directory, run)
|
||||
if args.verify_restore:
|
||||
import vergabe_pilot_operating_restore as recovery
|
||||
sealed = configure(directory, run)
|
||||
require(recovery.execute(directory, run, sealed, sources) == 0, "backup_published_but_restore_requires_review")
|
||||
summary = {"schema": "railiance.vergabe-pilot-attended-backup.v1", "status": "complete", "run_id": run,
|
||||
"provider_publication_verified": True, "independent_readback_verified": True,
|
||||
"decryption_verified": True, "session_cleanup_confirmed": True, "restore_verified": args.verify_restore,
|
||||
"provider_receipt": body["receipt_file"], "capture_requested": args.execute,
|
||||
"retention_execution_pending": bool(point.retention_plan(inventory(kit_root()))["candidates"])}
|
||||
write_receipt(directory / ("attended-backup-" + uuid.uuid4().hex[:12] + ".json"), summary)
|
||||
print(json.dumps(summary, indent=2))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
os.umask(0o077)
|
||||
try:
|
||||
raise SystemExit(main())
|
||||
except Exception as error: # noqa: BLE001 — keep private transport/parser output suppressed
|
||||
if "--contained" not in sys.argv:
|
||||
print(json.dumps({"status": "attended_backup_failed", "failure": str(error) if isinstance(error, Refused)
|
||||
else "attended_backup_error", "details_suppressed": True}))
|
||||
raise SystemExit(1)
|
||||
76
tools/vergabe_pilot_backup_node.py
Normal file
76
tools/vergabe_pilot_backup_node.py
Normal file
|
|
@ -0,0 +1,76 @@
|
|||
"""Attended capture/seal entry point; frozen capture and export implementations."""
|
||||
|
||||
import argparse
|
||||
import fcntl
|
||||
import json
|
||||
import os
|
||||
import pwd
|
||||
import signal
|
||||
|
||||
import vergabe_pilot_export as export
|
||||
import vergabe_pilot_point as point
|
||||
import vergabe_pilot_snapshot as snapshot
|
||||
from vergabe_pilot_recovery_store import Refused, private_directory, require
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--run", required=True)
|
||||
parser.add_argument("mode", choices=("check", "capture", "seal", "resume"))
|
||||
args = parser.parse_args()
|
||||
point.run_time(args.run)
|
||||
owner = pwd.getpwnam("tegwick")
|
||||
require(os.geteuid() == 0 and os.environ.get("SUDO_USER") == owner.pw_name
|
||||
and os.environ.get("SUDO_UID") == str(owner.pw_uid), "founder_node_sudo_required")
|
||||
if args.mode == "check":
|
||||
_, fixture, _ = snapshot.preflight(args.run)
|
||||
print(json.dumps({"status": "ready_for_attended_capture", "runtime_mutations": False,
|
||||
"fixture_present": fixture["fixture_present"], "helm_revision": snapshot.REVISION}))
|
||||
return 0
|
||||
if not snapshot.BASE.exists():
|
||||
snapshot.BASE.mkdir(mode=0o700)
|
||||
private_directory(snapshot.BASE)
|
||||
fd = os.open(snapshot.BASE / "operation.lock", os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, 0o600)
|
||||
with os.fdopen(fd, "w") as lock:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
if args.mode == "resume":
|
||||
private_directory(snapshot.BASE / args.run)
|
||||
require(point.read_private(snapshot.BASE / "active.json")["run_id"] == args.run,
|
||||
"resume_must_match_active_run")
|
||||
journal = point.read_private(snapshot.BASE / args.run / "journal.json")
|
||||
require(journal["run_id"] == args.run, "journal_run_mismatch")
|
||||
result = snapshot.resume(journal)
|
||||
if result["helper_cleanup_confirmed"]:
|
||||
snapshot.checkpoint(snapshot.BASE / "active.json", {"run_id": args.run, "status": "resumed_after_failure"})
|
||||
print(json.dumps({"status": "application_resumed", "run_id": args.run,
|
||||
"helper_cleanup_confirmed": result["helper_cleanup_confirmed"],
|
||||
"backup_acceptance": False}))
|
||||
return 0 if result["helper_cleanup_confirmed"] else 1
|
||||
if args.mode == "capture":
|
||||
require(args.run != point.BASELINE and not (snapshot.BASE / args.run).exists(), "fresh_run_required")
|
||||
# Refuse a forgotten older interrupted pause before creating any new one.
|
||||
active = snapshot.BASE / "active.json"
|
||||
if active.exists() or active.is_symlink():
|
||||
old = point.read_private(active)
|
||||
require(old["status"] in {"captured_and_resumed", "resumed_after_failure"}, "previous_capture_requires_review_or_resume")
|
||||
baseline, fixture, status = snapshot.preflight(args.run)
|
||||
code = snapshot.execute(args.run, baseline, fixture, status)
|
||||
if code:
|
||||
return code
|
||||
point.select_node(args.run)
|
||||
export.main()
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
os.umask(0o077)
|
||||
def interrupted(signum, frame):
|
||||
raise Refused("attended_backup_interrupted")
|
||||
signal.signal(signal.SIGHUP, interrupted)
|
||||
signal.signal(signal.SIGTERM, interrupted)
|
||||
try:
|
||||
raise SystemExit(main())
|
||||
except Exception as error: # noqa: BLE001 — never expose private subprocess/parser output
|
||||
print(json.dumps({"status": "attended_backup_failed", "failure": str(error) if isinstance(error, Refused)
|
||||
else "attended_backup_node_error", "details_suppressed": True}))
|
||||
raise SystemExit(1)
|
||||
120
tools/vergabe_pilot_operating_restore.py
Normal file
120
tools/vergabe_pilot_operating_restore.py
Normal file
|
|
@ -0,0 +1,120 @@
|
|||
"""Select a successful attended point for the already-proved isolated restore."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shlex
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
import vergabe_pilot_point as point
|
||||
import vergabe_pilot_restore as restore
|
||||
import vergabe_pilot_restore_remote as transport
|
||||
from vergabe_pilot_recovery_store import require
|
||||
|
||||
|
||||
def remote_command(sources, run, nonce, cleanup=False):
|
||||
point.run_time(run)
|
||||
require(re.fullmatch(r"[0-9a-f]{12}", nonce), "invalid_restore_nonce")
|
||||
bootstrap = "import sys,types;"
|
||||
for name in ("vergabe_pilot_recovery_store", "vergabe_pilot_export", "vergabe_pilot_point", "vergabe_pilot_restore"):
|
||||
bootstrap += ("m=types.ModuleType(" + repr(name) + ");sys.modules[m.__name__]=m;"
|
||||
"exec(compile(" + repr(sources[name]) + "," + repr(name + ".py") + ",'exec'),m.__dict__);")
|
||||
selection = '''
|
||||
import json,os,re,signal
|
||||
from pathlib import Path
|
||||
import vergabe_pilot_export as export
|
||||
import vergabe_pilot_point as point
|
||||
import vergabe_pilot_restore as restore
|
||||
from vergabe_pilot_recovery_store import Refused,require
|
||||
try:
|
||||
run,nonce,mode=sys.argv[1:]
|
||||
point.run_time(run)
|
||||
require(os.geteuid()==0 and os.environ.get('SUDO_USER')=='tegwick','founder_node_sudo_required')
|
||||
export.RUN_ID=run;export.NAME='vergabe-pilot-'+run
|
||||
if mode!='cleanup':
|
||||
stage=Path('/home/tegwick')/('vergabe-pilot-restore-'+nonce)
|
||||
info=stage.lstat()
|
||||
owner=int(os.environ['SUDO_UID'])
|
||||
require(stage.is_dir() and not stage.is_symlink() and info.st_uid==owner and not info.st_mode&0o077,'private_founder_stage_required')
|
||||
def read(name):
|
||||
path=stage/name
|
||||
info=export.regular(path,owner)
|
||||
require(0<info.st_size<=16384,'restore_metadata_bound')
|
||||
return json.loads(path.read_text())
|
||||
origin=read('origin.json');provider=read('provider.json')
|
||||
require(origin['run_id']==provider['run_id']==run,'restore_run_changed')
|
||||
point.select(run,origin['files'])
|
||||
require(all(isinstance(provider.get(k),str) and re.fullmatch('[0-9a-f]{64}',provider[k]) for k in ('ciphertext_sha256','plaintext_packet_sha256')),'restore_hash_shape')
|
||||
restore.CIPHER=provider['ciphertext_sha256'];restore.PLAIN=provider['plaintext_packet_sha256']
|
||||
def interrupted(signum,frame):raise Refused('restore_interrupted')
|
||||
signal.signal(signal.SIGHUP,interrupted);signal.signal(signal.SIGTERM,interrupted)
|
||||
sys.argv=['restore','--nonce',nonce,*(['--cleanup'] if mode=='cleanup' else [])]
|
||||
code=restore.main()
|
||||
except BaseException as error:
|
||||
print(json.dumps({'status':'operating_restore_failed','failure':str(error) if isinstance(error,Refused) else 'operating_restore_error','details_suppressed':True}))
|
||||
code=1
|
||||
raise SystemExit(code)
|
||||
'''
|
||||
bootstrap += "exec(" + repr(selection) + ")"
|
||||
command = shlex.join(["sudo", "/usr/bin/env", "KUBECONFIG=/etc/rancher/k3s/k3s.yaml", "/usr/bin/python3",
|
||||
"-c", bootstrap, run, nonce, "cleanup" if cleanup else "restore"])
|
||||
require(len(command.encode()) < 120000, "restore_transport_size_bound")
|
||||
return ["ssh", "-t", "railiance01", command]
|
||||
|
||||
|
||||
def selected_receipt(directory, run, sealed):
|
||||
paths = list(directory.glob("provider-receipt-*.json"))
|
||||
require(len(paths) <= 100, "provider_receipt_count_bound")
|
||||
accepted = []
|
||||
for path in paths:
|
||||
body = point.read_private(path)
|
||||
if point.successful_provider(body, run, sealed):
|
||||
accepted.append((path, body))
|
||||
require(accepted, "successful_provider_receipt_required")
|
||||
path, body = max(accepted, key=lambda item: item[0].stat().st_mtime_ns)
|
||||
restore.CIPHER = body["ciphertext_sha256"]
|
||||
restore.PLAIN = body["plaintext_packet_sha256"]
|
||||
# Full recovered ZIP/ciphertext/file verification, using the selected run.
|
||||
kit, recovered, files = transport.verify_input(path)
|
||||
return kit, recovered, files, path
|
||||
|
||||
|
||||
def execute(directory, run, sealed, sources):
|
||||
kit, recovered, files, receipt = selected_receipt(directory, run, sealed)
|
||||
nonce = uuid.uuid4().hex[:12]
|
||||
command = remote_command(sources, run, nonce)
|
||||
require(subprocess.run(transport.stage_command(nonce, "create"), check=False).returncode == 0,
|
||||
"restore_stage_creation_failed")
|
||||
print(json.dumps({"status": "restore_transport_started", "run_id": run, "nonce": nonce}), flush=True)
|
||||
with tempfile.TemporaryDirectory(prefix="restore-transfer-", dir=kit) as temporary:
|
||||
temporary = Path(temporary)
|
||||
for name in restore.FILES[:4]:
|
||||
shutil.copy2(recovered / name, temporary / name)
|
||||
shutil.copy2(receipt, temporary / "provider.json")
|
||||
(temporary / "origin.json").write_text(json.dumps({"run_id": run, "files": files}))
|
||||
(temporary / "origin.json").chmod(0o600)
|
||||
for name in restore.FILES:
|
||||
copied = subprocess.run(["scp", "-p", str(temporary / name),
|
||||
"railiance01:vergabe-pilot-restore-" + nonce + "/" + name],
|
||||
capture_output=True, timeout=120, check=False)
|
||||
require(copied.returncode == 0, "restore_private_transfer_failed")
|
||||
result = subprocess.run(command, check=False)
|
||||
removed = result.returncode == 0 and subprocess.run(transport.stage_command(nonce, "cleanup"), check=False).returncode == 0
|
||||
body = {"schema": "railiance.vergabe-pilot-native-restore-transport.v1", "run_id": run,
|
||||
"nonce": nonce, "worker_exit_code": result.returncode, "node_stage_removed": removed,
|
||||
"status": "complete" if result.returncode == 0 and removed else "requires_review"}
|
||||
with os.fdopen(os.open(kit / ("native-restore-transport-" + nonce + ".json"),
|
||||
os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600), "w") as stream:
|
||||
json.dump(body, stream, indent=2)
|
||||
print(json.dumps(body, indent=2))
|
||||
return 0 if body["status"] == "complete" else 1
|
||||
|
||||
|
||||
def cleanup(sources, run, nonce):
|
||||
result = subprocess.run(remote_command(sources, run, nonce, cleanup=True), check=False)
|
||||
staged = subprocess.run(transport.stage_command(nonce, "cleanup"), check=False)
|
||||
return 0 if result.returncode == staged.returncode == 0 else 1
|
||||
117
tools/vergabe_pilot_point.py
Normal file
117
tools/vergabe_pilot_point.py
Normal file
|
|
@ -0,0 +1,117 @@
|
|||
"""Validated point selection for attended backups of the admitted demo release.
|
||||
|
||||
The original acceptance modules remain byte-for-byte frozen. Only their point
|
||||
parameters are selected here; image, schema, custody, crypto and limits remain.
|
||||
"""
|
||||
|
||||
import datetime as dt
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
|
||||
import vergabe_pilot_export as export
|
||||
from vergabe_pilot_recovery_store import digest, private_directory, require
|
||||
|
||||
BASELINE = "20261105T233543Z-89bfbe04dece"
|
||||
PATTERN = r"[0-9]{8}T[0-9]{6}Z-[0-9a-f]{12}"
|
||||
FILES = {"database.dump", "stores.tar.gz", "manifest.private.json", "receipt.json"}
|
||||
|
||||
|
||||
def run_time(run):
|
||||
require(isinstance(run, str) and re.fullmatch(PATTERN, run), "invalid_backup_run")
|
||||
try:
|
||||
value = dt.datetime.strptime(run[:16], "%Y%m%dT%H%M%SZ").replace(tzinfo=dt.UTC)
|
||||
except ValueError:
|
||||
require(False, "invalid_backup_date")
|
||||
require(value >= dt.datetime(2026, 11, 5, tzinfo=dt.UTC)
|
||||
and value <= dt.datetime.now(dt.UTC) + dt.timedelta(minutes=5), "backup_date_outside_scope")
|
||||
return value
|
||||
|
||||
|
||||
def read_private(path, maximum=16384):
|
||||
info = export.regular(path, os.geteuid())
|
||||
require(0 < info.st_size <= maximum, "point_metadata_bound")
|
||||
return json.loads(path.read_text())
|
||||
|
||||
|
||||
def select(run, files):
|
||||
run_time(run)
|
||||
require(set(files) == FILES, "point_file_set_changed")
|
||||
for value in files.values():
|
||||
require(set(value) == {"bytes", "sha256"} and type(value["bytes"]) is int
|
||||
and 0 < value["bytes"] <= 1024**2 and isinstance(value["sha256"], str)
|
||||
and re.fullmatch(r"[0-9a-f]{64}", value["sha256"]), "point_file_metadata_invalid")
|
||||
export.RUN_ID = run
|
||||
export.NAME = "vergabe-pilot-" + run
|
||||
export.EXPECTED = {n: (files[n]["bytes"], files[n]["sha256"]) for n in ("database.dump", "stores.tar.gz")}
|
||||
|
||||
|
||||
def select_node(run):
|
||||
run_time(run)
|
||||
directory = export.BASE / run
|
||||
private_directory(export.BASE)
|
||||
private_directory(directory)
|
||||
receipt = read_private(directory / "receipt.json")
|
||||
require(receipt["run_id"] == run and receipt["status"] == "captured_and_resumed", "capture_not_complete")
|
||||
files = {}
|
||||
for name in FILES:
|
||||
path = directory / name
|
||||
info = export.regular(path, os.geteuid())
|
||||
require(0 < info.st_size <= 1024**2, "point_file_size_bound")
|
||||
files[name] = {"bytes": info.st_size, "sha256": digest(path)}
|
||||
select(run, files)
|
||||
# Reuses completed-writer/resume, revision, image and receipt-hash guards.
|
||||
require(export.capture_files(directory) == files, "node_capture_changed")
|
||||
return files
|
||||
|
||||
|
||||
def select_transfer(directory, run):
|
||||
run_time(run)
|
||||
private_directory(directory)
|
||||
metadata = read_private(directory / ("vergabe-pilot-" + run + ".transfer.json"))
|
||||
packet = metadata["packet"]
|
||||
require(metadata["schema"] == "railiance.vergabe-pilot-sealed-export.v1"
|
||||
and metadata["run_id"] == packet["run_id"] == run
|
||||
and metadata["recipient"] == export.RECIPIENT and packet["image"] == export.IMAGE
|
||||
and packet["database"] == "vergabe_demo_company", "point_binding_changed")
|
||||
require(type(metadata["ciphertext_bytes"]) is int and 0 < metadata["ciphertext_bytes"] <= 4 * 1024**2
|
||||
and re.fullmatch(r"[0-9a-f]{64}", metadata["ciphertext_sha256"]), "ciphertext_metadata_invalid")
|
||||
select(run, packet["files"])
|
||||
return metadata
|
||||
|
||||
|
||||
def successful_provider(body, run, sealed):
|
||||
return (body.get("run_id") == run and body.get("status") == "published_read_back_and_decrypted"
|
||||
and body.get("warden_exit_code") == 0
|
||||
and all(body.get(k) is True for k in ("provider_publication_verified", "independent_readback_verified",
|
||||
"decryption_verified", "session_cleanup_confirmed"))
|
||||
and body.get("ciphertext_sha256") == sealed["ciphertext_sha256"]
|
||||
and body.get("ciphertext_bytes") == sealed["ciphertext_bytes"]
|
||||
and body.get("object_path") == "r01-platform-pg/vergabe-pilot-" + run + ".zip.age")
|
||||
|
||||
|
||||
def retention_plan(points):
|
||||
"""Seven daily plus two weekly representatives; always protect baseline/newest.
|
||||
|
||||
Input is the owner's locally verified provider inventory. Unknown remote
|
||||
objects are never inferred eligible. Multiple points on a day don't consume
|
||||
extra daily slots; weekly representatives may overlap the daily set.
|
||||
"""
|
||||
require(len(points) <= 1000 and len({p["run_id"] for p in points}) == len(points), "inventory_bound_or_duplicate")
|
||||
ordered = sorted(points, key=lambda p: run_time(p["run_id"]), reverse=True)
|
||||
keep = {BASELINE}
|
||||
if ordered:
|
||||
keep.add(ordered[0]["run_id"])
|
||||
days, weeks = set(), set()
|
||||
for point in ordered:
|
||||
date = run_time(point["run_id"]).date()
|
||||
day, week = date.isoformat(), tuple(date.isocalendar()[:2])
|
||||
if day not in days and len(days) < 7:
|
||||
keep.add(point["run_id"])
|
||||
days.add(day)
|
||||
if week not in weeks and len(weeks) < 2:
|
||||
keep.add(point["run_id"])
|
||||
weeks.add(week)
|
||||
return {"keep": [p for p in ordered if p["run_id"] in keep],
|
||||
"candidates": [p for p in ordered if p["run_id"] not in keep],
|
||||
"baseline_protected": BASELINE, "unknown_objects_eligible": False}
|
||||
97
tools/vergabe_pilot_retention.py
Normal file
97
tools/vergabe_pilot_retention.py
Normal file
|
|
@ -0,0 +1,97 @@
|
|||
"""Contained owner retention for locally verified Vergabe provider points only."""
|
||||
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
import vergabe_pilot_point as point
|
||||
import vergabe_pilot_publish as publisher
|
||||
from vergabe_pilot_recovery_store import Refused, require
|
||||
|
||||
BASELINE_SHA = "4915288384e3ba8a87f1427b8e10106ee7241c04184adb97f439e57846bef76f"
|
||||
BASELINE_BYTES = 182521
|
||||
|
||||
|
||||
class Objects:
|
||||
def __init__(self, helper, auth):
|
||||
self.root = publisher.HOST + "/remote.php/dav/files/Backup/railiance-backups/" + publisher.PREFIX + "/"
|
||||
self.headers = {"Authorization": "Basic " + base64.b64encode((auth[0] + ":" + auth[1]).encode()).decode()}
|
||||
self.opener = urllib.request.build_opener(urllib.request.ProxyHandler({}), helper.NoRedirect())
|
||||
|
||||
def request(self, item, delete=False, etag=None):
|
||||
point.run_time(item["run_id"])
|
||||
url = self.root + "vergabe-pilot-" + item["run_id"] + ".zip.age"
|
||||
headers = dict(self.headers)
|
||||
if delete:
|
||||
require(isinstance(etag, str) and etag.startswith('"') and etag.endswith('"')
|
||||
and len(etag) < 1024 and not any(c in etag for c in "\r\n"), "strong_provider_etag_required")
|
||||
headers["If-Match"] = etag
|
||||
try:
|
||||
with self.opener.open(urllib.request.Request(url, method="DELETE" if delete else "GET", headers=headers), timeout=60) as response:
|
||||
content = response.read(1024 if delete else item["ciphertext_bytes"] + 1)
|
||||
return response.status, content, response.headers.get("ETag")
|
||||
except urllib.error.HTTPError as error:
|
||||
return error.code, b"", None
|
||||
|
||||
|
||||
def execute(plan, objects, removed):
|
||||
import hashlib
|
||||
require(0 < len(plan["keep"]) <= 10 and len(plan["candidates"]) <= 50, "retention_batch_bound")
|
||||
baseline = [p for p in plan["keep"] if p["run_id"] == point.BASELINE]
|
||||
require(len(baseline) == 1 and baseline[0]["ciphertext_sha256"] == BASELINE_SHA
|
||||
and baseline[0]["ciphertext_bytes"] == BASELINE_BYTES, "verified_baseline_required")
|
||||
require(not ({p["run_id"] for p in plan["keep"]} & {p["run_id"] for p in plan["candidates"]}),
|
||||
"retention_sets_overlap")
|
||||
etags = {}
|
||||
# Independently recheck every retained point before allowing any deletion.
|
||||
for item in [*plan["keep"], *plan["candidates"]]:
|
||||
code, content, etag = objects.request(item)
|
||||
require(code == 200 and len(content) == item["ciphertext_bytes"]
|
||||
and hashlib.sha256(content).hexdigest() == item["ciphertext_sha256"], "retention_provider_bytes_changed")
|
||||
require(isinstance(etag, str) and etag.startswith('"') and etag.endswith('"')
|
||||
and not etag.startswith('W/') and not any(c in etag for c in "\r\n"), "provider_etag_required")
|
||||
etags[item["run_id"]] = etag
|
||||
deleted = []
|
||||
for item in plan["candidates"]:
|
||||
require(item["run_id"] != point.BASELINE, "baseline_delete_refused")
|
||||
code, _, _ = objects.request(item, delete=True, etag=etags[item["run_id"]])
|
||||
require(code == 204, "conditional_retention_delete_failed")
|
||||
require(objects.request(item)[0] == 404, "retention_delete_readback_failed")
|
||||
# Durable per-point tombstones make partial batches safely resumable.
|
||||
removed(item)
|
||||
deleted.append(item["run_id"])
|
||||
return deleted
|
||||
|
||||
|
||||
def child(root, output, inventory):
|
||||
fd = os.open(output, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600)
|
||||
receipt = {"schema": "railiance.vergabe-pilot-provider-retention.v1", "status": "failed",
|
||||
"deleted": [], "application_changed": False, "local_copies_deleted": False}
|
||||
try:
|
||||
helper = publisher.owner_helpers()
|
||||
publisher.require_owner_handoff()
|
||||
credentials = publisher.custody(helper)
|
||||
plan = point.retention_plan(inventory(root))
|
||||
receipt["retained"] = [p["run_id"] for p in plan["keep"]]
|
||||
receipt["planned_candidates"] = [p["run_id"] for p in plan["candidates"]]
|
||||
receipt["quota_before"] = publisher.Provider(helper).quota(credentials["owner_auth"])
|
||||
def removed(item):
|
||||
path = root / item["run_id"] / "provider-removed.json"
|
||||
marker = {**item, "status": "provider_removed_and_readback_absent"}
|
||||
with os.fdopen(os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600), "w") as stream:
|
||||
json.dump(marker, stream)
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
receipt["deleted"].append(item["run_id"])
|
||||
execute(plan, Objects(helper, credentials["owner_auth"]), removed)
|
||||
receipt["status"] = "provider_retention_complete"
|
||||
except BaseException as error: # noqa: BLE001 — preserve partial progress without private diagnostics
|
||||
receipt["failure"] = str(error) if isinstance(error, Refused) else "provider_retention_failed"
|
||||
finally:
|
||||
with os.fdopen(fd, "w") as stream:
|
||||
json.dump(receipt, stream, indent=2)
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
return 0 if receipt["status"] == "provider_retention_complete" else 1
|
||||
|
|
@ -196,10 +196,10 @@ id: RAPPS-WP-0014-T03
|
|||
status: wait
|
||||
needs_human: true
|
||||
decision_id: d06307e5-2737-461a-a2f4-a12c3b492a9a
|
||||
intervention_note: "Provider publication, independent readback and escrow decryption pass; Warden cleanup confirmed. Prepared founder-run isolated native restore/configuration-inverse rehearsal from the provider-retrieved point with new namespace/claims, no live app pause. Native restore/cleanup evidence and operating acceptance remain; SSO/ingress and older-image/Helm rollback are not inferred."
|
||||
intervention_note: "Native three-store restore/configuration inverse and cleanup pass (215 seconds). Founder selected attended backups with himself as backup/incident contact, daily active-day/pre-change cadence, seven daily/two weekly points, one active day data-loss and one working day recovery targets. Repeatable fresh-point backup/retention/isolated-restore path prepared; 76 tests pass. Next founder command: tools/vergabe_pilot_backup.py --execute --verify-restore after --check. Native SSO/ingress and older-image/Helm rollback remain outside the proof."
|
||||
priority: high
|
||||
assignee: the-custodian
|
||||
blocking_reason: "Capture/restart, provider publication/readback and decryption pass. Await native isolated restore/workflow/access/configuration-inverse receipt and operating ownership/cadence/retention/RPO/RTO acceptance; broader restored SSO or version-rollback scope remains explicit."
|
||||
blocking_reason: "Recovery and clone configuration-inverse proof pass; attended operating model selected. Finish and verify repeatable fresh-point capture/publication and bounded retention procedure before claiming the accepted cadence active. Restored native SSO/ingress and broader version rollback remain explicit limits."
|
||||
state_hub_task_id: "dd069c6d-fcc1-5bac-b233-976f2f0d5cd1"
|
||||
```
|
||||
|
||||
|
|
@ -515,6 +515,49 @@ not yet evidenced. Session tests use force_login; no restored native SSO/MFA,
|
|||
ingress or older-image/schema/Helm rollback is claimed. T03 remains waiting for
|
||||
the reviewed execution and operating acceptance.
|
||||
|
||||
2026-11-06 native restore return: nonce `2d9a74da048c`, worker
|
||||
`restored_and_verified`, transport `complete` / exit 0. The provider-retrieved
|
||||
point matches all 39 PostgreSQL tables / 247 rows (logical SHA256
|
||||
`33191c43…3765`), one PDF, SQLite nine tables / seven rows and linked local issue.
|
||||
Both restored ordinary member sessions pass document hashes and cross-member
|
||||
task editing; anonymous, already-authenticated inactive-member and missing-CSRF
|
||||
requests are denied. Test transactions roll back. A clone-only committed marker
|
||||
survives two Pod replacements through bad configuration and its exact inverse.
|
||||
All new namespace/PVs and private staging are removed; production specification
|
||||
and readiness are unchanged. Restore/checks 55.41 seconds, inverse/recheck 55.24
|
||||
seconds, total worker 215 seconds; workstation authentication/transfer excluded.
|
||||
Receipts: `docs/evidence/2026-11-06-vergabe-native-restore-return.json` and
|
||||
`docs/evidence/2026-11-06-vergabe-native-restore-transport-return.json`.
|
||||
Captured tender status 10 is Gewonnen and not archived, explaining its exclusion
|
||||
from the deployed dashboard's status 1–9 range; no later live status is inferred.
|
||||
Restored sessions use test-client login; native SSO/ingress recovery and an older
|
||||
image/schema or Helm rollback remain untested.
|
||||
|
||||
The founder selected attended pilot backups: Bernd Worsch as backup execution
|
||||
and first incident/support contact, existing railiance-platform storage/custody
|
||||
lane, a fresh point after every active pilot day and before changes, seven daily
|
||||
and two weekly verified points, at most one active day's data loss and attended
|
||||
recovery within one working day. These are accepted operating targets, not an
|
||||
SLA inferred from the drill. `docs/vergabe-demo-company-operating-acceptance.proposed.md`
|
||||
records the arrangement and implementation gap: exact-point acceptance helpers
|
||||
are not yet a repeatable fresh-backup procedure. T03 remains wait until that path
|
||||
and bounded retention handling are ready and a fresh attended run passes.
|
||||
|
||||
Prepared `docs/proposals/2026-11-06-vergabe-attended-backups/README.md` and
|
||||
`tools/vergabe_pilot_backup.py`: the frozen capture, encryption, provider and
|
||||
native restore modules remain unchanged. Validated per-run selection supports
|
||||
fresh capture/seal/fetch, contained publication/readback/decryption, retries,
|
||||
guarded resume and optional full isolated restore for the new point. Provider
|
||||
retention previews only complete owner-verified kits, protects baseline/newest,
|
||||
and independently verifies retained/candidate ciphertext before separate
|
||||
ETag-guarded deletion/absence readback. Unknown provider objects are ineligible;
|
||||
local private copies remain. Seventeen new synthetic cases and 76 total pilot
|
||||
helper tests pass, including real age round-trips for two distinct fresh points,
|
||||
selected recovered-file checks and conditional retention races. No new native
|
||||
capture, provider object or retention deletion has been performed by the agent.
|
||||
T03 waits for the first attended operating run:
|
||||
`python3 tools/vergabe_pilot_backup.py --execute --verify-restore`, after `--check`.
|
||||
|
||||
|
||||
## Validate synthetic sizing against the exact release and shared contention
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue