From f77e2333afbc986611074e1dc9f93693257346c4 Mon Sep 17 00:00:00 2001 From: tegwick Date: Sat, 10 Oct 2026 13:39:49 +0200 Subject: [PATCH] Retain independent ledger recovery custody and activate verified backups Assistant: codex Assistant-Model: gpt-6.1-sol Assistant-Session: 01a11fd9-caf6-7a63-b578-9daaaa134ff2 --- WORK-RECORDS.md | 6 +- .../backup/daily-recovery-bucket-policy.json | 59 +++ .../email-connect-ledger-backup.service | 2 +- .../email-connect-ledger-freshness.service | 2 +- .../operating-acceptance-2026-10-10.json | 100 +++++ .../retained-recovery-bucket-policy.json | 30 ++ deploy/closure/plan.json | 13 +- deploy/closure/prepare.py | 8 +- docs/ledger-backup-review-2026-10-08.md | 81 ++++ docs/ledger-scaleway-custody-2026-10-10.json | 419 ++++++++++++++++++ scripts/backup-ledger.py | 27 +- scripts/mirror-ledger.py | 89 ++++ scripts/prepare-ledger-recovery-kit.py | 4 +- tests/test_closure_custody.py | 32 ++ tests/test_ledger_backup.py | 16 +- tests/test_ledger_mirror.py | 45 ++ .../EMAIL-WP-0007-ledger-backup-recovery.md | 58 ++- 17 files changed, 967 insertions(+), 24 deletions(-) create mode 100644 deploy/backup/daily-recovery-bucket-policy.json create mode 100644 deploy/backup/operating-acceptance-2026-10-10.json create mode 100644 deploy/backup/retained-recovery-bucket-policy.json create mode 100644 docs/ledger-scaleway-custody-2026-10-10.json create mode 100644 scripts/mirror-ledger.py create mode 100644 tests/test_ledger_mirror.py diff --git a/WORK-RECORDS.md b/WORK-RECORDS.md index 782591d..92c3e52 100644 --- a/WORK-RECORDS.md +++ b/WORK-RECORDS.md @@ -14,7 +14,7 @@ | workplan | EMAIL-WP-0004 | finished | — | workplans/EMAIL-WP-0004-transactional-mail-delivery-service.md | | workplan | EMAIL-WP-0005 | finished | — | workplans/EMAIL-WP-0005-test-mailbox-harness.md | | workplan | EMAIL-WP-0006 | finished | — | workplans/EMAIL-WP-0006-verification-retry-safety.md | -| workplan | EMAIL-WP-0007 | active | — | workplans/EMAIL-WP-0007-ledger-backup-recovery.md | +| workplan | EMAIL-WP-0007 | finished | — | workplans/EMAIL-WP-0007-ledger-backup-recovery.md | | workplan | EMAIL-WP-0008 | finished | — | workplans/EMAIL-WP-0008-scoped-transactional-callers.md | | task | EMAIL-WP-0001-T01 | done | — | workplans/EMAIL-WP-0001-repo-onboarding.md | | task | EMAIL-WP-0001-T02 | done | — | workplans/EMAIL-WP-0001-repo-onboarding.md | @@ -55,8 +55,8 @@ | task | EMAIL-WP-0006-T04 | done | — | workplans/EMAIL-WP-0006-verification-retry-safety.md | | task | EMAIL-WP-0007-T01 | done | — | workplans/EMAIL-WP-0007-ledger-backup-recovery.md | | task | EMAIL-WP-0007-T02 | done | — | workplans/EMAIL-WP-0007-ledger-backup-recovery.md | -| task | EMAIL-WP-0007-T03 | progress | — | workplans/EMAIL-WP-0007-ledger-backup-recovery.md | -| task | EMAIL-WP-0007-T04 | progress | — | workplans/EMAIL-WP-0007-ledger-backup-recovery.md | +| task | EMAIL-WP-0007-T03 | done | — | workplans/EMAIL-WP-0007-ledger-backup-recovery.md | +| task | EMAIL-WP-0007-T04 | done | — | workplans/EMAIL-WP-0007-ledger-backup-recovery.md | | task | EMAIL-WP-0008-T01 | done | — | workplans/EMAIL-WP-0008-scoped-transactional-callers.md | | task | EMAIL-WP-0008-T02 | done | — | workplans/EMAIL-WP-0008-scoped-transactional-callers.md | | task | EMAIL-WP-0008-T03 | done | — | workplans/EMAIL-WP-0008-scoped-transactional-callers.md | diff --git a/deploy/backup/daily-recovery-bucket-policy.json b/deploy/backup/daily-recovery-bucket-policy.json new file mode 100644 index 0000000..66110c5 --- /dev/null +++ b/deploy/backup/daily-recovery-bucket-policy.json @@ -0,0 +1,59 @@ +{ + "Version": "2023-04-17", + "Statement": [ + { + "Sid": "Operator", + "Effect": "Allow", + "Principal": { + "SCW": "user_id:3d029efb-4f70-4f82-a3bb-cefbe255232b" + }, + "Action": "s3:*", + "Resource": [ + "railiance-email-connect-archives", + "railiance-email-connect-archives/*" + ] + }, + { + "Sid": "ArchivePublisher", + "Effect": "Allow", + "Principal": { + "SCW": "application_id:0eb0f958-77a4-4c70-9aa7-14ba46411415" + }, + "Action": [ + "s3:GetObject", + "s3:GetObjectVersion", + "s3:PutObject" + ], + "Resource": "railiance-email-connect-archives/archives/*" + }, + { + "Sid": "ArchiveReader", + "Effect": "Allow", + "Principal": { + "SCW": "application_id:a84a18e2-3813-4918-9470-dbe0580a5ea6" + }, + "Action": [ + "s3:GetObject", + "s3:GetObjectVersion" + ], + "Resource": "railiance-email-connect-archives/archives/*" + }, + { + "Sid": "ArchiveList", + "Effect": "Allow", + "Principal": { + "SCW": "application_id:a84a18e2-3813-4918-9470-dbe0580a5ea6" + }, + "Action": [ + "s3:ListBucket", + "s3:ListBucketVersions" + ], + "Resource": "railiance-email-connect-archives", + "Condition": { + "StringLike": { + "s3:prefix": "archives/*" + } + } + } + ] +} diff --git a/deploy/backup/email-connect-ledger-backup.service b/deploy/backup/email-connect-ledger-backup.service index 86484fb..1c4db3d 100644 --- a/deploy/backup/email-connect-ledger-backup.service +++ b/deploy/backup/email-connect-ledger-backup.service @@ -11,7 +11,7 @@ UMask=0077 StateDirectory=email-connect-backup StateDirectoryMode=0700 EnvironmentFile=/etc/email-connect-backup/primary.env -ExecStart=/usr/bin/python3 /opt/email-connect-backup/backup-ledger.py backup --execute +ExecStart=/usr/bin/python3 /opt/email-connect-backup/backup-ledger.py backup --execute --mirror-required TimeoutStartSec=40min TimeoutStopSec=15min KillMode=mixed diff --git a/deploy/backup/email-connect-ledger-freshness.service b/deploy/backup/email-connect-ledger-freshness.service index c057bee..f76e08d 100644 --- a/deploy/backup/email-connect-ledger-freshness.service +++ b/deploy/backup/email-connect-ledger-freshness.service @@ -5,7 +5,7 @@ OnFailure=email-connect-ledger-alert@%n.service [Service] Type=oneshot User=root -ExecStart=/usr/bin/python3 /opt/email-connect-backup/backup-ledger.py freshness +ExecStart=/usr/bin/python3 /opt/email-connect-backup/backup-ledger.py freshness --mirror-required ExecStartPost=-/usr/bin/python3 /opt/email-connect-backup/notify-ledger-failure.py StateDirectory=email-connect-backup StateDirectoryMode=0700 diff --git a/deploy/backup/operating-acceptance-2026-10-10.json b/deploy/backup/operating-acceptance-2026-10-10.json new file mode 100644 index 0000000..3aa29ea --- /dev/null +++ b/deploy/backup/operating-acceptance-2026-10-10.json @@ -0,0 +1,100 @@ +{ + "schema": "email-connect.ledger-operating-acceptance.v1", + "date": "2026-10-10", + "status": "active", + "timers": { + "email-connect-ledger-backup.timer": { + "NextElapseUSecRealtime": "Sun 2026-10-11 04:15:29 UTC", + "ActiveState": "active", + "UnitFileState": "enabled" + }, + "email-connect-ledger-freshness.timer": { + "NextElapseUSecRealtime": "Sat 2026-10-10 12:00:00 UTC", + "ActiveState": "active", + "UnitFileState": "enabled" + } + }, + "freshness_service_passed": true, + "last_success": { + "schema": "platform.application-primary-archive.v1", + "status": "offsite_fetched_verified", + "application": "email-connect", + "captured_at": "2026-10-10T11:32:56Z", + "object_key": "platform-pg/application-archives/email-connect/2026/10/10/20261010T113256Z-e2efa67ecfb8154fd8bbcc59.tar.age", + "destination": "s3://railiance-platform-pg-backup/platform-pg/application-archives/email-connect/2026/10/10/20261010T113256Z-e2efa67ecfb8154fd8bbcc59.tar.age", + "ciphertext_sha256": "a007ab4f80924ecf5d69c52a91363298a68e004293f7e8f37bfae762c403ac5e", + "ciphertext_bytes": 71896, + "remote_retrieval_verified": true, + "stage": "transfer_verified", + "multipart_completed": true, + "version_id": "1791632035876582", + "immutable_create": true, + "image": "forgejo.coulomb.social/coulomb/email-connect@sha256:5d79fd1fec15d3a279383820f170d263de3a55f5308df01a0a34732ebaea6d3d", + "database_sha256": "ed6711a63cb0a4371c1b805e326ee6ec62e3b6a2358c37b0ba9152f4a5eaaecb", + "ledger_fingerprint": { + "schema_sha256": "13148e5a9c6e01cc593bb07278a8649b13e43b26c30b0ed1a6e15995f7b00732", + "user_version": 0, + "tables": { + "caller_buckets": { + "rows": 1, + "sha256": "d7def892f86c163aed9df55112b0e4551ff6004e337a6a2135c9b80d8cb8fdf7" + }, + "deliveries": { + "rows": 2, + "sha256": "a0387716815b2ec191187cb34bb3252b9cf51f5fb15a2019918a9cf1737c061e" + }, + "send_attempts": { + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "suppressions": { + "rows": 1, + "sha256": "ae26d10dfec16f1b626292b342a5e86afd1858fcaed7c29d4e68e7cf821183d7" + }, + "verification_attempts": { + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "verifications": { + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + } + } + }, + "recovery_mirror": { + "bucket": "railiance-email-connect-archives", + "object_key": "archives/platform-pg/application-archives/email-connect/2026/10/10/20261010T113256Z-e2efa67ecfb8154fd8bbcc59.tar.age", + "version_id": "1791632037091494", + "project_id": "1d37ea63-8c8f-44e1-80e1-2de862579ba7", + "hash_readback_verified": true, + "receipt_key": "archives/platform-pg/application-archives/email-connect/2026/10/10/20261010T113256Z-e2efa67ecfb8154fd8bbcc59.tar.age.1791632037091494.5aee83f185670ac3fd2058a76a5194306288042a6baac7aeb1a91903a4ec5242.receipt.json", + "receipt_version_id": "1791632037934451" + }, + "database_bytes": 57344, + "pause_and_capture_seconds": 57.55, + "local_archive": "20261010T113256Z-e2efa67ecfb8154fd8bbcc59.tar.age", + "application_restore_verified": false, + "completed_at": "2026-10-10T11:33:58.694614+00:00" + }, + "schedule_utc": "04:15", + "jitter_seconds": 300, + "rpo_target_hours": 24, + "freshness_threshold_hours": 26, + "attended_rto_target_minutes": 60, + "production_rto_measured": false, + "local_budget_bytes": 8589934592, + "minimum_free_bytes": 8589934592, + "retention_operator": "platform-root", + "owner_review_interval": "weekly", + "rolling_retention_days": 30, + "retained_kit_expires": false, + "owner_selection_daily": 14, + "owner_selection_weekly": 4, + "failure_recipient": "platform-root", + "source_sha256": { + "scripts/backup-ledger.py": "0bbef692208bd6b1f6e06f3ae969c01fb86cc2078529e2eb598f370b0363e99f", + "scripts/mirror-ledger.py": "abefb951dbdb4212f914a7701cfe0e7121e2c2643bf9f6892a0fc97fa6da760d", + "deploy/backup/email-connect-ledger-backup.service": "1190c30ccb4b6f7f2d0c7194907ffb1c7ccf6d5bf069f7cbd20fdbbce1a25534", + "deploy/backup/email-connect-ledger-freshness.service": "edbc0445fbeffdf108ef4a4f5ffeb0ab4f9909424d849fa55b021e92f6a25916" + } +} diff --git a/deploy/backup/retained-recovery-bucket-policy.json b/deploy/backup/retained-recovery-bucket-policy.json new file mode 100644 index 0000000..3ef270f --- /dev/null +++ b/deploy/backup/retained-recovery-bucket-policy.json @@ -0,0 +1,30 @@ +{ + "Version": "2023-04-17", + "Id": "email-connect-independent-recovery", + "Statement": [ + { + "Sid": "OperatorCustody", + "Effect": "Allow", + "Principal": { + "SCW": "user_id:3d029efb-4f70-4f82-a3bb-cefbe255232b" + }, + "Action": "s3:*", + "Resource": [ + "railiance-email-connect-recovery", + "railiance-email-connect-recovery/*" + ] + }, + { + "Sid": "RecoveryKitReadOnly", + "Effect": "Allow", + "Principal": { + "SCW": "application_id:a84a18e2-3813-4918-9470-dbe0580a5ea6" + }, + "Action": [ + "s3:GetObject", + "s3:GetObjectVersion" + ], + "Resource": "railiance-email-connect-recovery/kits/*" + } + ] +} diff --git a/deploy/closure/plan.json b/deploy/closure/plan.json index 87ce0a0..817fbfe 100644 --- a/deploy/closure/plan.json +++ b/deploy/closure/plan.json @@ -82,6 +82,14 @@ ] }, "backup": { + "recovery_mirror": { + "project_id": "1d37ea63-8c8f-44e1-80e1-2de862579ba7", + "bucket": "railiance-email-connect-archives", + "prefix": "archives/", + "verified_copy_required_before_success": true, + "versioned_recovery_receipt_required": true, + "retention_days": 30 + }, "schedule_utc": "04:15", "jitter_seconds": 300, "rpo_target_hours": 24, @@ -140,9 +148,10 @@ "scripts/admit-scoped-caller.py": "17bf68f85f0bb08b3ca89eee34e62e794ee4754fbbb21646c28c85a1c1dd0e0e", "scripts/scaleway-recovery-custody.py": "85874f0620dc9353a9aaae1194db054de166296c4385743e4d541bbe42486c15", "scripts/publish-ledger-recovery-kit.py": "cd7c6e8e6a4e8417ba55f8170d763d7c5a9e6dd9b5878dc3459021a0b64ea732", - "scripts/prepare-ledger-recovery-kit.py": "db92a29eb68e1b555cf18b17229d1e479b3147da7600ffa0616ac8b4d011ff55", + "scripts/prepare-ledger-recovery-kit.py": "8cf74c9ea300c5ff6a03282293ba76ea0516fbd16f841259ca70f563c9e82d2b", "scripts/rehearse-ledger.py": "5c43b655149d8ae577fab80e76358be35f90cb0357df9b726ac7cb23346946a0", - "scripts/backup-ledger.py": "e74c41948ca01dae09578e2abfba14b96fb1f158af0ec91a0fb1aa9ecaf01680", + "scripts/backup-ledger.py": "0bbef692208bd6b1f6e06f3ae969c01fb86cc2078529e2eb598f370b0363e99f", + "scripts/mirror-ledger.py": "abefb951dbdb4212f914a7701cfe0e7121e2c2643bf9f6892a0fc97fa6da760d", "scripts/capture-ledger.py": "7994fe7b4fbf72e28a4296212005a88d0ef9fcb626b066e6bcf2885500bc69d6" } } diff --git a/deploy/closure/prepare.py b/deploy/closure/prepare.py index 0c1ab19..7f0fa84 100644 --- a/deploy/closure/prepare.py +++ b/deploy/closure/prepare.py @@ -42,6 +42,12 @@ plan = { "reader_policy": "email-connect-scoped-caller-reader", "readers": readers, }, "backup": { + "recovery_mirror": { + "project_id": "1d37ea63-8c8f-44e1-80e1-2de862579ba7", + "bucket": "railiance-email-connect-archives", "prefix": "archives/", + "verified_copy_required_before_success": True, + "versioned_recovery_receipt_required": True, "retention_days": 30, + }, "schedule_utc": "04:15", "jitter_seconds": 300, "rpo_target_hours": 24, "stale_threshold_hours": 26, "attended_rto_target_minutes": 60, "minimum_node_free_bytes": 8 * 1024**3, "local_budget_bytes": 8 * 1024**3, @@ -130,7 +136,7 @@ artifacts = [ "deploy/closure/user-engine.patch.json", "scripts/admit-scoped-caller.py", "scripts/scaleway-recovery-custody.py", "scripts/publish-ledger-recovery-kit.py", "scripts/prepare-ledger-recovery-kit.py", "scripts/rehearse-ledger.py", - "scripts/backup-ledger.py", + "scripts/backup-ledger.py", "scripts/mirror-ledger.py", "scripts/capture-ledger.py", ] plan["artifacts"] = {name: hashlib.sha256((ROOT / name).read_bytes()).hexdigest() for name in artifacts} diff --git a/docs/ledger-backup-review-2026-10-08.md b/docs/ledger-backup-review-2026-10-08.md index 20b5b09..5e9b8d9 100644 --- a/docs/ledger-backup-review-2026-10-08.md +++ b/docs/ledger-backup-review-2026-10-08.md @@ -268,3 +268,84 @@ Ready. EMAIL-WP-0008 is finished; T03/T04 here remain progress for independent Scaleway custody/grants, retention/schedule and recurring operation. Timers remain disabled. The user was asked for an existing Scaleway operator lane or Console custody setup, since the existing publisher cannot administer escrow resources. + +## Independent custody and operating acceptance — 2026-10-10 + +The user completed an attended Scaleway browser login and authorized continuation +of the closure package. Native custody and recovery evidence is recorded in +`docs/ledger-scaleway-custody-2026-10-10.json`. No CCR-2026-0011 admission is +inferred from that separate operator login. + +Independent project `email-connect-recovery`, ID +`1d37ea63-8c8f-44e1-80e1-2de862579ba7`, is in nl-ams. Protected opaque Secret +Manager container `email-connect-ledger-age-recovery`, ID +`545a1f3c-8e21-4ed1-9ec9-33c9ea2e9272`, holds enabled, nonexpiring revision 1. +The private age identity was restored from Scaleway and matched the public +recipient; it was never installed on the backup node or placed in a kit. + +The private, versioned `railiance-email-connect-recovery` bucket has no lifecycle +expiry. Retained kit SHA256 +`3e786168328dc9c9a32f0b67e4d010ee9db0e0b855fc2460413fb85a16a2e884`, +version `1791631021870760`, includes the pinned scoped receiver image, ciphertext, +receipts and recovery tools. Provider retrieval, all-file hash checks, image load, +age decrypt and bundled-tool UID10001/network-none rehearsal passed. Kit assembly +now exports the public fingerprint library as 0644 so the container can read its +bind mount; the containing directory remains 0700, other files 0600. A regression +test covers this cold-kit failure found during the actual drill. + +Separate reader `email-connect-ledger-recovery-reader`, application +`a84a18e2-3813-4918-9470-dbe0580a5ea6`, can read kits and list/read the daily +archive prefix. Native checks prove write/delete, foreign-prefix, original-project +and unrelated-bucket denials. Native checks also prove the scheduled worker cannot +read the retained kit or age identity, or delete daily copies. Identity binding +was checked through IAM metadata, in addition to positive version/hash readback. + +A temporary read grant in the original project exposed metadata in an unrelated +bucket. It was revoked, its bucket-policy changes restored, and unrelated reads +now return 403. Scaleway Object Storage IAM grants apply across a project, and +resource conditions do not support Object Storage: +https://www.scaleway.com/en/docs/iam/policies-permissions/supported-products-resource-level/ +https://www.scaleway.com/en/docs/object-storage/api-cli/combining-iam-and-object-storage/ +No policy was installed on the unrelated bucket and no actual object data was read. + +Daily ciphertext is therefore copied into private, versioned bucket +`railiance-email-connect-archives` in the independent project. Its `archives/` +prefix expires current/noncurrent versions after 30 days. The existing +`email-connect-backup-primary` application remains the publisher, using its same +node-held pair with Scaleway's preferred-project suffix. New project IAM rights +and bucket whitelists admit only ciphertext/receipt PUT and GET; retained kits +and age escrow remain inaccessible. The original primary upload is preserved. +Success advances only after both ciphertext copies and a versioned, fully bound +recovery receipt are read back. Failed mirrors retain the pending capture for +ciphertext-only retry, without another receiver pause. Freshness requires mirror +evidence. Synthetic tests cover corruption, missing versions and mirror-failure +retry without advancing last-success. + +A recovery locator is also retained under `kits/recovery-reference-*.json`; its +exact version/hash is in the public acceptance evidence. Recovery after losing +both workstation and node starts with platform-root's Scaleway account login on +a replacement host. Mint an expiring API key for the reader application with +preferred project `1d37ea63-8c8f-44e1-80e1-2de862579ba7`; retrieve the kit/locator +and latest `archives/` ciphertext plus its versioned JSON receipt. Retrieve age +revision 1 separately through attended operator access into a private 0600 file. +Verify hashes, load the bundled image, then use bundled `backup-ledger.py recover` +and `rehearse-ledger.py --offline-image-binding /manifest.json`. No expired +drill credential, original workstation identity, node credential or forge pull +is required. Keep credentials and plaintext in private files and out of logs. + +The admitted operation is daily 04:15 UTC with up to five minutes jitter, hourly +freshness, a 24-hour RPO target and a 26-hour stale alert. The 60-minute attended +RTO is a target; the offline application rehearsals do not measure a production +restore. A fresh dual publication and independent daily-copy recovery passed. +The receiver resumed Ready after the bounded capture. Backup/freshness failures +route to the user-selected platform-root operations inbox, with private replay +if delivery is unavailable. Human acknowledgement is not claimed. + +Platform-root owns weekly retention/capacity review: select 14 daily/4 weekly +points, retire excess local ciphertext through an attended owner operation, and +refresh the nonexpiring retained kit when the image or recovery identity changes. +The worker never deletes retained objects. The 8 GiB free-space floor and 8 GiB +local budget fail before capture if custody needs review. Provider lifecycle is +30 days for rolling archives, while the protected kit is retained indefinitely. +Any production restore still requires delivery fencing and reconciliation of +post-capture operations and challenge state before traffic resumes. diff --git a/docs/ledger-scaleway-custody-2026-10-10.json b/docs/ledger-scaleway-custody-2026-10-10.json new file mode 100644 index 0000000..c60fe2c --- /dev/null +++ b/docs/ledger-scaleway-custody-2026-10-10.json @@ -0,0 +1,419 @@ +{ + "schema": "email-connect.scaleway-custody-acceptance.v1", + "date": "2026-10-10", + "operator_browser_login_completed_by_user": true, + "project_id": "1d37ea63-8c8f-44e1-80e1-2de862579ba7", + "organization_id": "e1a0dd0e-04b8-4ea9-8b30-d53f8c35c688", + "region": "nl-ams", + "age_escrow": { + "status": "scaleway_age_escrow_verified", + "secret_id": "545a1f3c-8e21-4ed1-9ec9-33c9ea2e9272", + "revision": 1, + "region": "nl-ams", + "public_recipient_matches": true, + "readback_equal": true, + "private_identity_printed": false, + "independent_account_login_verified": true, + "worker_access_denial_verified": true + }, + "retained_kit": { + "status": "scaleway_recovery_kit_readback_verified", + "bucket": "railiance-email-connect-recovery", + "object_key": "kits/3e786168328dc9c9a32f0b67e4d010ee9db0e0b855fc2460413fb85a16a2e884.tar.gz", + "version_id": "1791631021870760", + "kit_sha256": "3e786168328dc9c9a32f0b67e4d010ee9db0e0b855fc2460413fb85a16a2e884", + "kit_bytes": 47624244, + "separate_reader_readback_verified": true, + "private_identity_included": false, + "reader_write_denial_verified": true, + "worker_access_denial_verified": true, + "reader_application_binding_verified": true, + "restored_from_provider_kit": true + }, + "reader_boundaries": { + "schema": "email-connect.final-recovery-boundaries.v1", + "reader_application_id": "a84a18e2-3813-4918-9470-dbe0580a5ea6", + "daily_project": "1d37ea63-8c8f-44e1-80e1-2de862579ba7", + "primary_project_read_grant_revoked": true, + "primary_bucket_policy_restored": true, + "kit_version_hash_verified": true, + "first_mirror": { + "bucket": "railiance-email-connect-archives", + "object_key": "archives/platform-pg/application-archives/email-connect/2026/10/09/20261009T181751Z-7e1211057348f51569d8b9e4.tar.age", + "version_id": "1791631635943039", + "project_id": "1d37ea63-8c8f-44e1-80e1-2de862579ba7", + "hash_readback_verified": true, + "receipt_key": "archives/platform-pg/application-archives/email-connect/2026/10/09/20261009T181751Z-7e1211057348f51569d8b9e4.tar.age.1791631635943039.receipt.json", + "receipt_version_id": "1791631641289585" + }, + "daily_ciphertext_hash_verified": true, + "daily_recovery_receipt_verified": true, + "daily_prefix_list_verified": true, + "daily_write": { + "http_status": 403, + "denied": true + }, + "daily_delete": { + "http_status": 403, + "denied": true + }, + "daily_foreign_prefix_list": { + "http_status": 403, + "denied": true + }, + "kit_write": { + "http_status": 403, + "denied": true + }, + "kit_delete": { + "http_status": 403, + "denied": true + }, + "kit_foreign_prefix": { + "http_status": 403, + "denied": true + }, + "unrelated_bucket_list": { + "http_status": 403, + "denied": true + }, + "unrelated_bucket_get": { + "http_status": 403, + "denied": true + }, + "primary_project_get": { + "http_status": 403, + "denied": true + } + }, + "worker_boundaries": { + "schema": "email-connect.worker-boundaries.v1", + "retained_kit_get": { + "http_status": 403, + "denied": true + }, + "mirror_delete": { + "http_status": 403, + "denied": true + }, + "age_identity_get": { + "http_status": 403, + "denied": true + }, + "primary_versioning_preserved": true + }, + "buckets": { + "railiance-email-connect-recovery": { + "versioning": "Enabled", + "public_acl_absent": true, + "lifecycle_rules": [], + "policy": { + "Version": "2023-04-17", + "Id": "email-connect-independent-recovery", + "Statement": [ + { + "Sid": "OperatorCustody", + "Effect": "Allow", + "Principal": { + "SCW": "user_id:3d029efb-4f70-4f82-a3bb-cefbe255232b" + }, + "Action": "s3:*", + "Resource": [ + "railiance-email-connect-recovery", + "railiance-email-connect-recovery/*" + ] + }, + { + "Sid": "RecoveryKitReadOnly", + "Effect": "Allow", + "Principal": { + "SCW": "application_id:a84a18e2-3813-4918-9470-dbe0580a5ea6" + }, + "Action": [ + "s3:GetObject", + "s3:GetObjectVersion" + ], + "Resource": "railiance-email-connect-recovery/kits/*" + } + ] + } + }, + "railiance-email-connect-archives": { + "versioning": "Enabled", + "public_acl_absent": true, + "lifecycle_rules": [ + { + "Expiration": { + "Days": 30 + }, + "ID": "daily-30-days", + "Filter": { + "Prefix": "archives/" + }, + "Status": "Enabled", + "NoncurrentVersionExpiration": { + "NoncurrentDays": 30 + } + } + ], + "policy": { + "Version": "2023-04-17", + "Statement": [ + { + "Sid": "Operator", + "Effect": "Allow", + "Principal": { + "SCW": "user_id:3d029efb-4f70-4f82-a3bb-cefbe255232b" + }, + "Action": "s3:*", + "Resource": [ + "railiance-email-connect-archives", + "railiance-email-connect-archives/*" + ] + }, + { + "Sid": "ArchivePublisher", + "Effect": "Allow", + "Principal": { + "SCW": "application_id:0eb0f958-77a4-4c70-9aa7-14ba46411415" + }, + "Action": [ + "s3:GetObject", + "s3:GetObjectVersion", + "s3:PutObject" + ], + "Resource": "railiance-email-connect-archives/archives/*" + }, + { + "Sid": "ArchiveReader", + "Effect": "Allow", + "Principal": { + "SCW": "application_id:a84a18e2-3813-4918-9470-dbe0580a5ea6" + }, + "Action": [ + "s3:GetObject", + "s3:GetObjectVersion" + ], + "Resource": "railiance-email-connect-archives/archives/*" + }, + { + "Sid": "ArchiveList", + "Effect": "Allow", + "Principal": { + "SCW": "application_id:a84a18e2-3813-4918-9470-dbe0580a5ea6" + }, + "Action": [ + "s3:ListBucket", + "s3:ListBucketVersions" + ], + "Resource": "railiance-email-connect-archives", + "Condition": { + "StringLike": { + "s3:prefix": "archives/*" + } + } + } + ] + } + } + }, + "provider_only_retained_kit_rehearsal": { + "status": "application_restored_verified", + "application": "email-connect", + "object_key": "platform-pg/application-archives/email-connect/2026/10/09/20261009T181751Z-7e1211057348f51569d8b9e4.tar.age", + "version_id": "1791569930973785", + "ciphertext_sha256": "1a7ae09129d8252053ea210cfe37d8a27d25bc4ee615bfe85f65c207b3e5f38e", + "database_sha256": "6694bddc3b57b5918852fbdd79d569d5bf1393831ed55aca25bb19a59e601feb", + "ledger_fingerprint": { + "schema_sha256": "13148e5a9c6e01cc593bb07278a8649b13e43b26c30b0ed1a6e15995f7b00732", + "user_version": 0, + "tables": { + "caller_buckets": { + "rows": 1, + "sha256": "0ce5a0b678fdcf55de7a4b2314a02d12f168e0f08b2a0d42d82c3e1dc514778c" + }, + "deliveries": { + "rows": 2, + "sha256": "a0387716815b2ec191187cb34bb3252b9cf51f5fb15a2019918a9cf1737c061e" + }, + "send_attempts": { + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "suppressions": { + "rows": 1, + "sha256": "ae26d10dfec16f1b626292b342a5e86afd1858fcaed7c29d4e68e7cf821183d7" + }, + "verification_attempts": { + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "verifications": { + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + } + } + }, + "mail_sent": false, + "production_restored": false, + "application_restore_verified": true, + "recovery_image": "forgejo.coulomb.social/coulomb/email-connect@sha256:5d79fd1fec15d3a279383820f170d263de3a55f5308df01a0a34732ebaea6d3d", + "network": "disabled", + "checks": { + "accepted_verifications": 0, + "unknown_verifications": 0, + "invalidated_handles": 0, + "suppressions": 1, + "accepted_deliveries": 2, + "unknown_sends": 0 + }, + "uid": 10001, + "evidence_ceiling": "isolated_ledger_rehearsal", + "offline_image_binding_verified": true, + "elapsed_seconds": 3.8 + }, + "provider_only_daily_rehearsal": { + "status": "application_restored_verified", + "application": "email-connect", + "object_key": "platform-pg/application-archives/email-connect/2026/10/10/20261010T112917Z-e86c484a0918d1443984a10b.tar.age", + "version_id": "1791631827868341", + "ciphertext_sha256": "cba57bd57fe81adaa5594da5b57ea6a8b8f6d58b2a0718925325e3c6dd6cb302", + "database_sha256": "ed6711a63cb0a4371c1b805e326ee6ec62e3b6a2358c37b0ba9152f4a5eaaecb", + "ledger_fingerprint": { + "schema_sha256": "13148e5a9c6e01cc593bb07278a8649b13e43b26c30b0ed1a6e15995f7b00732", + "user_version": 0, + "tables": { + "caller_buckets": { + "rows": 1, + "sha256": "d7def892f86c163aed9df55112b0e4551ff6004e337a6a2135c9b80d8cb8fdf7" + }, + "deliveries": { + "rows": 2, + "sha256": "a0387716815b2ec191187cb34bb3252b9cf51f5fb15a2019918a9cf1737c061e" + }, + "send_attempts": { + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "suppressions": { + "rows": 1, + "sha256": "ae26d10dfec16f1b626292b342a5e86afd1858fcaed7c29d4e68e7cf821183d7" + }, + "verification_attempts": { + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "verifications": { + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + } + } + }, + "mail_sent": false, + "production_restored": false, + "application_restore_verified": true, + "recovery_image": "forgejo.coulomb.social/coulomb/email-connect@sha256:5d79fd1fec15d3a279383820f170d263de3a55f5308df01a0a34732ebaea6d3d", + "network": "disabled", + "checks": { + "accepted_verifications": 0, + "unknown_verifications": 0, + "invalidated_handles": 0, + "suppressions": 1, + "accepted_deliveries": 2, + "unknown_sends": 0 + }, + "uid": 10001, + "evidence_ceiling": "isolated_ledger_rehearsal", + "offline_image_binding_verified": true, + "elapsed_seconds": 2.87 + }, + "independent_recovery_reference": { + "bucket": "railiance-email-connect-recovery", + "key": "kits/recovery-reference-a8efafe91d9a80aa5ed5ca03f4a6536c26816d5345cbacdcb97b0c7e44cfafd5.json", + "version_id": "1791631821559658", + "sha256": "a8efafe91d9a80aa5ed5ca03f4a6536c26816d5345cbacdcb97b0c7e44cfafd5" + }, + "iam_policy_ids": { + "reader": "b83bbd41-1e57-44a0-8435-d24035497ef6", + "mirror_publisher": "6a9099b3-17ef-4a1f-a19a-e38419abe5c4", + "temporary_primary_reader_policy_revoked": "484e3064-7603-494c-910e-92226c0047c2" + }, + "privileged_test_keys": "Temporary operator and drill reader API key records deleted after proof; permanent reader app retained for attended reminting.", + "evidence_limits": { + "production_restore_performed": false, + "production_rto_measured": false, + "live_empty_tables": [ + "send_attempts", + "verification_attempts", + "verifications" + ], + "synthetic_semantics_cover_empty_live_tables": true, + "smtp_sent": false + }, + "regression_tests": { + "focused_passed": 47, + "skipped": 0 + }, + "final_provider_daily_rehearsal": { + "status": "application_restored_verified", + "application": "email-connect", + "object_key": "platform-pg/application-archives/email-connect/2026/10/10/20261010T113256Z-e2efa67ecfb8154fd8bbcc59.tar.age", + "version_id": "1791632035876582", + "ciphertext_sha256": "a007ab4f80924ecf5d69c52a91363298a68e004293f7e8f37bfae762c403ac5e", + "database_sha256": "ed6711a63cb0a4371c1b805e326ee6ec62e3b6a2358c37b0ba9152f4a5eaaecb", + "ledger_fingerprint": { + "schema_sha256": "13148e5a9c6e01cc593bb07278a8649b13e43b26c30b0ed1a6e15995f7b00732", + "user_version": 0, + "tables": { + "caller_buckets": { + "rows": 1, + "sha256": "d7def892f86c163aed9df55112b0e4551ff6004e337a6a2135c9b80d8cb8fdf7" + }, + "deliveries": { + "rows": 2, + "sha256": "a0387716815b2ec191187cb34bb3252b9cf51f5fb15a2019918a9cf1737c061e" + }, + "send_attempts": { + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "suppressions": { + "rows": 1, + "sha256": "ae26d10dfec16f1b626292b342a5e86afd1858fcaed7c29d4e68e7cf821183d7" + }, + "verification_attempts": { + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "verifications": { + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + } + } + }, + "mail_sent": false, + "production_restored": false, + "application_restore_verified": true, + "recovery_image": "forgejo.coulomb.social/coulomb/email-connect@sha256:5d79fd1fec15d3a279383820f170d263de3a55f5308df01a0a34732ebaea6d3d", + "network": "disabled", + "checks": { + "accepted_verifications": 0, + "unknown_verifications": 0, + "invalidated_handles": 0, + "suppressions": 1, + "accepted_deliveries": 2, + "unknown_sends": 0 + }, + "uid": 10001, + "evidence_ceiling": "isolated_ledger_rehearsal", + "offline_image_binding_verified": true, + "elapsed_seconds": 0.92 + }, + "operating_acceptance": "deploy/backup/operating-acceptance-2026-10-10.json", + "key_revocation": { + "reader_key_record_absent_http_status": 404, + "operator_key_record_absent_http_status": 404, + "permanent_reader_application_retained": true, + "operator_post_revoke_projects_http_status": 400 + } +} diff --git a/scripts/backup-ledger.py b/scripts/backup-ledger.py index e680e8c..c88fee6 100644 --- a/scripts/backup-ledger.py +++ b/scripts/backup-ledger.py @@ -95,7 +95,7 @@ def publish(encrypted, fetched, receipt_path, captured, tool, python): return receipt -def backup(state, primary_tool, primary_python, recipient, resume=False): +def backup(state, primary_tool, primary_python, recipient, resume=False, mirror_required=False): state.mkdir(mode=0o700, parents=True, exist_ok=True) private_directory(state) fd = os.open(state / "backup.lock", os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, 0o600) @@ -137,6 +137,17 @@ def backup(state, primary_tool, primary_python, recipient, resume=False): primary_tool, primary_python, ) + if mirror_required: + transfer.update(image=captured["image"], database_sha256=captured["database_sha256"], + ledger_fingerprint=captured["ledger_fingerprint"]) + checkpoint(stage / "primary-transfer.json", transfer) + command([str(primary_python), str(Path(__file__).with_name("mirror-ledger.py")), + "--archive", str(encrypted), "--primary-receipt", str(stage / "primary-transfer.json"), + "--output", str(stage / "mirror.json")]) + mirrored = json.loads((stage / "mirror.json").read_text()) + if mirrored.get("hash_readback_verified") is not True: + raise Refused("recovery_mirror_required") + transfer["recovery_mirror"] = mirrored filename = transfer["object_key"].rsplit("/", 1)[-1] if not NAME.fullmatch(filename): raise Refused("invalid_archive_name") @@ -173,7 +184,7 @@ def backup(state, primary_tool, primary_python, recipient, resume=False): return receipt -def freshness(state, now=None, hours=26): +def freshness(state, now=None, hours=26, mirror_required=False): now = now or datetime.now(UTC) private_directory(state) private_file(state / "last-success.json") @@ -195,6 +206,14 @@ def freshness(state, now=None, hours=26): private_file(state / name) if digest(state / name) != receipt["ciphertext_sha256"]: raise Refused("local_ciphertext_mismatch") + if mirror_required: + mirror = receipt.get("recovery_mirror", {}) + if (mirror.get("hash_readback_verified") is not True + or mirror.get("bucket") != "railiance-email-connect-archives" + or mirror.get("project_id") != "1d37ea63-8c8f-44e1-80e1-2de862579ba7" + or not mirror.get("version_id") or mirror["version_id"] == "null" + or not mirror.get("receipt_version_id") or mirror["receipt_version_id"] == "null"): + raise Refused("verified_recovery_mirror_required") return {"status": "fresh", "age_hours": round(age / 3600, 2), "remote_retrieval_verified": True} @@ -203,6 +222,7 @@ def main(): parser.add_argument("action", choices=["backup", "recover", "freshness"]) parser.add_argument("--execute", action="store_true") parser.add_argument("--resume-pending", action="store_true") + parser.add_argument("--mirror-required", action="store_true") parser.add_argument("--state-dir", type=Path, default=Path("/var/lib/email-connect-backup")) parser.add_argument("--primary-tool", type=Path, default=PRIMARY_TOOL) parser.add_argument("--primary-python", type=Path, default=PRIMARY_PYTHON) @@ -213,7 +233,7 @@ def main(): parser.add_argument("--destination", type=Path) args = parser.parse_args() if args.action == "freshness": - print(json.dumps(freshness(args.state_dir))) + print(json.dumps(freshness(args.state_dir, mirror_required=args.mirror_required))) elif args.action == "recover": if not args.execute: print(json.dumps({"status": "reviewed", "decryption_performed": False})) @@ -251,6 +271,7 @@ def main(): args.primary_python, args.age_recipient, args.resume_pending, + args.mirror_required, ) ) ) diff --git a/scripts/mirror-ledger.py b/scripts/mirror-ledger.py new file mode 100644 index 0000000..9aae1c9 --- /dev/null +++ b/scripts/mirror-ledger.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +"""Copy ciphertext into the isolated recovery project; never read an age key.""" +import argparse +import hashlib +import json +import os +from pathlib import Path + +from ledger_archive import Refused, checkpoint, digest, private_file + +PROJECT = "1d37ea63-8c8f-44e1-80e1-2de862579ba7" +BUCKET = "railiance-email-connect-archives" + + +def mirror(client, archive, receipt): + private_file(archive) + expected = digest(archive) + size = archive.stat().st_size + if expected != receipt["ciphertext_sha256"] or size != receipt["ciphertext_bytes"]: + raise Refused("mirror_source_differs") + key = "archives/" + receipt["object_key"] + try: + with archive.open("rb") as source: + created = client.put_object(Bucket=BUCKET, Key=key, Body=source, + IfNoneMatch="*", Metadata={"sha256": expected}) + version = created.get("VersionId") + except Exception as error: + if getattr(error, "response", {}).get("ResponseMetadata", {}).get("HTTPStatusCode") != 412: + raise + current = client.head_object(Bucket=BUCKET, Key=key) + if current.get("Metadata", {}).get("sha256") != expected or current.get("ContentLength") != size: + raise Refused("mirror_existing_object_differs") + version = current.get("VersionId") + if not version or version == "null": + raise Refused("versioned_mirror_required") + result = client.get_object(Bucket=BUCKET, Key=key, VersionId=version) + hashed, count = hashlib.sha256(), 0 + with result["Body"] as stream: + while block := stream.read(1024**2): + count += len(block) + if count > size: + raise Refused("mirror_readback_exceeds_size") + hashed.update(block) + if (result.get("VersionId") != version or count != size + or hashed.hexdigest() != expected): + raise Refused("mirror_readback_differs") + # Recovery needs a receipt even if both node and workstation are lost. + record = dict(receipt, recovery_mirror={"bucket": BUCKET, "object_key": key, + "version_id": version, "project_id": PROJECT, + "hash_readback_verified": True}) + payload = json.dumps(record, sort_keys=True).encode() + receipt_key = key + "." + version + "." + hashlib.sha256(payload).hexdigest() + ".receipt.json" + try: + saved = client.put_object(Bucket=BUCKET, Key=receipt_key, Body=payload, IfNoneMatch="*") + receipt_version = saved.get("VersionId") + except Exception as error: + if getattr(error, "response", {}).get("ResponseMetadata", {}).get("HTTPStatusCode") != 412: + raise + receipt_version = client.head_object(Bucket=BUCKET, Key=receipt_key).get("VersionId") + if not receipt_version or receipt_version == "null": + raise Refused("versioned_mirror_receipt_required") + saved = client.get_object(Bucket=BUCKET, Key=receipt_key, VersionId=receipt_version) + with saved["Body"] as stream: + if stream.read(len(payload) + 1) != payload: + raise Refused("mirror_receipt_readback_differs") + return dict(record["recovery_mirror"], receipt_key=receipt_key, + receipt_version_id=receipt_version) + + +def main(): + import boto3 + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--archive", type=Path, required=True) + parser.add_argument("--primary-receipt", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + private_file(args.primary_receipt) + key = os.environ["AWS_ACCESS_KEY_ID"].split("@", 1)[0] + "@" + PROJECT + client = boto3.client("s3", endpoint_url="https://s3.nl-ams.scw.cloud", region_name="nl-ams", + aws_access_key_id=key, aws_secret_access_key=os.environ["AWS_SECRET_ACCESS_KEY"]) + checkpoint(args.output, mirror(client, args.archive, json.loads(args.primary_receipt.read_text()))) + + +if __name__ == "__main__": + try: + main() + except Exception as error: + print(json.dumps({"status": "failed", "failure_type": type(error).__name__})) + raise SystemExit(1) from None diff --git a/scripts/prepare-ledger-recovery-kit.py b/scripts/prepare-ledger-recovery-kit.py index e9d1aca..0a4d5e9 100644 --- a/scripts/prepare-ledger-recovery-kit.py +++ b/scripts/prepare-ledger-recovery-kit.py @@ -42,7 +42,9 @@ def prepare(evidence, ciphertext, destination): (destination / "receiver-image.tar").chmod(0o600) for name in ("ledger_archive.py", "backup-ledger.py", "rehearse-ledger.py"): shutil.copyfile(ROOT / "scripts" / name, destination / name) - (destination / name).chmod(0o600) + # This public module is bind-mounted into the UID10001 rehearsal. + # The enclosing kit directory remains private; data stays mode 0600. + (destination / name).chmod(0o644 if name == "ledger_archive.py" else 0o600) shutil.copyfile(ciphertext, destination / "ledger.tar.age") (destination / "ledger.tar.age").chmod(0o600) checkpoint(destination / "transfer-receipt.json", transfer) diff --git a/tests/test_closure_custody.py b/tests/test_closure_custody.py index e6a7398..678db7b 100644 --- a/tests/test_closure_custody.py +++ b/tests/test_closure_custody.py @@ -4,6 +4,8 @@ import base64 import hashlib import importlib.util import json +import stat +import subprocess from pathlib import Path import pytest @@ -121,6 +123,36 @@ def test_caller_failure_receipt_allows_only_static_diagnostic_codes(monkeypatch) assert "synthetic-sensitive-value" not in json.dumps(receipt) +def test_recovery_kit_public_module_is_readable_by_rehearsal_uid(monkeypatch, tmp_path): + kit = module("prepare-ledger-recovery-kit", monkeypatch) + cipher = tmp_path / "ledger.tar.age" + cipher.write_bytes(b"synthetic-encrypted-ledger") + cipher.chmod(0o600) + hashed = hashlib.sha256(cipher.read_bytes()).hexdigest() + evidence = tmp_path / "evidence.json" + evidence.write_text(json.dumps({ + "publication": {"ciphertext_sha256": hashed, "ciphertext_bytes": cipher.stat().st_size, + "version_id": "synthetic-version"}, + "application_recovery": {"application_restore_verified": True, + "application": "email-connect", "ciphertext_sha256": hashed, + "version_id": "synthetic-version", "recovery_image": "synthetic-image"}, + })) + + def docker(args, **kwargs): + if args[2] == "inspect": + return subprocess.CompletedProcess(args, 0, json.dumps([{"Id": "sha256:" + "a" * 64}]).encode()) + Path(args[args.index("--output") + 1]).write_bytes(b"synthetic-image-archive") + return subprocess.CompletedProcess(args, 0, b"") + + monkeypatch.setattr(kit.subprocess, "run", docker) + destination = tmp_path / "kit" + kit.prepare(evidence, cipher, destination) + assert stat.S_IMODE(destination.stat().st_mode) == 0o700 + assert stat.S_IMODE((destination / "ledger_archive.py").stat().st_mode) == 0o644 + for name in ["ledger.tar.age", "receiver-image.tar", "transfer-receipt.json", "kit.tar.gz"]: + assert stat.S_IMODE((destination / name).stat().st_mode) == 0o600 + + @pytest.mark.parametrize("protected,ephemeral,corrupt", [ (True, None, False), (False, None, False), (True, {}, False), (True, None, True), ]) diff --git a/tests/test_ledger_backup.py b/tests/test_ledger_backup.py index b9081f9..9e6ccb6 100644 --- a/tests/test_ledger_backup.py +++ b/tests/test_ledger_backup.py @@ -298,7 +298,7 @@ def test_capture_resumes_unchanged_receiver_after_failures(modules, monkeypatch, assert receipt["status"] == ("captured" if failure is None else "failed") -@pytest.mark.parametrize("failure", [None, "publish", "encrypt"]) +@pytest.mark.parametrize("failure", [None, "publish", "encrypt", "mirror"]) def test_backup_success_requires_readback_and_preserves_one_pending_capture( modules, monkeypatch, tmp_path, failure ): @@ -308,6 +308,11 @@ def test_backup_success_requires_readback_and_preserves_one_pending_capture( state = tmp_path / "state" def command(argv, **kwargs): + if "--primary-receipt" in argv: + if failure == "mirror": + raise archive.Refused("synthetic mirror failure") + archive.checkpoint(Path(argv[-1]), {"hash_readback_verified": True}) + return captures.append(argv) captured(Path(argv[-1]), archive) @@ -352,16 +357,19 @@ def test_backup_success_requires_readback_and_preserves_one_pending_capture( monkeypatch.setattr(runner, "publish", publish) if failure: with pytest.raises(archive.Refused): - runner.backup(state, Path("synthetic-tool"), Path("synthetic-python"), public) + runner.backup(state, Path("synthetic-tool"), Path("synthetic-python"), public, + mirror_required=failure == "mirror") assert not (state / "last-success.json").exists() assert len(list(state.glob("pending-*"))) == 1 with pytest.raises(archive.Refused, match="pending_backup_requires_review"): runner.backup(state, Path("synthetic-tool"), Path("synthetic-python"), public) - if failure == "publish": + if failure in {"publish", "mirror"}: + mirror_required = failure == "mirror" ciphertext = publications[0] failure = None result = runner.backup( - state, Path("synthetic-tool"), Path("synthetic-python"), public, resume=True + state, Path("synthetic-tool"), Path("synthetic-python"), public, resume=True, + mirror_required=mirror_required, ) assert len(captures) == 1 and publications == [ciphertext, ciphertext] assert result["application_restore_verified"] is False diff --git a/tests/test_ledger_mirror.py b/tests/test_ledger_mirror.py new file mode 100644 index 0000000..3b8c366 --- /dev/null +++ b/tests/test_ledger_mirror.py @@ -0,0 +1,45 @@ +"""Synthetic immutable mirror/readback and recovery-receipt checks.""" +import hashlib +import importlib.util +import io +from pathlib import Path + +import pytest + + +@pytest.mark.parametrize("failure", [None, "ciphertext", "receipt", "unversioned"]) +def test_mirror_refuses_corruption_before_success(monkeypatch, tmp_path, failure): + scripts = Path(__file__).resolve().parents[1] / "scripts" + monkeypatch.syspath_prepend(str(scripts)) + spec = importlib.util.spec_from_file_location("mirror", scripts / "mirror-ledger.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + archive = tmp_path / "ledger.age" + archive.write_bytes(b"synthetic encrypted data") + archive.chmod(0o600) + objects = {} + + class Client: + def put_object(self, **kwargs): + assert kwargs["IfNoneMatch"] == "*" + body = kwargs["Body"] + objects[kwargs["Key"]] = body.read() if hasattr(body, "read") else body + return {"VersionId": "null" if failure == "unversioned" else "synthetic-version"} + + def get_object(self, **kwargs): + body = objects[kwargs["Key"]] + if ((failure == "ciphertext" and not kwargs["Key"].endswith("json")) + or (failure == "receipt" and kwargs["Key"].endswith("json"))): + body = b"corrupted" + return {"VersionId": kwargs["VersionId"], "Body": io.BytesIO(body)} + + receipt = {"object_key": "platform-pg/application-archives/email-connect/synthetic.age", + "ciphertext_sha256": hashlib.sha256(archive.read_bytes()).hexdigest(), + "ciphertext_bytes": archive.stat().st_size} + if failure: + with pytest.raises(module.Refused): + module.mirror(Client(), archive, receipt) + else: + result = module.mirror(Client(), archive, receipt) + assert result["hash_readback_verified"] + assert result["receipt_key"] in objects diff --git a/workplans/EMAIL-WP-0007-ledger-backup-recovery.md b/workplans/EMAIL-WP-0007-ledger-backup-recovery.md index 6d714da..92b9f45 100644 --- a/workplans/EMAIL-WP-0007-ledger-backup-recovery.md +++ b/workplans/EMAIL-WP-0007-ledger-backup-recovery.md @@ -4,11 +4,11 @@ type: workplan title: "Back up and recover the transactional delivery ledger" domain: infotech repo: email-connect -status: active +status: finished owner: codex topic_slug: email-connect created: "2026-10-07" -updated: "2026-10-09" +updated: "2026-10-10" origin: residual origin_ref: EMAIL-WP-0006 state_hub_workstream_id: "35be21a2-343b-52e1-ba5e-32fead7a7831" @@ -60,10 +60,9 @@ challenges before releasing traffic. Never auto-restore into a running service. ```task id: EMAIL-WP-0007-T03 -status: progress +status: done priority: high -needs_human: true -intervention_note: "Scaleway selected; email-connect-backup-primary remains the existing prefix-only publisher. Closure package and local offline kit are tested. Owner-provided durable age/kit custody and separate recovery reader grants are still absent; existing archive bucket expires objects after 30 days." +needs_human: false state_hub_task_id: "680fe4d6-12af-5296-acf9-15c9a01a0db3" ``` @@ -79,10 +78,9 @@ invented owner review or credential grant. ```task id: EMAIL-WP-0007-T04 -status: progress +status: done priority: high -needs_human: true -intervention_note: "First backup/recovery and platform-root inbox delivery pass. Daily schedule, retention and capacity targets are prepared in deploy/closure/plan.json. Independent Scaleway custody and recurring-operation acceptance remain open; timers stay disabled." +needs_human: false state_hub_task_id: "66401e5c-a5cf-5e67-beab-e503ae079964" ``` @@ -257,3 +255,47 @@ Ready. EMAIL-WP-0008 is finished; T03/T04 here remain progress for independent Scaleway custody/grants, retention/schedule and recurring operation. Timers remain disabled. The user was asked for an existing Scaleway operator lane or Console custody setup, since the existing publisher cannot administer escrow resources. + +## Completed independent custody and recurring operation — 2026-10-10 + +The user completed the attended Scaleway operator login and authorized finishing +the workplans. T03/T04 are done. Protected, nonexpiring age revision 1 and a +private, versioned, nonexpiring recovery kit now reside in independent project +email-connect-recovery (1d37ea63-8c8f-44e1-80e1-2de862579ba7), nl-ams. The +separate IAM reader is natively bound and tested; scheduled-worker kit/key reads +and retained-object deletes return 403. Recovery using only provider-retrieved +kit, bundled image/tools and restored escrow identity passes without network or +mail. Public kit library permissions were corrected after the cold drill found +UID10001 could not read its original 0600 bind mount; a regression covers it. + +A temporary original-project reader grant admitted metadata reads from an +unrelated bucket. It was removed and original bucket grants restored. Native +unrelated GET/LIST now deny 403. Daily archives and complete recovery receipts +are copied to private, versioned railiance-email-connect-archives in the isolated +project with 30-day expiry. The existing primary publisher remains in use. +Backup success/freshness require hash/version-verified mirror publication. +Failures preserve a pending capture for retry without another receiver pause. + +The final installed-code run captured/resumed in 57.55 seconds, independently +published/read back both copies, and restored through the separate recovery +reader and provider receipt. All six tables match, with one quota bucket, two +accepted deliveries and one suppression. Remaining live transactional tables +are empty; their behavior remains covered synthetically. Both deployments are +Ready, and daily 04:15 UTC (+ up to five-minute jitter) backup and hourly freshness +timers are enabled/active. Freshness passes; failures route to the user-selected +platform-root inbox, with pending delivery replay. Platform-root owns weekly +14-daily/4-weekly retention selection, local ciphertext expiry/capacity review and +retained-kit refresh. Worker deletion remains prohibited; 8 GiB local budget/free +floor guards remain installed. RPO target is 24 hours with a 26-hour stale alert. +60-minute attended RTO is a target, not a measured production restore. Production +recovery still requires fencing and post-capture/challenge reconciliation. + +Evidence: docs/ledger-scaleway-custody-2026-10-10.json, +deploy/backup/operating-acceptance-2026-10-10.json and +updated docs/ledger-backup-review-2026-10-08.md. Focused regression: 47 passed, +zero skipped. Syntax/undefined-name lint passes. The broader lint profile reports +existing shell/rehearsal exceptions and the deliberate credential-safe top-level +exception boundary; no broad clean-lint claim is made. Temporary operator/drill +API key records were deleted after proofs; the permanent reader application and +operator-account bootstrap path survive workstation/node loss. EMAIL-WP-0008 +remains finished. Earlier waiting/local-only entries above are historical.