Retain independent ledger recovery custody and activate verified backups
Assistant: codex Assistant-Model: gpt-6.1-sol Assistant-Session: 01a11fd9-caf6-7a63-b578-9daaaa134ff2
This commit is contained in:
parent
6c7ba61a05
commit
f77e2333af
17 changed files with 967 additions and 24 deletions
|
|
@ -14,7 +14,7 @@
|
|||
| workplan | EMAIL-WP-0004 | finished | — | workplans/EMAIL-WP-0004-transactional-mail-delivery-service.md |
|
||||
| workplan | EMAIL-WP-0005 | finished | — | workplans/EMAIL-WP-0005-test-mailbox-harness.md |
|
||||
| workplan | EMAIL-WP-0006 | finished | — | workplans/EMAIL-WP-0006-verification-retry-safety.md |
|
||||
| workplan | EMAIL-WP-0007 | active | — | workplans/EMAIL-WP-0007-ledger-backup-recovery.md |
|
||||
| workplan | EMAIL-WP-0007 | finished | — | workplans/EMAIL-WP-0007-ledger-backup-recovery.md |
|
||||
| workplan | EMAIL-WP-0008 | finished | — | workplans/EMAIL-WP-0008-scoped-transactional-callers.md |
|
||||
| task | EMAIL-WP-0001-T01 | done | — | workplans/EMAIL-WP-0001-repo-onboarding.md |
|
||||
| task | EMAIL-WP-0001-T02 | done | — | workplans/EMAIL-WP-0001-repo-onboarding.md |
|
||||
|
|
@ -55,8 +55,8 @@
|
|||
| task | EMAIL-WP-0006-T04 | done | — | workplans/EMAIL-WP-0006-verification-retry-safety.md |
|
||||
| task | EMAIL-WP-0007-T01 | done | — | workplans/EMAIL-WP-0007-ledger-backup-recovery.md |
|
||||
| task | EMAIL-WP-0007-T02 | done | — | workplans/EMAIL-WP-0007-ledger-backup-recovery.md |
|
||||
| task | EMAIL-WP-0007-T03 | progress | — | workplans/EMAIL-WP-0007-ledger-backup-recovery.md |
|
||||
| task | EMAIL-WP-0007-T04 | progress | — | workplans/EMAIL-WP-0007-ledger-backup-recovery.md |
|
||||
| task | EMAIL-WP-0007-T03 | done | — | workplans/EMAIL-WP-0007-ledger-backup-recovery.md |
|
||||
| task | EMAIL-WP-0007-T04 | done | — | workplans/EMAIL-WP-0007-ledger-backup-recovery.md |
|
||||
| task | EMAIL-WP-0008-T01 | done | — | workplans/EMAIL-WP-0008-scoped-transactional-callers.md |
|
||||
| task | EMAIL-WP-0008-T02 | done | — | workplans/EMAIL-WP-0008-scoped-transactional-callers.md |
|
||||
| task | EMAIL-WP-0008-T03 | done | — | workplans/EMAIL-WP-0008-scoped-transactional-callers.md |
|
||||
|
|
|
|||
59
deploy/backup/daily-recovery-bucket-policy.json
Normal file
59
deploy/backup/daily-recovery-bucket-policy.json
Normal file
|
|
@ -0,0 +1,59 @@
|
|||
{
|
||||
"Version": "2023-04-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Sid": "Operator",
|
||||
"Effect": "Allow",
|
||||
"Principal": {
|
||||
"SCW": "user_id:3d029efb-4f70-4f82-a3bb-cefbe255232b"
|
||||
},
|
||||
"Action": "s3:*",
|
||||
"Resource": [
|
||||
"railiance-email-connect-archives",
|
||||
"railiance-email-connect-archives/*"
|
||||
]
|
||||
},
|
||||
{
|
||||
"Sid": "ArchivePublisher",
|
||||
"Effect": "Allow",
|
||||
"Principal": {
|
||||
"SCW": "application_id:0eb0f958-77a4-4c70-9aa7-14ba46411415"
|
||||
},
|
||||
"Action": [
|
||||
"s3:GetObject",
|
||||
"s3:GetObjectVersion",
|
||||
"s3:PutObject"
|
||||
],
|
||||
"Resource": "railiance-email-connect-archives/archives/*"
|
||||
},
|
||||
{
|
||||
"Sid": "ArchiveReader",
|
||||
"Effect": "Allow",
|
||||
"Principal": {
|
||||
"SCW": "application_id:a84a18e2-3813-4918-9470-dbe0580a5ea6"
|
||||
},
|
||||
"Action": [
|
||||
"s3:GetObject",
|
||||
"s3:GetObjectVersion"
|
||||
],
|
||||
"Resource": "railiance-email-connect-archives/archives/*"
|
||||
},
|
||||
{
|
||||
"Sid": "ArchiveList",
|
||||
"Effect": "Allow",
|
||||
"Principal": {
|
||||
"SCW": "application_id:a84a18e2-3813-4918-9470-dbe0580a5ea6"
|
||||
},
|
||||
"Action": [
|
||||
"s3:ListBucket",
|
||||
"s3:ListBucketVersions"
|
||||
],
|
||||
"Resource": "railiance-email-connect-archives",
|
||||
"Condition": {
|
||||
"StringLike": {
|
||||
"s3:prefix": "archives/*"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -11,7 +11,7 @@ UMask=0077
|
|||
StateDirectory=email-connect-backup
|
||||
StateDirectoryMode=0700
|
||||
EnvironmentFile=/etc/email-connect-backup/primary.env
|
||||
ExecStart=/usr/bin/python3 /opt/email-connect-backup/backup-ledger.py backup --execute
|
||||
ExecStart=/usr/bin/python3 /opt/email-connect-backup/backup-ledger.py backup --execute --mirror-required
|
||||
TimeoutStartSec=40min
|
||||
TimeoutStopSec=15min
|
||||
KillMode=mixed
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ OnFailure=email-connect-ledger-alert@%n.service
|
|||
[Service]
|
||||
Type=oneshot
|
||||
User=root
|
||||
ExecStart=/usr/bin/python3 /opt/email-connect-backup/backup-ledger.py freshness
|
||||
ExecStart=/usr/bin/python3 /opt/email-connect-backup/backup-ledger.py freshness --mirror-required
|
||||
ExecStartPost=-/usr/bin/python3 /opt/email-connect-backup/notify-ledger-failure.py
|
||||
StateDirectory=email-connect-backup
|
||||
StateDirectoryMode=0700
|
||||
|
|
|
|||
100
deploy/backup/operating-acceptance-2026-10-10.json
Normal file
100
deploy/backup/operating-acceptance-2026-10-10.json
Normal file
|
|
@ -0,0 +1,100 @@
|
|||
{
|
||||
"schema": "email-connect.ledger-operating-acceptance.v1",
|
||||
"date": "2026-10-10",
|
||||
"status": "active",
|
||||
"timers": {
|
||||
"email-connect-ledger-backup.timer": {
|
||||
"NextElapseUSecRealtime": "Sun 2026-10-11 04:15:29 UTC",
|
||||
"ActiveState": "active",
|
||||
"UnitFileState": "enabled"
|
||||
},
|
||||
"email-connect-ledger-freshness.timer": {
|
||||
"NextElapseUSecRealtime": "Sat 2026-10-10 12:00:00 UTC",
|
||||
"ActiveState": "active",
|
||||
"UnitFileState": "enabled"
|
||||
}
|
||||
},
|
||||
"freshness_service_passed": true,
|
||||
"last_success": {
|
||||
"schema": "platform.application-primary-archive.v1",
|
||||
"status": "offsite_fetched_verified",
|
||||
"application": "email-connect",
|
||||
"captured_at": "2026-10-10T11:32:56Z",
|
||||
"object_key": "platform-pg/application-archives/email-connect/2026/10/10/20261010T113256Z-e2efa67ecfb8154fd8bbcc59.tar.age",
|
||||
"destination": "s3://railiance-platform-pg-backup/platform-pg/application-archives/email-connect/2026/10/10/20261010T113256Z-e2efa67ecfb8154fd8bbcc59.tar.age",
|
||||
"ciphertext_sha256": "a007ab4f80924ecf5d69c52a91363298a68e004293f7e8f37bfae762c403ac5e",
|
||||
"ciphertext_bytes": 71896,
|
||||
"remote_retrieval_verified": true,
|
||||
"stage": "transfer_verified",
|
||||
"multipart_completed": true,
|
||||
"version_id": "1791632035876582",
|
||||
"immutable_create": true,
|
||||
"image": "forgejo.coulomb.social/coulomb/email-connect@sha256:5d79fd1fec15d3a279383820f170d263de3a55f5308df01a0a34732ebaea6d3d",
|
||||
"database_sha256": "ed6711a63cb0a4371c1b805e326ee6ec62e3b6a2358c37b0ba9152f4a5eaaecb",
|
||||
"ledger_fingerprint": {
|
||||
"schema_sha256": "13148e5a9c6e01cc593bb07278a8649b13e43b26c30b0ed1a6e15995f7b00732",
|
||||
"user_version": 0,
|
||||
"tables": {
|
||||
"caller_buckets": {
|
||||
"rows": 1,
|
||||
"sha256": "d7def892f86c163aed9df55112b0e4551ff6004e337a6a2135c9b80d8cb8fdf7"
|
||||
},
|
||||
"deliveries": {
|
||||
"rows": 2,
|
||||
"sha256": "a0387716815b2ec191187cb34bb3252b9cf51f5fb15a2019918a9cf1737c061e"
|
||||
},
|
||||
"send_attempts": {
|
||||
"rows": 0,
|
||||
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
|
||||
},
|
||||
"suppressions": {
|
||||
"rows": 1,
|
||||
"sha256": "ae26d10dfec16f1b626292b342a5e86afd1858fcaed7c29d4e68e7cf821183d7"
|
||||
},
|
||||
"verification_attempts": {
|
||||
"rows": 0,
|
||||
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
|
||||
},
|
||||
"verifications": {
|
||||
"rows": 0,
|
||||
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
|
||||
}
|
||||
}
|
||||
},
|
||||
"recovery_mirror": {
|
||||
"bucket": "railiance-email-connect-archives",
|
||||
"object_key": "archives/platform-pg/application-archives/email-connect/2026/10/10/20261010T113256Z-e2efa67ecfb8154fd8bbcc59.tar.age",
|
||||
"version_id": "1791632037091494",
|
||||
"project_id": "1d37ea63-8c8f-44e1-80e1-2de862579ba7",
|
||||
"hash_readback_verified": true,
|
||||
"receipt_key": "archives/platform-pg/application-archives/email-connect/2026/10/10/20261010T113256Z-e2efa67ecfb8154fd8bbcc59.tar.age.1791632037091494.5aee83f185670ac3fd2058a76a5194306288042a6baac7aeb1a91903a4ec5242.receipt.json",
|
||||
"receipt_version_id": "1791632037934451"
|
||||
},
|
||||
"database_bytes": 57344,
|
||||
"pause_and_capture_seconds": 57.55,
|
||||
"local_archive": "20261010T113256Z-e2efa67ecfb8154fd8bbcc59.tar.age",
|
||||
"application_restore_verified": false,
|
||||
"completed_at": "2026-10-10T11:33:58.694614+00:00"
|
||||
},
|
||||
"schedule_utc": "04:15",
|
||||
"jitter_seconds": 300,
|
||||
"rpo_target_hours": 24,
|
||||
"freshness_threshold_hours": 26,
|
||||
"attended_rto_target_minutes": 60,
|
||||
"production_rto_measured": false,
|
||||
"local_budget_bytes": 8589934592,
|
||||
"minimum_free_bytes": 8589934592,
|
||||
"retention_operator": "platform-root",
|
||||
"owner_review_interval": "weekly",
|
||||
"rolling_retention_days": 30,
|
||||
"retained_kit_expires": false,
|
||||
"owner_selection_daily": 14,
|
||||
"owner_selection_weekly": 4,
|
||||
"failure_recipient": "platform-root",
|
||||
"source_sha256": {
|
||||
"scripts/backup-ledger.py": "0bbef692208bd6b1f6e06f3ae969c01fb86cc2078529e2eb598f370b0363e99f",
|
||||
"scripts/mirror-ledger.py": "abefb951dbdb4212f914a7701cfe0e7121e2c2643bf9f6892a0fc97fa6da760d",
|
||||
"deploy/backup/email-connect-ledger-backup.service": "1190c30ccb4b6f7f2d0c7194907ffb1c7ccf6d5bf069f7cbd20fdbbce1a25534",
|
||||
"deploy/backup/email-connect-ledger-freshness.service": "edbc0445fbeffdf108ef4a4f5ffeb0ab4f9909424d849fa55b021e92f6a25916"
|
||||
}
|
||||
}
|
||||
30
deploy/backup/retained-recovery-bucket-policy.json
Normal file
30
deploy/backup/retained-recovery-bucket-policy.json
Normal file
|
|
@ -0,0 +1,30 @@
|
|||
{
|
||||
"Version": "2023-04-17",
|
||||
"Id": "email-connect-independent-recovery",
|
||||
"Statement": [
|
||||
{
|
||||
"Sid": "OperatorCustody",
|
||||
"Effect": "Allow",
|
||||
"Principal": {
|
||||
"SCW": "user_id:3d029efb-4f70-4f82-a3bb-cefbe255232b"
|
||||
},
|
||||
"Action": "s3:*",
|
||||
"Resource": [
|
||||
"railiance-email-connect-recovery",
|
||||
"railiance-email-connect-recovery/*"
|
||||
]
|
||||
},
|
||||
{
|
||||
"Sid": "RecoveryKitReadOnly",
|
||||
"Effect": "Allow",
|
||||
"Principal": {
|
||||
"SCW": "application_id:a84a18e2-3813-4918-9470-dbe0580a5ea6"
|
||||
},
|
||||
"Action": [
|
||||
"s3:GetObject",
|
||||
"s3:GetObjectVersion"
|
||||
],
|
||||
"Resource": "railiance-email-connect-recovery/kits/*"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -82,6 +82,14 @@
|
|||
]
|
||||
},
|
||||
"backup": {
|
||||
"recovery_mirror": {
|
||||
"project_id": "1d37ea63-8c8f-44e1-80e1-2de862579ba7",
|
||||
"bucket": "railiance-email-connect-archives",
|
||||
"prefix": "archives/",
|
||||
"verified_copy_required_before_success": true,
|
||||
"versioned_recovery_receipt_required": true,
|
||||
"retention_days": 30
|
||||
},
|
||||
"schedule_utc": "04:15",
|
||||
"jitter_seconds": 300,
|
||||
"rpo_target_hours": 24,
|
||||
|
|
@ -140,9 +148,10 @@
|
|||
"scripts/admit-scoped-caller.py": "17bf68f85f0bb08b3ca89eee34e62e794ee4754fbbb21646c28c85a1c1dd0e0e",
|
||||
"scripts/scaleway-recovery-custody.py": "85874f0620dc9353a9aaae1194db054de166296c4385743e4d541bbe42486c15",
|
||||
"scripts/publish-ledger-recovery-kit.py": "cd7c6e8e6a4e8417ba55f8170d763d7c5a9e6dd9b5878dc3459021a0b64ea732",
|
||||
"scripts/prepare-ledger-recovery-kit.py": "db92a29eb68e1b555cf18b17229d1e479b3147da7600ffa0616ac8b4d011ff55",
|
||||
"scripts/prepare-ledger-recovery-kit.py": "8cf74c9ea300c5ff6a03282293ba76ea0516fbd16f841259ca70f563c9e82d2b",
|
||||
"scripts/rehearse-ledger.py": "5c43b655149d8ae577fab80e76358be35f90cb0357df9b726ac7cb23346946a0",
|
||||
"scripts/backup-ledger.py": "e74c41948ca01dae09578e2abfba14b96fb1f158af0ec91a0fb1aa9ecaf01680",
|
||||
"scripts/backup-ledger.py": "0bbef692208bd6b1f6e06f3ae969c01fb86cc2078529e2eb598f370b0363e99f",
|
||||
"scripts/mirror-ledger.py": "abefb951dbdb4212f914a7701cfe0e7121e2c2643bf9f6892a0fc97fa6da760d",
|
||||
"scripts/capture-ledger.py": "7994fe7b4fbf72e28a4296212005a88d0ef9fcb626b066e6bcf2885500bc69d6"
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -42,6 +42,12 @@ plan = {
|
|||
"reader_policy": "email-connect-scoped-caller-reader", "readers": readers,
|
||||
},
|
||||
"backup": {
|
||||
"recovery_mirror": {
|
||||
"project_id": "1d37ea63-8c8f-44e1-80e1-2de862579ba7",
|
||||
"bucket": "railiance-email-connect-archives", "prefix": "archives/",
|
||||
"verified_copy_required_before_success": True,
|
||||
"versioned_recovery_receipt_required": True, "retention_days": 30,
|
||||
},
|
||||
"schedule_utc": "04:15", "jitter_seconds": 300, "rpo_target_hours": 24,
|
||||
"stale_threshold_hours": 26, "attended_rto_target_minutes": 60,
|
||||
"minimum_node_free_bytes": 8 * 1024**3, "local_budget_bytes": 8 * 1024**3,
|
||||
|
|
@ -130,7 +136,7 @@ artifacts = [
|
|||
"deploy/closure/user-engine.patch.json", "scripts/admit-scoped-caller.py",
|
||||
"scripts/scaleway-recovery-custody.py", "scripts/publish-ledger-recovery-kit.py",
|
||||
"scripts/prepare-ledger-recovery-kit.py", "scripts/rehearse-ledger.py",
|
||||
"scripts/backup-ledger.py",
|
||||
"scripts/backup-ledger.py", "scripts/mirror-ledger.py",
|
||||
"scripts/capture-ledger.py",
|
||||
]
|
||||
plan["artifacts"] = {name: hashlib.sha256((ROOT / name).read_bytes()).hexdigest() for name in artifacts}
|
||||
|
|
|
|||
|
|
@ -268,3 +268,84 @@ Ready. EMAIL-WP-0008 is finished; T03/T04 here remain progress for independent
|
|||
Scaleway custody/grants, retention/schedule and recurring operation. Timers remain
|
||||
disabled. The user was asked for an existing Scaleway operator lane or Console
|
||||
custody setup, since the existing publisher cannot administer escrow resources.
|
||||
|
||||
## Independent custody and operating acceptance — 2026-10-10
|
||||
|
||||
The user completed an attended Scaleway browser login and authorized continuation
|
||||
of the closure package. Native custody and recovery evidence is recorded in
|
||||
`docs/ledger-scaleway-custody-2026-10-10.json`. No CCR-2026-0011 admission is
|
||||
inferred from that separate operator login.
|
||||
|
||||
Independent project `email-connect-recovery`, ID
|
||||
`1d37ea63-8c8f-44e1-80e1-2de862579ba7`, is in nl-ams. Protected opaque Secret
|
||||
Manager container `email-connect-ledger-age-recovery`, ID
|
||||
`545a1f3c-8e21-4ed1-9ec9-33c9ea2e9272`, holds enabled, nonexpiring revision 1.
|
||||
The private age identity was restored from Scaleway and matched the public
|
||||
recipient; it was never installed on the backup node or placed in a kit.
|
||||
|
||||
The private, versioned `railiance-email-connect-recovery` bucket has no lifecycle
|
||||
expiry. Retained kit SHA256
|
||||
`3e786168328dc9c9a32f0b67e4d010ee9db0e0b855fc2460413fb85a16a2e884`,
|
||||
version `1791631021870760`, includes the pinned scoped receiver image, ciphertext,
|
||||
receipts and recovery tools. Provider retrieval, all-file hash checks, image load,
|
||||
age decrypt and bundled-tool UID10001/network-none rehearsal passed. Kit assembly
|
||||
now exports the public fingerprint library as 0644 so the container can read its
|
||||
bind mount; the containing directory remains 0700, other files 0600. A regression
|
||||
test covers this cold-kit failure found during the actual drill.
|
||||
|
||||
Separate reader `email-connect-ledger-recovery-reader`, application
|
||||
`a84a18e2-3813-4918-9470-dbe0580a5ea6`, can read kits and list/read the daily
|
||||
archive prefix. Native checks prove write/delete, foreign-prefix, original-project
|
||||
and unrelated-bucket denials. Native checks also prove the scheduled worker cannot
|
||||
read the retained kit or age identity, or delete daily copies. Identity binding
|
||||
was checked through IAM metadata, in addition to positive version/hash readback.
|
||||
|
||||
A temporary read grant in the original project exposed metadata in an unrelated
|
||||
bucket. It was revoked, its bucket-policy changes restored, and unrelated reads
|
||||
now return 403. Scaleway Object Storage IAM grants apply across a project, and
|
||||
resource conditions do not support Object Storage:
|
||||
https://www.scaleway.com/en/docs/iam/policies-permissions/supported-products-resource-level/
|
||||
https://www.scaleway.com/en/docs/object-storage/api-cli/combining-iam-and-object-storage/
|
||||
No policy was installed on the unrelated bucket and no actual object data was read.
|
||||
|
||||
Daily ciphertext is therefore copied into private, versioned bucket
|
||||
`railiance-email-connect-archives` in the independent project. Its `archives/`
|
||||
prefix expires current/noncurrent versions after 30 days. The existing
|
||||
`email-connect-backup-primary` application remains the publisher, using its same
|
||||
node-held pair with Scaleway's preferred-project suffix. New project IAM rights
|
||||
and bucket whitelists admit only ciphertext/receipt PUT and GET; retained kits
|
||||
and age escrow remain inaccessible. The original primary upload is preserved.
|
||||
Success advances only after both ciphertext copies and a versioned, fully bound
|
||||
recovery receipt are read back. Failed mirrors retain the pending capture for
|
||||
ciphertext-only retry, without another receiver pause. Freshness requires mirror
|
||||
evidence. Synthetic tests cover corruption, missing versions and mirror-failure
|
||||
retry without advancing last-success.
|
||||
|
||||
A recovery locator is also retained under `kits/recovery-reference-*.json`; its
|
||||
exact version/hash is in the public acceptance evidence. Recovery after losing
|
||||
both workstation and node starts with platform-root's Scaleway account login on
|
||||
a replacement host. Mint an expiring API key for the reader application with
|
||||
preferred project `1d37ea63-8c8f-44e1-80e1-2de862579ba7`; retrieve the kit/locator
|
||||
and latest `archives/` ciphertext plus its versioned JSON receipt. Retrieve age
|
||||
revision 1 separately through attended operator access into a private 0600 file.
|
||||
Verify hashes, load the bundled image, then use bundled `backup-ledger.py recover`
|
||||
and `rehearse-ledger.py --offline-image-binding <kit>/manifest.json`. No expired
|
||||
drill credential, original workstation identity, node credential or forge pull
|
||||
is required. Keep credentials and plaintext in private files and out of logs.
|
||||
|
||||
The admitted operation is daily 04:15 UTC with up to five minutes jitter, hourly
|
||||
freshness, a 24-hour RPO target and a 26-hour stale alert. The 60-minute attended
|
||||
RTO is a target; the offline application rehearsals do not measure a production
|
||||
restore. A fresh dual publication and independent daily-copy recovery passed.
|
||||
The receiver resumed Ready after the bounded capture. Backup/freshness failures
|
||||
route to the user-selected platform-root operations inbox, with private replay
|
||||
if delivery is unavailable. Human acknowledgement is not claimed.
|
||||
|
||||
Platform-root owns weekly retention/capacity review: select 14 daily/4 weekly
|
||||
points, retire excess local ciphertext through an attended owner operation, and
|
||||
refresh the nonexpiring retained kit when the image or recovery identity changes.
|
||||
The worker never deletes retained objects. The 8 GiB free-space floor and 8 GiB
|
||||
local budget fail before capture if custody needs review. Provider lifecycle is
|
||||
30 days for rolling archives, while the protected kit is retained indefinitely.
|
||||
Any production restore still requires delivery fencing and reconciliation of
|
||||
post-capture operations and challenge state before traffic resumes.
|
||||
|
|
|
|||
419
docs/ledger-scaleway-custody-2026-10-10.json
Normal file
419
docs/ledger-scaleway-custody-2026-10-10.json
Normal file
|
|
@ -0,0 +1,419 @@
|
|||
{
|
||||
"schema": "email-connect.scaleway-custody-acceptance.v1",
|
||||
"date": "2026-10-10",
|
||||
"operator_browser_login_completed_by_user": true,
|
||||
"project_id": "1d37ea63-8c8f-44e1-80e1-2de862579ba7",
|
||||
"organization_id": "e1a0dd0e-04b8-4ea9-8b30-d53f8c35c688",
|
||||
"region": "nl-ams",
|
||||
"age_escrow": {
|
||||
"status": "scaleway_age_escrow_verified",
|
||||
"secret_id": "545a1f3c-8e21-4ed1-9ec9-33c9ea2e9272",
|
||||
"revision": 1,
|
||||
"region": "nl-ams",
|
||||
"public_recipient_matches": true,
|
||||
"readback_equal": true,
|
||||
"private_identity_printed": false,
|
||||
"independent_account_login_verified": true,
|
||||
"worker_access_denial_verified": true
|
||||
},
|
||||
"retained_kit": {
|
||||
"status": "scaleway_recovery_kit_readback_verified",
|
||||
"bucket": "railiance-email-connect-recovery",
|
||||
"object_key": "kits/3e786168328dc9c9a32f0b67e4d010ee9db0e0b855fc2460413fb85a16a2e884.tar.gz",
|
||||
"version_id": "1791631021870760",
|
||||
"kit_sha256": "3e786168328dc9c9a32f0b67e4d010ee9db0e0b855fc2460413fb85a16a2e884",
|
||||
"kit_bytes": 47624244,
|
||||
"separate_reader_readback_verified": true,
|
||||
"private_identity_included": false,
|
||||
"reader_write_denial_verified": true,
|
||||
"worker_access_denial_verified": true,
|
||||
"reader_application_binding_verified": true,
|
||||
"restored_from_provider_kit": true
|
||||
},
|
||||
"reader_boundaries": {
|
||||
"schema": "email-connect.final-recovery-boundaries.v1",
|
||||
"reader_application_id": "a84a18e2-3813-4918-9470-dbe0580a5ea6",
|
||||
"daily_project": "1d37ea63-8c8f-44e1-80e1-2de862579ba7",
|
||||
"primary_project_read_grant_revoked": true,
|
||||
"primary_bucket_policy_restored": true,
|
||||
"kit_version_hash_verified": true,
|
||||
"first_mirror": {
|
||||
"bucket": "railiance-email-connect-archives",
|
||||
"object_key": "archives/platform-pg/application-archives/email-connect/2026/10/09/20261009T181751Z-7e1211057348f51569d8b9e4.tar.age",
|
||||
"version_id": "1791631635943039",
|
||||
"project_id": "1d37ea63-8c8f-44e1-80e1-2de862579ba7",
|
||||
"hash_readback_verified": true,
|
||||
"receipt_key": "archives/platform-pg/application-archives/email-connect/2026/10/09/20261009T181751Z-7e1211057348f51569d8b9e4.tar.age.1791631635943039.receipt.json",
|
||||
"receipt_version_id": "1791631641289585"
|
||||
},
|
||||
"daily_ciphertext_hash_verified": true,
|
||||
"daily_recovery_receipt_verified": true,
|
||||
"daily_prefix_list_verified": true,
|
||||
"daily_write": {
|
||||
"http_status": 403,
|
||||
"denied": true
|
||||
},
|
||||
"daily_delete": {
|
||||
"http_status": 403,
|
||||
"denied": true
|
||||
},
|
||||
"daily_foreign_prefix_list": {
|
||||
"http_status": 403,
|
||||
"denied": true
|
||||
},
|
||||
"kit_write": {
|
||||
"http_status": 403,
|
||||
"denied": true
|
||||
},
|
||||
"kit_delete": {
|
||||
"http_status": 403,
|
||||
"denied": true
|
||||
},
|
||||
"kit_foreign_prefix": {
|
||||
"http_status": 403,
|
||||
"denied": true
|
||||
},
|
||||
"unrelated_bucket_list": {
|
||||
"http_status": 403,
|
||||
"denied": true
|
||||
},
|
||||
"unrelated_bucket_get": {
|
||||
"http_status": 403,
|
||||
"denied": true
|
||||
},
|
||||
"primary_project_get": {
|
||||
"http_status": 403,
|
||||
"denied": true
|
||||
}
|
||||
},
|
||||
"worker_boundaries": {
|
||||
"schema": "email-connect.worker-boundaries.v1",
|
||||
"retained_kit_get": {
|
||||
"http_status": 403,
|
||||
"denied": true
|
||||
},
|
||||
"mirror_delete": {
|
||||
"http_status": 403,
|
||||
"denied": true
|
||||
},
|
||||
"age_identity_get": {
|
||||
"http_status": 403,
|
||||
"denied": true
|
||||
},
|
||||
"primary_versioning_preserved": true
|
||||
},
|
||||
"buckets": {
|
||||
"railiance-email-connect-recovery": {
|
||||
"versioning": "Enabled",
|
||||
"public_acl_absent": true,
|
||||
"lifecycle_rules": [],
|
||||
"policy": {
|
||||
"Version": "2023-04-17",
|
||||
"Id": "email-connect-independent-recovery",
|
||||
"Statement": [
|
||||
{
|
||||
"Sid": "OperatorCustody",
|
||||
"Effect": "Allow",
|
||||
"Principal": {
|
||||
"SCW": "user_id:3d029efb-4f70-4f82-a3bb-cefbe255232b"
|
||||
},
|
||||
"Action": "s3:*",
|
||||
"Resource": [
|
||||
"railiance-email-connect-recovery",
|
||||
"railiance-email-connect-recovery/*"
|
||||
]
|
||||
},
|
||||
{
|
||||
"Sid": "RecoveryKitReadOnly",
|
||||
"Effect": "Allow",
|
||||
"Principal": {
|
||||
"SCW": "application_id:a84a18e2-3813-4918-9470-dbe0580a5ea6"
|
||||
},
|
||||
"Action": [
|
||||
"s3:GetObject",
|
||||
"s3:GetObjectVersion"
|
||||
],
|
||||
"Resource": "railiance-email-connect-recovery/kits/*"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"railiance-email-connect-archives": {
|
||||
"versioning": "Enabled",
|
||||
"public_acl_absent": true,
|
||||
"lifecycle_rules": [
|
||||
{
|
||||
"Expiration": {
|
||||
"Days": 30
|
||||
},
|
||||
"ID": "daily-30-days",
|
||||
"Filter": {
|
||||
"Prefix": "archives/"
|
||||
},
|
||||
"Status": "Enabled",
|
||||
"NoncurrentVersionExpiration": {
|
||||
"NoncurrentDays": 30
|
||||
}
|
||||
}
|
||||
],
|
||||
"policy": {
|
||||
"Version": "2023-04-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Sid": "Operator",
|
||||
"Effect": "Allow",
|
||||
"Principal": {
|
||||
"SCW": "user_id:3d029efb-4f70-4f82-a3bb-cefbe255232b"
|
||||
},
|
||||
"Action": "s3:*",
|
||||
"Resource": [
|
||||
"railiance-email-connect-archives",
|
||||
"railiance-email-connect-archives/*"
|
||||
]
|
||||
},
|
||||
{
|
||||
"Sid": "ArchivePublisher",
|
||||
"Effect": "Allow",
|
||||
"Principal": {
|
||||
"SCW": "application_id:0eb0f958-77a4-4c70-9aa7-14ba46411415"
|
||||
},
|
||||
"Action": [
|
||||
"s3:GetObject",
|
||||
"s3:GetObjectVersion",
|
||||
"s3:PutObject"
|
||||
],
|
||||
"Resource": "railiance-email-connect-archives/archives/*"
|
||||
},
|
||||
{
|
||||
"Sid": "ArchiveReader",
|
||||
"Effect": "Allow",
|
||||
"Principal": {
|
||||
"SCW": "application_id:a84a18e2-3813-4918-9470-dbe0580a5ea6"
|
||||
},
|
||||
"Action": [
|
||||
"s3:GetObject",
|
||||
"s3:GetObjectVersion"
|
||||
],
|
||||
"Resource": "railiance-email-connect-archives/archives/*"
|
||||
},
|
||||
{
|
||||
"Sid": "ArchiveList",
|
||||
"Effect": "Allow",
|
||||
"Principal": {
|
||||
"SCW": "application_id:a84a18e2-3813-4918-9470-dbe0580a5ea6"
|
||||
},
|
||||
"Action": [
|
||||
"s3:ListBucket",
|
||||
"s3:ListBucketVersions"
|
||||
],
|
||||
"Resource": "railiance-email-connect-archives",
|
||||
"Condition": {
|
||||
"StringLike": {
|
||||
"s3:prefix": "archives/*"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"provider_only_retained_kit_rehearsal": {
|
||||
"status": "application_restored_verified",
|
||||
"application": "email-connect",
|
||||
"object_key": "platform-pg/application-archives/email-connect/2026/10/09/20261009T181751Z-7e1211057348f51569d8b9e4.tar.age",
|
||||
"version_id": "1791569930973785",
|
||||
"ciphertext_sha256": "1a7ae09129d8252053ea210cfe37d8a27d25bc4ee615bfe85f65c207b3e5f38e",
|
||||
"database_sha256": "6694bddc3b57b5918852fbdd79d569d5bf1393831ed55aca25bb19a59e601feb",
|
||||
"ledger_fingerprint": {
|
||||
"schema_sha256": "13148e5a9c6e01cc593bb07278a8649b13e43b26c30b0ed1a6e15995f7b00732",
|
||||
"user_version": 0,
|
||||
"tables": {
|
||||
"caller_buckets": {
|
||||
"rows": 1,
|
||||
"sha256": "0ce5a0b678fdcf55de7a4b2314a02d12f168e0f08b2a0d42d82c3e1dc514778c"
|
||||
},
|
||||
"deliveries": {
|
||||
"rows": 2,
|
||||
"sha256": "a0387716815b2ec191187cb34bb3252b9cf51f5fb15a2019918a9cf1737c061e"
|
||||
},
|
||||
"send_attempts": {
|
||||
"rows": 0,
|
||||
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
|
||||
},
|
||||
"suppressions": {
|
||||
"rows": 1,
|
||||
"sha256": "ae26d10dfec16f1b626292b342a5e86afd1858fcaed7c29d4e68e7cf821183d7"
|
||||
},
|
||||
"verification_attempts": {
|
||||
"rows": 0,
|
||||
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
|
||||
},
|
||||
"verifications": {
|
||||
"rows": 0,
|
||||
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
|
||||
}
|
||||
}
|
||||
},
|
||||
"mail_sent": false,
|
||||
"production_restored": false,
|
||||
"application_restore_verified": true,
|
||||
"recovery_image": "forgejo.coulomb.social/coulomb/email-connect@sha256:5d79fd1fec15d3a279383820f170d263de3a55f5308df01a0a34732ebaea6d3d",
|
||||
"network": "disabled",
|
||||
"checks": {
|
||||
"accepted_verifications": 0,
|
||||
"unknown_verifications": 0,
|
||||
"invalidated_handles": 0,
|
||||
"suppressions": 1,
|
||||
"accepted_deliveries": 2,
|
||||
"unknown_sends": 0
|
||||
},
|
||||
"uid": 10001,
|
||||
"evidence_ceiling": "isolated_ledger_rehearsal",
|
||||
"offline_image_binding_verified": true,
|
||||
"elapsed_seconds": 3.8
|
||||
},
|
||||
"provider_only_daily_rehearsal": {
|
||||
"status": "application_restored_verified",
|
||||
"application": "email-connect",
|
||||
"object_key": "platform-pg/application-archives/email-connect/2026/10/10/20261010T112917Z-e86c484a0918d1443984a10b.tar.age",
|
||||
"version_id": "1791631827868341",
|
||||
"ciphertext_sha256": "cba57bd57fe81adaa5594da5b57ea6a8b8f6d58b2a0718925325e3c6dd6cb302",
|
||||
"database_sha256": "ed6711a63cb0a4371c1b805e326ee6ec62e3b6a2358c37b0ba9152f4a5eaaecb",
|
||||
"ledger_fingerprint": {
|
||||
"schema_sha256": "13148e5a9c6e01cc593bb07278a8649b13e43b26c30b0ed1a6e15995f7b00732",
|
||||
"user_version": 0,
|
||||
"tables": {
|
||||
"caller_buckets": {
|
||||
"rows": 1,
|
||||
"sha256": "d7def892f86c163aed9df55112b0e4551ff6004e337a6a2135c9b80d8cb8fdf7"
|
||||
},
|
||||
"deliveries": {
|
||||
"rows": 2,
|
||||
"sha256": "a0387716815b2ec191187cb34bb3252b9cf51f5fb15a2019918a9cf1737c061e"
|
||||
},
|
||||
"send_attempts": {
|
||||
"rows": 0,
|
||||
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
|
||||
},
|
||||
"suppressions": {
|
||||
"rows": 1,
|
||||
"sha256": "ae26d10dfec16f1b626292b342a5e86afd1858fcaed7c29d4e68e7cf821183d7"
|
||||
},
|
||||
"verification_attempts": {
|
||||
"rows": 0,
|
||||
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
|
||||
},
|
||||
"verifications": {
|
||||
"rows": 0,
|
||||
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
|
||||
}
|
||||
}
|
||||
},
|
||||
"mail_sent": false,
|
||||
"production_restored": false,
|
||||
"application_restore_verified": true,
|
||||
"recovery_image": "forgejo.coulomb.social/coulomb/email-connect@sha256:5d79fd1fec15d3a279383820f170d263de3a55f5308df01a0a34732ebaea6d3d",
|
||||
"network": "disabled",
|
||||
"checks": {
|
||||
"accepted_verifications": 0,
|
||||
"unknown_verifications": 0,
|
||||
"invalidated_handles": 0,
|
||||
"suppressions": 1,
|
||||
"accepted_deliveries": 2,
|
||||
"unknown_sends": 0
|
||||
},
|
||||
"uid": 10001,
|
||||
"evidence_ceiling": "isolated_ledger_rehearsal",
|
||||
"offline_image_binding_verified": true,
|
||||
"elapsed_seconds": 2.87
|
||||
},
|
||||
"independent_recovery_reference": {
|
||||
"bucket": "railiance-email-connect-recovery",
|
||||
"key": "kits/recovery-reference-a8efafe91d9a80aa5ed5ca03f4a6536c26816d5345cbacdcb97b0c7e44cfafd5.json",
|
||||
"version_id": "1791631821559658",
|
||||
"sha256": "a8efafe91d9a80aa5ed5ca03f4a6536c26816d5345cbacdcb97b0c7e44cfafd5"
|
||||
},
|
||||
"iam_policy_ids": {
|
||||
"reader": "b83bbd41-1e57-44a0-8435-d24035497ef6",
|
||||
"mirror_publisher": "6a9099b3-17ef-4a1f-a19a-e38419abe5c4",
|
||||
"temporary_primary_reader_policy_revoked": "484e3064-7603-494c-910e-92226c0047c2"
|
||||
},
|
||||
"privileged_test_keys": "Temporary operator and drill reader API key records deleted after proof; permanent reader app retained for attended reminting.",
|
||||
"evidence_limits": {
|
||||
"production_restore_performed": false,
|
||||
"production_rto_measured": false,
|
||||
"live_empty_tables": [
|
||||
"send_attempts",
|
||||
"verification_attempts",
|
||||
"verifications"
|
||||
],
|
||||
"synthetic_semantics_cover_empty_live_tables": true,
|
||||
"smtp_sent": false
|
||||
},
|
||||
"regression_tests": {
|
||||
"focused_passed": 47,
|
||||
"skipped": 0
|
||||
},
|
||||
"final_provider_daily_rehearsal": {
|
||||
"status": "application_restored_verified",
|
||||
"application": "email-connect",
|
||||
"object_key": "platform-pg/application-archives/email-connect/2026/10/10/20261010T113256Z-e2efa67ecfb8154fd8bbcc59.tar.age",
|
||||
"version_id": "1791632035876582",
|
||||
"ciphertext_sha256": "a007ab4f80924ecf5d69c52a91363298a68e004293f7e8f37bfae762c403ac5e",
|
||||
"database_sha256": "ed6711a63cb0a4371c1b805e326ee6ec62e3b6a2358c37b0ba9152f4a5eaaecb",
|
||||
"ledger_fingerprint": {
|
||||
"schema_sha256": "13148e5a9c6e01cc593bb07278a8649b13e43b26c30b0ed1a6e15995f7b00732",
|
||||
"user_version": 0,
|
||||
"tables": {
|
||||
"caller_buckets": {
|
||||
"rows": 1,
|
||||
"sha256": "d7def892f86c163aed9df55112b0e4551ff6004e337a6a2135c9b80d8cb8fdf7"
|
||||
},
|
||||
"deliveries": {
|
||||
"rows": 2,
|
||||
"sha256": "a0387716815b2ec191187cb34bb3252b9cf51f5fb15a2019918a9cf1737c061e"
|
||||
},
|
||||
"send_attempts": {
|
||||
"rows": 0,
|
||||
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
|
||||
},
|
||||
"suppressions": {
|
||||
"rows": 1,
|
||||
"sha256": "ae26d10dfec16f1b626292b342a5e86afd1858fcaed7c29d4e68e7cf821183d7"
|
||||
},
|
||||
"verification_attempts": {
|
||||
"rows": 0,
|
||||
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
|
||||
},
|
||||
"verifications": {
|
||||
"rows": 0,
|
||||
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
|
||||
}
|
||||
}
|
||||
},
|
||||
"mail_sent": false,
|
||||
"production_restored": false,
|
||||
"application_restore_verified": true,
|
||||
"recovery_image": "forgejo.coulomb.social/coulomb/email-connect@sha256:5d79fd1fec15d3a279383820f170d263de3a55f5308df01a0a34732ebaea6d3d",
|
||||
"network": "disabled",
|
||||
"checks": {
|
||||
"accepted_verifications": 0,
|
||||
"unknown_verifications": 0,
|
||||
"invalidated_handles": 0,
|
||||
"suppressions": 1,
|
||||
"accepted_deliveries": 2,
|
||||
"unknown_sends": 0
|
||||
},
|
||||
"uid": 10001,
|
||||
"evidence_ceiling": "isolated_ledger_rehearsal",
|
||||
"offline_image_binding_verified": true,
|
||||
"elapsed_seconds": 0.92
|
||||
},
|
||||
"operating_acceptance": "deploy/backup/operating-acceptance-2026-10-10.json",
|
||||
"key_revocation": {
|
||||
"reader_key_record_absent_http_status": 404,
|
||||
"operator_key_record_absent_http_status": 404,
|
||||
"permanent_reader_application_retained": true,
|
||||
"operator_post_revoke_projects_http_status": 400
|
||||
}
|
||||
}
|
||||
|
|
@ -95,7 +95,7 @@ def publish(encrypted, fetched, receipt_path, captured, tool, python):
|
|||
return receipt
|
||||
|
||||
|
||||
def backup(state, primary_tool, primary_python, recipient, resume=False):
|
||||
def backup(state, primary_tool, primary_python, recipient, resume=False, mirror_required=False):
|
||||
state.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
private_directory(state)
|
||||
fd = os.open(state / "backup.lock", os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, 0o600)
|
||||
|
|
@ -137,6 +137,17 @@ def backup(state, primary_tool, primary_python, recipient, resume=False):
|
|||
primary_tool,
|
||||
primary_python,
|
||||
)
|
||||
if mirror_required:
|
||||
transfer.update(image=captured["image"], database_sha256=captured["database_sha256"],
|
||||
ledger_fingerprint=captured["ledger_fingerprint"])
|
||||
checkpoint(stage / "primary-transfer.json", transfer)
|
||||
command([str(primary_python), str(Path(__file__).with_name("mirror-ledger.py")),
|
||||
"--archive", str(encrypted), "--primary-receipt", str(stage / "primary-transfer.json"),
|
||||
"--output", str(stage / "mirror.json")])
|
||||
mirrored = json.loads((stage / "mirror.json").read_text())
|
||||
if mirrored.get("hash_readback_verified") is not True:
|
||||
raise Refused("recovery_mirror_required")
|
||||
transfer["recovery_mirror"] = mirrored
|
||||
filename = transfer["object_key"].rsplit("/", 1)[-1]
|
||||
if not NAME.fullmatch(filename):
|
||||
raise Refused("invalid_archive_name")
|
||||
|
|
@ -173,7 +184,7 @@ def backup(state, primary_tool, primary_python, recipient, resume=False):
|
|||
return receipt
|
||||
|
||||
|
||||
def freshness(state, now=None, hours=26):
|
||||
def freshness(state, now=None, hours=26, mirror_required=False):
|
||||
now = now or datetime.now(UTC)
|
||||
private_directory(state)
|
||||
private_file(state / "last-success.json")
|
||||
|
|
@ -195,6 +206,14 @@ def freshness(state, now=None, hours=26):
|
|||
private_file(state / name)
|
||||
if digest(state / name) != receipt["ciphertext_sha256"]:
|
||||
raise Refused("local_ciphertext_mismatch")
|
||||
if mirror_required:
|
||||
mirror = receipt.get("recovery_mirror", {})
|
||||
if (mirror.get("hash_readback_verified") is not True
|
||||
or mirror.get("bucket") != "railiance-email-connect-archives"
|
||||
or mirror.get("project_id") != "1d37ea63-8c8f-44e1-80e1-2de862579ba7"
|
||||
or not mirror.get("version_id") or mirror["version_id"] == "null"
|
||||
or not mirror.get("receipt_version_id") or mirror["receipt_version_id"] == "null"):
|
||||
raise Refused("verified_recovery_mirror_required")
|
||||
return {"status": "fresh", "age_hours": round(age / 3600, 2), "remote_retrieval_verified": True}
|
||||
|
||||
|
||||
|
|
@ -203,6 +222,7 @@ def main():
|
|||
parser.add_argument("action", choices=["backup", "recover", "freshness"])
|
||||
parser.add_argument("--execute", action="store_true")
|
||||
parser.add_argument("--resume-pending", action="store_true")
|
||||
parser.add_argument("--mirror-required", action="store_true")
|
||||
parser.add_argument("--state-dir", type=Path, default=Path("/var/lib/email-connect-backup"))
|
||||
parser.add_argument("--primary-tool", type=Path, default=PRIMARY_TOOL)
|
||||
parser.add_argument("--primary-python", type=Path, default=PRIMARY_PYTHON)
|
||||
|
|
@ -213,7 +233,7 @@ def main():
|
|||
parser.add_argument("--destination", type=Path)
|
||||
args = parser.parse_args()
|
||||
if args.action == "freshness":
|
||||
print(json.dumps(freshness(args.state_dir)))
|
||||
print(json.dumps(freshness(args.state_dir, mirror_required=args.mirror_required)))
|
||||
elif args.action == "recover":
|
||||
if not args.execute:
|
||||
print(json.dumps({"status": "reviewed", "decryption_performed": False}))
|
||||
|
|
@ -251,6 +271,7 @@ def main():
|
|||
args.primary_python,
|
||||
args.age_recipient,
|
||||
args.resume_pending,
|
||||
args.mirror_required,
|
||||
)
|
||||
)
|
||||
)
|
||||
|
|
|
|||
89
scripts/mirror-ledger.py
Normal file
89
scripts/mirror-ledger.py
Normal file
|
|
@ -0,0 +1,89 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Copy ciphertext into the isolated recovery project; never read an age key."""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from ledger_archive import Refused, checkpoint, digest, private_file
|
||||
|
||||
PROJECT = "1d37ea63-8c8f-44e1-80e1-2de862579ba7"
|
||||
BUCKET = "railiance-email-connect-archives"
|
||||
|
||||
|
||||
def mirror(client, archive, receipt):
|
||||
private_file(archive)
|
||||
expected = digest(archive)
|
||||
size = archive.stat().st_size
|
||||
if expected != receipt["ciphertext_sha256"] or size != receipt["ciphertext_bytes"]:
|
||||
raise Refused("mirror_source_differs")
|
||||
key = "archives/" + receipt["object_key"]
|
||||
try:
|
||||
with archive.open("rb") as source:
|
||||
created = client.put_object(Bucket=BUCKET, Key=key, Body=source,
|
||||
IfNoneMatch="*", Metadata={"sha256": expected})
|
||||
version = created.get("VersionId")
|
||||
except Exception as error:
|
||||
if getattr(error, "response", {}).get("ResponseMetadata", {}).get("HTTPStatusCode") != 412:
|
||||
raise
|
||||
current = client.head_object(Bucket=BUCKET, Key=key)
|
||||
if current.get("Metadata", {}).get("sha256") != expected or current.get("ContentLength") != size:
|
||||
raise Refused("mirror_existing_object_differs")
|
||||
version = current.get("VersionId")
|
||||
if not version or version == "null":
|
||||
raise Refused("versioned_mirror_required")
|
||||
result = client.get_object(Bucket=BUCKET, Key=key, VersionId=version)
|
||||
hashed, count = hashlib.sha256(), 0
|
||||
with result["Body"] as stream:
|
||||
while block := stream.read(1024**2):
|
||||
count += len(block)
|
||||
if count > size:
|
||||
raise Refused("mirror_readback_exceeds_size")
|
||||
hashed.update(block)
|
||||
if (result.get("VersionId") != version or count != size
|
||||
or hashed.hexdigest() != expected):
|
||||
raise Refused("mirror_readback_differs")
|
||||
# Recovery needs a receipt even if both node and workstation are lost.
|
||||
record = dict(receipt, recovery_mirror={"bucket": BUCKET, "object_key": key,
|
||||
"version_id": version, "project_id": PROJECT,
|
||||
"hash_readback_verified": True})
|
||||
payload = json.dumps(record, sort_keys=True).encode()
|
||||
receipt_key = key + "." + version + "." + hashlib.sha256(payload).hexdigest() + ".receipt.json"
|
||||
try:
|
||||
saved = client.put_object(Bucket=BUCKET, Key=receipt_key, Body=payload, IfNoneMatch="*")
|
||||
receipt_version = saved.get("VersionId")
|
||||
except Exception as error:
|
||||
if getattr(error, "response", {}).get("ResponseMetadata", {}).get("HTTPStatusCode") != 412:
|
||||
raise
|
||||
receipt_version = client.head_object(Bucket=BUCKET, Key=receipt_key).get("VersionId")
|
||||
if not receipt_version or receipt_version == "null":
|
||||
raise Refused("versioned_mirror_receipt_required")
|
||||
saved = client.get_object(Bucket=BUCKET, Key=receipt_key, VersionId=receipt_version)
|
||||
with saved["Body"] as stream:
|
||||
if stream.read(len(payload) + 1) != payload:
|
||||
raise Refused("mirror_receipt_readback_differs")
|
||||
return dict(record["recovery_mirror"], receipt_key=receipt_key,
|
||||
receipt_version_id=receipt_version)
|
||||
|
||||
|
||||
def main():
|
||||
import boto3
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--archive", type=Path, required=True)
|
||||
parser.add_argument("--primary-receipt", type=Path, required=True)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
private_file(args.primary_receipt)
|
||||
key = os.environ["AWS_ACCESS_KEY_ID"].split("@", 1)[0] + "@" + PROJECT
|
||||
client = boto3.client("s3", endpoint_url="https://s3.nl-ams.scw.cloud", region_name="nl-ams",
|
||||
aws_access_key_id=key, aws_secret_access_key=os.environ["AWS_SECRET_ACCESS_KEY"])
|
||||
checkpoint(args.output, mirror(client, args.archive, json.loads(args.primary_receipt.read_text())))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
except Exception as error:
|
||||
print(json.dumps({"status": "failed", "failure_type": type(error).__name__}))
|
||||
raise SystemExit(1) from None
|
||||
|
|
@ -42,7 +42,9 @@ def prepare(evidence, ciphertext, destination):
|
|||
(destination / "receiver-image.tar").chmod(0o600)
|
||||
for name in ("ledger_archive.py", "backup-ledger.py", "rehearse-ledger.py"):
|
||||
shutil.copyfile(ROOT / "scripts" / name, destination / name)
|
||||
(destination / name).chmod(0o600)
|
||||
# This public module is bind-mounted into the UID10001 rehearsal.
|
||||
# The enclosing kit directory remains private; data stays mode 0600.
|
||||
(destination / name).chmod(0o644 if name == "ledger_archive.py" else 0o600)
|
||||
shutil.copyfile(ciphertext, destination / "ledger.tar.age")
|
||||
(destination / "ledger.tar.age").chmod(0o600)
|
||||
checkpoint(destination / "transfer-receipt.json", transfer)
|
||||
|
|
|
|||
|
|
@ -4,6 +4,8 @@ import base64
|
|||
import hashlib
|
||||
import importlib.util
|
||||
import json
|
||||
import stat
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
|
@ -121,6 +123,36 @@ def test_caller_failure_receipt_allows_only_static_diagnostic_codes(monkeypatch)
|
|||
assert "synthetic-sensitive-value" not in json.dumps(receipt)
|
||||
|
||||
|
||||
def test_recovery_kit_public_module_is_readable_by_rehearsal_uid(monkeypatch, tmp_path):
|
||||
kit = module("prepare-ledger-recovery-kit", monkeypatch)
|
||||
cipher = tmp_path / "ledger.tar.age"
|
||||
cipher.write_bytes(b"synthetic-encrypted-ledger")
|
||||
cipher.chmod(0o600)
|
||||
hashed = hashlib.sha256(cipher.read_bytes()).hexdigest()
|
||||
evidence = tmp_path / "evidence.json"
|
||||
evidence.write_text(json.dumps({
|
||||
"publication": {"ciphertext_sha256": hashed, "ciphertext_bytes": cipher.stat().st_size,
|
||||
"version_id": "synthetic-version"},
|
||||
"application_recovery": {"application_restore_verified": True,
|
||||
"application": "email-connect", "ciphertext_sha256": hashed,
|
||||
"version_id": "synthetic-version", "recovery_image": "synthetic-image"},
|
||||
}))
|
||||
|
||||
def docker(args, **kwargs):
|
||||
if args[2] == "inspect":
|
||||
return subprocess.CompletedProcess(args, 0, json.dumps([{"Id": "sha256:" + "a" * 64}]).encode())
|
||||
Path(args[args.index("--output") + 1]).write_bytes(b"synthetic-image-archive")
|
||||
return subprocess.CompletedProcess(args, 0, b"")
|
||||
|
||||
monkeypatch.setattr(kit.subprocess, "run", docker)
|
||||
destination = tmp_path / "kit"
|
||||
kit.prepare(evidence, cipher, destination)
|
||||
assert stat.S_IMODE(destination.stat().st_mode) == 0o700
|
||||
assert stat.S_IMODE((destination / "ledger_archive.py").stat().st_mode) == 0o644
|
||||
for name in ["ledger.tar.age", "receiver-image.tar", "transfer-receipt.json", "kit.tar.gz"]:
|
||||
assert stat.S_IMODE((destination / name).stat().st_mode) == 0o600
|
||||
|
||||
|
||||
@pytest.mark.parametrize("protected,ephemeral,corrupt", [
|
||||
(True, None, False), (False, None, False), (True, {}, False), (True, None, True),
|
||||
])
|
||||
|
|
|
|||
|
|
@ -298,7 +298,7 @@ def test_capture_resumes_unchanged_receiver_after_failures(modules, monkeypatch,
|
|||
assert receipt["status"] == ("captured" if failure is None else "failed")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("failure", [None, "publish", "encrypt"])
|
||||
@pytest.mark.parametrize("failure", [None, "publish", "encrypt", "mirror"])
|
||||
def test_backup_success_requires_readback_and_preserves_one_pending_capture(
|
||||
modules, monkeypatch, tmp_path, failure
|
||||
):
|
||||
|
|
@ -308,6 +308,11 @@ def test_backup_success_requires_readback_and_preserves_one_pending_capture(
|
|||
state = tmp_path / "state"
|
||||
|
||||
def command(argv, **kwargs):
|
||||
if "--primary-receipt" in argv:
|
||||
if failure == "mirror":
|
||||
raise archive.Refused("synthetic mirror failure")
|
||||
archive.checkpoint(Path(argv[-1]), {"hash_readback_verified": True})
|
||||
return
|
||||
captures.append(argv)
|
||||
captured(Path(argv[-1]), archive)
|
||||
|
||||
|
|
@ -352,16 +357,19 @@ def test_backup_success_requires_readback_and_preserves_one_pending_capture(
|
|||
monkeypatch.setattr(runner, "publish", publish)
|
||||
if failure:
|
||||
with pytest.raises(archive.Refused):
|
||||
runner.backup(state, Path("synthetic-tool"), Path("synthetic-python"), public)
|
||||
runner.backup(state, Path("synthetic-tool"), Path("synthetic-python"), public,
|
||||
mirror_required=failure == "mirror")
|
||||
assert not (state / "last-success.json").exists()
|
||||
assert len(list(state.glob("pending-*"))) == 1
|
||||
with pytest.raises(archive.Refused, match="pending_backup_requires_review"):
|
||||
runner.backup(state, Path("synthetic-tool"), Path("synthetic-python"), public)
|
||||
if failure == "publish":
|
||||
if failure in {"publish", "mirror"}:
|
||||
mirror_required = failure == "mirror"
|
||||
ciphertext = publications[0]
|
||||
failure = None
|
||||
result = runner.backup(
|
||||
state, Path("synthetic-tool"), Path("synthetic-python"), public, resume=True
|
||||
state, Path("synthetic-tool"), Path("synthetic-python"), public, resume=True,
|
||||
mirror_required=mirror_required,
|
||||
)
|
||||
assert len(captures) == 1 and publications == [ciphertext, ciphertext]
|
||||
assert result["application_restore_verified"] is False
|
||||
|
|
|
|||
45
tests/test_ledger_mirror.py
Normal file
45
tests/test_ledger_mirror.py
Normal file
|
|
@ -0,0 +1,45 @@
|
|||
"""Synthetic immutable mirror/readback and recovery-receipt checks."""
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import io
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.mark.parametrize("failure", [None, "ciphertext", "receipt", "unversioned"])
|
||||
def test_mirror_refuses_corruption_before_success(monkeypatch, tmp_path, failure):
|
||||
scripts = Path(__file__).resolve().parents[1] / "scripts"
|
||||
monkeypatch.syspath_prepend(str(scripts))
|
||||
spec = importlib.util.spec_from_file_location("mirror", scripts / "mirror-ledger.py")
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
archive = tmp_path / "ledger.age"
|
||||
archive.write_bytes(b"synthetic encrypted data")
|
||||
archive.chmod(0o600)
|
||||
objects = {}
|
||||
|
||||
class Client:
|
||||
def put_object(self, **kwargs):
|
||||
assert kwargs["IfNoneMatch"] == "*"
|
||||
body = kwargs["Body"]
|
||||
objects[kwargs["Key"]] = body.read() if hasattr(body, "read") else body
|
||||
return {"VersionId": "null" if failure == "unversioned" else "synthetic-version"}
|
||||
|
||||
def get_object(self, **kwargs):
|
||||
body = objects[kwargs["Key"]]
|
||||
if ((failure == "ciphertext" and not kwargs["Key"].endswith("json"))
|
||||
or (failure == "receipt" and kwargs["Key"].endswith("json"))):
|
||||
body = b"corrupted"
|
||||
return {"VersionId": kwargs["VersionId"], "Body": io.BytesIO(body)}
|
||||
|
||||
receipt = {"object_key": "platform-pg/application-archives/email-connect/synthetic.age",
|
||||
"ciphertext_sha256": hashlib.sha256(archive.read_bytes()).hexdigest(),
|
||||
"ciphertext_bytes": archive.stat().st_size}
|
||||
if failure:
|
||||
with pytest.raises(module.Refused):
|
||||
module.mirror(Client(), archive, receipt)
|
||||
else:
|
||||
result = module.mirror(Client(), archive, receipt)
|
||||
assert result["hash_readback_verified"]
|
||||
assert result["receipt_key"] in objects
|
||||
|
|
@ -4,11 +4,11 @@ type: workplan
|
|||
title: "Back up and recover the transactional delivery ledger"
|
||||
domain: infotech
|
||||
repo: email-connect
|
||||
status: active
|
||||
status: finished
|
||||
owner: codex
|
||||
topic_slug: email-connect
|
||||
created: "2026-10-07"
|
||||
updated: "2026-10-09"
|
||||
updated: "2026-10-10"
|
||||
origin: residual
|
||||
origin_ref: EMAIL-WP-0006
|
||||
state_hub_workstream_id: "35be21a2-343b-52e1-ba5e-32fead7a7831"
|
||||
|
|
@ -60,10 +60,9 @@ challenges before releasing traffic. Never auto-restore into a running service.
|
|||
|
||||
```task
|
||||
id: EMAIL-WP-0007-T03
|
||||
status: progress
|
||||
status: done
|
||||
priority: high
|
||||
needs_human: true
|
||||
intervention_note: "Scaleway selected; email-connect-backup-primary remains the existing prefix-only publisher. Closure package and local offline kit are tested. Owner-provided durable age/kit custody and separate recovery reader grants are still absent; existing archive bucket expires objects after 30 days."
|
||||
needs_human: false
|
||||
state_hub_task_id: "680fe4d6-12af-5296-acf9-15c9a01a0db3"
|
||||
```
|
||||
|
||||
|
|
@ -79,10 +78,9 @@ invented owner review or credential grant.
|
|||
|
||||
```task
|
||||
id: EMAIL-WP-0007-T04
|
||||
status: progress
|
||||
status: done
|
||||
priority: high
|
||||
needs_human: true
|
||||
intervention_note: "First backup/recovery and platform-root inbox delivery pass. Daily schedule, retention and capacity targets are prepared in deploy/closure/plan.json. Independent Scaleway custody and recurring-operation acceptance remain open; timers stay disabled."
|
||||
needs_human: false
|
||||
state_hub_task_id: "66401e5c-a5cf-5e67-beab-e503ae079964"
|
||||
```
|
||||
|
||||
|
|
@ -257,3 +255,47 @@ Ready. EMAIL-WP-0008 is finished; T03/T04 here remain progress for independent
|
|||
Scaleway custody/grants, retention/schedule and recurring operation. Timers remain
|
||||
disabled. The user was asked for an existing Scaleway operator lane or Console
|
||||
custody setup, since the existing publisher cannot administer escrow resources.
|
||||
|
||||
## Completed independent custody and recurring operation — 2026-10-10
|
||||
|
||||
The user completed the attended Scaleway operator login and authorized finishing
|
||||
the workplans. T03/T04 are done. Protected, nonexpiring age revision 1 and a
|
||||
private, versioned, nonexpiring recovery kit now reside in independent project
|
||||
email-connect-recovery (1d37ea63-8c8f-44e1-80e1-2de862579ba7), nl-ams. The
|
||||
separate IAM reader is natively bound and tested; scheduled-worker kit/key reads
|
||||
and retained-object deletes return 403. Recovery using only provider-retrieved
|
||||
kit, bundled image/tools and restored escrow identity passes without network or
|
||||
mail. Public kit library permissions were corrected after the cold drill found
|
||||
UID10001 could not read its original 0600 bind mount; a regression covers it.
|
||||
|
||||
A temporary original-project reader grant admitted metadata reads from an
|
||||
unrelated bucket. It was removed and original bucket grants restored. Native
|
||||
unrelated GET/LIST now deny 403. Daily archives and complete recovery receipts
|
||||
are copied to private, versioned railiance-email-connect-archives in the isolated
|
||||
project with 30-day expiry. The existing primary publisher remains in use.
|
||||
Backup success/freshness require hash/version-verified mirror publication.
|
||||
Failures preserve a pending capture for retry without another receiver pause.
|
||||
|
||||
The final installed-code run captured/resumed in 57.55 seconds, independently
|
||||
published/read back both copies, and restored through the separate recovery
|
||||
reader and provider receipt. All six tables match, with one quota bucket, two
|
||||
accepted deliveries and one suppression. Remaining live transactional tables
|
||||
are empty; their behavior remains covered synthetically. Both deployments are
|
||||
Ready, and daily 04:15 UTC (+ up to five-minute jitter) backup and hourly freshness
|
||||
timers are enabled/active. Freshness passes; failures route to the user-selected
|
||||
platform-root inbox, with pending delivery replay. Platform-root owns weekly
|
||||
14-daily/4-weekly retention selection, local ciphertext expiry/capacity review and
|
||||
retained-kit refresh. Worker deletion remains prohibited; 8 GiB local budget/free
|
||||
floor guards remain installed. RPO target is 24 hours with a 26-hour stale alert.
|
||||
60-minute attended RTO is a target, not a measured production restore. Production
|
||||
recovery still requires fencing and post-capture/challenge reconciliation.
|
||||
|
||||
Evidence: docs/ledger-scaleway-custody-2026-10-10.json,
|
||||
deploy/backup/operating-acceptance-2026-10-10.json and
|
||||
updated docs/ledger-backup-review-2026-10-08.md. Focused regression: 47 passed,
|
||||
zero skipped. Syntax/undefined-name lint passes. The broader lint profile reports
|
||||
existing shell/rehearsal exceptions and the deliberate credential-safe top-level
|
||||
exception boundary; no broad clean-lint claim is made. Temporary operator/drill
|
||||
API key records were deleted after proofs; the permanent reader application and
|
||||
operator-account bootstrap path survive workstation/node loss. EMAIL-WP-0008
|
||||
remains finished. Earlier waiting/local-only entries above are historical.
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue