coulomb/email-connect
5
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Prepare immutable caller rollout and independent Scaleway recovery custody
Some checks failed
Transactional mail acceptance / mail (push) Failing after 32s
Details
CI Smoke / host-smoke (push) Successful in 0s
Details
CI Smoke / container-smoke (push) Successful in 1s
Details

Browse source 
Assistant: codex
Assistant-Model: gpt-6.1-sol
Assistant-Session: 01a11fd9-caf6-7a63-b578-9daaaa134ff2
This commit is contained in:
tegwick 2026-10-09 19:49:24 +02:00
parent 3e18f11afe
commit 9a2cf213d1
19 changed files with 1543 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

170
deploy/closure/caller-delivery.json Normal file
View file

@ -0,0 +1,170 @@
{
"apiVersion": "v1",
"kind": "List",
"items": [
{
"apiVersion": "v1",
"kind": "ServiceAccount",
"metadata": {
"name": "email-connect-scoped-caller-eso",
"namespace": "email-connect"
},
"automountServiceAccountToken": false
},
{
"apiVersion": "external-secrets.io/v1",
"kind": "ClusterSecretStore",
"metadata": {
"name": "openbao-email-connect-scoped-caller"
},
"spec": {
"conditions": [
{
"namespaces": [
"email-connect"
]
}
],
"provider": {
"vault": {
"server": "http://openbao.openbao.svc:8200",
"path": "platform",
"version": "v2",
"auth": {
"kubernetes": {
"mountPath": "kubernetes",
"role": "email-connect-scoped-caller-eso",
"serviceAccountRef": {
"name": "email-connect-scoped-caller-eso",
"namespace": "email-connect",
"audiences": [
"openbao"
]
}
}
}
}
}
}
},
{
"apiVersion": "external-secrets.io/v1",
"kind": "ExternalSecret",
"metadata": {
"name": "email-connect-caller-user-engine",
"namespace": "email-connect"
},
"spec": {
"refreshInterval": "5m",
"secretStoreRef": {
"kind": "ClusterSecretStore",
"name": "openbao-email-connect-scoped-caller"
},
"target": {
"name": "email-connect-caller-user-engine",
"creationPolicy": "Owner",
"deletionPolicy": "Retain",
"template": {
"metadata": {
"labels": {
"email-connect.io/caller": "user-engine"
},
"annotations": {}
}
}
},
"data": [
{
"secretKey": "token",
"remoteRef": {
"key": "platform/workloads/email-connect/callers/user-engine",
"property": "TOKEN"
}
}
]
}
},
{
"apiVersion": "v1",
"kind": "ServiceAccount",
"metadata": {
"name": "user-engine-scoped-mail-eso",
"namespace": "user-engine"
},
"automountServiceAccountToken": false
},
{
"apiVersion": "external-secrets.io/v1",
"kind": "ClusterSecretStore",
"metadata": {
"name": "openbao-user-engine-scoped-mail"
},
"spec": {
"conditions": [
{
"namespaces": [
"user-engine"
]
}
],
"provider": {
"vault": {
"server": "http://openbao.openbao.svc:8200",
"path": "platform",
"version": "v2",
"auth": {
"kubernetes": {
"mountPath": "kubernetes",
"role": "user-engine-scoped-mail-eso",
"serviceAccountRef": {
"name": "user-engine-scoped-mail-eso",
"namespace": "user-engine",
"audiences": [
"openbao"
]
}
}
}
}
}
}
},
{
"apiVersion": "external-secrets.io/v1",
"kind": "ExternalSecret",
"metadata": {
"name": "user-engine-scoped-mail",
"namespace": "user-engine"
},
"spec": {
"refreshInterval": "5m",
"secretStoreRef": {
"kind": "ClusterSecretStore",
"name": "openbao-user-engine-scoped-mail"
},
"target": {
"name": "user-engine-scoped-mail",
"creationPolicy": "Owner",
"deletionPolicy": "Retain",
"template": {
"metadata": {
"labels": {
"email-connect.io/caller": "user-engine"
},
"annotations": {}
}
}
},
"data": [
{
"secretKey": "token",
"remoteRef": {
"key": "platform/workloads/email-connect/callers/user-engine",
"property": "TOKEN"
}
}
]
}
}
]
}

6
deploy/closure/caller-reader.hcl Normal file
View file

@ -0,0 +1,6 @@
path "platform/data/workloads/email-connect/callers/user-engine" {
capabilities = ["read"]
}
path "auth/token/lookup-self" { capabilities = ["read"] }
path "auth/token/revoke-self" { capabilities = ["update"] }

10
deploy/closure/caller-writer.hcl Normal file
View file

@ -0,0 +1,10 @@
path "platform/data/workloads/email-connect/callers/user-engine" {
capabilities = ["create", "read", "update"]
}
path "auth/token/lookup-self" { capabilities = ["read"] }
path "auth/token/revoke-self" { capabilities = ["update"] }
path "platform/metadata/workloads/email-connect/callers/user-engine" {
capabilities = ["read"]
}

63
deploy/closure/evidence-2026-10-09.json Normal file
View file

@ -0,0 +1,63 @@
{
"schema": "email-connect.closure-preparation.v1",
"date": "2026-10-09",
"status": "prepared_not_activated",
"plan_sha256": "8e529e2ffdb6ee3d53a768d65b2d025b95e788734c48a29f2f8b975825dbaed7",
"receiver_image": "forgejo.coulomb.social/coulomb/email-connect@sha256:5d79fd1fec15d3a279383820f170d263de3a55f5308df01a0a34732ebaea6d3d",
"user_engine_image": "forgejo.coulomb.social/coulomb/user-engine@sha256:312dcb062b237c518148e1b4f09798a4e6ad08bc60d4add6834aab3f246f2002",
"user_engine_baseline": "0d0751d",
"user_engine_patch_source": "adf6e99",
"tests": {
"email_connect_passed": 193,
"email_connect_skipped": 0,
"user_engine_baseline_passed": 302,
"user_engine_baseline_skipped": 20,
"packaged_binding_tests_passed": 6,
"focused_ruff": "passed"
},
"cluster_server_dry_run": {
"caller_delivery_objects": 6,
"receiver_patch": "passed",
"user_engine_patch": "passed"
},
"primary_bucket_metadata": {
"versioning": "Enabled",
"current_expiration_days": 30,
"noncurrent_expiration_days": 30,
"lifecycle_filter_prefix": "",
"provider_mutations": false
},
"recovery_kit": {
"status": "recovery_kit_prepared",
"kit_sha256": "97d1a4d4b55729dd226277601637d64b086de21b21c9951c1d7da05c914c350b",
"kit_bytes": 47541825,
"image": "forgejo.coulomb.social/coulomb/email-connect@sha256:eb3e7b68d21209e9eb71b928a5b0bc987e8e3164f21dca4ff712f946d0c103f2",
"image_id": "sha256:eb3e7b68d21209e9eb71b928a5b0bc987e8e3164f21dca4ff712f946d0c103f2",
"private_identity_included": false,
"provider_upload_verified": false,
"protected_version": "1791537739370570"
},
"offline_recovery": {
"status": "application_restored_verified",
"mail_sent": false,
"production_restored": false,
"application_restore_verified": true,
"network": "disabled",
"uid": 10001,
"offline_image_binding_verified": true,
"elapsed_seconds": 2.31
},
"live_changes": {
"caller_credentials_issued": false,
"new_images_deployed": false,
"scaleway_escrow_created": false,
"backup_timers_enabled": false,
"node_helpers_updated": false
},
"remaining": [
"Attended caller custody and live reader boundary proofs",
"Coordinated scoped rollout and no-send continuity/denial proofs",
"Independent Scaleway identity/kit custody with separate recovery reader and effective denial proofs",
"Schedule and retention admission; recurring operation/freshness evidence"
]
}

147
deploy/closure/plan.json Normal file
View file

@ -0,0 +1,147 @@
{
"schema": "email-connect.closure-plan.v1",
"status": "prepared",
"existing_backup_publisher": {
"application": "email-connect-backup-primary",
"application_id": "0eb0f958-77a4-4c70-9aa7-14ba46411415",
"bucket": "railiance-platform-pg-backup",
"prefix": "platform-pg/application-archives/email-connect/",
"retain_for_routine_backups": true,
"escrow_or_administration_grant_inferred": false
},
"receiver_image": "forgejo.coulomb.social/coulomb/email-connect@sha256:5d79fd1fec15d3a279383820f170d263de3a55f5308df01a0a34732ebaea6d3d",
"user_engine_image": "forgejo.coulomb.social/coulomb/user-engine@sha256:312dcb062b237c518148e1b4f09798a4e6ad08bc60d4add6834aab3f246f2002",
"user_engine_live_baseline": "0d0751d",
"user_engine_patch_sha256": "1fd705f1917f6c14cce69ae75b53ccbf1a2d9eee45561a72ae524a497d9f036e",
"caller_policy": {
"schema_version": 1,
"callers": [
{
"caller_id": "user-engine",
"source": "user-engine",
"token_file": "/var/run/secrets/email-connect-callers/user-engine",
"bindings": [
[
"tenant:coulomb",
"coulomb-social"
]
],
"binding_patterns": [
[
"tenant:*-family",
"coulomb-social"
],
[
"tenant:family-*",
"coulomb-social"
]
],
"operations": [
"send",
"delivery-status",
"status",
"verify",
"consume",
"cancel"
],
"templates": [
"family_member.invited",
"family_invitation.resent"
],
"rate_per_minute": 10,
"burst": 5,
"max_payload_bytes": 8192,
"adopt_legacy_verifications": true,
"adopt_legacy_invitations": true
}
]
},
"caller_custody": {
"kv_path": "platform/workloads/email-connect/callers/user-engine",
"field": "TOKEN",
"initial_cas": 0,
"writer_policy": "email-connect-scoped-caller-writer",
"reader_policy": "email-connect-scoped-caller-reader",
"readers": [
{
"namespace": "email-connect",
"service_account": "email-connect-scoped-caller-eso",
"role": "email-connect-scoped-caller-eso",
"store": "openbao-email-connect-scoped-caller",
"secret": "email-connect-caller-user-engine",
"secret_key": "token"
},
{
"namespace": "user-engine",
"service_account": "user-engine-scoped-mail-eso",
"role": "user-engine-scoped-mail-eso",
"store": "openbao-user-engine-scoped-mail",
"secret": "user-engine-scoped-mail",
"secret_key": "token"
}
]
},
"backup": {
"schedule_utc": "04:15",
"jitter_seconds": 300,
"rpo_target_hours": 24,
"stale_threshold_hours": 26,
"attended_rto_target_minutes": 60,
"minimum_node_free_bytes": 8589934592,
"local_budget_bytes": 8589934592,
"daily_generations": 14,
"weekly_generations": 4,
"provider_archive_retention_days": 30,
"retained_recovery_point_copied_to_escrow_kit": true,
"automatic_retained_object_deletion": false,
"failure_recipient": "platform-root",
"retention_operator": "platform-root",
"retention_review_interval": "weekly",
"retention_execution": "Separate owner-operated expiry; publisher never deletes retained objects",
"incident_restore_requires_fencing_and_post_capture_reconciliation": true
},
"scaleway_recovery": {
"region": "nl-ams",
"secret_manager_name": "email-connect-ledger-age-recovery",
"secret_manager_protected": true,
"secret_manager_ephemeral_policy": null,
"kit_bucket": "railiance-email-connect-recovery",
"kit_prefix": "kits/",
"kit_lifecycle": null,
"kit_versioning": "Enabled",
"kit_contents": [
"age-encrypted-ledger",
"public-transfer-and-recovery-receipts",
"pinned-receiver-image",
"recovery-tools",
"SHA256SUMS"
],
"private_age_identity_in_kit": false,
"scheduled_worker_escrow_access": false,
"recovery_caller": "Separate read-only IAM application; effective prefix/bucket denial proofs required",
"independent_entry": "platform-root Scaleway account; credentials held only on railiance01 do not qualify"
},
"activation_order": [
"Approve exact plan and perform attended caller custody and Scaleway admission",
"Deliver two bounded ESO projections and privately verify matching caller credentials",
"Freeze user-engine dispatch, capture current ledger and resume receiver",
"Apply coordinated immutable receiver/user-engine images and scoped policy",
"Verify health, denial, limits, accepted/unknown continuity and original handles without mail",
"Repin capture/rehearsal together; refresh and independently read back recovery kit",
"Enable daily backup/hourly freshness after independent custody and retention proofs",
"Finish workplans only after all admitted custody and operating proofs are present"
],
"artifacts": {
"deploy/closure/caller-reader.hcl": "cbc57e731c4fd9dbbaf98364de8ff87ed2966df18c2235cf298ffda963bf7553",
"deploy/closure/caller-writer.hcl": "a56013092fa6f0dff9282676ba1e94d5bbf37c2dc481232188f68a1ce543510c",
"deploy/closure/caller-delivery.json": "6fed12e204533ec679e80b3d9d0ba814cbf1d6f91a5a02211f0334617a92b0eb",
"deploy/closure/receiver.patch.json": "449ce8456323347eca81190e2ccca334c90397c6699498466308bf8bfe63a9de",
"deploy/closure/user-engine.patch.json": "2cae6ea6259ac1031fb7b8b11f011686baad8cb576cd72ee3f74461924daef46",
"scripts/admit-scoped-caller.py": "592bd46740f9eea4ad1ae78aa674f6c381ce05ab6044cedf0af968a1c340adf2",
"scripts/scaleway-recovery-custody.py": "85874f0620dc9353a9aaae1194db054de166296c4385743e4d541bbe42486c15",
"scripts/publish-ledger-recovery-kit.py": "cd7c6e8e6a4e8417ba55f8170d763d7c5a9e6dd9b5878dc3459021a0b64ea732",
"scripts/prepare-ledger-recovery-kit.py": "db92a29eb68e1b555cf18b17229d1e479b3147da7600ffa0616ac8b4d011ff55",
"scripts/rehearse-ledger.py": "d3da88972e44dac3800afb48d21e525dc1476ca58de3e7702f7751aea33eda53",
"scripts/backup-ledger.py": "e74c41948ca01dae09578e2abfba14b96fb1f158af0ec91a0fb1aa9ecaf01680"
}
}

136
deploy/closure/prepare.py Normal file
View file

@ -0,0 +1,136 @@
"""Render public closure artifacts. No credentials, network calls or activation."""
import hashlib
import json
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
OUT = Path(__file__).resolve().parent
def save(name, value):
(OUT / name).write_text(json.dumps(value, indent=2) + "\n")
policy = json.loads((ROOT / "config/callers.user-engine.draft.json").read_text())
readers = [
dict(namespace="email-connect", service_account="email-connect-scoped-caller-eso",
role="email-connect-scoped-caller-eso", store="openbao-email-connect-scoped-caller",
secret="email-connect-caller-user-engine", secret_key="token"),
dict(namespace="user-engine", service_account="user-engine-scoped-mail-eso",
role="user-engine-scoped-mail-eso", store="openbao-user-engine-scoped-mail",
secret="user-engine-scoped-mail", secret_key="token"),
]
plan = {
"schema": "email-connect.closure-plan.v1", "status": "prepared",
"existing_backup_publisher": {
"application": "email-connect-backup-primary",
"application_id": "0eb0f958-77a4-4c70-9aa7-14ba46411415",
"bucket": "railiance-platform-pg-backup",
"prefix": "platform-pg/application-archives/email-connect/",
"retain_for_routine_backups": True,
"escrow_or_administration_grant_inferred": False,
},
"receiver_image": "forgejo.coulomb.social/coulomb/email-connect@sha256:5d79fd1fec15d3a279383820f170d263de3a55f5308df01a0a34732ebaea6d3d",
"user_engine_image": "forgejo.coulomb.social/coulomb/user-engine@sha256:312dcb062b237c518148e1b4f09798a4e6ad08bc60d4add6834aab3f246f2002",
"user_engine_live_baseline": "0d0751d",
"user_engine_patch_sha256": hashlib.sha256((ROOT / "deploy/caller-rollout/user-engine-mail-bindings.patch").read_bytes()).hexdigest(),
"caller_policy": policy,
"caller_custody": {
"kv_path": "platform/workloads/email-connect/callers/user-engine", "field": "TOKEN",
"initial_cas": 0, "writer_policy": "email-connect-scoped-caller-writer",
"reader_policy": "email-connect-scoped-caller-reader", "readers": readers,
},
"backup": {
"schedule_utc": "04:15", "jitter_seconds": 300, "rpo_target_hours": 24,
"stale_threshold_hours": 26, "attended_rto_target_minutes": 60,
"minimum_node_free_bytes": 8 * 1024**3, "local_budget_bytes": 8 * 1024**3,
"daily_generations": 14, "weekly_generations": 4, "provider_archive_retention_days": 30,
"retained_recovery_point_copied_to_escrow_kit": True,
"automatic_retained_object_deletion": False, "failure_recipient": "platform-root",
"retention_operator": "platform-root", "retention_review_interval": "weekly",
"retention_execution": "Separate owner-operated expiry; publisher never deletes retained objects",
"incident_restore_requires_fencing_and_post_capture_reconciliation": True,
},
"scaleway_recovery": {
"region": "nl-ams", "secret_manager_name": "email-connect-ledger-age-recovery",
"secret_manager_protected": True,
"secret_manager_ephemeral_policy": None, "kit_bucket": "railiance-email-connect-recovery",
"kit_prefix": "kits/", "kit_lifecycle": None, "kit_versioning": "Enabled",
"kit_contents": ["age-encrypted-ledger", "public-transfer-and-recovery-receipts",
"pinned-receiver-image", "recovery-tools", "SHA256SUMS"],
"private_age_identity_in_kit": False, "scheduled_worker_escrow_access": False,
"recovery_caller": "Separate read-only IAM application; effective prefix/bucket denial proofs required",
"independent_entry": "platform-root Scaleway account; credentials held only on railiance01 do not qualify",
},
"activation_order": [
"Approve exact plan and perform attended caller custody and Scaleway admission",
"Deliver two bounded ESO projections and privately verify matching caller credentials",
"Freeze user-engine dispatch, capture current ledger and resume receiver",
"Apply coordinated immutable receiver/user-engine images and scoped policy",
"Verify health, denial, limits, accepted/unknown continuity and original handles without mail",
"Repin capture/rehearsal together; refresh and independently read back recovery kit",
"Enable daily backup/hourly freshness after independent custody and retention proofs",
"Finish workplans only after all admitted custody and operating proofs are present",
],
}
save("plan.json", plan)
reader = '''path "platform/data/workloads/email-connect/callers/user-engine" {
capabilities = ["read"]
}
path "auth/token/lookup-self" { capabilities = ["read"] }
path "auth/token/revoke-self" { capabilities = ["update"] }
'''
writer = reader.replace('capabilities = ["read"]', 'capabilities = ["create", "read", "update"]', 1)
writer += '''
path "platform/metadata/workloads/email-connect/callers/user-engine" {
capabilities = ["read"]
}
'''
(OUT / "caller-reader.hcl").write_text(reader)
(OUT / "caller-writer.hcl").write_text(writer)
items = []
for row in readers:
ns, sa = row["namespace"], row["service_account"]
items.append({"apiVersion": "v1", "kind": "ServiceAccount",
"metadata": {"name": sa, "namespace": ns}, "automountServiceAccountToken": False})
vault = {
"server": "http://openbao.openbao.svc:8200", "path": "platform", "version": "v2",
"auth": {"kubernetes": {"mountPath": "kubernetes", "role": row["role"],
"serviceAccountRef": {"name": sa, "namespace": ns, "audiences": ["openbao"]}}},
}
items.append({"apiVersion": "external-secrets.io/v1", "kind": "ClusterSecretStore",
"metadata": {"name": row["store"]},
"spec": {"conditions": [{"namespaces": [ns]}], "provider": {"vault": vault}}})
items.append({
"apiVersion": "external-secrets.io/v1", "kind": "ExternalSecret",
"metadata": {"name": row["secret"], "namespace": ns},
"spec": {
"refreshInterval": "5m", "secretStoreRef": {"kind": "ClusterSecretStore", "name": row["store"]},
"target": {"name": row["secret"], "creationPolicy": "Owner", "deletionPolicy": "Retain",
"template": {"metadata": {"labels": {"email-connect.io/caller": "user-engine"}, "annotations": {}}}},
"data": [{"secretKey": row["secret_key"],
"remoteRef": {"key": plan["caller_custody"]["kv_path"], "property": "TOKEN"}}],
},
})
save("caller-delivery.json", {"apiVersion": "v1", "kind": "List", "items": items})
user = {"name": "portal", "image": plan["user_engine_image"], "env": [
{"name": "USER_ENGINE_MAIL_CLIENT_BINDINGS", "value": json.dumps(json.loads((ROOT / "config/mail-client-bindings.user-engine.draft.json").read_text()), separators=(",", ":"))},
*[{"name": name, "valueFrom": {"secretKeyRef": {"name": "user-engine-scoped-mail", "key": "token"}}}
for name in ("USER_ENGINE_MAIL_TOKEN", "USER_ENGINE_REGISTRATION_VERIFICATION_TOKEN")],
]}
save("user-engine.patch.json", {"spec": {"template": {"spec": {"containers": [user]}}}})
receiver = json.loads((ROOT / "deploy/caller-rollout/receiver-policy.patch.json").read_text())
receiver["spec"]["template"]["spec"]["containers"][0]["image"] = plan["receiver_image"]
save("receiver.patch.json", receiver)
artifacts = [
"deploy/closure/caller-reader.hcl", "deploy/closure/caller-writer.hcl",
"deploy/closure/caller-delivery.json", "deploy/closure/receiver.patch.json",
"deploy/closure/user-engine.patch.json", "scripts/admit-scoped-caller.py",
"scripts/scaleway-recovery-custody.py", "scripts/publish-ledger-recovery-kit.py",
"scripts/prepare-ledger-recovery-kit.py", "scripts/rehearse-ledger.py",
"scripts/backup-ledger.py",
]
plan["artifacts"] = {name: hashlib.sha256((ROOT / name).read_bytes()).hexdigest() for name in artifacts}
save("plan.json", plan)

54
deploy/closure/receiver.patch.json Normal file
View file

@ -0,0 +1,54 @@
{
"spec": {
"template": {
"spec": {
"containers": [
{
"name": "email-connect",
"env": [
{
"name": "EMAIL_CONNECT_CALLERS_FILE",
"value": "/etc/email-connect-callers/callers.json"
}
],
"volumeMounts": [
{
"name": "caller-policy",
"mountPath": "/etc/email-connect-callers",
"readOnly": true
},
{
"name": "caller-credentials",
"mountPath": "/var/run/secrets/email-connect-callers",
"readOnly": true
}
],
"image": "forgejo.coulomb.social/coulomb/email-connect@sha256:5d79fd1fec15d3a279383820f170d263de3a55f5308df01a0a34732ebaea6d3d"
}
],
"volumes": [
{
"name": "caller-policy",
"configMap": {
"name": "email-connect-callers-user-engine-276257849bf2",
"defaultMode": 292
}
},
{
"name": "caller-credentials",
"secret": {
"secretName": "email-connect-caller-user-engine",
"defaultMode": 288,
"items": [
{
"key": "token",
"path": "user-engine"
}
]
}
}
]
}
}
}
}

38
deploy/closure/user-engine.patch.json Normal file
View file

@ -0,0 +1,38 @@
{
"spec": {
"template": {
"spec": {
"containers": [
{
"name": "portal",
"image": "forgejo.coulomb.social/coulomb/user-engine@sha256:312dcb062b237c518148e1b4f09798a4e6ad08bc60d4add6834aab3f246f2002",
"env": [
{
"name": "USER_ENGINE_MAIL_CLIENT_BINDINGS",
"value": "[[\"tenant:coulomb\",\"coulomb-social\"],[\"tenant:*-family\",\"coulomb-social\"],[\"tenant:family-*\",\"coulomb-social\"]]"
},
{
"name": "USER_ENGINE_MAIL_TOKEN",
"valueFrom": {
"secretKeyRef": {
"name": "user-engine-scoped-mail",
"key": "token"
}
}
},
{
"name": "USER_ENGINE_REGISTRATION_VERIFICATION_TOKEN",
"valueFrom": {
"secretKeyRef": {
"name": "user-engine-scoped-mail",
"key": "token"
}
}
}
]
}
]
}
}
}
}

46
docs/caller-policy-draft-2026-10-08.md
View file

@ -86,3 +86,49 @@ must be repinned together only when a new receiver image is actually deployed.
Policy narrowing/removal preserves ledger evidence but can deny existing links;
review rollback with the compatible image and both caller credentials before
releasing traffic. Source preparation does not issue a token or select this policy.
## Published closure candidates — 2026-10-09
The immutable receiver candidate is
forgejo.coulomb.social/coulomb/email-connect@sha256:5d79fd1fec15d3a279383820f170d263de3a55f5308df01a0a34732ebaea6d3d.
The coordinated user-engine candidate is
forgejo.coulomb.social/coulomb/user-engine@sha256:312dcb062b237c518148e1b4f09798a4e6ad08bc60d4add6834aab3f246f2002.
It applies only the retained binding patch to the verified live source baseline
0d0751d, avoiding unrelated owner-repository changes. Baseline regression passes
302 tests with 20 baseline optional skips; the six packaged binding tests pass
under UID10001/network-none. A read-only live query found no historical family
invitation outbox rows. This does not replace scope checks at release time.
Six delivery objects and both immutable-image deployment patches in deploy/closure
pass actual cluster server dry-run. Caller custody is a separate OpenBao leaf,
platform/workloads/email-connect/callers/user-engine, field TOKEN, initial CAS0.
The two dedicated namespace/ServiceAccount readers have exact leaf-only policies;
no legacy ingest token is copied. scripts/admit-scoped-caller.py previews by
default and silently installs exact policies/roles, creates or verifies the leaf
with a five-minute bounded writer, and self-revokes that writer before its receipt.
It refuses changed artifacts, differing existing roles/policies and unreviewed
plan decisions. It does not apply workloads or create Kubernetes Secrets.
The reviewed plan hash is 8e529e2ffdb6ee3d53a768d65b2d025b95e788734c48a29f2f8b975825dbaed7. For an attended execution, bind a resolved
User decision to this exact plan and use a fresh receipt under a mode-0700 parent:
```text
BAO_ADDR=http://127.0.0.1:18200 VAULT_ADDR=http://127.0.0.1:18200 \
warden access openbao-platform-admin-login --exec -- \
python3 /home/worsch/email-connect/scripts/admit-scoped-caller.py --execute \
--reviewed-sha256 8e529e2ffdb6ee3d53a768d65b2d025b95e788734c48a29f2f8b975825dbaed7 \
--decision-id RESOLVED_USER_DECISION_UUID --receipt PRIVATE_RECEIPT_PATH
```
The environment orientation §5 requires the founder to run this in their own
terminal because OIDC/MFA needs a browser and TTY:
/home/worsch/the-custodian/docs/agent-environment-orientation.md.
The child captures all secret output. Operator login and session revocation must
be verified separately; Warden's exit code alone is insufficient.
After custody/reader proof, project both bounded secrets, freeze user-engine
dispatch, capture the ledger, deploy the coordinated scoped pair, and prove health,
denials/quotas and legacy handle continuity without sending mail. Repin backup
capture/rehearsal together only after actual receiver activation and refresh the
recovery kit. No new credential, image rollout or policy activation has occurred.
T03 remains progress. Full email regression: 193 passed, zero skips.

51
docs/ledger-backup-review-2026-10-08.md
View file

@ -185,3 +185,54 @@ using the loopback GreenMail harness; focused syntax checks pass. Retention,
capacity, independent receipt/key/image custody, separate recovery caller and
recurring schedule/RPO admission retain EMAIL-WP-0007-T03/T04. Production restore
still requires fencing and post-capture reconciliation before traffic release.
## Closure package and Scaleway custody — 2026-10-09
User selected Scaleway, then named `email-connect-backup-primary`. Retain that
existing IAM application (0eb0f958-77a4-4c70-9aa7-14ba46411415) for routine archive
publication; CCR-2026-0037 admits only the email-connect archive prefix and excludes
escrow and administration. This selection is not evidence of additional grants.
Actual native metadata shows versioning Enabled and a bucket-wide 30-day expiry
for current and noncurrent objects. No provider configuration was changed.
The public closure contract is deploy/closure/plan.json; preparation evidence is
deploy/closure/evidence-2026-10-09.json. Proposed schedule is daily 04:15 UTC with
five-minute jitter, 24-hour RPO target and 26-hour freshness threshold. Proposed
local capacity floor/budget are both 8 GiB, with 14 daily/4 weekly generations and
weekly owner-operated expiry by platform-root. The new staged capacity guard
refuses before capture at low free space or when local retained files approach
that budget; it never deletes retained objects. These targets are prepared, not
measured production RTO or enabled operation. Node-installed helpers are unchanged.
A private 47,541,825-byte recovery kit binds the existing protected first backup,
receiver image, recovery tools and public receipts. It excludes the private age
identity. SHA256: 97d1a4d4b55729dd226277601637d64b086de21b21c9951c1d7da05c914c350b.
Rehearsal using its hash-bound bundled image passes without pulling an image,
under UID10001/network-none, preserving all six tables; elapsed 2.31 seconds is
application rehearsal only. The kit remains local and has not been uploaded.
Prepared Scaleway custody uses a protected opaque Secret Manager container
`email-connect-ledger-age-recovery` in nl-ams with no ephemeral policy, and private
versioned bucket `railiance-email-connect-recovery`, prefix `kits/`, with no expiry.
This preserves the protected recovery point beyond the primary archive lifecycle.
Provider containers, project/secret references, separate recovery application and
effective grants still need owner admission. Existing publisher access cannot
create these resources. A different durable arrangement is acceptable after its
actual effective permissions and retention are reconciled with the contract.
scripts/scaleway-recovery-custody.py defaults to preview. Attended execution takes
--secret-id, --project-id, --identity and a fresh --receipt; SCW_SECRET_KEY stays
in the operator environment. It checks exact protected metadata, writes/reuses
only revision 1 and silently verifies the matching age recipient by readback.
scripts/publish-ledger-recovery-kit.py defaults to preview; execution accepts
private owner/reader credential files, --kit, --kit-receipt, --output and --receipt.
It refuses disabled versioning or lifecycle rules, publishes conditionally and
checks size/hash through a separate reader's version-pinned GET. Different key
IDs do not prove different application bindings or denied writes. Receipts keep
those fields false pending native proof, independent account entry and a drill
from the provider-retrieved kit and escrow identity. No secret values belong in
chat, argv, evidence, Git or the operations inbox.
Full email regression: 193 passed, zero skips; focused syntax checks pass.
Timers remain disabled. T03/T04 stay progress until durable independent custody,
retention/schedule admission and actual recurring operation are evidenced.

182
scripts/admit-scoped-caller.py Normal file
View file

@ -0,0 +1,182 @@
#!/usr/bin/env python3
"""Silent, attended exact-plan caller custody. Preview is the default."""
from __future__ import annotations
import argparse
import hashlib
import hmac
import json
import os
import re
import secrets
import subprocess
import uuid
from pathlib import Path
from urllib.request import urlopen
from ledger_archive import Refused, checkpoint, private_directory
ROOT = Path(__file__).resolve().parents[1]
PLAN = ROOT / "deploy/closure/plan.json"
KV = "platform/workloads/email-connect/callers/user-engine"
WRITER = "email-connect-scoped-caller-writer"
READER = "email-connect-scoped-caller-reader"
READERS = [
("email-connect", "email-connect-scoped-caller-eso"),
("user-engine", "user-engine-scoped-mail-eso"),
]
def role(namespace, account):
return {
"bound_service_account_names": [account],
"bound_service_account_namespaces": [namespace], "audience": "openbao",
"policies": [READER], "ttl": "15m", "token_max_ttl": "15m",
"token_explicit_max_ttl": "15m", "token_no_default_policy": True,
"token_type": "service",
}
def bao(args, payload=None, token=None, missing=False):
env = os.environ.copy()
if token:
env.update(BAO_TOKEN=token, VAULT_TOKEN=token)
raw = payload if isinstance(payload, bytes) else None if payload is None else json.dumps(payload).encode()
result = subprocess.run(["bao", *args], input=raw, env=env, capture_output=True, timeout=60)
if result.returncode:
if missing and (b"Code: 404" in result.stderr or b"No value found" in result.stderr):
return None
raise Refused("attended_bao_operation_failed")
return result.stdout
def approved_plan(plan, reviewed, decision, base):
expected = hashlib.sha256(PLAN.read_bytes()).hexdigest()
if reviewed != expected:
raise Refused("reviewed_plan_hash_required")
for name, expected_hash in plan["artifacts"].items():
path = ROOT / name
if not path.resolve().is_relative_to(ROOT) or hashlib.sha256(path.read_bytes()).hexdigest() != expected_hash:
raise Refused("reviewed_artifact_changed")
uuid.UUID(decision)
with urlopen(base.rstrip("/") + "/decisions/" + decision, timeout=15) as response:
record = json.loads(response.read(65536))
if (record.get("status") != "resolved" or record.get("decision_type") != "made"
or record.get("decided_by") != "User"
or expected not in (record.get("description", "") or "")):
raise Refused("founder_exact_plan_decision_required")
custody = plan["caller_custody"]
if (custody["kv_path"] != KV or custody["field"] != "TOKEN"
or custody["writer_policy"] != WRITER or custody["reader_policy"] != READER
or [(r["namespace"], r["service_account"]) for r in custody["readers"]] != READERS):
raise Refused("exact_caller_contract_required")
return expected
def install_policy(name, path):
intended = path.read_bytes()
raw = bao(["read", "-format=json", "sys/policies/acl/" + name], missing=True)
observed = None if raw is None else json.loads(raw)["data"]["policy"].encode()
if observed is not None and observed.strip() != intended.strip():
raise Refused("existing_policy_differs")
if observed is None:
bao(["write", "sys/policies/acl/" + name, "-"], {"policy": intended.decode()})
observed = json.loads(bao(["read", "-format=json", "sys/policies/acl/" + name]))["data"]["policy"]
if observed.strip() != intended.decode().strip():
raise Refused("policy_readback_differs")
def provision(transport=bao):
auth = json.loads(transport([
"token", "create", "-format=json", "-policy=" + WRITER,
"-no-default-policy", "-ttl=5m", "-explicit-max-ttl=5m", "-use-limit=8",
]))["auth"]
token = auth["client_token"]
try:
if (auth.get("policies") != [WRITER] or auth.get("identity_policies", [])
or not 0 < auth.get("lease_duration", 0) <= 300):
raise Refused("bounded_writer_required")
metadata = transport(["read", "-format=json", "platform/metadata/workloads/email-connect/callers/user-engine"],
token=token, missing=True)
if metadata is None:
value = secrets.token_urlsafe(48)
written = json.loads(transport(["write", "-format=json", "platform/data/workloads/email-connect/callers/user-engine", "-"],
{"options": {"cas": 0}, "data": {"TOKEN": value}}, token=token))
version = written["data"]["version"]
if version != 1:
raise Refused("initial_cas_write_required")
created = True
else:
version = json.loads(metadata)["data"]["current_version"]
value = None
created = False
readback = json.loads(transport(["read", "-format=json", "platform/data/workloads/email-connect/callers/user-engine"],
token=token))["data"]
if set(readback["data"]) != {"TOKEN"} or readback["metadata"]["version"] != version:
raise Refused("caller_record_changed")
stored = readback["data"]["TOKEN"]
if not isinstance(stored, str) or not re.fullmatch(r"[!-~]{16,512}", stored):
raise Refused("caller_credential_invalid")
if value is not None and not hmac.compare_digest(value, stored):
raise Refused("caller_readback_differs")
return {"kv_version": version, "created": created, "readback_equal": True,
"credential_values_printed": False}
finally:
transport(["write", "auth/token/revoke-self"], token=token)
def execute(plan, reviewed, decision, base):
binding = approved_plan(plan, reviewed, decision, base)
for name, filename in ((WRITER, "caller-writer.hcl"), (READER, "caller-reader.hcl")):
install_policy(name, ROOT / "deploy/closure" / filename)
for namespace, account in READERS:
wanted = role(namespace, account)
observed = bao(["read", "-format=json", "auth/kubernetes/role/" + account], missing=True)
if observed is None:
bao(["write", "auth/kubernetes/role/" + account, "-"], wanted)
observed = bao(["read", "-format=json", "auth/kubernetes/role/" + account])
data = json.loads(observed)["data"]
if (data.get("bound_service_account_names") != [account]
or data.get("bound_service_account_namespaces") != [namespace]
or data.get("audience") != "openbao"
or (data.get("token_policies") or data.get("policies")) != [READER]
or data.get("token_no_default_policy") is not True
or data.get("token_max_ttl") not in (900, "15m")
or data.get("token_explicit_max_ttl") not in (900, "15m")):
raise Refused("existing_reader_role_differs")
return dict(provision(), status="caller_custody_prepared", plan_sha256=binding,
decision_id=decision, workloads_changed=False)
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--execute", action="store_true")
parser.add_argument("--reviewed-sha256")
parser.add_argument("--decision-id")
parser.add_argument("--api-base", default="http://127.0.0.1:8000")
parser.add_argument("--receipt", type=Path)
args = parser.parse_args()
plan = json.loads(PLAN.read_text())
if not args.execute:
print(json.dumps({"status": "prepared", "plan_sha256": hashlib.sha256(PLAN.read_bytes()).hexdigest(),
"credential_generated": False, "workloads_changed": False}))
return
if not args.receipt or not args.receipt.is_absolute():
raise Refused("private_receipt_path_required")
private_directory(args.receipt.parent)
if args.receipt.exists() or args.receipt.is_symlink():
raise Refused("fresh_receipt_required")
try:
receipt = execute(plan, args.reviewed_sha256, args.decision_id, args.api_base)
except Exception as error:
checkpoint(args.receipt, {"status": "failed", "failure_type": type(error).__name__})
raise
checkpoint(args.receipt, receipt)
if __name__ == "__main__":
try:
main()
except Exception:
raise SystemExit(1) from None

24
scripts/backup-ledger.py
View file

@ -30,9 +30,32 @@ from ledger_archive import (
NAME = re.compile(r"\d{8}T\d{6}Z-[a-f0-9]{24}\.tar\.age")
PRIMARY_TOOL = Path("/opt/railiance-platform/scripts/primary_application_archive.py")
PRIMARY_PYTHON = Path("/opt/railiance-platform/backup-venv/bin/python")
MINIMUM_FREE_BYTES = 8 * 1024**3
LOCAL_BUDGET_BYTES = 8 * 1024**3
RECIPIENT = "age1zvryunvjhvpkmasskauga2heeg0ztnte9ymgppvjge36ekumk50syr3tsz"
def capacity(state, minimum_free=MINIMUM_FREE_BYTES, budget=LOCAL_BUDGET_BYTES):
ancestor = state
while not ancestor.exists():
ancestor = ancestor.parent
if shutil.disk_usage(ancestor).free < minimum_free:
raise Refused("backup_disk_headroom_required")
used = 0
if state.exists():
private_directory(state)
for path in state.rglob("*"):
if path.is_symlink():
raise Refused("backup_state_symlink_requires_review")
if path.is_file():
used += path.stat().st_size
# Reserve room for capture, ciphertext, readback and a failure packet.
if used + 512 * 1024**2 > budget:
raise Refused("backup_local_retention_review_required")
return {"status": "headroom_verified", "local_bytes": used,
"minimum_free_bytes": minimum_free, "local_budget_bytes": budget}
def publish(encrypted, fetched, receipt_path, captured, tool, python):
command(
[
@ -209,6 +232,7 @@ def main():
or not args.primary_python.is_file()
):
raise Refused("invalid_backup_configuration")
capacity(args.state_dir)
command(["age", "--version"])
command(
[

86
scripts/prepare-ledger-recovery-kit.py Normal file
View file

@ -0,0 +1,86 @@
#!/usr/bin/env python3
"""Build an explicit offline recovery kit; never include an age private identity."""
from __future__ import annotations
import argparse
import json
import os
import shutil
import subprocess
import tarfile
from pathlib import Path
from ledger_archive import Refused, checkpoint, digest, private_directory, private_file
ROOT = Path(__file__).resolve().parents[1]
def prepare(evidence, ciphertext, destination):
proof = json.loads(evidence.read_text())
transfer, recovery = proof["publication"], proof["application_recovery"]
if (recovery.get("application_restore_verified") is not True
or recovery.get("application") != "email-connect"
or recovery.get("ciphertext_sha256") != transfer.get("ciphertext_sha256")
or recovery.get("version_id") != transfer.get("version_id")):
raise Refused("verified_recovery_point_required")
private_file(ciphertext)
if (digest(ciphertext) != transfer["ciphertext_sha256"]
or ciphertext.stat().st_size != transfer["ciphertext_bytes"]):
raise Refused("protected_ciphertext_binding_changed")
destination.mkdir(mode=0o700)
private_directory(destination)
image = recovery["recovery_image"]
inspected = subprocess.run(["docker", "image", "inspect", image], capture_output=True, timeout=30)
if inspected.returncode:
raise Refused("cached_recovery_image_required")
image_id = json.loads(inspected.stdout)[0]["Id"]
saved = subprocess.run(["docker", "image", "save", "--output", str(destination / "receiver-image.tar"), image],
capture_output=True, timeout=180)
if saved.returncode:
raise Refused("recovery_image_export_failed")
(destination / "receiver-image.tar").chmod(0o600)
for name in ("ledger_archive.py", "backup-ledger.py", "rehearse-ledger.py"):
shutil.copyfile(ROOT / "scripts" / name, destination / name)
(destination / name).chmod(0o600)
shutil.copyfile(ciphertext, destination / "ledger.tar.age")
(destination / "ledger.tar.age").chmod(0o600)
checkpoint(destination / "transfer-receipt.json", transfer)
checkpoint(destination / "application-recovery.json", recovery)
files = {p.name: {"sha256": digest(p), "bytes": p.stat().st_size} for p in destination.iterdir()}
manifest = {"schema": "email-connect.ledger-recovery-kit.v1", "image": image, "image_id": image_id,
"files": files, "private_identity_included": False,
"protected_version": transfer["version_id"], "protected_ciphertext_sha256": transfer["ciphertext_sha256"]}
checkpoint(destination / "manifest.json", manifest)
with tarfile.open(destination / "kit.tar.gz", "w:gz") as archive:
for name in sorted([*files, "manifest.json"]):
archive.add(destination / name, arcname=name, recursive=False)
(destination / "kit.tar.gz").chmod(0o600)
receipt = {"status": "recovery_kit_prepared", "kit_sha256": digest(destination / "kit.tar.gz"),
"kit_bytes": (destination / "kit.tar.gz").stat().st_size,
"image": image, "image_id": image_id, "private_identity_included": False,
"provider_upload_verified": False, "protected_version": transfer["version_id"]}
checkpoint(destination / "kit-receipt.json", receipt)
return receipt
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--execute", action="store_true")
parser.add_argument("--evidence", type=Path, required=True)
parser.add_argument("--ciphertext", type=Path, required=True)
parser.add_argument("--destination", type=Path, required=True)
args = parser.parse_args()
if not args.execute:
print(json.dumps({"status": "prepared", "image_exported": False, "provider_contacted": False}))
return
os.umask(0o077)
print(json.dumps(prepare(args.evidence, args.ciphertext, args.destination)))
if __name__ == "__main__":
try:
main()
except Exception as error:
print(json.dumps({"status": "failed", "failure_type": type(error).__name__}))
raise SystemExit(1)

135
scripts/publish-ledger-recovery-kit.py Normal file
View file

@ -0,0 +1,135 @@
#!/usr/bin/env python3
"""Attended immutable kit upload and read-only recovery-caller readback."""
from __future__ import annotations
import argparse
import hashlib
import json
import os
from pathlib import Path
from ledger_archive import Refused, checkpoint, digest, private_directory, private_file
BUCKET = "railiance-email-connect-recovery"
ENDPOINT = "https://s3.nl-ams.scw.cloud"
def client(credentials):
import boto3
from botocore.config import Config
return boto3.client("s3", endpoint_url=ENDPOINT, region_name="nl-ams",
aws_access_key_id=credentials["AWS_ACCESS_KEY_ID"],
aws_secret_access_key=credentials["AWS_SECRET_ACCESS_KEY"],
config=Config(signature_version="s3v4", connect_timeout=15, read_timeout=180,
s3={"addressing_style": "path"}))
def credentials(path):
private_file(path)
value = json.loads(path.read_text())
if (set(value) != {"AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"}
or any(not isinstance(v, str) or not v.strip() for v in value.values())):
raise Refused("exact_private_s3_pair_required")
return value
def version(value):
if not isinstance(value, str) or not value or value == "null" or any(c in value for c in "\r\n"):
raise Refused("versioned_kit_required")
return value
def publish(owner, reader, kit, expected, output):
if (expected.get("status") != "recovery_kit_prepared"
or expected.get("private_identity_included") is not False
or digest(kit) != expected.get("kit_sha256")
or kit.stat().st_size != expected.get("kit_bytes")
or not 0 < kit.stat().st_size <= 1024**3):
raise Refused("verified_bounded_kit_required")
if owner.get_bucket_versioning(Bucket=BUCKET).get("Status") != "Enabled":
raise Refused("versioned_recovery_bucket_required")
try:
lifecycle = owner.get_bucket_lifecycle_configuration(Bucket=BUCKET)
except Exception as error:
if getattr(error, "response", {}).get("Error", {}).get("Code") != "NoSuchLifecycleConfiguration":
raise
else:
if lifecycle.get("Rules"):
raise Refused("nonexpiring_recovery_bucket_required")
key = "kits/" + expected["kit_sha256"] + ".tar.gz"
private_directory(output.parent)
if output.exists() or output.is_symlink():
raise Refused("fresh_kit_readback_required")
# Conditional creation preserves an existing immutable kit. Readback must
# use a separately admitted recovery identity, never the uploader.
try:
with kit.open("rb") as source:
created = owner.put_object(Bucket=BUCKET, Key=key, Body=source, IfNoneMatch="*",
Metadata={"sha256": expected["kit_sha256"]})
selected = version(created.get("VersionId"))
except Exception as error:
if getattr(error, "response", {}).get("ResponseMetadata", {}).get("HTTPStatusCode") != 412:
raise
head = owner.head_object(Bucket=BUCKET, Key=key)
if (head.get("Metadata", {}).get("sha256") != expected["kit_sha256"]
or head.get("ContentLength") != expected["kit_bytes"]):
raise Refused("existing_kit_binding_changed")
selected = version(head.get("VersionId"))
response = reader.get_object(Bucket=BUCKET, Key=key, VersionId=selected)
if response.get("VersionId") != selected or response.get("ContentLength") != expected["kit_bytes"]:
response["Body"].close()
raise Refused("kit_readback_version_changed")
hashed, size = hashlib.sha256(), 0
with response["Body"] as body, output.open("xb") as target:
output.chmod(0o600)
while block := body.read(8 * 1024**2):
size += len(block)
if size > expected["kit_bytes"]:
raise Refused("kit_readback_exceeds_expected_size")
hashed.update(block)
target.write(block)
if size != expected["kit_bytes"] or hashed.hexdigest() != expected["kit_sha256"]:
raise Refused("kit_readback_hash_changed")
return {"status": "scaleway_recovery_kit_readback_verified", "bucket": BUCKET,
"object_key": key, "version_id": selected, "kit_sha256": hashed.hexdigest(), "kit_bytes": size,
"separate_reader_readback_verified": True, "private_identity_included": False,
"reader_write_denial_verified": False, "worker_access_denial_verified": False,
"reader_application_binding_verified": False, "restored_from_provider_kit": False}
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--execute", action="store_true")
for name in ("kit", "kit-receipt", "owner-credentials", "reader-credentials", "output", "receipt"):
parser.add_argument("--" + name, type=Path)
args = parser.parse_args()
if not args.execute:
print(json.dumps({"status": "prepared", "bucket": BUCKET, "provider_contacted": False,
"caller_credentials_read": False, "private_identity_included": False}))
return
if not all((args.kit, args.kit_receipt, args.owner_credentials, args.reader_credentials, args.output, args.receipt)):
raise Refused("attended_kit_arguments_required")
private_directory(args.receipt.parent)
if args.receipt.exists() or args.receipt.is_symlink():
raise Refused("fresh_receipt_required")
private_file(args.kit)
private_file(args.kit_receipt)
owner, reader = credentials(args.owner_credentials), credentials(args.reader_credentials)
if owner["AWS_ACCESS_KEY_ID"] == reader["AWS_ACCESS_KEY_ID"]:
raise Refused("separate_recovery_identity_required")
os.umask(0o077)
try:
result = publish(client(owner), client(reader), args.kit,
json.loads(args.kit_receipt.read_text()), args.output)
except Exception as error:
checkpoint(args.receipt, {"status": "failed", "failure_type": type(error).__name__})
raise
checkpoint(args.receipt, result)
if __name__ == "__main__":
try:
main()
except Exception:
raise SystemExit(1) from None

33
scripts/rehearse-ledger.py
View file

@ -5,6 +5,7 @@ from __future__ import annotations
import argparse
import json
import re
import secrets
import subprocess
import time
@ -13,6 +14,7 @@ from pathlib import Path
from ledger_archive import (
Refused,
checkpoint,
digest,
private_directory,
private_file,
validate_capture,
@ -89,7 +91,26 @@ print(json.dumps({'status':'isolated_ledger_verified','ledger_fingerprint':befor
"""
def rehearse(recovered, receipt_path):
def recovery_image(offline_binding=None):
if offline_binding is None:
return IMAGE
private_file(offline_binding)
binding = json.loads(offline_binding.read_text())
image_id = binding.get("image_id", "")
if (binding.get("schema") != "email-connect.ledger-recovery-kit.v1"
or binding.get("image") != IMAGE
or not isinstance(image_id, str)
or not re.fullmatch(r"sha256:[a-f0-9]{64}", image_id)):
raise Refused("offline_image_binding_required")
archive = offline_binding.parent / "receiver-image.tar"
private_file(archive)
expected = binding["files"]["receiver-image.tar"]
if archive.stat().st_size != expected["bytes"] or digest(archive) != expected["sha256"]:
raise Refused("offline_image_archive_changed")
return image_id
def rehearse(recovered, receipt_path, offline_binding=None):
private_directory(recovered)
private_directory(receipt_path.parent)
if receipt_path.exists() or receipt_path.is_symlink():
@ -104,8 +125,9 @@ def rehearse(recovered, receipt_path):
):
raise Refused("pinned_recovered_ledger_required")
# Never pull a recovery image from the production forge during this rehearsal.
selected_image = recovery_image(offline_binding)
inspected = subprocess.run(
["docker", "image", "inspect", IMAGE], capture_output=True, timeout=30
["docker", "image", "inspect", selected_image], capture_output=True, timeout=30
)
if inspected.returncode:
raise Refused("cached_pinned_recovery_image_required")
@ -119,6 +141,7 @@ def rehearse(recovered, receipt_path):
"docker",
"run",
"--rm",
"--pull=never",
"--name",
name,
"-i",
@ -145,7 +168,7 @@ def rehearse(recovered, receipt_path):
"PYTHONDONTWRITEBYTECODE=1",
"--entrypoint",
"python",
IMAGE,
selected_image,
"-c",
CODE,
],
@ -171,6 +194,7 @@ def rehearse(recovered, receipt_path):
uid=verification["uid"],
mail_sent=False,
evidence_ceiling="isolated_ledger_rehearsal",
offline_image_binding_verified=offline_binding is not None,
elapsed_seconds=round(time.monotonic() - started, 2),
)
checkpoint(receipt_path, receipt)
@ -182,6 +206,7 @@ def main():
parser.add_argument("--recovered", required=True, type=Path)
parser.add_argument("--receipt", required=True, type=Path)
parser.add_argument("--execute", action="store_true")
parser.add_argument("--offline-image-binding", type=Path)
args = parser.parse_args()
if not args.execute:
print(
@ -190,7 +215,7 @@ def main():
)
)
return 0
print(json.dumps(rehearse(args.recovered, args.receipt)))
print(json.dumps(rehearse(args.recovered, args.receipt, args.offline_image_binding)))
return 0

120
scripts/scaleway-recovery-custody.py Normal file
View file

@ -0,0 +1,120 @@
#!/usr/bin/env python3
"""Attended age escrow to a protected Scaleway Secret Manager secret, no key output."""
from __future__ import annotations
import argparse
import base64
import hmac
import json
import os
import subprocess
import uuid
from pathlib import Path
from urllib.request import Request, urlopen
from ledger_archive import Refused, checkpoint, private_directory, private_file
NAME = "email-connect-ledger-age-recovery"
REGION = "nl-ams"
RECIPIENT = "age1zvryunvjhvpkmasskauga2heeg0ztnte9ymgppvjge36ekumk50syr3tsz"
BASE = "https://api.scaleway.com/secret-manager/v1beta1/regions/nl-ams/secrets/"
def request(secret_id, suffix="", payload=None):
key = os.environ.get("SCW_SECRET_KEY", "")
if not key:
raise Refused("attended_scaleway_identity_required")
req = Request(BASE + secret_id + suffix,
data=None if payload is None else json.dumps(payload).encode(),
headers={"X-Auth-Token": key, "Content-Type": "application/json"},
method="GET" if payload is None else "POST")
with urlopen(req, timeout=30) as response:
return json.loads(response.read(262144))
def recipient(identity):
result = subprocess.run(["age-keygen", "-y", "/dev/stdin"], input=identity,
capture_output=True, timeout=30)
if result.returncode:
raise Refused("age_identity_invalid")
return result.stdout.decode().strip()
def escrow(secret_id, project_id, identity, transport=request):
uuid.UUID(secret_id)
uuid.UUID(project_id)
if recipient(identity) != RECIPIENT or not 0 < len(identity) <= 65536:
raise Refused("configured_age_recipient_required")
metadata = transport(secret_id)
if (metadata.get("name") != NAME or metadata.get("project_id") != project_id
or metadata.get("region") != REGION or metadata.get("type") != "opaque"
or metadata.get("protected") is not True
or metadata.get("ephemeral_policy") is not None
or metadata.get("deletion_requested_at") is not None):
raise Refused("protected_nonexpiring_exact_secret_required")
versions = transport(secret_id, "/versions?page_size=100")
count = versions.get("total_count")
if type(count) is not int or count != len(versions.get("versions", [])) or count > 1:
raise Refused("existing_versions_require_custodian_review")
if count == 0:
written = transport(secret_id, "/versions", {
"data": base64.b64encode(identity).decode(),
"description": "Existing email-connect age identity; attended recovery only.",
"disable_previous": False,
})
revision = written["revision"]
if revision != 1:
raise Refused("initial_secret_version_required")
else:
version = versions["versions"][0]
properties = version.get("ephemeral_properties") or {}
if (version.get("status") != "enabled" or properties.get("expires_at") is not None
or properties.get("expires_once_accessed") is True
or version.get("revision") != 1):
raise Refused("nonexpiring_enabled_version_required")
revision = 1
returned = transport(secret_id, "/versions/" + str(revision) + "/access")
restored = base64.b64decode(returned["data"], validate=True)
if (returned.get("secret_id") != secret_id or returned.get("revision") != revision
or not hmac.compare_digest(restored, identity) or recipient(restored) != RECIPIENT):
raise Refused("independent_escrow_readback_differs")
return {"status": "scaleway_age_escrow_verified", "secret_id": secret_id,
"revision": revision, "region": REGION, "public_recipient_matches": True,
"readback_equal": True, "private_identity_printed": False,
"independent_account_login_verified": False,
"worker_access_denial_verified": False}
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--execute", action="store_true")
parser.add_argument("--secret-id")
parser.add_argument("--project-id")
parser.add_argument("--identity", type=Path)
parser.add_argument("--receipt", type=Path)
args = parser.parse_args()
if not args.execute:
print(json.dumps({"status": "prepared", "name": NAME, "region": REGION,
"protected": True, "ephemeral_policy": None,
"provider_contacted": False, "private_identity_read": False}))
return
if not args.identity or not args.receipt or not args.receipt.is_absolute():
raise Refused("private_identity_and_receipt_paths_required")
private_directory(args.receipt.parent)
private_file(args.identity)
if args.receipt.exists() or args.receipt.is_symlink():
raise Refused("fresh_receipt_required")
try:
result = escrow(args.secret_id, args.project_id, args.identity.read_bytes())
except Exception as error:
checkpoint(args.receipt, {"status": "failed", "failure_type": type(error).__name__})
raise
checkpoint(args.receipt, result)
if __name__ == "__main__":
try:
main()
except Exception:
raise SystemExit(1) from None

183
tests/test_closure_custody.py Normal file
View file

@ -0,0 +1,183 @@
"""Synthetic exact-custody and offline-image guards; no operator/provider access."""
import base64
import hashlib
import importlib.util
import json
from pathlib import Path
import pytest
def module(name, monkeypatch):
scripts = Path(__file__).resolve().parents[1] / "scripts"
monkeypatch.syspath_prepend(str(scripts))
spec = importlib.util.spec_from_file_location(name.replace("-", "_"), scripts / (name + ".py"))
result = importlib.util.module_from_spec(spec)
spec.loader.exec_module(result)
return result
@pytest.mark.parametrize("failure", [None, "write", "readback"])
def test_caller_writer_uses_cas0_and_revokes_on_every_outcome(monkeypatch, failure):
caller = module("admit-scoped-caller", monkeypatch)
value = "synthetic-caller-credential"
monkeypatch.setattr(caller.secrets, "token_urlsafe", lambda *args: value)
calls = []
def transport(args, payload=None, token=None, missing=False):
calls.append((args, payload, token))
if args[0] == "token":
return json.dumps({"auth": {"client_token": "synthetic-writer",
"policies": [caller.WRITER], "lease_duration": 300}}).encode()
if "auth/token/revoke-self" in args:
return b"{}"
if "platform/metadata/workloads/email-connect/callers/user-engine" in args:
return None
if args[0] == "write":
assert payload == {"options": {"cas": 0}, "data": {"TOKEN": value}}
if failure == "write":
raise caller.Refused("synthetic_failure")
return b'{"data":{"version":1}}'
if failure == "readback":
raise caller.Refused("synthetic_failure")
return json.dumps({"data": {"data": {"TOKEN": value}, "metadata": {"version": 1}}}).encode()
if failure:
with pytest.raises(caller.Refused):
caller.provision(transport)
else:
receipt = caller.provision(transport)
assert receipt["kv_version"] == 1 and receipt["readback_equal"]
assert value not in json.dumps(receipt)
assert calls[-1][0] == ["write", "auth/token/revoke-self"]
assert calls[-1][2] == "synthetic-writer"
def test_caller_writer_rejects_inherited_authority_before_any_secret_access(monkeypatch):
caller = module("admit-scoped-caller", monkeypatch)
seen = []
def transport(args, *rest, **kwargs):
seen.append(args)
if args[0] == "token":
return json.dumps({"auth": {"client_token": "synthetic-writer",
"policies": [caller.WRITER], "identity_policies": ["platform-admin"],
"lease_duration": 300}}).encode()
return b"{}"
with pytest.raises(caller.Refused, match="bounded_writer_required"):
caller.provision(transport)
assert len(seen) == 2 and seen[-1] == ["write", "auth/token/revoke-self"]
def test_unreviewed_plan_refuses_before_decision_network_call(monkeypatch):
caller = module("admit-scoped-caller", monkeypatch)
monkeypatch.setattr(caller, "urlopen", lambda *a, **kw: pytest.fail("network forbidden"))
with pytest.raises(caller.Refused, match="reviewed_plan_hash_required"):
caller.approved_plan({}, "unreviewed", None, "http://synthetic.invalid")
@pytest.mark.parametrize("protected,ephemeral,corrupt", [
(True, None, False), (False, None, False), (True, {}, False), (True, None, True),
])
def test_scaleway_key_escrow_requires_exact_protected_nonexpiring_readback(monkeypatch, protected, ephemeral, corrupt):
escrow = module("scaleway-recovery-custody", monkeypatch)
identity = b"synthetic-only-private-identity"
secret_id = "00000000-0000-0000-0000-000000000001"
project_id = "00000000-0000-0000-0000-000000000002"
monkeypatch.setattr(escrow, "recipient", lambda value: escrow.RECIPIENT)
seen = []
def transport(secret, suffix="", payload=None):
seen.append((secret, suffix, payload))
if suffix == "":
return {"name": escrow.NAME, "project_id": project_id, "region": escrow.REGION,
"type": "opaque", "protected": protected, "ephemeral_policy": ephemeral}
if "?" in suffix:
return {"total_count": 0, "versions": []}
if payload is not None:
assert base64.b64decode(payload["data"]) == identity
assert payload["disable_previous"] is False
return {"revision": 1}
return {"secret_id": secret_id, "revision": 1,
"data": base64.b64encode(b"synthetic-corrupt" if corrupt else identity).decode()}
if protected and ephemeral is None and not corrupt:
result = escrow.escrow(secret_id, project_id, identity, transport)
assert result["readback_equal"] and not result["worker_access_denial_verified"]
assert identity.decode() not in json.dumps(result)
else:
with pytest.raises(escrow.Refused):
escrow.escrow(secret_id, project_id, identity, transport)
if not protected or ephemeral is not None:
assert len(seen) == 1
def test_offline_recovery_image_is_bound_to_export_and_capture_image(monkeypatch, tmp_path):
rehearsal = module("rehearse-ledger", monkeypatch)
image = tmp_path / "receiver-image.tar"
image.write_bytes(b"synthetic-image-export")
image.chmod(0o600)
manifest = tmp_path / "manifest.json"
manifest.write_text(json.dumps({"schema": "email-connect.ledger-recovery-kit.v1",
"image": rehearsal.IMAGE, "image_id": "sha256:" + "a" * 64,
"files": {"receiver-image.tar": {"sha256": hashlib.sha256(image.read_bytes()).hexdigest(),
"bytes": image.stat().st_size}}}))
manifest.chmod(0o600)
assert rehearsal.recovery_image(manifest) == "sha256:" + "a" * 64
image.write_bytes(b"synthetic-changed-export")
with pytest.raises(rehearsal.Refused, match="offline_image_archive_changed"):
rehearsal.recovery_image(manifest)
@pytest.mark.parametrize("lifecycle,corrupt", [(False, False), (True, False), (False, True)])
def test_kit_requires_no_expiry_and_separate_version_hash_readback(monkeypatch, tmp_path, lifecycle, corrupt):
import io
publisher = module("publish-ledger-recovery-kit", monkeypatch)
kit = tmp_path / "kit.tar.gz"
content = b"synthetic-kit-with-no-private-identity"
kit.write_bytes(content)
expected = {"status": "recovery_kit_prepared", "private_identity_included": False,
"kit_sha256": hashlib.sha256(content).hexdigest(), "kit_bytes": len(content)}
monkeypatch.setattr(publisher, "private_directory", lambda path: None)
calls = []
class Owner:
def get_bucket_versioning(self, **kwargs): return {"Status": "Enabled"}
def get_bucket_lifecycle_configuration(self, **kwargs):
return {"Rules": [{"Expiration": {"Days": 30}}]} if lifecycle else {"Rules": []}
def put_object(self, **kwargs):
calls.append(kwargs)
assert kwargs["IfNoneMatch"] == "*"
return {"VersionId": "synthetic-kit-version"}
class Reader:
def get_object(self, **kwargs):
assert kwargs["VersionId"] == "synthetic-kit-version"
return {"VersionId": "synthetic-kit-version", "ContentLength": len(content),
"Body": io.BytesIO(b"synthetic-corrupt" if corrupt else content)}
if lifecycle or corrupt:
with pytest.raises(publisher.Refused):
publisher.publish(Owner(), Reader(), kit, expected, tmp_path / "readback")
if lifecycle:
assert calls == []
else:
result = publisher.publish(Owner(), Reader(), kit, expected, tmp_path / "readback")
assert result["separate_reader_readback_verified"]
assert not result["reader_application_binding_verified"]
assert not result["private_identity_included"]
def test_low_disk_headroom_refuses_before_capture(monkeypatch, tmp_path):
from types import SimpleNamespace
backup = module("backup-ledger", monkeypatch)
monkeypatch.setattr(backup.shutil, "disk_usage", lambda *args: SimpleNamespace(free=1023))
with pytest.raises(backup.Refused, match="backup_disk_headroom_required"):
backup.capacity(tmp_path / "not-created", minimum_free=1024)
assert not (tmp_path / "not-created").exists()
def test_retention_budget_refuses_without_deleting_ciphertext(monkeypatch, tmp_path):
from types import SimpleNamespace
backup = module("backup-ledger", monkeypatch)
state = tmp_path / "state"
state.mkdir(mode=0o700)
archive = state / "synthetic.age"
archive.write_bytes(b"synthetic-protected-ciphertext")
monkeypatch.setattr(backup.shutil, "disk_usage", lambda *args: SimpleNamespace(free=10**10))
with pytest.raises(backup.Refused, match="backup_local_retention_review_required"):
backup.capacity(state, budget=512 * 1024**2)
assert archive.read_bytes() == b"synthetic-protected-ciphertext"

51
workplans/EMAIL-WP-0007-ledger-backup-recovery.md
View file

@ -174,3 +174,54 @@ both timers are disabled. T04 is progress, with first-run/recovery/delivery prov
T03 retains generation retention/capacity, independently retained receipt/key/image
escrow and a separate recovery caller. Schedule/RPO admission and incident
post-capture reconciliation remain required before workplan completion.
## Closure package and Scaleway custody — 2026-10-09
User selected Scaleway, then named `email-connect-backup-primary`. Retain that
existing IAM application (0eb0f958-77a4-4c70-9aa7-14ba46411415) for routine archive
publication; CCR-2026-0037 admits only the email-connect archive prefix and excludes
escrow and administration. This selection is not evidence of additional grants.
Actual native metadata shows versioning Enabled and a bucket-wide 30-day expiry
for current and noncurrent objects. No provider configuration was changed.
The public closure contract is deploy/closure/plan.json; preparation evidence is
deploy/closure/evidence-2026-10-09.json. Proposed schedule is daily 04:15 UTC with
five-minute jitter, 24-hour RPO target and 26-hour freshness threshold. Proposed
local capacity floor/budget are both 8 GiB, with 14 daily/4 weekly generations and
weekly owner-operated expiry by platform-root. The new staged capacity guard
refuses before capture at low free space or when local retained files approach
that budget; it never deletes retained objects. These targets are prepared, not
measured production RTO or enabled operation. Node-installed helpers are unchanged.
A private 47,541,825-byte recovery kit binds the existing protected first backup,
receiver image, recovery tools and public receipts. It excludes the private age
identity. SHA256: 97d1a4d4b55729dd226277601637d64b086de21b21c9951c1d7da05c914c350b.
Rehearsal using its hash-bound bundled image passes without pulling an image,
under UID10001/network-none, preserving all six tables; elapsed 2.31 seconds is
application rehearsal only. The kit remains local and has not been uploaded.
Prepared Scaleway custody uses a protected opaque Secret Manager container
`email-connect-ledger-age-recovery` in nl-ams with no ephemeral policy, and private
versioned bucket `railiance-email-connect-recovery`, prefix `kits/`, with no expiry.
This preserves the protected recovery point beyond the primary archive lifecycle.
Provider containers, project/secret references, separate recovery application and
effective grants still need owner admission. Existing publisher access cannot
create these resources. A different durable arrangement is acceptable after its
actual effective permissions and retention are reconciled with the contract.
scripts/scaleway-recovery-custody.py defaults to preview. Attended execution takes
--secret-id, --project-id, --identity and a fresh --receipt; SCW_SECRET_KEY stays
in the operator environment. It checks exact protected metadata, writes/reuses
only revision 1 and silently verifies the matching age recipient by readback.
scripts/publish-ledger-recovery-kit.py defaults to preview; execution accepts
private owner/reader credential files, --kit, --kit-receipt, --output and --receipt.
It refuses disabled versioning or lifecycle rules, publishes conditionally and
checks size/hash through a separate reader's version-pinned GET. Different key
IDs do not prove different application bindings or denied writes. Receipts keep
those fields false pending native proof, independent account entry and a drill
from the provider-retrieved kit and escrow identity. No secret values belong in
chat, argv, evidence, Git or the operations inbox.
Full email regression: 193 passed, zero skips; focused syntax checks pass.
Timers remain disabled. T03/T04 stay progress until durable independent custody,
retention/schedule admission and actual recurring operation are evidenced.

12
workplans/EMAIL-WP-0008-scoped-transactional-callers.md
View file

@ -158,3 +158,15 @@ credential delivery, exact scope/image and coordinated activation remain open.
Final email regression after caller/notification preparation: 178 passed, zero
skips with the loopback-only GreenMail harness. Focused syntax checks pass.
User-engine source preparation is recorded at adf6e99 in its owner repository.
## Closure preparation — 2026-10-09
Published both immutable candidates and dry-ran two deployment patches plus six
caller delivery objects. The user-engine candidate patches only live baseline
0d0751d; 302 regression tests pass with 20 baseline optional skips and all six
packaged binding tests pass. Full email suite: 193 passed, zero skips.
Exact custody/activation contract: deploy/closure/plan.json; public preparation
evidence: deploy/closure/evidence-2026-10-09.json. Reviewed plan SHA256: 8e529e2ffdb6ee3d53a768d65b2d025b95e788734c48a29f2f8b975825dbaed7.
Attended caller custody, native reader proofs and coordinated scoped activation
remain open. T03 stays progress; live receiver and caller mode are unchanged.
Execution handoff: docs/caller-policy-draft-2026-10-08.md.