coulomb/email-connect
5
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Prepare immutable caller rollout and independent Scaleway recovery custody
Some checks failed
Transactional mail acceptance / mail (push) Failing after 32s
Details
CI Smoke / host-smoke (push) Successful in 0s
Details
CI Smoke / container-smoke (push) Successful in 1s
Details

Browse source 
Assistant: codex
Assistant-Model: gpt-6.1-sol
Assistant-Session: 01a11fd9-caf6-7a63-b578-9daaaa134ff2
This commit is contained in:
tegwick 2026-10-09 19:49:24 +02:00
parent 3e18f11afe
commit 9a2cf213d1
19 changed files with 1543 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

170
deploy/closure/caller-delivery.json Normal file
View file

@ -0,0 +1,170 @@
{
"apiVersion": "v1",
"kind": "List",
"items": [
{
"apiVersion": "v1",
"kind": "ServiceAccount",
"metadata": {
"name": "email-connect-scoped-caller-eso",
"namespace": "email-connect"
},
"automountServiceAccountToken": false
},
{
"apiVersion": "external-secrets.io/v1",
"kind": "ClusterSecretStore",
"metadata": {
"name": "openbao-email-connect-scoped-caller"
},
"spec": {
"conditions": [
{
"namespaces": [
"email-connect"
]
}
],
"provider": {
"vault": {
"server": "http://openbao.openbao.svc:8200",
"path": "platform",
"version": "v2",
"auth": {
"kubernetes": {
"mountPath": "kubernetes",
"role": "email-connect-scoped-caller-eso",
"serviceAccountRef": {
"name": "email-connect-scoped-caller-eso",
"namespace": "email-connect",
"audiences": [
"openbao"
]
}
}
}
}
}
}
},
{
"apiVersion": "external-secrets.io/v1",
"kind": "ExternalSecret",
"metadata": {
"name": "email-connect-caller-user-engine",
"namespace": "email-connect"
},
"spec": {
"refreshInterval": "5m",
"secretStoreRef": {
"kind": "ClusterSecretStore",
"name": "openbao-email-connect-scoped-caller"
},
"target": {
"name": "email-connect-caller-user-engine",
"creationPolicy": "Owner",
"deletionPolicy": "Retain",
"template": {
"metadata": {
"labels": {
"email-connect.io/caller": "user-engine"
},
"annotations": {}
}
}
},
"data": [
{
"secretKey": "token",
"remoteRef": {
"key": "platform/workloads/email-connect/callers/user-engine",
"property": "TOKEN"
}
}
]
}
},
{
"apiVersion": "v1",
"kind": "ServiceAccount",
"metadata": {
"name": "user-engine-scoped-mail-eso",
"namespace": "user-engine"
},
"automountServiceAccountToken": false
},
{
"apiVersion": "external-secrets.io/v1",
"kind": "ClusterSecretStore",
"metadata": {
"name": "openbao-user-engine-scoped-mail"
},
"spec": {
"conditions": [
{
"namespaces": [
"user-engine"
]
}
],
"provider": {
"vault": {
"server": "http://openbao.openbao.svc:8200",
"path": "platform",
"version": "v2",
"auth": {
"kubernetes": {
"mountPath": "kubernetes",
"role": "user-engine-scoped-mail-eso",
"serviceAccountRef": {
"name": "user-engine-scoped-mail-eso",
"namespace": "user-engine",
"audiences": [
"openbao"
]
}
}
}
}
}
}
},
{
"apiVersion": "external-secrets.io/v1",
"kind": "ExternalSecret",
"metadata": {
"name": "user-engine-scoped-mail",
"namespace": "user-engine"
},
"spec": {
"refreshInterval": "5m",
"secretStoreRef": {
"kind": "ClusterSecretStore",
"name": "openbao-user-engine-scoped-mail"
},
"target": {
"name": "user-engine-scoped-mail",
"creationPolicy": "Owner",
"deletionPolicy": "Retain",
"template": {
"metadata": {
"labels": {
"email-connect.io/caller": "user-engine"
},
"annotations": {}
}
}
},
"data": [
{
"secretKey": "token",
"remoteRef": {
"key": "platform/workloads/email-connect/callers/user-engine",
"property": "TOKEN"
}
}
]
}
}
]
}

6
deploy/closure/caller-reader.hcl Normal file
View file

@ -0,0 +1,6 @@
path "platform/data/workloads/email-connect/callers/user-engine" {
capabilities = ["read"]
}
path "auth/token/lookup-self" { capabilities = ["read"] }
path "auth/token/revoke-self" { capabilities = ["update"] }

10
deploy/closure/caller-writer.hcl Normal file
View file

@ -0,0 +1,10 @@
path "platform/data/workloads/email-connect/callers/user-engine" {
capabilities = ["create", "read", "update"]
}
path "auth/token/lookup-self" { capabilities = ["read"] }
path "auth/token/revoke-self" { capabilities = ["update"] }
path "platform/metadata/workloads/email-connect/callers/user-engine" {
capabilities = ["read"]
}

63
deploy/closure/evidence-2026-10-09.json Normal file
View file

@ -0,0 +1,63 @@
{
"schema": "email-connect.closure-preparation.v1",
"date": "2026-10-09",
"status": "prepared_not_activated",
"plan_sha256": "8e529e2ffdb6ee3d53a768d65b2d025b95e788734c48a29f2f8b975825dbaed7",
"receiver_image": "forgejo.coulomb.social/coulomb/email-connect@sha256:5d79fd1fec15d3a279383820f170d263de3a55f5308df01a0a34732ebaea6d3d",
"user_engine_image": "forgejo.coulomb.social/coulomb/user-engine@sha256:312dcb062b237c518148e1b4f09798a4e6ad08bc60d4add6834aab3f246f2002",
"user_engine_baseline": "0d0751d",
"user_engine_patch_source": "adf6e99",
"tests": {
"email_connect_passed": 193,
"email_connect_skipped": 0,
"user_engine_baseline_passed": 302,
"user_engine_baseline_skipped": 20,
"packaged_binding_tests_passed": 6,
"focused_ruff": "passed"
},
"cluster_server_dry_run": {
"caller_delivery_objects": 6,
"receiver_patch": "passed",
"user_engine_patch": "passed"
},
"primary_bucket_metadata": {
"versioning": "Enabled",
"current_expiration_days": 30,
"noncurrent_expiration_days": 30,
"lifecycle_filter_prefix": "",
"provider_mutations": false
},
"recovery_kit": {
"status": "recovery_kit_prepared",
"kit_sha256": "97d1a4d4b55729dd226277601637d64b086de21b21c9951c1d7da05c914c350b",
"kit_bytes": 47541825,
"image": "forgejo.coulomb.social/coulomb/email-connect@sha256:eb3e7b68d21209e9eb71b928a5b0bc987e8e3164f21dca4ff712f946d0c103f2",
"image_id": "sha256:eb3e7b68d21209e9eb71b928a5b0bc987e8e3164f21dca4ff712f946d0c103f2",
"private_identity_included": false,
"provider_upload_verified": false,
"protected_version": "1791537739370570"
},
"offline_recovery": {
"status": "application_restored_verified",
"mail_sent": false,
"production_restored": false,
"application_restore_verified": true,
"network": "disabled",
"uid": 10001,
"offline_image_binding_verified": true,
"elapsed_seconds": 2.31
},
"live_changes": {
"caller_credentials_issued": false,
"new_images_deployed": false,
"scaleway_escrow_created": false,
"backup_timers_enabled": false,
"node_helpers_updated": false
},
"remaining": [
"Attended caller custody and live reader boundary proofs",
"Coordinated scoped rollout and no-send continuity/denial proofs",
"Independent Scaleway identity/kit custody with separate recovery reader and effective denial proofs",
"Schedule and retention admission; recurring operation/freshness evidence"
]
}

147
deploy/closure/plan.json Normal file
View file

@ -0,0 +1,147 @@
{
"schema": "email-connect.closure-plan.v1",
"status": "prepared",
"existing_backup_publisher": {
"application": "email-connect-backup-primary",
"application_id": "0eb0f958-77a4-4c70-9aa7-14ba46411415",
"bucket": "railiance-platform-pg-backup",
"prefix": "platform-pg/application-archives/email-connect/",
"retain_for_routine_backups": true,
"escrow_or_administration_grant_inferred": false
},
"receiver_image": "forgejo.coulomb.social/coulomb/email-connect@sha256:5d79fd1fec15d3a279383820f170d263de3a55f5308df01a0a34732ebaea6d3d",
"user_engine_image": "forgejo.coulomb.social/coulomb/user-engine@sha256:312dcb062b237c518148e1b4f09798a4e6ad08bc60d4add6834aab3f246f2002",
"user_engine_live_baseline": "0d0751d",
"user_engine_patch_sha256": "1fd705f1917f6c14cce69ae75b53ccbf1a2d9eee45561a72ae524a497d9f036e",
"caller_policy": {
"schema_version": 1,
"callers": [
{
"caller_id": "user-engine",
"source": "user-engine",
"token_file": "/var/run/secrets/email-connect-callers/user-engine",
"bindings": [
[
"tenant:coulomb",
"coulomb-social"
]
],
"binding_patterns": [
[
"tenant:*-family",
"coulomb-social"
],
[
"tenant:family-*",
"coulomb-social"
]
],
"operations": [
"send",
"delivery-status",
"status",
"verify",
"consume",
"cancel"
],
"templates": [
"family_member.invited",
"family_invitation.resent"
],
"rate_per_minute": 10,
"burst": 5,
"max_payload_bytes": 8192,
"adopt_legacy_verifications": true,
"adopt_legacy_invitations": true
}
]
},
"caller_custody": {
"kv_path": "platform/workloads/email-connect/callers/user-engine",
"field": "TOKEN",
"initial_cas": 0,
"writer_policy": "email-connect-scoped-caller-writer",
"reader_policy": "email-connect-scoped-caller-reader",
"readers": [
{
"namespace": "email-connect",
"service_account": "email-connect-scoped-caller-eso",
"role": "email-connect-scoped-caller-eso",
"store": "openbao-email-connect-scoped-caller",
"secret": "email-connect-caller-user-engine",
"secret_key": "token"
},
{
"namespace": "user-engine",
"service_account": "user-engine-scoped-mail-eso",
"role": "user-engine-scoped-mail-eso",
"store": "openbao-user-engine-scoped-mail",
"secret": "user-engine-scoped-mail",
"secret_key": "token"
}
]
},
"backup": {
"schedule_utc": "04:15",
"jitter_seconds": 300,
"rpo_target_hours": 24,
"stale_threshold_hours": 26,
"attended_rto_target_minutes": 60,
"minimum_node_free_bytes": 8589934592,
"local_budget_bytes": 8589934592,
"daily_generations": 14,
"weekly_generations": 4,
"provider_archive_retention_days": 30,
"retained_recovery_point_copied_to_escrow_kit": true,
"automatic_retained_object_deletion": false,
"failure_recipient": "platform-root",
"retention_operator": "platform-root",
"retention_review_interval": "weekly",
"retention_execution": "Separate owner-operated expiry; publisher never deletes retained objects",
"incident_restore_requires_fencing_and_post_capture_reconciliation": true
},
"scaleway_recovery": {
"region": "nl-ams",
"secret_manager_name": "email-connect-ledger-age-recovery",
"secret_manager_protected": true,
"secret_manager_ephemeral_policy": null,
"kit_bucket": "railiance-email-connect-recovery",
"kit_prefix": "kits/",
"kit_lifecycle": null,
"kit_versioning": "Enabled",
"kit_contents": [
"age-encrypted-ledger",
"public-transfer-and-recovery-receipts",
"pinned-receiver-image",
"recovery-tools",
"SHA256SUMS"
],
"private_age_identity_in_kit": false,
"scheduled_worker_escrow_access": false,
"recovery_caller": "Separate read-only IAM application; effective prefix/bucket denial proofs required",
"independent_entry": "platform-root Scaleway account; credentials held only on railiance01 do not qualify"
},
"activation_order": [
"Approve exact plan and perform attended caller custody and Scaleway admission",
"Deliver two bounded ESO projections and privately verify matching caller credentials",
"Freeze user-engine dispatch, capture current ledger and resume receiver",
"Apply coordinated immutable receiver/user-engine images and scoped policy",
"Verify health, denial, limits, accepted/unknown continuity and original handles without mail",
"Repin capture/rehearsal together; refresh and independently read back recovery kit",
"Enable daily backup/hourly freshness after independent custody and retention proofs",
"Finish workplans only after all admitted custody and operating proofs are present"
],
"artifacts": {
"deploy/closure/caller-reader.hcl": "cbc57e731c4fd9dbbaf98364de8ff87ed2966df18c2235cf298ffda963bf7553",
"deploy/closure/caller-writer.hcl": "a56013092fa6f0dff9282676ba1e94d5bbf37c2dc481232188f68a1ce543510c",
"deploy/closure/caller-delivery.json": "6fed12e204533ec679e80b3d9d0ba814cbf1d6f91a5a02211f0334617a92b0eb",
"deploy/closure/receiver.patch.json": "449ce8456323347eca81190e2ccca334c90397c6699498466308bf8bfe63a9de",
"deploy/closure/user-engine.patch.json": "2cae6ea6259ac1031fb7b8b11f011686baad8cb576cd72ee3f74461924daef46",
"scripts/admit-scoped-caller.py": "592bd46740f9eea4ad1ae78aa674f6c381ce05ab6044cedf0af968a1c340adf2",
"scripts/scaleway-recovery-custody.py": "85874f0620dc9353a9aaae1194db054de166296c4385743e4d541bbe42486c15",
"scripts/publish-ledger-recovery-kit.py": "cd7c6e8e6a4e8417ba55f8170d763d7c5a9e6dd9b5878dc3459021a0b64ea732",
"scripts/prepare-ledger-recovery-kit.py": "db92a29eb68e1b555cf18b17229d1e479b3147da7600ffa0616ac8b4d011ff55",
"scripts/rehearse-ledger.py": "d3da88972e44dac3800afb48d21e525dc1476ca58de3e7702f7751aea33eda53",
"scripts/backup-ledger.py": "e74c41948ca01dae09578e2abfba14b96fb1f158af0ec91a0fb1aa9ecaf01680"
}
}

136
deploy/closure/prepare.py Normal file
View file

@ -0,0 +1,136 @@
"""Render public closure artifacts. No credentials, network calls or activation."""
import hashlib
import json
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
OUT = Path(__file__).resolve().parent
def save(name, value):
(OUT / name).write_text(json.dumps(value, indent=2) + "\n")
policy = json.loads((ROOT / "config/callers.user-engine.draft.json").read_text())
readers = [
dict(namespace="email-connect", service_account="email-connect-scoped-caller-eso",
role="email-connect-scoped-caller-eso", store="openbao-email-connect-scoped-caller",
secret="email-connect-caller-user-engine", secret_key="token"),
dict(namespace="user-engine", service_account="user-engine-scoped-mail-eso",
role="user-engine-scoped-mail-eso", store="openbao-user-engine-scoped-mail",
secret="user-engine-scoped-mail", secret_key="token"),
]
plan = {
"schema": "email-connect.closure-plan.v1", "status": "prepared",
"existing_backup_publisher": {
"application": "email-connect-backup-primary",
"application_id": "0eb0f958-77a4-4c70-9aa7-14ba46411415",
"bucket": "railiance-platform-pg-backup",
"prefix": "platform-pg/application-archives/email-connect/",
"retain_for_routine_backups": True,
"escrow_or_administration_grant_inferred": False,
},
"receiver_image": "forgejo.coulomb.social/coulomb/email-connect@sha256:5d79fd1fec15d3a279383820f170d263de3a55f5308df01a0a34732ebaea6d3d",
"user_engine_image": "forgejo.coulomb.social/coulomb/user-engine@sha256:312dcb062b237c518148e1b4f09798a4e6ad08bc60d4add6834aab3f246f2002",
"user_engine_live_baseline": "0d0751d",
"user_engine_patch_sha256": hashlib.sha256((ROOT / "deploy/caller-rollout/user-engine-mail-bindings.patch").read_bytes()).hexdigest(),
"caller_policy": policy,
"caller_custody": {
"kv_path": "platform/workloads/email-connect/callers/user-engine", "field": "TOKEN",
"initial_cas": 0, "writer_policy": "email-connect-scoped-caller-writer",
"reader_policy": "email-connect-scoped-caller-reader", "readers": readers,
},
"backup": {
"schedule_utc": "04:15", "jitter_seconds": 300, "rpo_target_hours": 24,
"stale_threshold_hours": 26, "attended_rto_target_minutes": 60,
"minimum_node_free_bytes": 8 * 1024**3, "local_budget_bytes": 8 * 1024**3,
"daily_generations": 14, "weekly_generations": 4, "provider_archive_retention_days": 30,
"retained_recovery_point_copied_to_escrow_kit": True,
"automatic_retained_object_deletion": False, "failure_recipient": "platform-root",
"retention_operator": "platform-root", "retention_review_interval": "weekly",
"retention_execution": "Separate owner-operated expiry; publisher never deletes retained objects",
"incident_restore_requires_fencing_and_post_capture_reconciliation": True,
},
"scaleway_recovery": {
"region": "nl-ams", "secret_manager_name": "email-connect-ledger-age-recovery",
"secret_manager_protected": True,
"secret_manager_ephemeral_policy": None, "kit_bucket": "railiance-email-connect-recovery",
"kit_prefix": "kits/", "kit_lifecycle": None, "kit_versioning": "Enabled",
"kit_contents": ["age-encrypted-ledger", "public-transfer-and-recovery-receipts",
"pinned-receiver-image", "recovery-tools", "SHA256SUMS"],
"private_age_identity_in_kit": False, "scheduled_worker_escrow_access": False,
"recovery_caller": "Separate read-only IAM application; effective prefix/bucket denial proofs required",
"independent_entry": "platform-root Scaleway account; credentials held only on railiance01 do not qualify",
},
"activation_order": [
"Approve exact plan and perform attended caller custody and Scaleway admission",
"Deliver two bounded ESO projections and privately verify matching caller credentials",
"Freeze user-engine dispatch, capture current ledger and resume receiver",
"Apply coordinated immutable receiver/user-engine images and scoped policy",
"Verify health, denial, limits, accepted/unknown continuity and original handles without mail",
"Repin capture/rehearsal together; refresh and independently read back recovery kit",
"Enable daily backup/hourly freshness after independent custody and retention proofs",
"Finish workplans only after all admitted custody and operating proofs are present",
],
}
save("plan.json", plan)
reader = '''path "platform/data/workloads/email-connect/callers/user-engine" {
capabilities = ["read"]
}
path "auth/token/lookup-self" { capabilities = ["read"] }
path "auth/token/revoke-self" { capabilities = ["update"] }
'''
writer = reader.replace('capabilities = ["read"]', 'capabilities = ["create", "read", "update"]', 1)
writer += '''
path "platform/metadata/workloads/email-connect/callers/user-engine" {
capabilities = ["read"]
}
'''
(OUT / "caller-reader.hcl").write_text(reader)
(OUT / "caller-writer.hcl").write_text(writer)
items = []
for row in readers:
ns, sa = row["namespace"], row["service_account"]
items.append({"apiVersion": "v1", "kind": "ServiceAccount",
"metadata": {"name": sa, "namespace": ns}, "automountServiceAccountToken": False})
vault = {
"server": "http://openbao.openbao.svc:8200", "path": "platform", "version": "v2",
"auth": {"kubernetes": {"mountPath": "kubernetes", "role": row["role"],
"serviceAccountRef": {"name": sa, "namespace": ns, "audiences": ["openbao"]}}},
}
items.append({"apiVersion": "external-secrets.io/v1", "kind": "ClusterSecretStore",
"metadata": {"name": row["store"]},
"spec": {"conditions": [{"namespaces": [ns]}], "provider": {"vault": vault}}})
items.append({
"apiVersion": "external-secrets.io/v1", "kind": "ExternalSecret",
"metadata": {"name": row["secret"], "namespace": ns},
"spec": {
"refreshInterval": "5m", "secretStoreRef": {"kind": "ClusterSecretStore", "name": row["store"]},
"target": {"name": row["secret"], "creationPolicy": "Owner", "deletionPolicy": "Retain",
"template": {"metadata": {"labels": {"email-connect.io/caller": "user-engine"}, "annotations": {}}}},
"data": [{"secretKey": row["secret_key"],
"remoteRef": {"key": plan["caller_custody"]["kv_path"], "property": "TOKEN"}}],
},
})
save("caller-delivery.json", {"apiVersion": "v1", "kind": "List", "items": items})
user = {"name": "portal", "image": plan["user_engine_image"], "env": [
{"name": "USER_ENGINE_MAIL_CLIENT_BINDINGS", "value": json.dumps(json.loads((ROOT / "config/mail-client-bindings.user-engine.draft.json").read_text()), separators=(",", ":"))},
*[{"name": name, "valueFrom": {"secretKeyRef": {"name": "user-engine-scoped-mail", "key": "token"}}}
for name in ("USER_ENGINE_MAIL_TOKEN", "USER_ENGINE_REGISTRATION_VERIFICATION_TOKEN")],
]}
save("user-engine.patch.json", {"spec": {"template": {"spec": {"containers": [user]}}}})
receiver = json.loads((ROOT / "deploy/caller-rollout/receiver-policy.patch.json").read_text())
receiver["spec"]["template"]["spec"]["containers"][0]["image"] = plan["receiver_image"]
save("receiver.patch.json", receiver)
artifacts = [
"deploy/closure/caller-reader.hcl", "deploy/closure/caller-writer.hcl",
"deploy/closure/caller-delivery.json", "deploy/closure/receiver.patch.json",
"deploy/closure/user-engine.patch.json", "scripts/admit-scoped-caller.py",
"scripts/scaleway-recovery-custody.py", "scripts/publish-ledger-recovery-kit.py",
"scripts/prepare-ledger-recovery-kit.py", "scripts/rehearse-ledger.py",
"scripts/backup-ledger.py",
]
plan["artifacts"] = {name: hashlib.sha256((ROOT / name).read_bytes()).hexdigest() for name in artifacts}
save("plan.json", plan)

54
deploy/closure/receiver.patch.json Normal file
View file

@ -0,0 +1,54 @@
{
"spec": {
"template": {
"spec": {
"containers": [
{
"name": "email-connect",
"env": [
{
"name": "EMAIL_CONNECT_CALLERS_FILE",
"value": "/etc/email-connect-callers/callers.json"
}
],
"volumeMounts": [
{
"name": "caller-policy",
"mountPath": "/etc/email-connect-callers",
"readOnly": true
},
{
"name": "caller-credentials",
"mountPath": "/var/run/secrets/email-connect-callers",
"readOnly": true
}
],
"image": "forgejo.coulomb.social/coulomb/email-connect@sha256:5d79fd1fec15d3a279383820f170d263de3a55f5308df01a0a34732ebaea6d3d"
}
],
"volumes": [
{
"name": "caller-policy",
"configMap": {
"name": "email-connect-callers-user-engine-276257849bf2",
"defaultMode": 292
}
},
{
"name": "caller-credentials",
"secret": {
"secretName": "email-connect-caller-user-engine",
"defaultMode": 288,
"items": [
{
"key": "token",
"path": "user-engine"
}
]
}
}
]
}
}
}
}

38
deploy/closure/user-engine.patch.json Normal file
View file

@ -0,0 +1,38 @@
{
"spec": {
"template": {
"spec": {
"containers": [
{
"name": "portal",
"image": "forgejo.coulomb.social/coulomb/user-engine@sha256:312dcb062b237c518148e1b4f09798a4e6ad08bc60d4add6834aab3f246f2002",
"env": [
{
"name": "USER_ENGINE_MAIL_CLIENT_BINDINGS",
"value": "[[\"tenant:coulomb\",\"coulomb-social\"],[\"tenant:*-family\",\"coulomb-social\"],[\"tenant:family-*\",\"coulomb-social\"]]"
},
{
"name": "USER_ENGINE_MAIL_TOKEN",
"valueFrom": {
"secretKeyRef": {
"name": "user-engine-scoped-mail",
"key": "token"
}
}
},
{
"name": "USER_ENGINE_REGISTRATION_VERIFICATION_TOKEN",
"valueFrom": {
"secretKeyRef": {
"name": "user-engine-scoped-mail",
"key": "token"
}
}
}
]
}
]
}
}
}
}