Prepare glob-scoped callers and prove offsite ledger recovery

Assistant: codex Assistant-Model: gpt-6.1-sol Assistant-Session: 01a11fd9-caf6-7a63-b578-9daaaa134ff2
2026-10-09 11:43:11 +02:00 · 2026-10-09 11:43:11 +02:00 · 93b6ba2aae
commit 93b6ba2aae
parent 6fb206b359
21 changed files with 1080 additions and 19 deletions
--- a/deploy/backup/email-connect-ledger-alert@.service
+++ b/deploy/backup/email-connect-ledger-alert@.service
@ -0,0 +1,18 @@
+[Unit]
+Description=Notify platform-root of email-connect ledger failure in %i
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=oneshot
+User=root
+UMask=0077
+StateDirectory=email-connect-backup
+StateDirectoryMode=0700
+ExecStart=/usr/bin/python3 /opt/email-connect-backup/notify-ledger-failure.py --unit %i
+TimeoutStartSec=2min
+NoNewPrivileges=true
+PrivateTmp=true
+ProtectSystem=strict
+ReadWritePaths=/var/lib/email-connect-backup
+ProtectHome=true
--- a/deploy/backup/email-connect-ledger-backup.service
+++ b/deploy/backup/email-connect-ledger-backup.service
@ -2,6 +2,7 @@
 Description=Coherent encrypted email-connect transactional ledger backup
 After=network-online.target k3s.service
 Wants=network-online.target
+OnFailure=email-connect-ledger-alert@%n.service

 [Service]
 Type=oneshot
--- a/deploy/backup/email-connect-ledger-freshness.service
+++ b/deploy/backup/email-connect-ledger-freshness.service
@ -1,10 +1,16 @@
 [Unit]
 Description=Check email-connect ledger archive age and local ciphertext hash
+OnFailure=email-connect-ledger-alert@%n.service

 [Service]
 Type=oneshot
 User=root
 ExecStart=/usr/bin/python3 /opt/email-connect-backup/backup-ledger.py freshness
+ExecStartPost=-/usr/bin/python3 /opt/email-connect-backup/notify-ledger-failure.py
+StateDirectory=email-connect-backup
+StateDirectoryMode=0700
+UMask=0077
+ReadWritePaths=/var/lib/email-connect-backup
 NoNewPrivileges=true
 ProtectSystem=strict
 ProtectHome=true
--- a/deploy/backup/runtime-manifest-2026-10-09.json
+++ b/deploy/backup/runtime-manifest-2026-10-09.json
@ -0,0 +1,29 @@
+{
+  "schema": "email-connect.ledger-runtime-bundle.v1",
+  "files": {
+    "email-connect-ledger-freshness.service": "a9ce8d1e92fadcda8bb125bf952c42f338514545b6a980cfd8be0dd95bc5eb06",
+    "backup-primary.lock": "7142d3690855a6fcfdd2dac02113e7a76b6731988d52b4b013a50d97570f2aaa",
+    "email-connect-ledger-backup.service": "e2f41257ac7df0aac6250822a794289946f4282e68ee61d751c2c84a1caeb1e5",
+    "notify-ledger-failure.py": "d5bc735963896b2e909660e678b8ed1b220ec8f1038014399c5b995a577c90b6",
+    "primary_application_archive.py": "a3f0a61e869a6012edb4b5288be14e3b0c4a9a1273b090320ab146d12e5aa94d",
+    "rehearse-ledger.py": "3ee32b035b6c6c2f7c994346d1db04622daeb9260b8ed723bc89661f3632eaba",
+    "backup-ledger.py": "8a89aa01c0c22e46352f1ad7a6f30b9b5db792639c1acc9a60579f50b7c8c369",
+    "capture-ledger.py": "df09d972be74db89a80f4899ef782f5412e655dfa12317a07cc7f35e9b1063da",
+    "email-connect-ledger-alert@.service": "4ee43ffab27478556c75153e89668a3bb36e6c56d6dd3b52d0db8d8603b9263c",
+    "email-connect-ledger-backup.timer": "ffb64bf39763c5e3d52cea5a13aa3fdffe243c3ff5a81555eb47048746527782",
+    "ledger_archive.py": "92d9d5e1908c7d433eb3f4ada6460f7d9b639017a4c2311cb7ffaacd262f33c3",
+    "email-connect-ledger-freshness.timer": "9103655972b13c25dcbb6a8d53f3076fe8b9b7eb04a91e85c1cd1a875bf3310f"
+  },
+  "receiver_image": "eb3e7b68d21209e9eb71b928a5b0bc987e8e3164f21dca4ff712f946d0c103f2",
+  "timers_enabled": false,
+  "failure_recipient": "platform-root",
+  "notification_delivery_verified": true,
+  "runtime_installed": true,
+  "sdk_hash_locked": true,
+  "hardened_preflight_verified": true,
+  "notification_message_id": "adcebf95-26c4-4adb-99b0-4446c8505a42",
+  "failure_hook_test_verified": true,
+  "freshness_verified": true,
+  "human_acknowledged": false,
+  "units_installed": true
+}
--- a/deploy/caller-rollout/packaged-evidence-2026-10-09.json
+++ b/deploy/caller-rollout/packaged-evidence-2026-10-09.json
@ -0,0 +1,13 @@
+{
+  "status": "packaged_caller_contract_verified",
+  "uid": 10001,
+  "mail_sent": false,
+  "legacy_unknown_blocked": true,
+  "legacy_accepted_preserved": true,
+  "glob_denials_verified": true,
+  "module_sha256": "06fe961b9030dba2c0b26b92f8097e8fbd24e205d5fa79642aa9af60382886dc",
+  "local_image": "email-connect:email-wp-0008-globs-20261009",
+  "image_id": "sha256:5d79fd1fec15d3a279383820f170d263de3a55f5308df01a0a34732ebaea6d3d",
+  "network": "disabled",
+  "runtime_activated": false
+}
--- a/deploy/caller-rollout/receiver-policy.patch.json
+++ b/deploy/caller-rollout/receiver-policy.patch.json
@ -0,0 +1,53 @@
+{
+  "spec": {
+    "template": {
+      "spec": {
+        "containers": [
+          {
+            "name": "email-connect",
+            "env": [
+              {
+                "name": "EMAIL_CONNECT_CALLERS_FILE",
+                "value": "/etc/email-connect-callers/callers.json"
+              }
+            ],
+            "volumeMounts": [
+              {
+                "name": "caller-policy",
+                "mountPath": "/etc/email-connect-callers",
+                "readOnly": true
+              },
+              {
+                "name": "caller-credentials",
+                "mountPath": "/var/run/secrets/email-connect-callers",
+                "readOnly": true
+              }
+            ]
+          }
+        ],
+        "volumes": [
+          {
+            "name": "caller-policy",
+            "configMap": {
+              "name": "email-connect-callers-user-engine-276257849bf2",
+              "defaultMode": 292
+            }
+          },
+          {
+            "name": "caller-credentials",
+            "secret": {
+              "secretName": "email-connect-caller-user-engine",
+              "defaultMode": 288,
+              "items": [
+                {
+                  "key": "token",
+                  "path": "user-engine"
+                }
+              ]
+            }
+          }
+        ]
+      }
+    }
+  }
+}
--- a/deploy/caller-rollout/user-engine-mail-bindings.patch
+++ b/deploy/caller-rollout/user-engine-mail-bindings.patch
@ -0,0 +1,171 @@
+--- /dev/null
+++ b/src/user_engine/mail_bindings.py
+@@ -0,0 +1,40 @@
+"""Explicit tenant globs for the scoped invitation mail transport."""
+
+from __future__ import annotations
+
+import json
+import re
+from fnmatch import fnmatchcase
+
+IDENTIFIER = re.compile(r"[A-Za-z0-9][A-Za-z0-9._:-]{0,127}\Z")
+PATTERN = re.compile(r"[A-Za-z0-9*?][A-Za-z0-9._:*?-]{0,127}\Z")
+
+
+def load_mail_client_bindings(raw):
+    if raw is None:
+        return None
+    try:
+        rows = json.loads(raw)
+        if not isinstance(rows, list) or not 1 <= len(rows) <= 128:
+            raise ValueError
+        result = []
+        for row in rows:
+            if (not isinstance(row, list) or len(row) != 2
+                    or not isinstance(row[0], str) or not PATTERN.fullmatch(row[0])
+                    or not isinstance(row[1], str) or not IDENTIFIER.fullmatch(row[1])):
+                raise ValueError
+            result.append(tuple(row))
+        if len({row[0] for row in result}) != len(result):
+            raise ValueError
+        return tuple(result)
+    except (ValueError, TypeError):
+        raise ValueError("mail_client_bindings_invalid") from None
+
+
+def mail_client_for(tenant, bindings):
+    if not isinstance(tenant, str) or not IDENTIFIER.fullmatch(tenant):
+        raise ValueError("mail_binding_denied")
+    matches = {client for pattern, client in bindings if fnmatchcase(tenant, pattern)}
+    if len(matches) != 1:
+        raise ValueError("mail_binding_denied")
+    return matches.pop()
+--- a/src/user_engine/adapters/delivery.py
+++ b/src/user_engine/adapters/delivery.py
+@@ -19,6 +19,7 @@
+ 
+ 
+ from user_engine.domain import OutboxEvent
+from user_engine.mail_bindings import load_mail_client_bindings, mail_client_for
+ 
+ _MAIL_EVENTS = {"family_member.invited", "family_invitation.resent"}
+ 
+@@ -34,12 +35,14 @@
+         mail_url: str | None = None,
+         mail_bearer_token: str | None = None,
+         timeout_seconds: float = 5.0,
+        mail_client_bindings: str | None = None,
+     ) -> None:
+         self.event_url = event_url
+         self.mail_url = mail_url
+         self.event_bearer_token = event_bearer_token
+         self.mail_bearer_token = mail_bearer_token
+         self.timeout_seconds = timeout_seconds
+        self.mail_client_bindings = load_mail_client_bindings(mail_client_bindings)
+ 
+     def __call__(self, event: OutboxEvent) -> None:
+         envelope = {
+@@ -57,7 +60,15 @@
+         if self.mail_url and event.event_type in _MAIL_EVENTS:
+             if not self.mail_bearer_token:
+                 raise RuntimeError("mail delivery token is required")
+-            self._post(self.mail_url, envelope, self.mail_bearer_token)
+            mail_envelope = dict(envelope)
+            if self.mail_client_bindings is not None:
+                try:
+                    mail_envelope["client_id"] = mail_client_for(
+                        event.tenant, self.mail_client_bindings
+                    )
+                except ValueError:
+                    raise DeliveryError("mail_binding_denied") from None
+            self._post(self.mail_url, mail_envelope, self.mail_bearer_token)
+         event_envelope = dict(envelope)
+         event_data = dict(envelope["data"])
+         if "primary_email" in event_data:
+--- a/src/user_engine/runtime.py
+++ b/src/user_engine/runtime.py
+@@ -65,6 +65,7 @@
+             else None
+         ),
+         timeout_seconds=float(os.environ.get("USER_ENGINE_DELIVERY_TIMEOUT", "5")),
+        mail_client_bindings=os.environ.get("USER_ENGINE_MAIL_CLIENT_BINDINGS"),
+     )
+     app = PortalApplication(
+         service,
+--- /dev/null
+++ b/tests/test_mail_bindings.py
+@@ -0,0 +1,73 @@
+"""Scoped mail migration uses synthetic identities and a mocked HTTP boundary."""
+
+import json
+import unittest
+from unittest.mock import patch
+
+from user_engine.adapters.delivery import HTTPOutboxDeliveryAdapter, DeliveryError
+from user_engine.domain import OutboxEvent
+from user_engine.mail_bindings import load_mail_client_bindings, mail_client_for
+
+
+class MailBindingsTests(unittest.TestCase):
+    def test_whole_case_sensitive_globs_and_literal_clients(self):
+        bindings = load_mail_client_bindings('[ ["tenant:*-family", "coulomb-social"] ]')
+        self.assertEqual(mail_client_for("tenant:synthetic-family", bindings), "coulomb-social")
+        for tenant in ("tenant:platform", "tenant:Synthetic-Family", "tenant:x-family/path"):
+            with self.assertRaisesRegex(ValueError, "mail_binding_denied"):
+                mail_client_for(tenant, bindings)
+
+    def test_invalid_configuration_never_falls_back(self):
+        for raw in ('', '[]', '{}', '[["*", "*"]]', '[["[a]", "client"]]',
+                    '[["*", "one"], ["*", "two"]]', '[[true, "client"]]'):
+            with self.assertRaisesRegex(ValueError, "mail_client_bindings_invalid"):
+                load_mail_client_bindings(raw)
+        self.assertIsNone(load_mail_client_bindings(None))
+
+    def test_conflicting_overlaps_are_denied(self):
+        bindings = load_mail_client_bindings('[["tenant:*", "one"], ["tenant:x", "two"]]')
+        with self.assertRaisesRegex(ValueError, "mail_binding_denied"):
+            mail_client_for("tenant:x", bindings)
+
+    def adapter(self, bindings=None):
+        return HTTPOutboxDeliveryAdapter(event_url="http://events.example.test",
+            event_bearer_token="synthetic-event-token", mail_url="http://mail.example.test",
+            mail_bearer_token="synthetic-mail-token", mail_client_bindings=bindings)
+
+    def event(self, kind="family_member.invited", tenant="tenant:synthetic-family"):
+        return OutboxEvent(event_id="evt-synthetic", event_type=kind, aggregate_id="inv-synthetic",
+            payload={"primary_email": "member@example.test", "invitation_id": "inv-synthetic"},
+            tenant=tenant, correlation_id="trace-synthetic")
+
+    def test_binding_only_changes_mail_transport_not_event_or_outbox(self):
+        event = self.event()
+        adapter = self.adapter('[["tenant:*-family", "coulomb-social"]]')
+        with patch.object(adapter, "_post") as post:
+            adapter(event)
+        mail, audit = [call.args[1] for call in post.call_args_list]
+        self.assertEqual(mail["client_id"], "coulomb-social")
+        self.assertNotIn("client_id", audit)
+        self.assertNotIn("primary_email", audit["data"])
+        self.assertNotIn("client_id", event.payload)
+        self.assertEqual(mail["id"], event.event_id)
+
+    def test_legacy_mode_and_non_mail_events_preserve_envelopes(self):
+        adapter = self.adapter()
+        with patch.object(adapter, "_post") as post:
+            adapter(self.event())
+        self.assertNotIn("client_id", post.call_args_list[0].args[1])
+        adapter = self.adapter('[["tenant:*-family", "coulomb-social"]]')
+        with patch.object(adapter, "_post") as post:
+            adapter(self.event("membership.added", "tenant:platform"))
+        self.assertEqual(post.call_count, 1)
+
+    def test_unadmitted_mail_tenant_contacts_neither_lane(self):
+        adapter = self.adapter('[["tenant:*-family", "coulomb-social"]]')
+        with patch.object(adapter, "_post") as post:
+            with self.assertRaisesRegex(DeliveryError, "mail_binding_denied"):
+                adapter(self.event(tenant="tenant:platform"))
+        post.assert_not_called()
+
+
+if __name__ == "__main__":
+    unittest.main()
--- a/deploy/caller-rollout/user-engine-policy.configmap.json
+++ b/deploy/caller-rollout/user-engine-policy.configmap.json
@ -0,0 +1,12 @@
+{
+  "apiVersion": "v1",
+  "kind": "ConfigMap",
+  "immutable": true,
+  "metadata": {
+    "name": "email-connect-callers-user-engine-276257849bf2",
+    "namespace": "email-connect"
+  },
+  "data": {
+    "callers.json": "{\n  \"callers\": [\n    {\n      \"adopt_legacy_invitations\": true,\n      \"adopt_legacy_verifications\": true,\n      \"binding_patterns\": [\n        [\n          \"tenant:*-family\",\n          \"coulomb-social\"\n        ],\n        [\n          \"tenant:family-*\",\n          \"coulomb-social\"\n        ]\n      ],\n      \"bindings\": [\n        [\n          \"tenant:coulomb\",\n          \"coulomb-social\"\n        ]\n      ],\n      \"burst\": 5,\n      \"caller_id\": \"user-engine\",\n      \"max_payload_bytes\": 8192,\n      \"operations\": [\n        \"send\",\n        \"delivery-status\",\n        \"status\",\n        \"verify\",\n        \"consume\",\n        \"cancel\"\n      ],\n      \"rate_per_minute\": 10,\n      \"source\": \"user-engine\",\n      \"templates\": [\n        \"family_member.invited\",\n        \"family_invitation.resent\"\n      ],\n      \"token_file\": \"/var/run/secrets/email-connect-callers/user-engine\"\n    }\n  ],\n  \"schema_version\": 1\n}\n"
+  }
+}
--- a/deploy/caller-rollout/verify-packaged.py
+++ b/deploy/caller-rollout/verify-packaged.py
@ -0,0 +1,59 @@
+"""Check the packaged caller module and transition without credentials or mail."""
+
+import hashlib
+import io
+import json
+import os
+import sys
+from pathlib import Path
+
+import email_connect.callers as callers
+from email_connect.transactional import SQLiteDeliveryStore
+
+assert os.getuid() == 10001
+assert hashlib.sha256(Path(callers.__file__).read_bytes()).hexdigest() == sys.argv[1]
+assert len(Path('/proc/net/route').read_text().splitlines()) == 1
+token = "synthetic-packaged-caller-token"
+Path('/tmp/token').write_text(token)
+document = json.loads(Path('/review/config/callers.user-engine.draft.json').read_text())
+document['callers'][0]['token_file'] = '/tmp/token'
+Path('/tmp/policy.json').write_text(json.dumps(document))
+policies = callers.load_callers('/tmp/policy.json')
+assert ('tenant:synthetic-family', 'coulomb-social') in policies[0]
+assert ('tenant:platform', 'coulomb-social') not in policies[0]
+store = SQLiteDeliveryStore('/tmp/ledger.sqlite3')
+
+
+class ForbiddenProvider:
+    def send(self, *args):
+        raise AssertionError('provider_call_forbidden')
+
+
+app = callers.ScopedTransactionalApplication(store, ForbiddenProvider(),
+                                             'https://issuer.example.test', policies)
+old = {'id': 'evt-synthetic', 'source': 'user-engine', 'type': 'family_member.invited',
+       'tenant': 'tenant:synthetic-family',
+       'data': {'primary_email': 'member@example.test', 'invitation_id': 'inv-synthetic'}}
+fingerprint = hashlib.sha256(json.dumps(old, sort_keys=True, separators=(',', ':')).encode()).hexdigest()
+store.reserve_send(old['id'], fingerprint)
+
+
+def invoke(payload):
+    raw = json.dumps(payload).encode()
+    response = {}
+    body = json.loads(b''.join(app({'PATH_INFO': '/v1/send', 'REQUEST_METHOD': 'POST',
+        'CONTENT_LENGTH': str(len(raw)), 'wsgi.input': io.BytesIO(raw),
+        'HTTP_AUTHORIZATION': 'Bearer ' + token, 'HTTP_IDEMPOTENCY_KEY': payload['id']},
+        lambda status, headers: response.update(status=int(status.split()[0])))))
+    return response['status'], body
+
+
+bound = dict(old, client_id='coulomb-social')
+assert invoke(bound)[1]['error'] == 'delivery_outcome_unknown'
+store.record(old['id'], 'synthetic-old-reference')
+assert invoke(bound)[1]['status'] == 'duplicate'
+assert invoke(dict(bound, tenant='tenant:platform'))[1]['error'] == 'caller_scope_denied'
+assert invoke(dict(bound, data=dict(old['data'], primary_email='changed@example.test')))[1]['error'] == 'idempotency_payload_mismatch'
+print(json.dumps({'status': 'packaged_caller_contract_verified', 'uid': os.getuid(),
+                  'mail_sent': False, 'legacy_unknown_blocked': True,
+                  'legacy_accepted_preserved': True, 'glob_denials_verified': True}))