From 93b6ba2aaeb4c7aa04d8011fb4b4594869f9234f Mon Sep 17 00:00:00 2001 From: tegwick Date: Fri, 9 Oct 2026 11:43:11 +0200 Subject: [PATCH] Prepare glob-scoped callers and prove offsite ledger recovery Assistant: codex Assistant-Model: gpt-6.1-sol Assistant-Session: 01a11fd9-caf6-7a63-b578-9daaaa134ff2 --- config/callers.user-engine.draft.json | 7 +- ...ail-client-bindings.user-engine.draft.json | 5 + .../email-connect-ledger-alert@.service | 18 ++ .../email-connect-ledger-backup.service | 1 + .../email-connect-ledger-freshness.service | 6 + .../backup/runtime-manifest-2026-10-09.json | 29 +++ .../packaged-evidence-2026-10-09.json | 13 ++ .../caller-rollout/receiver-policy.patch.json | 53 ++++++ .../user-engine-mail-bindings.patch | 171 ++++++++++++++++++ .../user-engine-policy.configmap.json | 12 ++ deploy/caller-rollout/verify-packaged.py | 59 ++++++ docs/caller-policy-draft-2026-10-08.md | 46 +++++ docs/ledger-backup-review-2026-10-08.md | 38 ++++ docs/ledger-first-backup-2026-10-09.json | 131 ++++++++++++++ scripts/notify-ledger-failure.py | 100 ++++++++++ src/email_connect/callers.py | 80 ++++++-- tests/test_caller_policy_draft.py | 24 +++ tests/test_callers.py | 149 +++++++++++++++ tests/test_ledger_notifications.py | 67 +++++++ .../EMAIL-WP-0007-ledger-backup-recovery.md | 40 +++- ...IL-WP-0008-scoped-transactional-callers.md | 50 ++++- 21 files changed, 1080 insertions(+), 19 deletions(-) create mode 100644 config/mail-client-bindings.user-engine.draft.json create mode 100644 deploy/backup/email-connect-ledger-alert@.service create mode 100644 deploy/backup/runtime-manifest-2026-10-09.json create mode 100644 deploy/caller-rollout/packaged-evidence-2026-10-09.json create mode 100644 deploy/caller-rollout/receiver-policy.patch.json create mode 100644 deploy/caller-rollout/user-engine-mail-bindings.patch create mode 100644 deploy/caller-rollout/user-engine-policy.configmap.json create mode 100644 deploy/caller-rollout/verify-packaged.py create mode 100644 docs/ledger-first-backup-2026-10-09.json create mode 100644 scripts/notify-ledger-failure.py create mode 100644 tests/test_ledger_notifications.py diff --git a/config/callers.user-engine.draft.json b/config/callers.user-engine.draft.json index bfb6bb1..df27544 100644 --- a/config/callers.user-engine.draft.json +++ b/config/callers.user-engine.draft.json @@ -8,6 +8,10 @@ "bindings": [ ["tenant:coulomb", "coulomb-social"] ], + "binding_patterns": [ + ["tenant:*-family", "coulomb-social"], + ["tenant:family-*", "coulomb-social"] + ], "operations": [ "send", "delivery-status", @@ -23,7 +27,8 @@ "rate_per_minute": 10, "burst": 5, "max_payload_bytes": 8192, - "adopt_legacy_verifications": true + "adopt_legacy_verifications": true, + "adopt_legacy_invitations": true } ] } diff --git a/config/mail-client-bindings.user-engine.draft.json b/config/mail-client-bindings.user-engine.draft.json new file mode 100644 index 0000000..7060529 --- /dev/null +++ b/config/mail-client-bindings.user-engine.draft.json @@ -0,0 +1,5 @@ +[ + ["tenant:coulomb", "coulomb-social"], + ["tenant:*-family", "coulomb-social"], + ["tenant:family-*", "coulomb-social"] +] diff --git a/deploy/backup/email-connect-ledger-alert@.service b/deploy/backup/email-connect-ledger-alert@.service new file mode 100644 index 0000000..76f00e3 --- /dev/null +++ b/deploy/backup/email-connect-ledger-alert@.service @@ -0,0 +1,18 @@ +[Unit] +Description=Notify platform-root of email-connect ledger failure in %i +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +User=root +UMask=0077 +StateDirectory=email-connect-backup +StateDirectoryMode=0700 +ExecStart=/usr/bin/python3 /opt/email-connect-backup/notify-ledger-failure.py --unit %i +TimeoutStartSec=2min +NoNewPrivileges=true +PrivateTmp=true +ProtectSystem=strict +ReadWritePaths=/var/lib/email-connect-backup +ProtectHome=true diff --git a/deploy/backup/email-connect-ledger-backup.service b/deploy/backup/email-connect-ledger-backup.service index 2394b12..86484fb 100644 --- a/deploy/backup/email-connect-ledger-backup.service +++ b/deploy/backup/email-connect-ledger-backup.service @@ -2,6 +2,7 @@ Description=Coherent encrypted email-connect transactional ledger backup After=network-online.target k3s.service Wants=network-online.target +OnFailure=email-connect-ledger-alert@%n.service [Service] Type=oneshot diff --git a/deploy/backup/email-connect-ledger-freshness.service b/deploy/backup/email-connect-ledger-freshness.service index 5d1ab71..c057bee 100644 --- a/deploy/backup/email-connect-ledger-freshness.service +++ b/deploy/backup/email-connect-ledger-freshness.service @@ -1,10 +1,16 @@ [Unit] Description=Check email-connect ledger archive age and local ciphertext hash +OnFailure=email-connect-ledger-alert@%n.service [Service] Type=oneshot User=root ExecStart=/usr/bin/python3 /opt/email-connect-backup/backup-ledger.py freshness +ExecStartPost=-/usr/bin/python3 /opt/email-connect-backup/notify-ledger-failure.py +StateDirectory=email-connect-backup +StateDirectoryMode=0700 +UMask=0077 +ReadWritePaths=/var/lib/email-connect-backup NoNewPrivileges=true ProtectSystem=strict ProtectHome=true diff --git a/deploy/backup/runtime-manifest-2026-10-09.json b/deploy/backup/runtime-manifest-2026-10-09.json new file mode 100644 index 0000000..64b0962 --- /dev/null +++ b/deploy/backup/runtime-manifest-2026-10-09.json @@ -0,0 +1,29 @@ +{ + "schema": "email-connect.ledger-runtime-bundle.v1", + "files": { + "email-connect-ledger-freshness.service": "a9ce8d1e92fadcda8bb125bf952c42f338514545b6a980cfd8be0dd95bc5eb06", + "backup-primary.lock": "7142d3690855a6fcfdd2dac02113e7a76b6731988d52b4b013a50d97570f2aaa", + "email-connect-ledger-backup.service": "e2f41257ac7df0aac6250822a794289946f4282e68ee61d751c2c84a1caeb1e5", + "notify-ledger-failure.py": "d5bc735963896b2e909660e678b8ed1b220ec8f1038014399c5b995a577c90b6", + "primary_application_archive.py": "a3f0a61e869a6012edb4b5288be14e3b0c4a9a1273b090320ab146d12e5aa94d", + "rehearse-ledger.py": "3ee32b035b6c6c2f7c994346d1db04622daeb9260b8ed723bc89661f3632eaba", + "backup-ledger.py": "8a89aa01c0c22e46352f1ad7a6f30b9b5db792639c1acc9a60579f50b7c8c369", + "capture-ledger.py": "df09d972be74db89a80f4899ef782f5412e655dfa12317a07cc7f35e9b1063da", + "email-connect-ledger-alert@.service": "4ee43ffab27478556c75153e89668a3bb36e6c56d6dd3b52d0db8d8603b9263c", + "email-connect-ledger-backup.timer": "ffb64bf39763c5e3d52cea5a13aa3fdffe243c3ff5a81555eb47048746527782", + "ledger_archive.py": "92d9d5e1908c7d433eb3f4ada6460f7d9b639017a4c2311cb7ffaacd262f33c3", + "email-connect-ledger-freshness.timer": "9103655972b13c25dcbb6a8d53f3076fe8b9b7eb04a91e85c1cd1a875bf3310f" + }, + "receiver_image": "eb3e7b68d21209e9eb71b928a5b0bc987e8e3164f21dca4ff712f946d0c103f2", + "timers_enabled": false, + "failure_recipient": "platform-root", + "notification_delivery_verified": true, + "runtime_installed": true, + "sdk_hash_locked": true, + "hardened_preflight_verified": true, + "notification_message_id": "adcebf95-26c4-4adb-99b0-4446c8505a42", + "failure_hook_test_verified": true, + "freshness_verified": true, + "human_acknowledged": false, + "units_installed": true +} diff --git a/deploy/caller-rollout/packaged-evidence-2026-10-09.json b/deploy/caller-rollout/packaged-evidence-2026-10-09.json new file mode 100644 index 0000000..c43409e --- /dev/null +++ b/deploy/caller-rollout/packaged-evidence-2026-10-09.json @@ -0,0 +1,13 @@ +{ + "status": "packaged_caller_contract_verified", + "uid": 10001, + "mail_sent": false, + "legacy_unknown_blocked": true, + "legacy_accepted_preserved": true, + "glob_denials_verified": true, + "module_sha256": "06fe961b9030dba2c0b26b92f8097e8fbd24e205d5fa79642aa9af60382886dc", + "local_image": "email-connect:email-wp-0008-globs-20261009", + "image_id": "sha256:5d79fd1fec15d3a279383820f170d263de3a55f5308df01a0a34732ebaea6d3d", + "network": "disabled", + "runtime_activated": false +} diff --git a/deploy/caller-rollout/receiver-policy.patch.json b/deploy/caller-rollout/receiver-policy.patch.json new file mode 100644 index 0000000..bea1db3 --- /dev/null +++ b/deploy/caller-rollout/receiver-policy.patch.json @@ -0,0 +1,53 @@ +{ + "spec": { + "template": { + "spec": { + "containers": [ + { + "name": "email-connect", + "env": [ + { + "name": "EMAIL_CONNECT_CALLERS_FILE", + "value": "/etc/email-connect-callers/callers.json" + } + ], + "volumeMounts": [ + { + "name": "caller-policy", + "mountPath": "/etc/email-connect-callers", + "readOnly": true + }, + { + "name": "caller-credentials", + "mountPath": "/var/run/secrets/email-connect-callers", + "readOnly": true + } + ] + } + ], + "volumes": [ + { + "name": "caller-policy", + "configMap": { + "name": "email-connect-callers-user-engine-276257849bf2", + "defaultMode": 292 + } + }, + { + "name": "caller-credentials", + "secret": { + "secretName": "email-connect-caller-user-engine", + "defaultMode": 288, + "items": [ + { + "key": "token", + "path": "user-engine" + } + ] + } + } + ] + } + } + } +} diff --git a/deploy/caller-rollout/user-engine-mail-bindings.patch b/deploy/caller-rollout/user-engine-mail-bindings.patch new file mode 100644 index 0000000..5e96ef9 --- /dev/null +++ b/deploy/caller-rollout/user-engine-mail-bindings.patch @@ -0,0 +1,171 @@ +--- /dev/null ++++ b/src/user_engine/mail_bindings.py +@@ -0,0 +1,40 @@ ++"""Explicit tenant globs for the scoped invitation mail transport.""" ++ ++from __future__ import annotations ++ ++import json ++import re ++from fnmatch import fnmatchcase ++ ++IDENTIFIER = re.compile(r"[A-Za-z0-9][A-Za-z0-9._:-]{0,127}\Z") ++PATTERN = re.compile(r"[A-Za-z0-9*?][A-Za-z0-9._:*?-]{0,127}\Z") ++ ++ ++def load_mail_client_bindings(raw): ++ if raw is None: ++ return None ++ try: ++ rows = json.loads(raw) ++ if not isinstance(rows, list) or not 1 <= len(rows) <= 128: ++ raise ValueError ++ result = [] ++ for row in rows: ++ if (not isinstance(row, list) or len(row) != 2 ++ or not isinstance(row[0], str) or not PATTERN.fullmatch(row[0]) ++ or not isinstance(row[1], str) or not IDENTIFIER.fullmatch(row[1])): ++ raise ValueError ++ result.append(tuple(row)) ++ if len({row[0] for row in result}) != len(result): ++ raise ValueError ++ return tuple(result) ++ except (ValueError, TypeError): ++ raise ValueError("mail_client_bindings_invalid") from None ++ ++ ++def mail_client_for(tenant, bindings): ++ if not isinstance(tenant, str) or not IDENTIFIER.fullmatch(tenant): ++ raise ValueError("mail_binding_denied") ++ matches = {client for pattern, client in bindings if fnmatchcase(tenant, pattern)} ++ if len(matches) != 1: ++ raise ValueError("mail_binding_denied") ++ return matches.pop() +--- a/src/user_engine/adapters/delivery.py ++++ b/src/user_engine/adapters/delivery.py +@@ -19,6 +19,7 @@ + + + from user_engine.domain import OutboxEvent ++from user_engine.mail_bindings import load_mail_client_bindings, mail_client_for + + _MAIL_EVENTS = {"family_member.invited", "family_invitation.resent"} + +@@ -34,12 +35,14 @@ + mail_url: str | None = None, + mail_bearer_token: str | None = None, + timeout_seconds: float = 5.0, ++ mail_client_bindings: str | None = None, + ) -> None: + self.event_url = event_url + self.mail_url = mail_url + self.event_bearer_token = event_bearer_token + self.mail_bearer_token = mail_bearer_token + self.timeout_seconds = timeout_seconds ++ self.mail_client_bindings = load_mail_client_bindings(mail_client_bindings) + + def __call__(self, event: OutboxEvent) -> None: + envelope = { +@@ -57,7 +60,15 @@ + if self.mail_url and event.event_type in _MAIL_EVENTS: + if not self.mail_bearer_token: + raise RuntimeError("mail delivery token is required") +- self._post(self.mail_url, envelope, self.mail_bearer_token) ++ mail_envelope = dict(envelope) ++ if self.mail_client_bindings is not None: ++ try: ++ mail_envelope["client_id"] = mail_client_for( ++ event.tenant, self.mail_client_bindings ++ ) ++ except ValueError: ++ raise DeliveryError("mail_binding_denied") from None ++ self._post(self.mail_url, mail_envelope, self.mail_bearer_token) + event_envelope = dict(envelope) + event_data = dict(envelope["data"]) + if "primary_email" in event_data: +--- a/src/user_engine/runtime.py ++++ b/src/user_engine/runtime.py +@@ -65,6 +65,7 @@ + else None + ), + timeout_seconds=float(os.environ.get("USER_ENGINE_DELIVERY_TIMEOUT", "5")), ++ mail_client_bindings=os.environ.get("USER_ENGINE_MAIL_CLIENT_BINDINGS"), + ) + app = PortalApplication( + service, +--- /dev/null ++++ b/tests/test_mail_bindings.py +@@ -0,0 +1,73 @@ ++"""Scoped mail migration uses synthetic identities and a mocked HTTP boundary.""" ++ ++import json ++import unittest ++from unittest.mock import patch ++ ++from user_engine.adapters.delivery import HTTPOutboxDeliveryAdapter, DeliveryError ++from user_engine.domain import OutboxEvent ++from user_engine.mail_bindings import load_mail_client_bindings, mail_client_for ++ ++ ++class MailBindingsTests(unittest.TestCase): ++ def test_whole_case_sensitive_globs_and_literal_clients(self): ++ bindings = load_mail_client_bindings('[ ["tenant:*-family", "coulomb-social"] ]') ++ self.assertEqual(mail_client_for("tenant:synthetic-family", bindings), "coulomb-social") ++ for tenant in ("tenant:platform", "tenant:Synthetic-Family", "tenant:x-family/path"): ++ with self.assertRaisesRegex(ValueError, "mail_binding_denied"): ++ mail_client_for(tenant, bindings) ++ ++ def test_invalid_configuration_never_falls_back(self): ++ for raw in ('', '[]', '{}', '[["*", "*"]]', '[["[a]", "client"]]', ++ '[["*", "one"], ["*", "two"]]', '[[true, "client"]]'): ++ with self.assertRaisesRegex(ValueError, "mail_client_bindings_invalid"): ++ load_mail_client_bindings(raw) ++ self.assertIsNone(load_mail_client_bindings(None)) ++ ++ def test_conflicting_overlaps_are_denied(self): ++ bindings = load_mail_client_bindings('[["tenant:*", "one"], ["tenant:x", "two"]]') ++ with self.assertRaisesRegex(ValueError, "mail_binding_denied"): ++ mail_client_for("tenant:x", bindings) ++ ++ def adapter(self, bindings=None): ++ return HTTPOutboxDeliveryAdapter(event_url="http://events.example.test", ++ event_bearer_token="synthetic-event-token", mail_url="http://mail.example.test", ++ mail_bearer_token="synthetic-mail-token", mail_client_bindings=bindings) ++ ++ def event(self, kind="family_member.invited", tenant="tenant:synthetic-family"): ++ return OutboxEvent(event_id="evt-synthetic", event_type=kind, aggregate_id="inv-synthetic", ++ payload={"primary_email": "member@example.test", "invitation_id": "inv-synthetic"}, ++ tenant=tenant, correlation_id="trace-synthetic") ++ ++ def test_binding_only_changes_mail_transport_not_event_or_outbox(self): ++ event = self.event() ++ adapter = self.adapter('[["tenant:*-family", "coulomb-social"]]') ++ with patch.object(adapter, "_post") as post: ++ adapter(event) ++ mail, audit = [call.args[1] for call in post.call_args_list] ++ self.assertEqual(mail["client_id"], "coulomb-social") ++ self.assertNotIn("client_id", audit) ++ self.assertNotIn("primary_email", audit["data"]) ++ self.assertNotIn("client_id", event.payload) ++ self.assertEqual(mail["id"], event.event_id) ++ ++ def test_legacy_mode_and_non_mail_events_preserve_envelopes(self): ++ adapter = self.adapter() ++ with patch.object(adapter, "_post") as post: ++ adapter(self.event()) ++ self.assertNotIn("client_id", post.call_args_list[0].args[1]) ++ adapter = self.adapter('[["tenant:*-family", "coulomb-social"]]') ++ with patch.object(adapter, "_post") as post: ++ adapter(self.event("membership.added", "tenant:platform")) ++ self.assertEqual(post.call_count, 1) ++ ++ def test_unadmitted_mail_tenant_contacts_neither_lane(self): ++ adapter = self.adapter('[["tenant:*-family", "coulomb-social"]]') ++ with patch.object(adapter, "_post") as post: ++ with self.assertRaisesRegex(DeliveryError, "mail_binding_denied"): ++ adapter(self.event(tenant="tenant:platform")) ++ post.assert_not_called() ++ ++ ++if __name__ == "__main__": ++ unittest.main() diff --git a/deploy/caller-rollout/user-engine-policy.configmap.json b/deploy/caller-rollout/user-engine-policy.configmap.json new file mode 100644 index 0000000..0a9fc80 --- /dev/null +++ b/deploy/caller-rollout/user-engine-policy.configmap.json @@ -0,0 +1,12 @@ +{ + "apiVersion": "v1", + "kind": "ConfigMap", + "immutable": true, + "metadata": { + "name": "email-connect-callers-user-engine-276257849bf2", + "namespace": "email-connect" + }, + "data": { + "callers.json": "{\n \"callers\": [\n {\n \"adopt_legacy_invitations\": true,\n \"adopt_legacy_verifications\": true,\n \"binding_patterns\": [\n [\n \"tenant:*-family\",\n \"coulomb-social\"\n ],\n [\n \"tenant:family-*\",\n \"coulomb-social\"\n ]\n ],\n \"bindings\": [\n [\n \"tenant:coulomb\",\n \"coulomb-social\"\n ]\n ],\n \"burst\": 5,\n \"caller_id\": \"user-engine\",\n \"max_payload_bytes\": 8192,\n \"operations\": [\n \"send\",\n \"delivery-status\",\n \"status\",\n \"verify\",\n \"consume\",\n \"cancel\"\n ],\n \"rate_per_minute\": 10,\n \"source\": \"user-engine\",\n \"templates\": [\n \"family_member.invited\",\n \"family_invitation.resent\"\n ],\n \"token_file\": \"/var/run/secrets/email-connect-callers/user-engine\"\n }\n ],\n \"schema_version\": 1\n}\n" + } +} diff --git a/deploy/caller-rollout/verify-packaged.py b/deploy/caller-rollout/verify-packaged.py new file mode 100644 index 0000000..15cb568 --- /dev/null +++ b/deploy/caller-rollout/verify-packaged.py @@ -0,0 +1,59 @@ +"""Check the packaged caller module and transition without credentials or mail.""" + +import hashlib +import io +import json +import os +import sys +from pathlib import Path + +import email_connect.callers as callers +from email_connect.transactional import SQLiteDeliveryStore + +assert os.getuid() == 10001 +assert hashlib.sha256(Path(callers.__file__).read_bytes()).hexdigest() == sys.argv[1] +assert len(Path('/proc/net/route').read_text().splitlines()) == 1 +token = "synthetic-packaged-caller-token" +Path('/tmp/token').write_text(token) +document = json.loads(Path('/review/config/callers.user-engine.draft.json').read_text()) +document['callers'][0]['token_file'] = '/tmp/token' +Path('/tmp/policy.json').write_text(json.dumps(document)) +policies = callers.load_callers('/tmp/policy.json') +assert ('tenant:synthetic-family', 'coulomb-social') in policies[0] +assert ('tenant:platform', 'coulomb-social') not in policies[0] +store = SQLiteDeliveryStore('/tmp/ledger.sqlite3') + + +class ForbiddenProvider: + def send(self, *args): + raise AssertionError('provider_call_forbidden') + + +app = callers.ScopedTransactionalApplication(store, ForbiddenProvider(), + 'https://issuer.example.test', policies) +old = {'id': 'evt-synthetic', 'source': 'user-engine', 'type': 'family_member.invited', + 'tenant': 'tenant:synthetic-family', + 'data': {'primary_email': 'member@example.test', 'invitation_id': 'inv-synthetic'}} +fingerprint = hashlib.sha256(json.dumps(old, sort_keys=True, separators=(',', ':')).encode()).hexdigest() +store.reserve_send(old['id'], fingerprint) + + +def invoke(payload): + raw = json.dumps(payload).encode() + response = {} + body = json.loads(b''.join(app({'PATH_INFO': '/v1/send', 'REQUEST_METHOD': 'POST', + 'CONTENT_LENGTH': str(len(raw)), 'wsgi.input': io.BytesIO(raw), + 'HTTP_AUTHORIZATION': 'Bearer ' + token, 'HTTP_IDEMPOTENCY_KEY': payload['id']}, + lambda status, headers: response.update(status=int(status.split()[0]))))) + return response['status'], body + + +bound = dict(old, client_id='coulomb-social') +assert invoke(bound)[1]['error'] == 'delivery_outcome_unknown' +store.record(old['id'], 'synthetic-old-reference') +assert invoke(bound)[1]['status'] == 'duplicate' +assert invoke(dict(bound, tenant='tenant:platform'))[1]['error'] == 'caller_scope_denied' +assert invoke(dict(bound, data=dict(old['data'], primary_email='changed@example.test')))[1]['error'] == 'idempotency_payload_mismatch' +print(json.dumps({'status': 'packaged_caller_contract_verified', 'uid': os.getuid(), + 'mail_sent': False, 'legacy_unknown_blocked': True, + 'legacy_accepted_preserved': True, 'glob_denials_verified': True})) diff --git a/docs/caller-policy-draft-2026-10-08.md b/docs/caller-policy-draft-2026-10-08.md index 01f49f5..e2ffd00 100644 --- a/docs/caller-policy-draft-2026-10-08.md +++ b/docs/caller-policy-draft-2026-10-08.md @@ -1,5 +1,8 @@ # User-engine caller policy draft — 2026-10-08 +This is the historical draft review. The compatible rollout preparation below +updates its scope and resolves the source compatibility gap; activation remains open. + This file proposes the only caller that exists today. It is not mounted, `EMAIL_CONNECT_CALLERS_FILE` is not set, and no token file was created. EMAIL-WP-0008-T03 stays waiting until the bindings, quotas and token custody @@ -40,3 +43,46 @@ admitted tenant/client pair, or a separate reviewed contract change. The token path in the draft does not exist. Selecting this file on the running receiver would fail startup before serving traffic. Do not copy the shared ingest token into that path as part of this draft. + +## Compatible rollout preparation — 2026-10-09 + +User selected the proposed limits and requested glob or regex support. The draft +now admits exact tenant:coulomb/coulomb-social plus the proposed case-sensitive +tenant:*-family and tenant:family-* globs with literal client coulomb-social. +Only * and ? glob operators are supported; all patterns and actual identifiers +are bounded to 128 characters, with at most 128 total exact/pattern pairs. +These family naming patterns are proposals rather than a complete tenant inventory. +They apply to verification creation and handle consumption/cancellation as well +as invitation sends. Suppression remains excluded. + +User-engine source accepts opt-in USER_ENGINE_MAIL_CLIENT_BINDINGS, a JSON list +matching config/mail-client-bindings.user-engine.draft.json. It resolves one +literal client for the actual tenant; missing or ambiguous matches refuse before +any HTTP delivery. Only the mail transport gains client_id: audit envelopes and +stored outbox payloads retain their original shape. Default unset configuration +retains current live behavior. The reviewed patch is retained in +deploy/caller-rollout/user-engine-mail-bindings.patch. + +adopt_legacy_invitations explicitly reserves original invitation ids for this +user-engine caller. A matching old fingerprint can tolerate the added transport +client_id, preserving accepted evidence and terminal unknown/permanent outcomes. +Other immutable changes still refuse; admission checks precede legacy matching. +New events bind the full payload. Other callers cannot opt into this adoption. + +The immutable ConfigMap and strategic deployment patch in deploy/caller-rollout +pass server dry-run; packaged-evidence-2026-10-09.json binds the local candidate +image to the exact caller module under UID10001 and disabled networking. +Before activation, deliver a caller-specific token to the separately admitted +Secret email-connect-caller-user-engine (key token) and corresponding user-engine +custody lane. Do not copy the legacy shared ingest token into that projection. +Publish/admit the new receiver image, retain the current ledger/recovery receipt, +fence outbox dispatch during coordinated token/policy transition, and set both +the receiver policy and user-engine binding configuration. Pending outbox tenants +must match the admitted scope before traffic resumes; narrow or add explicit +bindings when their names do not match these proposed globs. + +The live receiver remains eb3e7b68 in single-caller mode. Capture and rehearsal +must be repinned together only when a new receiver image is actually deployed. +Policy narrowing/removal preserves ledger evidence but can deny existing links; +review rollback with the compatible image and both caller credentials before +releasing traffic. Source preparation does not issue a token or select this policy. diff --git a/docs/ledger-backup-review-2026-10-08.md b/docs/ledger-backup-review-2026-10-08.md index 3732f00..39725a7 100644 --- a/docs/ledger-backup-review-2026-10-08.md +++ b/docs/ledger-backup-review-2026-10-08.md @@ -1,5 +1,8 @@ # Email ledger backup/recovery candidate — 2026-10-08 +Historical candidate review. The admitted first run and monitoring proof on +2026-10-09 are recorded below; scheduling and independent custody remain open. + EMAIL-WP-0007 implements the independent transactional SQLite recovery scope requested by CSOC-WP-0010-T04/T05. The verification retry release remains live; this candidate does not change that deployment, SMTP/caller credentials or inbox @@ -147,3 +150,38 @@ T01/T02 implementation/rehearsal can close on this evidence; T03/T04 remain open for actual custody, offsite capture/recovery, post-capture reconciliation contract, monitored operation and founder activation. CSOC-WP-0010 remains active for those and caller/sender/rate-limit, reply/bounce/inbox and cross-owner journey acceptance. + +## Admitted first run — 2026-10-09 + +The user authorized one first capture/upload/retrieval/isolated recovery with +schedules disabled (decision 0dfcdd94-50ee-4219-9659-5c4c445721f0). +Installed source and hash-locked SDK pass hardened node preflight. The receiver +resumed after 56.41 seconds; primary version 1791537739370570 passed size/hash +readback. A separate version-pinned GET and workstation decrypt/application +rehearsal preserve all six tables, two accepted deliveries and one suppression, +under UID10001/network-none with no provider calls. Other state tables are empty +in this live snapshot; their invariants retain synthetic recovery evidence. +The isolated drill interval was 636.29 seconds including intervening operator +work, rather than a production restore/RTO measurement. + +Public transfer/recovery evidence is docs/ledger-first-backup-2026-10-09.json. +The existing mode-0600 workstation identity matches the configured public age +recipient; separate escrow admission is still open. The admitted node publisher +credential performed retrieval and was not copied to the workstation. This does +not prove a separately scoped recovery caller or node/workstation-loss custody. + +User selected the platform-root operations inbox. OnFailure hooks now deliver +redacted notices to its State Hub inbox. A synthetic failed unit delivered +adcebf95-26c4-4adb-99b0-4446c8505a42, verified by an independent inbox read. +Failures to deliver retain private pending records for at-least-once replay with +stable event ids; duplicates are possible if acceptance succeeded but its reply +was lost. Successful freshness checks replay pending notices, while failures +invoke the same delivery hook. No notification includes journal output, +credentials, message contents or ledger rows. Human response is not claimed. +Both timers are installed and disabled; one attended freshness check passed. + +Full email regression after caller/notification changes: 178 passed, zero skips +using the loopback GreenMail harness; focused syntax checks pass. Retention, +capacity, independent receipt/key/image custody, separate recovery caller and +recurring schedule/RPO admission retain EMAIL-WP-0007-T03/T04. Production restore +still requires fencing and post-capture reconciliation before traffic release. diff --git a/docs/ledger-first-backup-2026-10-09.json b/docs/ledger-first-backup-2026-10-09.json new file mode 100644 index 0000000..528f11a --- /dev/null +++ b/docs/ledger-first-backup-2026-10-09.json @@ -0,0 +1,131 @@ +{ + "schema": "email-connect.first-primary-ledger-recovery.v1", + "decision_id": "0dfcdd94-50ee-4219-9659-5c4c445721f0", + "publication": { + "schema": "platform.application-primary-archive.v1", + "status": "offsite_fetched_verified", + "application": "email-connect", + "captured_at": "2026-10-09T09:21:20Z", + "object_key": "platform-pg/application-archives/email-connect/2026/10/09/20261009T092120Z-004a5e182fd57e5005787bb8.tar.age", + "destination": "s3://railiance-platform-pg-backup/platform-pg/application-archives/email-connect/2026/10/09/20261009T092120Z-004a5e182fd57e5005787bb8.tar.age", + "ciphertext_sha256": "8053f6dfd9763fbcdef4e9a005c79c2cab3d43ff31d37e8decd690ad07b4f5eb", + "ciphertext_bytes": 71896, + "remote_retrieval_verified": true, + "stage": "transfer_verified", + "multipart_completed": true, + "version_id": "1791537739370570", + "immutable_create": true, + "image": "forgejo.coulomb.social/coulomb/email-connect@sha256:eb3e7b68d21209e9eb71b928a5b0bc987e8e3164f21dca4ff712f946d0c103f2", + "database_sha256": "b6020ffa13a097f0e4f24ce3a273cdf7a3eac5ab6b8c0770af8971ce3c951d58", + "database_bytes": 57344, + "ledger_fingerprint": { + "schema_sha256": "13148e5a9c6e01cc593bb07278a8649b13e43b26c30b0ed1a6e15995f7b00732", + "user_version": 0, + "tables": { + "caller_buckets": { + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "deliveries": { + "rows": 2, + "sha256": "a0387716815b2ec191187cb34bb3252b9cf51f5fb15a2019918a9cf1737c061e" + }, + "send_attempts": { + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "suppressions": { + "rows": 1, + "sha256": "ae26d10dfec16f1b626292b342a5e86afd1858fcaed7c29d4e68e7cf821183d7" + }, + "verification_attempts": { + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "verifications": { + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + } + } + }, + "pause_and_capture_seconds": 56.41, + "local_archive": "20261009T092120Z-004a5e182fd57e5005787bb8.tar.age", + "application_restore_verified": false, + "completed_at": "2026-10-09T09:22:20.500704+00:00" + }, + "application_recovery": { + "status": "application_restored_verified", + "application": "email-connect", + "object_key": "platform-pg/application-archives/email-connect/2026/10/09/20261009T092120Z-004a5e182fd57e5005787bb8.tar.age", + "version_id": "1791537739370570", + "ciphertext_sha256": "8053f6dfd9763fbcdef4e9a005c79c2cab3d43ff31d37e8decd690ad07b4f5eb", + "database_sha256": "b6020ffa13a097f0e4f24ce3a273cdf7a3eac5ab6b8c0770af8971ce3c951d58", + "ledger_fingerprint": { + "schema_sha256": "13148e5a9c6e01cc593bb07278a8649b13e43b26c30b0ed1a6e15995f7b00732", + "user_version": 0, + "tables": { + "caller_buckets": { + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "deliveries": { + "rows": 2, + "sha256": "a0387716815b2ec191187cb34bb3252b9cf51f5fb15a2019918a9cf1737c061e" + }, + "send_attempts": { + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "suppressions": { + "rows": 1, + "sha256": "ae26d10dfec16f1b626292b342a5e86afd1858fcaed7c29d4e68e7cf821183d7" + }, + "verification_attempts": { + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "verifications": { + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + } + } + }, + "mail_sent": false, + "production_restored": false, + "application_restore_verified": true, + "recovery_image": "forgejo.coulomb.social/coulomb/email-connect@sha256:eb3e7b68d21209e9eb71b928a5b0bc987e8e3164f21dca4ff712f946d0c103f2", + "network": "disabled", + "checks": { + "accepted_verifications": 0, + "unknown_verifications": 0, + "invalidated_handles": 0, + "suppressions": 1, + "accepted_deliveries": 2, + "unknown_sends": 0 + }, + "uid": 10001, + "evidence_ceiling": "isolated_ledger_rehearsal", + "elapsed_seconds": 0.93 + }, + "independent_retrieval": { + "separate_provider_GET": true, + "version_and_hash_verified": true, + "publisher_credential_used_on_node": true, + "credential_copied_to_workstation": false, + "separate_recovery_caller_proved": false + }, + "isolated_drill_wall_seconds": 636.29, + "interval_includes_operator_work": true, + "recovery_proof_at": "2026-10-09T09:31:56.290620+00:00", + "production_rto_measured": false, + "independent_identity_escrow_admitted": false, + "schedules_enabled": false, + "failure_notification": [ + { + "id": "adcebf95-26c4-4adb-99b0-4446c8505a42", + "to_agent": "platform-root", + "subject": "email-connect ledger notification test", + "read": null + } + ] +} diff --git a/scripts/notify-ledger-failure.py b/scripts/notify-ledger-failure.py new file mode 100644 index 0000000..252464c --- /dev/null +++ b/scripts/notify-ledger-failure.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +"""Redacted ledger failure notices to the platform-root State Hub inbox.""" + +from __future__ import annotations + +import argparse +import fcntl +import json +import os +import uuid +from datetime import UTC, datetime +from pathlib import Path +from urllib.request import Request, urlopen + +from ledger_archive import Refused, checkpoint, private_directory, private_file + +UNITS = {"email-connect-ledger-backup.service", "email-connect-ledger-freshness.service"} +TEST_UNIT = "email-connect-ledger-alert-test.service" +RECIPIENT = "platform-root" + + +def post(base, message): + request = Request(base.rstrip("/") + "/messages/", data=json.dumps(message).encode(), + headers={"Content-Type": "application/json"}, method="POST") + with urlopen(request, timeout=15) as response: + result = json.loads(response.read(65536)) + # A queue receipt is pending, never evidence of central inbox delivery. + if (result.get("to_agent") != RECIPIENT or result.get("from_agent") != "email-connect" + or result.get("subject") != message["subject"] + or result.get("body") != message["body"]): + raise Refused("inbox_receipt_not_verified") + uuid.UUID(result["id"]) + return result["id"] + + +def notify(state, base, unit=None, test=False, sender=post): + if unit not in UNITS | {None, TEST_UNIT} or (test and unit is None): + raise Refused("invalid_notification_unit") + test = test or unit == TEST_UNIT + state.mkdir(mode=0o700, parents=True, exist_ok=True) + private_directory(state) + fd = os.open(state / "notify.lock", os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, 0o600) + with os.fdopen(fd, "w") as lock: + fcntl.flock(lock, fcntl.LOCK_EX) + if unit: + event = str(uuid.uuid4()) + record = { + "event_id": event, "unit": unit, "test": test, + "observed_at": datetime.now(UTC).isoformat(), "status": "pending", + "message": { + "from_agent": "email-connect", "to_agent": RECIPIENT, + "subject": "email-connect ledger " + ("notification test" if test else "backup failure"), + "body": json.dumps({ + "event_id": event, "unit": unit, "test": test, + "workplan": "EMAIL-WP-0007", "host": "railiance01", + "action": "Confirm inbox delivery only." if test else + "Inspect receiver readiness, backup pending evidence and archive freshness; " + "do not automatically restore, resend or discard a pending capture.", + }, sort_keys=True), + }, + } + checkpoint(state / (event + ".json"), record) + delivered = [] + for path in sorted(state.glob("*.json")): + private_file(path) + record = json.loads(path.read_text()) + if record["status"] != "pending": + continue + message = record["message"] + if message["from_agent"] != "email-connect" or message["to_agent"] != RECIPIENT: + raise Refused("invalid_pending_notification") + # Delivery is at least once: if the hub accepted a request but the + # reply was lost, replay may duplicate its stable event_id. + message_id = sender(base, message) + record.update(status="delivered", message_id=message_id, + delivered_at=datetime.now(UTC).isoformat()) + checkpoint(path, record) + delivered.append(message_id) + return {"status": "inbox_delivery_verified", "recipient": RECIPIENT, + "message_ids": delivered, "human_acknowledged": False} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--unit", choices=sorted(UNITS | {TEST_UNIT})) + parser.add_argument("--test", action="store_true") + parser.add_argument("--state-dir", type=Path, + default=Path("/var/lib/email-connect-backup/notifications")) + parser.add_argument("--api-base", default="http://10.43.68.154:8000") + args = parser.parse_args() + print(json.dumps(notify(args.state_dir, args.api_base, args.unit, args.test))) + + +if __name__ == "__main__": + try: + main() + except Exception as error: + print(json.dumps({"status": "pending_notification_requires_replay", + "failure_type": type(error).__name__})) + raise SystemExit(1) diff --git a/src/email_connect/callers.py b/src/email_connect/callers.py index e49b100..9ed5504 100644 --- a/src/email_connect/callers.py +++ b/src/email_connect/callers.py @@ -15,6 +15,7 @@ import re import secrets import time from dataclasses import dataclass, field +from fnmatch import fnmatchcase from http import HTTPStatus from pathlib import Path @@ -63,20 +64,26 @@ class CallerPolicy: burst: int max_payload_bytes: int adopt_legacy_verifications: bool = False + adopt_legacy_invitations: bool = False legacy_invitation_binding: tuple[str, str] | None = None + binding_patterns: frozenset[tuple[str, str]] = frozenset() def __post_init__(self): - if not isinstance(self.caller_id, str) or not IDENTIFIER.fullmatch(self.caller_id): + if not isinstance(self.caller_id, str) or not IDENTIFIER.fullmatch( + self.caller_id + ): raise ValueError("invalid_caller_policy") if not isinstance(self.source, str) or not IDENTIFIER.fullmatch(self.source): raise ValueError("invalid_caller_policy") - if not isinstance(self.token, str) or not re.fullmatch(r"[!-~]{16,512}", self.token): + if not isinstance(self.token, str) or not re.fullmatch( + r"[!-~]{16,512}", self.token + ): raise ValueError("invalid_caller_credential") if ( not self.operations or not self.operations <= set(OPERATIONS.values()) or not self.templates <= TEMPLATES - or not self.bindings + or not (self.bindings or self.binding_patterns) ): raise ValueError("invalid_caller_policy") for binding in self.bindings: @@ -84,6 +91,15 @@ class CallerPolicy: not isinstance(v, str) or not IDENTIFIER.fullmatch(v) for v in binding ): raise ValueError("invalid_caller_binding") + if len(self.bindings) + len(self.binding_patterns) > 128: + raise ValueError("too_many_caller_bindings") + for binding in self.binding_patterns: + if len(binding) != 2 or any( + not isinstance(v, str) + or not re.fullmatch(r"[A-Za-z0-9*?][A-Za-z0-9._:*?-]{0,127}", v) + for v in binding + ): + raise ValueError("invalid_caller_binding_pattern") for value, maximum in ( (self.rate_per_minute, 6000), (self.burst, 600), @@ -91,17 +107,40 @@ class CallerPolicy: ): if type(value) is not int or not 0 < value <= maximum: raise ValueError("invalid_caller_limits") - if type(self.adopt_legacy_verifications) is not bool: + if any( + type(value) is not bool + for value in ( + self.adopt_legacy_verifications, + self.adopt_legacy_invitations, + ) + ): raise ValueError("invalid_legacy_admission") - if self.adopt_legacy_verifications or self.legacy_invitation_binding is not None: - if self.caller_id != "user-engine" or self.source != "user-engine": - raise ValueError("legacy_admission_requires_original_caller") + if ( + self.adopt_legacy_verifications + or self.adopt_legacy_invitations + or self.legacy_invitation_binding is not None + ) and (self.caller_id != "user-engine" or self.source != "user-engine"): + raise ValueError("legacy_admission_requires_original_caller") if ( self.legacy_invitation_binding is not None and self.legacy_invitation_binding not in self.bindings ): raise ValueError("invalid_legacy_binding") + def __contains__(self, binding): + if ( + not isinstance(binding, tuple) + or len(binding) != 2 + or any( + not isinstance(v, str) or not IDENTIFIER.fullmatch(v) for v in binding + ) + ): + return False + return binding in self.bindings or any( + fnmatchcase(binding[0], tenant) and fnmatchcase(binding[1], client) + for tenant, client in self.binding_patterns + ) + def load_callers(path): """Owner-controlled non-secret policy plus independently delivered token files.""" @@ -127,6 +166,9 @@ def load_callers(path): token = raw_token.decode("ascii").strip() values["token"] = token values["bindings"] = frozenset(tuple(v) for v in values["bindings"]) + values["binding_patterns"] = frozenset( + tuple(v) for v in values.get("binding_patterns", []) + ) for key in ("operations", "templates"): values[key] = frozenset(values[key]) if values.get("legacy_invitation_binding") is not None: @@ -161,7 +203,7 @@ class ScopedTransactionalApplication: self.internal_token, portal_url, verification_namespace="" if p.adopt_legacy_verifications else p.caller_id, - verification_bindings=p.bindings, + verification_bindings=p, ) for p in policies } @@ -187,7 +229,7 @@ class ScopedTransactionalApplication: or event_id.startswith("ec2_") ): raise ValueError("invalid_event_id") - if policy.legacy_invitation_binding is not None: + if policy.adopt_legacy_invitations or policy.legacy_invitation_binding is not None: return event_id return ( "ec2_" + hashlib.sha256(json.dumps([policy.caller_id, event_id]).encode()).hexdigest() @@ -319,7 +361,7 @@ class ScopedTransactionalApplication: binding = (payload.get("tenant"), payload.get("client_id")) if binding == (None, None) and policy.legacy_invitation_binding is not None: binding = policy.legacy_invitation_binding - if not all(isinstance(v, str) for v in binding) or binding not in policy.bindings: + if binding not in policy: raise ValueError("caller_scope_denied") data = payload.get("data") if ( @@ -332,9 +374,25 @@ class ScopedTransactionalApplication: if not isinstance(recipient, str) or len(recipient) > 254: raise ValueError("invalid_recipient") payload = dict(payload, id=event_key, source="user-engine") + if policy.adopt_legacy_invitations: + # The original caller adds client_id at the transport boundary. + # Match an existing pre-transition fingerprint before removing + # that field; all other immutable fields still have to match. + legacy = dict(payload) + legacy.pop("client_id", None) + expected = hashlib.sha256( + json.dumps(legacy, sort_keys=True, separators=(",", ":")).encode() + ).hexdigest() + with self.store.lock: + row = self.store.db.execute( + "SELECT fingerprint FROM send_attempts WHERE event_id=?", + (event_key,), + ).fetchone() + if row and hmac.compare_digest(row[0], expected): + payload = legacy if path == "/v1/registration-verifications": binding = (payload.get("tenant"), payload.get("client_id")) - if not all(isinstance(v, str) for v in binding) or binding not in policy.bindings: + if binding not in policy: raise ValueError("caller_scope_denied") raw = json.dumps(payload).encode() internal = dict(environ, CONTENT_LENGTH=str(len(raw))) diff --git a/tests/test_caller_policy_draft.py b/tests/test_caller_policy_draft.py index 02d0507..b0d8ca4 100644 --- a/tests/test_caller_policy_draft.py +++ b/tests/test_caller_policy_draft.py @@ -60,6 +60,9 @@ def test_draft_admits_only_the_live_registration_pair(tmp_path): assert policy.caller_id == "user-engine" assert policy.source == "user-engine" assert policy.bindings == frozenset({("tenant:coulomb", "coulomb-social")}) + assert policy.binding_patterns == frozenset({ + ("tenant:*-family", "coulomb-social"), ("tenant:family-*", "coulomb-social") + }) assert policy.operations == frozenset( {"send", "delivery-status", "status", "verify", "consume", "cancel"} ) @@ -67,6 +70,7 @@ def test_draft_admits_only_the_live_registration_pair(tmp_path): assert policy.templates == frozenset({"family_member.invited", "family_invitation.resent"}) assert (policy.rate_per_minute, policy.burst, policy.max_payload_bytes) == (10, 5, 8192) assert policy.adopt_legacy_verifications is True + assert policy.adopt_legacy_invitations is True assert policy.legacy_invitation_binding is None assert policy.token == TOKEN assert TOKEN not in DRAFT.read_text(encoding="utf-8") @@ -105,3 +109,23 @@ def test_current_invitation_shape_is_denied_and_registration_pair_is_allowed(tmp status, body = invoke(app, {"recipient": "member@example.test"}, "/v1/suppressions") assert status == 403 and body["error"] == "operation_not_allowed" + + +def test_transport_draft_matches_receiver_bindings_and_accepts_family_invitation(tmp_path): + transport = json.loads((ROOT / "config" / "mail-client-bindings.user-engine.draft.json").read_text()) + policies = load_draft(tmp_path) + policy = policies[0] + assert {tuple(pair) for pair in transport} == policy.bindings | policy.binding_patterns + provider = Provider() + app = ScopedTransactionalApplication( + SQLiteDeliveryStore(str(tmp_path / "ledger.db")), provider, + "https://issuer.example.test", policies, + ) + invitation = { + "id": "evt-compatible", "type": "family_member.invited", "source": "user-engine", + "tenant": "tenant:synthetic-family", "client_id": "coulomb-social", + "data": {"primary_email": "member@example.test", "invitation_id": "invite-compatible"}, + } + status, body = invoke(app, invitation, "/v1/send") + assert status == 202 and body["status"] == "accepted" + assert len(provider.calls) == 1 diff --git a/tests/test_callers.py b/tests/test_callers.py index 03a1282..5a96a9f 100644 --- a/tests/test_callers.py +++ b/tests/test_callers.py @@ -412,3 +412,152 @@ def test_invalid_policy_refuses_startup_before_database_mutation(tmp_path, monke with pytest.raises(ValueError, match="caller_configuration_invalid"): main() assert not (tmp_path / "not-created.db").exists() + + +@pytest.mark.parametrize("outcome", ["accepted", "unknown", "permanent"]) +def test_original_caller_adds_binding_without_replaying_legacy_mail(tmp_path, outcome): + import hashlib + + store = SQLiteDeliveryStore(str(tmp_path / "ledger.db")) + provider = Provider() + old = event("user-engine") + old.pop("client_id") + fingerprint = hashlib.sha256( + json.dumps(old, sort_keys=True, separators=(",", ":")).encode() + ).hexdigest() + store.reserve_send(old["id"], fingerprint) + if outcome == "accepted": + store.record(old["id"], "synthetic-legacy-reference") + elif outcome == "permanent": + store.send_failed( + old["id"], True, ProviderError("permanent_rejection", retryable=False) + ) + admitted = policy("user-engine", adopt_legacy_invitations=True) + scoped = ScopedTransactionalApplication( + store, provider, "https://issuer.example.test", [admitted] + ) + bound = dict(old, client_id="client:a") + status, body, _ = invoke(scoped, bound, caller="user-engine") + if outcome == "accepted": + assert status == 200 and body["status"] == "duplicate" + else: + assert status == 422 + assert body["error"] == ( + "delivery_outcome_unknown" + if outcome == "unknown" + else "permanent_rejection" + ) + assert provider.calls == [] + assert store.db.execute("SELECT fingerprint FROM send_attempts").fetchall() == [ + (fingerprint,) + ] + status, body, _ = invoke( + scoped, + dict(bound, data=dict(bound["data"], primary_email="changed@example.test")), + caller="user-engine", + ) + assert status == 400 and body["error"] == "idempotency_payload_mismatch" + status, body, _ = invoke( + scoped, dict(bound, tenant="tenant:denied"), caller="user-engine" + ) + assert status == 400 and body["error"] == "caller_scope_denied" + status, body, _ = invoke( + scoped, + {"event_id": old["id"]}, + caller="user-engine", + path="/v1/delivery-status", + ) + assert status == 200 + assert ( + body["state"] + == { + "accepted": "provider_accepted", + "unknown": "unknown", + "permanent": "failed", + }[outcome] + ) + assert provider.calls == [] + + +def test_invitation_adoption_requires_original_caller_and_boolean(): + with pytest.raises(ValueError, match="legacy_admission_requires_original_caller"): + policy(adopt_legacy_invitations=True) + with pytest.raises(ValueError, match="invalid_legacy_admission"): + policy("user-engine", adopt_legacy_invitations="true") + + +@pytest.mark.parametrize( + "tenant,client,allowed", + [ + ("tenant:family-one", "client:app", True), + ("tenant:family-two", "client:app", True), + ("tenant:platform", "client:app", False), + ("tenant:family-one", "client:other", False), + ("tenant:Family-one", "client:app", False), + ("tenant:family-one/foreign", "client:app", False), + ], +) +def test_binding_globs_match_whole_case_sensitive_valid_identifiers( + tmp_path, tenant, client, allowed +): + admitted = policy(binding_patterns=frozenset({("tenant:family-*", "client:app")})) + app, _, provider = application(tmp_path, [admitted]) + status, body, _ = invoke(app, event(tenant=tenant, client_id=client)) + assert status == (202 if allowed else 400) + assert len(provider.calls) == int(allowed) + if not allowed: + assert body["error"] == "caller_scope_denied" + + +def test_glob_binding_applies_to_verification_and_consumption(tmp_path): + admitted = policy( + bindings=frozenset(), + binding_patterns=frozenset({("tenant:family-*", "client:?")}), + ) + app, _, provider = application(tmp_path, [admitted]) + status, body, _ = invoke( + app, + verification(tenant="tenant:family-one"), + path="/v1/registration-verifications", + ) + assert status == 202 + handle = provider.calls[0][2].split("handle=", 1)[1].splitlines()[0] + status, _, _ = invoke( + app, {"handle": handle}, path="/v1/registration-verifications/consume" + ) + assert status == 200 + + +@pytest.mark.parametrize( + "pattern", ["(tenant:.*)", "tenant:[ab]", "tenant:/other", "*" * 129] +) +def test_invalid_or_unbounded_binding_patterns_refuse_startup(pattern): + with pytest.raises(ValueError, match="invalid_caller_binding_pattern"): + policy(binding_patterns=frozenset({(pattern, "client:a")})) + + +def test_new_adopted_invitation_keeps_bound_fingerprint_across_restart(tmp_path): + admitted = policy("user-engine", adopt_legacy_invitations=True) + app, store, _ = application(tmp_path, [admitted]) + assert invoke(app, event("user-engine"), caller="user-engine")[0] == 202 + store.db.close() + recovered, _, provider = application(tmp_path, [admitted]) + assert ( + invoke(recovered, event("user-engine"), caller="user-engine")[1]["status"] + == "duplicate" + ) + assert ( + invoke( + recovered, + event( + "user-engine", + data={ + "primary_email": "changed@example.test", + "invitation_id": "invite-1", + }, + ), + caller="user-engine", + )[0] + == 400 + ) + assert provider.calls == [] diff --git a/tests/test_ledger_notifications.py b/tests/test_ledger_notifications.py new file mode 100644 index 0000000..d3b39f4 --- /dev/null +++ b/tests/test_ledger_notifications.py @@ -0,0 +1,67 @@ +"""Inbox notification failure/replay with synthetic receipts; no live messages.""" + +import importlib.util +import json +from pathlib import Path + +import pytest + + +@pytest.fixture +def notifier(monkeypatch): + scripts = Path(__file__).resolve().parents[1] / "scripts" + monkeypatch.syspath_prepend(str(scripts)) + spec = importlib.util.spec_from_file_location("ledger_notifier", scripts / "notify-ledger-failure.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_outage_preserves_private_pending_notice_and_replay_delivers(notifier, tmp_path): + state = tmp_path / "outbox" + def down(base, message): + raise OSError("synthetic HTTP failure") + with pytest.raises(OSError): + notifier.notify(state, "http://synthetic.invalid", next(iter(notifier.UNITS)), sender=down) + path, = state.glob("*.json") + pending = json.loads(path.read_text()) + assert pending["status"] == "pending" + assert state.stat().st_mode & 0o777 == 0o700 + assert path.stat().st_mode & 0o777 == 0o600 + sent = [] + def accepted(base, message): + sent.append(message) + return "00000000-0000-0000-0000-000000000001" + receipt = notifier.notify(state, "http://synthetic.invalid", sender=accepted) + assert receipt["recipient"] == "platform-root" + assert receipt["human_acknowledged"] is False + assert json.loads(path.read_text())["status"] == "delivered" + assert json.loads(sent[0]["body"])["event_id"] == pending["event_id"] + notifier.notify(state, "http://synthetic.invalid", sender=accepted) + assert len(sent) == 1 + + +def test_unexpected_unit_is_rejected_before_outbox_mutation(notifier, tmp_path): + with pytest.raises(notifier.Refused, match="invalid_notification_unit"): + notifier.notify(tmp_path / "unused", "http://synthetic.invalid", "other.service") + assert not (tmp_path / "unused").exists() + + +def test_failure_hook_test_is_labelled_without_real_failure_claim(notifier, tmp_path): + sent = [] + def accepted(base, message): + sent.append(message) + return "00000000-0000-0000-0000-000000000001" + notifier.notify(tmp_path / "outbox", "http://synthetic.invalid", notifier.TEST_UNIT, sender=accepted) + assert json.loads(sent[0]["body"])["test"] is True + assert sent[0]["subject"].endswith("notification test") + + +def test_queued_response_is_not_central_inbox_delivery(notifier, monkeypatch): + class Response: + def __enter__(self): return self + def __exit__(self, *args): pass + def read(self, *args): return b'{"status":"queued","receipt_id":"synthetic"}' + monkeypatch.setattr(notifier, "urlopen", lambda *args, **kwargs: Response()) + with pytest.raises(notifier.Refused, match="inbox_receipt_not_verified"): + notifier.post("http://synthetic.invalid", {"subject": "synthetic", "body": "synthetic"}) diff --git a/workplans/EMAIL-WP-0007-ledger-backup-recovery.md b/workplans/EMAIL-WP-0007-ledger-backup-recovery.md index 7849daf..0253dac 100644 --- a/workplans/EMAIL-WP-0007-ledger-backup-recovery.md +++ b/workplans/EMAIL-WP-0007-ledger-backup-recovery.md @@ -8,7 +8,7 @@ status: active owner: codex topic_slug: email-connect created: "2026-10-07" -updated: "2026-10-08" +updated: "2026-10-09" origin: residual origin_ref: EMAIL-WP-0006 state_hub_workstream_id: "35be21a2-343b-52e1-ba5e-32fead7a7831" @@ -63,7 +63,7 @@ id: EMAIL-WP-0007-T03 status: progress priority: high needs_human: true -intervention_note: "The email-connect reader is accepted and primary.env is delivered. Retention, capacity, independent receipt custody and recovery escrow remain open." +intervention_note: "First provider capture/retrieval and workstation recovery are proved. Retention/capacity, independent receipt/key/image escrow and separate recovery caller admission remain open." state_hub_task_id: "680fe4d6-12af-5296-acf9-15c9a01a0db3" ``` @@ -79,10 +79,10 @@ invented owner review or credential grant. ```task id: EMAIL-WP-0007-T04 -status: wait +status: progress priority: high needs_human: true -intervention_note: "Needs T03 admission and the exact first receiver pause/upload/schedule decision; real offsite retrieval, attended recovery and failure reception remain unproved." +intervention_note: "First admitted backup/recovery and platform-root State Hub inbox failure delivery pass. Recurring schedule, retention/capacity, independent recovery custody and operator response acceptance remain open; timers stay disabled." state_hub_task_id: "66401e5c-a5cf-5e67-beab-e503ae079964" ``` @@ -142,3 +142,35 @@ policies and Kubernetes roles are installed. email-connect directory, KV version 2, token revoked before publication, readback equal. Coulomb's reader is applied and its node file was not created. No receiver pause, upload, escrow, retention decision or timer. T04 stays wait. + +## First admitted offsite recovery and failure delivery — 2026-10-09 + +User authorized one guarded capture, encrypted primary upload, separate provider +retrieval and isolated workstation recovery; schedules explicitly remain disabled. +Decision: 0dfcdd94-50ee-4219-9659-5c4c445721f0. Runtime scripts and the platform +hash-locked SDK are installed under /opt; actual credential presence/model check +and capture preview pass under the service filesystem restrictions. File hashes: +deploy/backup/runtime-manifest-2026-10-09.json. + +Capture/resume took 56.41 seconds. Provider version 1791537739370570 and +ciphertext size/hash readback pass. A separate GET retrieves that version with +the admitted node publisher credential, then ciphertext and the public receipt +move to the workstation; no credential or private identity moves to the node. +The existing private workstation identity matches the configured public recipient. +Attended decrypt plus UID10001/network-none application rehearsal preserve all six +tables/schema/rows, two accepted deliveries and one suppression without sending. +Other transactional state and quota tables are empty in this live snapshot; +their recovery behavior remains supported by the earlier synthetic tests, not a +claim of populated live recovery coverage. Production restore/RTO are not measured. +Public proof: docs/ledger-first-backup-2026-10-09.json. Private recovery artifacts +remain mode 0700/0600 under /tmp/email-connect-first-recovery-20261009. + +Both units now have OnFailure hooks to the platform-root State Hub inbox, selected +by the user. Notification failures retain private pending records with stable +event ids; freshness runs replay them. A synthetic systemd failure delivered +message adcebf95-26c4-4adb-99b0-4446c8505a42, independently found in that inbox. +Human acknowledgement/response is not claimed. Installed freshness passes and +both timers are disabled. T04 is progress, with first-run/recovery/delivery proved. +T03 retains generation retention/capacity, independently retained receipt/key/image +escrow and a separate recovery caller. Schedule/RPO admission and incident +post-capture reconciliation remain required before workplan completion. diff --git a/workplans/EMAIL-WP-0008-scoped-transactional-callers.md b/workplans/EMAIL-WP-0008-scoped-transactional-callers.md index 4c2424d..94b5b46 100644 --- a/workplans/EMAIL-WP-0008-scoped-transactional-callers.md +++ b/workplans/EMAIL-WP-0008-scoped-transactional-callers.md @@ -8,7 +8,7 @@ status: active owner: codex topic_slug: netkingdom created: "2026-10-08" -updated: "2026-10-08" +updated: "2026-10-09" origin: residual origin_ref: CSOC-WP-0010 state_hub_workstream_id: "3a0f264d-e441-5b12-964e-dab8fe5b3828" @@ -57,10 +57,10 @@ receiver and caller module/tests. T03 remains waiting for runtime admission. ```task id: EMAIL-WP-0008-T03 -status: wait +status: progress priority: high needs_human: true -intervention_note: "Actual callers/scopes/quotas and separate token custody, platform sender/reply/bounce ownership, schema rollout/rollback and independent ledger recovery admission remain unapproved; source tests do not activate them." +intervention_note: "User approved compatible rollout preparation, proposed limits and pattern support. Glob allow-list, legacy invitation adoption, user-engine transport, candidate image and manifests are prepared. Separate token custody and coordinated runtime activation remain open; productive sender/reply/bounce acceptance stays in CSOC-WP-0015." state_hub_task_id: "9b4c7260-a6bf-54f9-82f8-2e98c368d294" ``` @@ -114,3 +114,47 @@ No suppress operation and no token value. Current invitation envelopes omit client_id, so this draft does not admit them. The token path is not created and EMAIL_CONNECT_CALLERS_FILE is not set. T03 stays wait for approval of these bindings, quotas and separate token custody. + +## Compatible original-caller rollout prepared — 2026-10-09 + +User requested compatible invitation transport with the draft limits and an +allow-list supporting globs or regexes. Receiver policies now accept optional +binding_patterns pairs using case-sensitive whole-identifier * and ? globs. +Identifiers/patterns are bounded to 128 characters and 128 total binding pairs; +regex and character-class syntax are refused. Exact bindings remain supported. +The same matcher constrains send, verification, consume and cancel. + +Draft retains tenant:coulomb/coulomb-social and proposes tenant:*-family and +tenant:family-* with literal client coulomb-social. The latter patterns are +reviewable family-name conventions, not discovery of every live tenant. Original +user-engine can explicitly adopt legacy invitations while retaining their event +ids, accepted receipts and unknown/permanent restrictions. Matching an existing +legacy fingerprint permits only the added transport client_id to be omitted from +its fingerprint; changed recipients/payloads still refuse and scope checks run +first. New bound invitations retain their full fingerprint across restarts. + +User-engine source now has opt-in USER_ENGINE_MAIL_CLIENT_BINDINGS: a bounded JSON +list of tenant-glob/literal-client pairs. Ambiguous or missing matches deny mail +before either delivery lane is contacted. It adds client_id only to the mail +envelope, leaving persisted outbox and audit envelopes unchanged. Default legacy +mode is preserved. Owner patch: deploy/caller-rollout/user-engine-mail-bindings.patch; +applied to /home/worsch/user-engine with its six new synthetic tests passing. +Full user-engine regression: 302 tests, eight optional skips, no failures. + +Local candidate email-connect:email-wp-0008-globs-20261009 is built and tested +under UID10001/network-none/read-only root with its exact caller module hash. +Proof: deploy/caller-rollout/packaged-evidence-2026-10-09.json. Immutable policy +ConfigMap and receiver mount/env patch in deploy/caller-rollout pass node server +dry-run. They require separately delivered email-connect-caller-user-engine/token, +an admitted immutable registry image and coordinated user-engine rollout; they +are excluded from the live kustomization and have not been applied. + +The receiver remains on eb3e7b68 in single-caller mode; capture/rehearsal pins +remain correctly aligned with that live image. EMAIL-WP-0007 now has an actual +offsite recovery drill and inbox failure receipt. Its remaining recurring backup +and independent custody obligations are not waived. T03 is progress while +credential delivery, exact scope/image and coordinated activation remain open. + +Final email regression after caller/notification preparation: 178 passed, zero +skips with the loopback-only GreenMail harness. Focused syntax checks pass. +User-engine source preparation is recorded at adf6e99 in its owner repository.