diff --git a/WORK-RECORDS.md b/WORK-RECORDS.md index 54aacc4..e3dc311 100644 --- a/WORK-RECORDS.md +++ b/WORK-RECORDS.md @@ -65,6 +65,6 @@ | task | EMAIL-WP-0008-T04 | done | — | workplans/EMAIL-WP-0008-scoped-transactional-callers.md | | task | EMAIL-WP-0009-T01 | done | — | workplans/EMAIL-WP-0009-wave2-resource-right-sizing.md | | task | EMAIL-WP-0010-T01 | done | — | workplans/EMAIL-WP-0010-platform-sender-transition.md | -| task | EMAIL-WP-0010-T02 | progress | — | workplans/EMAIL-WP-0010-platform-sender-transition.md | +| task | EMAIL-WP-0010-T02 | wait | — | workplans/EMAIL-WP-0010-platform-sender-transition.md | | task | EMAIL-WP-0010-T03 | wait | — | workplans/EMAIL-WP-0010-platform-sender-transition.md | | task | EMAIL-WP-0010-T04 | todo | — | workplans/EMAIL-WP-0010-platform-sender-transition.md | diff --git a/docs/sender-attended-entry-2026-10-31.json b/docs/sender-attended-entry-2026-10-31.json new file mode 100644 index 0000000..2026802 --- /dev/null +++ b/docs/sender-attended-entry-2026-10-31.json @@ -0,0 +1,61 @@ +{ + "schema": "email-connect.sender-attended-entry-preparation.v1", + "recorded_at": "2026-10-31T16:16:52.410337+00:00", + "status": "entry_implemented_owner_return_pending", + "owner_task_id": "da56d175-3126-5eaa-90c5-dd1e9da050d5", + "owner_task_observed_status": "todo", + "live_preparation_result": { + "status": "sender_release_refused", + "reason": "verified_credential_owner_return_required", + "retry_automatically": false + }, + "live_proposal_sealed": false, + "node_contacted_by_preparation": false, + "application_changed": false, + "mail_sent": false, + "new_assembly_tests_passed": 21, + "suite_passed": 270, + "suite_existing_optional_skips": 21, + "suite_seconds": 36.83, + "native_kubernetes_effects_in_assembly": "modeled", + "sqlite_and_runtime_probes": "actual_synthetic_subprocess_and_local_wsgi", + "sources": [ + { + "path": "scripts/release-sender.py", + "sha256": "98a59ebefc36094a01beaa7359f2508f54e2d59f87646e5219deeaae4b767c5a" + }, + { + "path": "scripts/sender_release.py", + "sha256": "3415b03757fcfcaf4fce69e7b7c2e99702f4b238e19d9b2f8d021b01b3de68fc" + }, + { + "path": "tests/test_sender_release.py", + "sha256": "d353f78da47c8d4d64b0faba3af5fa79d6756f9df9256bfd9812b3818d83e843" + }, + { + "path": "scripts/sender_native.py", + "sha256": "41722d23f7d04a41c7fb428cb25a24438a144684cfbc5dcd875d5299ec6ffc71" + }, + { + "path": "scripts/sender_transition.py", + "sha256": "fa80b428b6231b74578baf23ff5977b6ce871a84321718adb2ba15d314f7abe1" + }, + { + "path": "scripts/capture-ledger.py", + "sha256": "7994fe7b4fbf72e28a4296212005a88d0ef9fcb626b066e6bcf2885500bc69d6" + }, + { + "path": "scripts/ledger_archive.py", + "sha256": "92d9d5e1908c7d433eb3f4ada6460f7d9b639017a4c2311cb7ffaacd262f33c3" + }, + { + "path": "scripts/inspect-sender-ledger.py", + "sha256": "262d6d52d5c10fb8583e0828ed34945b647f1d53bf9de5286374fef85cf76a8d" + }, + { + "path": "scripts/rehearse-ledger.py", + "sha256": "5c43b655149d8ae577fab80e76358be35f90cb0357df9b726ac7cb23346946a0" + } + ], + "all_projection_resource_uids_bound": true +} diff --git a/docs/sender-attended-entry-2026-10-31.md b/docs/sender-attended-entry-2026-10-31.md new file mode 100644 index 0000000..d02a265 --- /dev/null +++ b/docs/sender-attended-entry-2026-10-31.md @@ -0,0 +1,110 @@ +# Attended sender release preparation + +The source-bound launcher and node coordinator are implemented. Twenty-one new tests +pass; the full owner suite reports **270 passed, 21 existing optional harness +skips in 36.83s**. This does not establish live sender deployment. No runnable +proposal has been sealed: the exact credential owner task RPF-WP-0068-T04 is still +`todo`, and the preparation command refuses before contacting the node. + +EMAIL-WP-0010-T02 now waits for that owner return so the live proposal and its +exact digest can be prepared and reviewed. T03/T04 retain attended execution and +independent acceptance. The existing native adapter's five server dry-runs and +pinned-image snapshot rehearsal remain source-valid; those sources did not change. + +## Source binding and admission + +`scripts/release-sender.py` is the workstation entry point. It defaults to no +action and requires an explicit prepare/show/inspect/execute/recover selection. +Preparation requires a private fresh output path and the complete successful +foundation journal. It checks that the exact RPF-WP-0068-T04 UUID and workplan UUID +are marked done in State Hub before any node call. The owner handoff must include +the review's Warden session revocation/cleanup, CCR verification and rotation +inventory obligations; the consumer never marks that task done itself. + +The node repeats the owner check, validates the foundation receipt's fixed digest +and proof fields, verifies current projection specs/Ready/ownership and the +admitted service-account UID, then performs the five native server dry-runs. Its +proposal binds complete before/after observations, the exact foundation contract, +all four projection resource UIDs, successful owner receipt, completed task +identity and eight source hashes. A replaced projection refuses forward execution +even if its spec is unchanged; explicit configuration recovery remains available. +The capsule contains those eight exact files, including launcher and coordinator. +It is saved exclusively as mode 0600 in a private operator directory. + +Execution requires the retained capsule, unchanged reviewed source bytes, the +full proposal digest, a human terminal and the exact `DEPLOY ` response. +It rechecks the owner return and native preimages. Source transfer uses a fixed +SSH target and a quoted Python bootstrap; payload data never enters shell code. +The bootstrap verifies source hashes and uses a temporary private directory with +isolated Python imports. No generic archive extraction or persistent installation +is involved. + +## Durable execution and recovery + +The node creates one exclusive attempt directory under +`/var/lib/email-connect-sender-release//`. It retains the capsule, +private append-only journal, process identity and ledger snapshots. The journal +is exclusively locked; the existing ledger capture lock excludes concurrent +backup capture. The coordinator invokes the actual native adapter and protocol +for pause, fence, capture/replay, configuration change, resume, runtime checks, +postflight comparison and UID-bound helper cleanup. + +The successful forward result is `sender_deployed_verified`; successful explicit +recovery is `configuration_recovered_verified`. The final journal phase is the +durable evidence if the launcher's response is lost. A failed response or timeout +is **unconfirmed**, not proof that the node stopped. The launcher never retries. + +`--inspect` checks the retained capsule, process PID plus Linux start ticks, the +journal and current recognized configuration. It checks recovery processes as +well as the original process. A running attempt is reported as running. After +the process exits, it reports the journal phase and actual sender/replica count; +it does not mutate or resume the application. Inspection refuses a truncated +journal or unrelated native drift instead of reporting success. + +`--recover` requires the same capsule/digest and a human `RECOVER ` +confirmation. It does not depend on a still-healthy new projection or a renewed +owner approval: it restores the already reviewed original configuration after +capturing the **current** ledger. It retains subsequent accepted and uncertain +sends, suppression and consumed/canceled handles. It never restores an older +database, clears unknown reservations or sends a message. Repeated forward +execution and completed recovery refuse. If source files have changed, restore +the exact reviewed capsule sources before invoking its launcher; never edit or +reseal a retained attempt to bypass a mismatch. + +Some ambiguous states require separate inspection: a truncated journal, a crash +during initial metadata creation, an unconfirmed helper creation without a +recorded UID, or unrelated configuration drift. The tool does not silently adopt +pods, discard evidence or invent a recovery from those states. A missing local +capsule must be recovered from the retained private node copy, not reconstructed +from current configuration. + +## Validation scope + +The assembly tests invoke the actual coordinator, protocol and native adapter. +Kubernetes transport is modeled with synthetic objects; SQLite capture/replay +runs in real subprocesses, and runtime probes contact the actual scoped WSGI +application on local loopback with zero synthetic provider sends. Host path/UID +substitutions are explicit in that harness. It is not a real Kubernetes rollout +or a new pinned-image claim; the separate isolated-image rehearsal covers that +snapshot path. + +Tests cover complete forward/recovery, a lost native configuration reply, later +accepted and uncertain sends, capture-lock contention, retained exclusive attempts, +source/proposal/foundation/owner drift, private complete final foundation receipts, +terminal and exact confirmation gates, bootstrap hash refusal, unconfirmed SSH +timeouts and real process lifetime observation. The earlier protocol tests retain +their ten real abrupt child exits around phase boundaries. + +Ruff passes. Suite log: `/tmp/email-sender-release-final-check-20261031.log`. +Source hashes and the live owner-gate refusal are in +`sender-attended-entry-2026-10-31.json`. + +## Next owner handoff + +After RPF-WP-0068-T04 has its verified native return, prepare a fresh private +capsule using `--prepare --packet --foundation-receipt +`. Preparation is read-only for production. Review +the resulting digest, native preimages and bound sources; publish the concrete +execution/recovery packet before asking for attended execution. The actual +`--execute` and `--recover` calls then use that retained `--packet` and its exact +`--expect-digest`. No live digest or execution instruction is claimed ready here. diff --git a/docs/sender-native-adapter-2026-10-31.md b/docs/sender-native-adapter-2026-10-31.md index 0d4d4f1..01c5360 100644 --- a/docs/sender-native-adapter-2026-10-31.md +++ b/docs/sender-native-adapter-2026-10-31.md @@ -1,10 +1,11 @@ # Native sender adapter preparation -EMAIL-WP-0010-T02 remains in progress. `scripts/sender_native.py` connects the +The native adapter is implemented. `scripts/sender_native.py` connects the sender transition protocol to the existing Kubernetes, ledger capture and isolated replay tools. It has no CLI; default construction permits reads and -server dry-runs only. The attended entry point and final source-bound release -packet still need preparation. No production sender change occurred. +server dry-runs only. The attended entry point is now implemented separately; +EMAIL-WP-0010-T02 waits for the verified owner return before sealing the final +live release packet. No production sender change occurred. ## Implemented native checks @@ -68,6 +69,11 @@ tests. Ruff passes. Log: `/tmp/email-sender-native-check-20261031.log`. ## Remaining T02 work +Update: the launcher, source binding and assembled protocol/adapter tests are now +implemented in [the attended entry review](sender-attended-entry-2026-10-31.md). +T02 waits for the verified owner receipt before sealing the live proposal and +publishing its concrete execution/recovery digest. + Bind the complete executor and dependencies into the final capsule, implement the attended entry point with exact proposal digest and owner-return verification, and prove the assembled forward/recovery path. Prepare the concrete execution and diff --git a/docs/sender-transition-protocol-2026-10-31.md b/docs/sender-transition-protocol-2026-10-31.md index bea9c76..6f01aaf 100644 --- a/docs/sender-transition-protocol-2026-10-31.md +++ b/docs/sender-transition-protocol-2026-10-31.md @@ -1,8 +1,10 @@ # Sender transition protocol preparation -EMAIL-WP-0010-T02 is in progress. `scripts/sender_transition.py` is a tested +`scripts/sender_transition.py` is a tested protocol core, with no production CLI or Kubernetes credentials. It does not establish native fencing, credential admission or an execution-ready release. +The adapter and attended entry point are now implemented separately; T02 waits +for the verified owner return before sealing the final live proposal. ## Concrete behavior diff --git a/scripts/release-sender.py b/scripts/release-sender.py new file mode 100644 index 0000000..f617225 --- /dev/null +++ b/scripts/release-sender.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python3 +"""Prepare, inspect or attend the fixed email-connect sender transition.""" + +from __future__ import annotations + +import argparse +import json +import shlex +import subprocess +import sys +from pathlib import Path + +import sender_release as release +from sender_transition import Refused, require + +ROOT = Path(__file__).resolve().parent +FOUNDATION = Path("/home/worsch/railiance-platform/openbao/email-connect-platform-smtp/foundation.json") + +# This bootstrap is itself bound by the launcher's source hash. No archive +# extraction, arbitrary filenames, shell interpolation of data or persistent code. +BOOTSTRAP = r''' +import hashlib,json,os,sys,tempfile +from pathlib import Path +sys.dont_write_bytecode=True +request=json.load(sys.stdin) +sources=request['sources'] +expected=request['source_hashes'] +assert __debug__ and set(sources)==set(expected) +assert all('/' not in n and '\\' not in n and n.endswith('.py') for n in sources) +assert sum(len(v.encode()) for v in sources.values())<=2*1024**2 +assert {n:hashlib.sha256(v.encode()).hexdigest() for n,v in sources.items()}==expected +with tempfile.TemporaryDirectory(prefix='email-sender-capsule-') as directory: + for name,source in sources.items(): + fd=os.open(Path(directory)/name,os.O_WRONLY|os.O_CREAT|os.O_EXCL,0o600) + with os.fdopen(fd,'w') as stream:stream.write(source) + sys.path.insert(0,directory) + import sender_release as release + try: + operation=request['operation'] + if operation=='prepare': + result=release.prepare(sources,request['foundation'],request['foundation_receipt']) + else: + packet=request['packet'] + assert packet['sources']==sources + release.validate(packet) + if operation=='inspect':result=release.inspect(packet) + elif operation in ('execute','recover'):result=release.execute(packet,recover=operation=='recover') + else:raise ValueError('unknown_operation') + print(json.dumps(result)) + except Exception as error: + print(json.dumps({'status':'sender_release_requires_inspection' if request['operation'] in ('execute','recover') + else 'sender_release_refused','failure_type':type(error).__name__, + 'retry_automatically':False})) + sys.exit(1) +''' + + +def remote(request): + try: + response = subprocess.run([ + "ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=10", "railiance01", + "sudo -n python3 -I -c " + shlex.quote(BOOTSTRAP)], + input=json.dumps(request).encode(), capture_output=True, timeout=900, check=False) + result = json.loads(response.stdout) + except (subprocess.TimeoutExpired, ValueError): + # A timeout/lost SSH reply says nothing about whether the node stopped. + return {"status": "remote_outcome_unconfirmed", "retry_automatically": False}, 1 + return result, int(response.returncode != 0) + + +def load_packet(path): + release.private(path) + require(path.stat().st_size <= 4 * 1024**2, "capsule_too_large") + packet = json.loads(path.read_text()) + release.validate(packet) + current = {name: (ROOT / name).read_text() for name in release.FILES} + require(current == packet["sources"], "reviewed_source_changed_restore_capsule_sources") + return packet + + +def read_foundation_receipt(path): + release.private(path) + require(path.stat().st_size <= 2 * 1024**2, "foundation_journal_too_large") + data = path.read_bytes() + require(data.endswith(b"\n"), "incomplete_foundation_journal") + entries = [json.loads(line) for line in data.splitlines()] + require(bool(entries), "foundation_receipt_missing") + release.foundation_receipt(entries[-1]) + return entries[-1] + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + actions = parser.add_mutually_exclusive_group(required=True) + for action in ("prepare", "show", "inspect", "execute", "recover"): + actions.add_argument("--" + action, action="store_true") + parser.add_argument("--packet", type=Path, required=True, + help="Private new output for preparation; retained sealed packet otherwise") + parser.add_argument("--foundation-receipt", type=Path) + parser.add_argument("--expect-digest") + args = parser.parse_args(argv) + require(__debug__, "optimized_python_refused") + if args.prepare: + require(args.foundation_receipt is not None and not args.packet.exists(), "fresh_packet_and_receipt_required") + release.private(args.packet.parent, directory=True) + release.owner_return("http://127.0.0.1:8000") + receipt = read_foundation_receipt(args.foundation_receipt) + sources = {name: (ROOT / name).read_text() for name in release.FILES} + request = {"operation": "prepare", "sources": sources, + "source_hashes": release.source_hashes(sources), + "foundation": json.loads(FOUNDATION.read_text()), "foundation_receipt": receipt} + packet, code = remote(request) + if code: + print(json.dumps(packet)) + return code + release.validate(packet) + require(packet["sources"] == sources, "returned_sources_changed") + release.write(args.packet, packet) + print(json.dumps({"status": "sender_proposal_prepared", "proposal_digest": packet["proposal_digest"], + "native_server_dry_runs": 5, "executed": False})) + return 0 + require(args.foundation_receipt is None, "receipt_only_for_preparation") + packet = load_packet(args.packet) + value = packet["proposal_digest"] + if args.expect_digest: + require(args.expect_digest == value, "reviewed_digest_changed") + if args.show: + print(json.dumps({"proposal_digest": value, "source_files": len(packet["sources"]), + "old_sender": packet["proposal"]["before"]["config_data"]["EMAIL_CONNECT_SENDER"], + "new_sender": packet["proposal"]["after"]["config_data"]["EMAIL_CONNECT_SENDER"], + "ledger_restore": False, "mail_send": False, "executed": False})) + return 0 + if args.execute or args.recover: + require(args.expect_digest == value and sys.stdin.isatty(), "founder_terminal_and_exact_digest_required") + if args.execute: + release.owner_return("http://127.0.0.1:8000") + verb = "RECOVER" if args.recover else "DEPLOY" + print("Recover original configuration using the current ledger." if args.recover else + "Switch email-connect to the admitted platform sender; retain the current ledger.") + require(input("Type " + verb + " followed by the full digest: ").strip() == verb + " " + value, + "not_confirmed") + request = {"operation": "recover" if args.recover else "execute" if args.execute else "inspect", + "packet": packet, "sources": packet["sources"], "source_hashes": packet["proposal"]["sources"]} + result, code = remote(request) + print(json.dumps(result)) + return code + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except Refused as error: + print(json.dumps({"status": "sender_release_refused", "reason": str(error), "retry_automatically": False})) + raise SystemExit(1) from None + except (OSError, ValueError, KeyError, TypeError) as error: + print(json.dumps({"status": "sender_release_refused", "failure_type": type(error).__name__, + "retry_automatically": False})) + raise SystemExit(1) from None diff --git a/scripts/sender_release.py b/scripts/sender_release.py new file mode 100644 index 0000000..d96dce3 --- /dev/null +++ b/scripts/sender_release.py @@ -0,0 +1,214 @@ +"""Source-bound node coordinator for the attended sender release.""" + +from __future__ import annotations + +import hashlib +import json +import os +import stat +import urllib.request +from pathlib import Path + +from sender_native import FOUNDATION, FOUNDATION_SPEC +from sender_native import Native as BaseNative +from sender_transition import Journal, desired, digest, known, require, run + +SCHEMA = "email-connect.sender-release-proposal.v1" +TASK = "da56d175-3126-5eaa-90c5-dd1e9da050d5" +WORKPLAN = "c042789b-4d7f-5c55-8d49-39dd16500f41" +STATE = Path("/var/lib/email-connect-sender-release") +FILES = {"release-sender.py", "sender_release.py", "sender_native.py", "sender_transition.py", + "capture-ledger.py", "ledger_archive.py", "inspect-sender-ledger.py", "rehearse-ledger.py"} + + +class Native(BaseNative): + """Bind projection UIDs observed at sealing, as well as the adapter's specs.""" + + def projection(self, proposal): + pins = super().projection(proposal) + if "foundation_pins" in proposal: + require(pins == proposal["foundation_pins"], "reviewed_projection_identity_changed") + return pins + + +def source_hashes(sources): + require(set(sources) == FILES and all(isinstance(v, str) for v in sources.values()), + "source_closure_changed") + require(sum(len(v.encode()) for v in sources.values()) <= 2 * 1024**2, "source_capsule_too_large") + return {name: hashlib.sha256(value.encode()).hexdigest() for name, value in sources.items()} + + +def owner_return(base): + # A consumer may observe this handoff, never mark the owner task done itself. + with urllib.request.urlopen(base + "/tasks/" + TASK, timeout=15) as response: + value = json.load(response) + require(value.get("id") == TASK and value.get("workplan_id") == WORKPLAN + and value.get("status") == "done", "verified_credential_owner_return_required") + return {key: value[key] for key in ("id", "workplan_id", "status")} + + +def foundation_receipt(value): + require(value.get("schema") == "platform.email-connect-smtp-foundation.v1" + and value.get("proposal_digest") == FOUNDATION + and value.get("status") == "foundation_admitted_projection_ready" + and value.get("application_sender_changed") is False + and isinstance(value.get("service_account_uid"), str) + and bool(value["service_account_uid"]), "successful_foundation_receipt_required") + for key in ("native_binding_verified", "native_consumer_auth_verified", "exact_read_verified", + "mutation_sibling_metadata_denied", "bounded_ttl", "other_service_account_denied", + "wrong_audience_denied", "workload_token_revocation_verified", "projection_ready"): + require(value.get(key) is True, "foundation_proof_incomplete") + + +def validate(packet): + require(__debug__, "optimized_python_refused") + plan = packet["proposal"] + require(plan["schema"] == SCHEMA and packet["proposal_digest"] == "sha256:" + digest(plan), + "proposal_digest_mismatch") + require(plan["sources"] == source_hashes(packet["sources"]), "source_digest_mismatch") + require(plan["after"] == desired(plan["before"]), "unreviewed_configuration_delta") + require(digest(plan["foundation"]) == FOUNDATION_SPEC, "foundation_contract_changed") + pins = plan["foundation_pins"] + require(set(pins) == {"ServiceAccount", "SecretStore", "ExternalSecret", "Secret"} + and all(isinstance(v, str) and bool(v) for v in pins.values()), "projection_identity_binding_required") + foundation_receipt(plan["foundation_receipt"]) + require(plan["owner_return"] == {"id": TASK, "workplan_id": WORKPLAN, "status": "done"}, + "verified_credential_owner_return_required") + return plan + + +def prepare(sources, foundation, receipt): + hashes = source_hashes(sources) + owner = owner_return("http://10.43.68.154:8000") + require(digest(foundation) == FOUNDATION_SPEC, "foundation_contract_changed") + foundation_receipt(receipt) + adapter = Native() + pins = adapter.projection({"foundation": foundation}) + require(pins["ServiceAccount"] == receipt["service_account_uid"], "admitted_identity_changed") + native = adapter.prepare() + require(adapter.projection({"foundation": foundation}) == pins, "projection_changed_during_preparation") + plan = {"schema": SCHEMA, "before": native["before"], "after": native["after"], + "sources": hashes, "foundation": foundation, "foundation_receipt": receipt, + "foundation_pins": pins, "owner_return": owner} + return {"proposal": plan, "proposal_digest": "sha256:" + digest(plan), "sources": sources} + + +def private(path, *, directory=False): + require(not any(p.is_symlink() for p in (path, *path.parents)), "real_private_path_required") + info = path.stat() + require(info.st_uid == os.geteuid() and not info.st_mode & 0o077 + and (stat.S_ISDIR(info.st_mode) if directory else + stat.S_ISREG(info.st_mode) and info.st_nlink == 1), "private_owned_path_required") + + +def write(path, value): + private(path.parent, directory=True) + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) + with os.fdopen(fd, "w") as stream: + json.dump(value, stream, sort_keys=True) + stream.write("\n") + stream.flush() + os.fsync(stream.fileno()) + sync_directory(path.parent) + + +def sync_directory(path): + fd = os.open(path, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + try: + os.fsync(fd) + finally: + os.close(fd) + + +def process_identity(pid): + try: + # Field 22 follows the parenthesized command; that name may contain spaces. + fields = Path(f"/proc/{pid}/stat").read_text().rsplit(")", 1)[1].split() + if fields[0] == "Z": + return None + return {"pid": pid, "start_ticks": fields[19]} + except FileNotFoundError: + return None + + +def attempt(packet): + validate(packet) + return STATE / packet["proposal_digest"].removeprefix("sha256:") + + +def inspect(packet): + directory = attempt(packet) + if not directory.exists(): + return {"status": "attempt_absent", "proposal_digest": packet["proposal_digest"]} + private(directory, directory=True) + private(directory / "capsule.json") + require(json.loads((directory / "capsule.json").read_text()) == packet, "retained_capsule_changed") + running = False + for process in [directory / "process.json", *directory.glob("recovery-process-*.json")]: + if not process.exists(): + continue + private(process) + recorded = json.loads(process.read_text()) + running = process_identity(recorded["pid"]) == recorded + if running: + break + # Do not read a partial journal as a success or infer death from a stale file. + if running: + return {"status": "attempt_process_running", "proposal_digest": packet["proposal_digest"], + "pid": recorded["pid"], "retry_automatically": False} + journal = Journal(directory / "journal.jsonl") + try: + require(journal.entries[0]["proposal"] == packet["proposal"], "journal_proposal_changed") + observed = Native().observe() + known(observed, packet["proposal"]["before"], packet["proposal"]["after"]) + return {"status": "attempt_inspected", "proposal_digest": packet["proposal_digest"], + "last_phase": journal.entries[-1]["phase"], "process_running": False, + "sender": observed["config_data"]["EMAIL_CONNECT_SENDER"], + "replicas": observed["deployment_spec"]["replicas"], "retry_automatically": False} + finally: + journal.close() + + +def execute(packet, *, recover=False): + plan = validate(packet) + directory = attempt(packet) + if not recover: + require(owner_return("http://10.43.68.154:8000") == plan["owner_return"], "owner_return_changed") + # Verify live preimages and projection before creating an attempt. + adapter = Native() + require(adapter.observe() == plan["before"], "preflight_changed") + require(adapter.projection(plan)["ServiceAccount"] == plan["foundation_receipt"]["service_account_uid"], + "admitted_identity_changed") + if not STATE.exists(): + STATE.mkdir(mode=0o700) + sync_directory(STATE.parent) + private(STATE, directory=True) + directory.mkdir(mode=0o700) # Exclusive: no forward replay. + sync_directory(STATE) + write(directory / "capsule.json", packet) + journal = Journal(directory / "journal.jsonl", proposal=plan) + else: + private(directory, directory=True) + private(directory / "capsule.json") + require(json.loads((directory / "capsule.json").read_text()) == packet, "retained_capsule_changed") + journal = Journal(directory / "journal.jsonl") + adapter = None + try: + require(journal.entries[0]["proposal"] == plan, "journal_proposal_changed") + # Journal flock excludes a live attempt even if the launcher lost its reply. + # Keep the original process record; recovery gets a separately numbered one. + write(directory / ("recovery-process-" + str(len(journal.entries)) + ".json" if recover + else "process.json"), process_identity(os.getpid())) + adapter = Native(execute=True, journal=journal) + adapter.acquire() + result = run(adapter, journal, recover=recover) + return {"schema": "email-connect.sender-release-result.v1", + "proposal_digest": packet["proposal_digest"], + "status": "configuration_recovered_verified" if recover else "sender_deployed_verified", + "sender": journal.entries[-1]["sender"], "ledger_restored": False, + "helper_cleanup_verified": True, "continuity": result, + "journal": str(journal.path), "mail_sent": False} + finally: + if adapter is not None: + adapter.close() + journal.close() diff --git a/tests/test_sender_release.py b/tests/test_sender_release.py new file mode 100644 index 0000000..b078d4b --- /dev/null +++ b/tests/test_sender_release.py @@ -0,0 +1,321 @@ +"""Sealed release and complete native-adapter assembly, with synthetic API effects.""" + +import copy +import importlib.util +import json +import os +import subprocess +import sys +import threading +from pathlib import Path +from wsgiref.simple_server import make_server + +import pytest +from test_callers import application +from test_sender_native import API, obj + +SCRIPTS = Path(__file__).resolve().parents[1] / "scripts" +sys.path.insert(0, str(SCRIPTS)) +import sender_native as native +import sender_release as release +from sender_transition import Refused, digest + +spec = importlib.util.spec_from_file_location("sender_launcher", SCRIPTS / "release-sender.py") +launcher = importlib.util.module_from_spec(spec) +spec.loader.exec_module(launcher) + + +def successful_receipt(): + value = {"schema": "platform.email-connect-smtp-foundation.v1", "proposal_digest": native.FOUNDATION, + "status": "foundation_admitted_projection_ready", "application_sender_changed": False, + "service_account_uid": "sa-uid"} + for key in ("native_binding_verified", "native_consumer_auth_verified", "exact_read_verified", + "mutation_sibling_metadata_denied", "bounded_ttl", "other_service_account_denied", + "wrong_audience_denied", "workload_token_revocation_verified", "projection_ready"): + value[key] = True + return value + + +class Assembly(API, release.Native): + """Only the transport is replaced; all Native operations run unchanged. + + API objects model Kubernetes. Helper Python runs in a subprocess against a + private synthetic ledger. Host UID/path substitutions are explicit here; + pinned-image isolation is separately covered by rehearse-sender-native.py. + """ + + def __init__(self, root): + super().__init__() + self.root = root + self.crash_config = False + self.port = None + self.projection_meta = {"uid": "new-secret", "owners": [{"uid": "projection-uid", "controller": True}]} + self.objects[("deployment", native.NS)]["spec"]["template"]["spec"]["volumes"].append( + {"name": "data", "persistentVolumeClaim": {"claimName": native.PVC}}) + + def metadata(self, name): + return copy.deepcopy(self.projection_meta) if name == "email-connect-platform-smtp" else super().metadata(name) + + def k(self, *args, body=None, timeout=180): + if args[0] == "exec": + code = args[-1] + code = code.replace("file:/data/email-connect.db", "file:" + str(self.root / "ledger.db")) + code = code.replace("/tmp/ledger.sqlite3", str(self.root / "helper-snapshot.sqlite3")) + code = code.replace("/tmp/recovered.sqlite3", str(self.root / "helper-recovered.sqlite3")) + code = code.replace("assert os.getuid()==10001", "assert os.getuid()==" + str(os.getuid())) + code = code.replace("127.0.0.1:8080", f"127.0.0.1:{self.port}") + sender = self.objects[("configmap", "email-connect-config")]["data"]["EMAIL_CONNECT_SENDER"] + result = subprocess.run([sys.executable, "-c", code], input=body, capture_output=True, + timeout=30, check=False, env={**os.environ, "PYTHONPATH": str(SCRIPTS.parent / "src"), + "EMAIL_CONNECT_SENDER": sender, "EMAIL_CONNECT_SMTP_USERNAME": sender, + "EMAIL_CONNECT_SMTP_PASSWORD": "synthetic-password", + "EMAIL_CONNECT_CALLERS_FILE": "/etc/email-connect-callers/callers.json"}) + assert result.returncode == 0, result.stderr.decode() + return result.stdout + result = super().k(*args, body=body, timeout=timeout) + if args[0] == "patch" and "--dry-run=server" not in args: + if args[1] == "configmap" and self.crash_config: + self.crash_config = False + raise OSError("lost config response") + if args[1] == "deployment": + self.pods.pop("receiver", None) + dep = self.objects[("deployment", native.NS)] + if dep["spec"]["replicas"] == 1: + pod = obj("Pod", "receiver", uid="resumed-receiver-uid", spec=copy.deepcopy(dep["spec"]["template"]["spec"])) + pod["metadata"].update(labels={"app.kubernetes.io/name": native.NS}, + ownerReferences=[{"kind": "ReplicaSet", "name": "rs", "uid": "rs-uid", "controller": True}]) + pod["status"] = {"phase": "Running", "containerStatuses": [{"ready": True, "imageID": native.IMAGE}]} + self.pods["receiver"] = pod + rs = obj("ReplicaSet", "rs", uid="rs-uid") + rs["metadata"]["ownerReferences"] = [{"uid": native.DEPLOYMENT, "controller": True}] + self.objects[("replicaset", "rs")] = rs + return result + + +@pytest.fixture +def assembled(tmp_path, monkeypatch): + root = tmp_path / "fixture" + root.mkdir(mode=0o700) + api = Assembly(root) + foundation = {"resources": [ + {"apiVersion": "v1", "kind": "ServiceAccount", "metadata": {"name": "sa"}}, + {"apiVersion": "external-secrets.io/v1", "kind": "SecretStore", "metadata": {"name": "store"}, "spec": {"fixture": True}}, + {"apiVersion": "external-secrets.io/v1", "kind": "ExternalSecret", "metadata": {"name": "projection"}, "spec": {"fixture": True}}, + ]} + for wanted in foundation["resources"]: + value = obj(wanted["kind"], wanted["metadata"]["name"], spec=wanted.get("spec")) + value["apiVersion"] = wanted["apiVersion"] + value["automountServiceAccountToken"] = False + value["status"] = {"conditions": [{"type": "Ready", "status": "True"}], "refreshTime": "fixture"} + api.objects[(wanted["kind"], wanted["metadata"]["name"])] = value + monkeypatch.setattr(native, "FOUNDATION_SPEC", digest(foundation)) + monkeypatch.setattr(release, "FOUNDATION_SPEC", digest(foundation)) + monkeypatch.setattr(release, "STATE", root / "attempts") + lock = root / "capture.lock" + monkeypatch.setattr(native, "CAPTURE_LOCK", lock) + def factory(**kwargs): + api.execute = kwargs.get("execute", False) + api.journal = kwargs.get("journal") + api.foundation_pins = None + api.receiver_uid = None + api.lock_path = lock + return api + monkeypatch.setattr(release, "Native", factory) + owner = {"id": release.TASK, "workplan_id": release.WORKPLAN, "status": "done"} + monkeypatch.setattr(release, "owner_return", lambda base: owner) + sources = {name: (SCRIPTS / name).read_text() for name in release.FILES} + packet = release.prepare(sources, foundation, successful_receipt()) + app, store, provider = application(root) + store.reserve_send("before", "fixed-fingerprint") + store.record("before", "accepted-before") + server = make_server("127.0.0.1", 0, app) + api.port = server.server_port + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield api, packet, store, provider + finally: + server.shutdown() + thread.join(timeout=5) + server.server_close() + store.db.close() + + +def test_complete_native_assembly_forward_and_current_ledger_recovery(assembled): + api, packet, store, provider = assembled + result = release.execute(packet) + assert result["status"] == "sender_deployed_verified" + assert result["continuity"]["all_six_tables_checked"] and not result["ledger_restored"] + assert list(api.pods) == ["receiver"] and not provider.calls + with pytest.raises((FileExistsError, Refused)): + release.execute(packet) + store.reserve_send("after", "later-fingerprint") + store.record("after", "accepted-after") + store.reserve_send("uncertain", "unknown-fingerprint") + before = api.observer.inspect(api.root / "ledger.db")["ledger_fingerprint"] + recovered = release.execute(packet, recover=True) + assert recovered["status"] == "configuration_recovered_verified" + assert recovered["sender"] == "forgejo@coulomb.social" + assert api.observer.inspect(api.root / "ledger.db")["ledger_fingerprint"] == before + assert list(api.pods) == ["receiver"] and not provider.calls + with pytest.raises(Refused, match="already_completed"): + release.execute(packet, recover=True) + + +def test_lost_native_config_reply_retains_attempt_and_recovers(assembled): + api, packet, store, provider = assembled + api.crash_config = True + with pytest.raises(OSError, match="lost config"): + release.execute(packet) + directory = release.attempt(packet) + entries = [json.loads(line) for line in (directory / "journal.jsonl").read_text().splitlines()] + assert entries[-1]["phase"] == "requires_inspection" + assert api.objects[("deployment", native.NS)]["spec"]["replicas"] == 0 + assert api.objects[("configmap", "email-connect-config")]["data"]["EMAIL_CONNECT_SENDER"] == "platform@coulomb.social" + store.reserve_send("late-unknown", "unknown-fingerprint") + result = release.execute(packet, recover=True) + assert result["status"] == "configuration_recovered_verified" and not provider.calls + assert store.delivery_status("late-unknown")["state"] == "unknown" + + +@pytest.mark.parametrize("change", ["source", "delta", "owner", "receipt", "foundation", "digest"]) +def test_seal_refuses_tampering_before_native_effects(assembled, change): + api, original, _, _ = assembled + packet = copy.deepcopy(original) + if change == "source": packet["sources"]["sender_native.py"] += "\n# changed\n" + if change == "delta": packet["proposal"]["after"]["deployment_spec"]["replicas"] = 2 + if change == "owner": packet["proposal"]["owner_return"]["status"] = "todo" + if change == "receipt": packet["proposal"]["foundation_receipt"]["workload_token_revocation_verified"] = False + if change == "foundation": packet["proposal"]["foundation"]["resources"].pop() + if change != "digest": packet["proposal_digest"] = "sha256:" + digest(packet["proposal"]) + else: packet["proposal_digest"] = "sha256:wrong" + calls = len(api.calls) + with pytest.raises(Refused): release.execute(packet) + assert len(api.calls) == calls and not release.STATE.exists() + + +def test_replaced_projection_refuses_before_pause_but_does_not_block_recovery(assembled): + api, packet, _, _ = assembled + api.projection_meta["uid"] = "replacement-secret" + with pytest.raises(Refused, match="projection_identity"): + release.execute(packet) + assert not release.STATE.exists() + api.projection_meta["uid"] = packet["proposal"]["foundation_pins"]["Secret"] + release.execute(packet) + api.projection_meta["uid"] = "replacement-secret" + assert release.execute(packet, recover=True)["status"] == "configuration_recovered_verified" + + +def test_execute_and_recover_require_founder_terminal_before_transport(assembled, monkeypatch): + api, packet, _, _ = assembled + path = api.root / "packet.json" + release.write(path, packet) + monkeypatch.setattr(launcher.sys.stdin, "isatty", lambda: False) + monkeypatch.setattr(launcher, "remote", lambda request: pytest.fail("must not contact node")) + for action in ("execute", "recover"): + with pytest.raises(Refused, match="founder_terminal"): + launcher.main(["--" + action, "--packet", str(path), "--expect-digest", packet["proposal_digest"]]) + + +def test_owner_task_must_be_exact_completed_handoff(monkeypatch): + import io + for value in ({"id": release.TASK, "workplan_id": release.WORKPLAN, "status": "todo"}, + {"id": "different", "workplan_id": release.WORKPLAN, "status": "done"}): + monkeypatch.setattr(release.urllib.request, "urlopen", lambda *a, value=value, **kw: io.BytesIO(json.dumps(value).encode())) + with pytest.raises(Refused, match="owner_return"): + release.owner_return("http://fixture.invalid") + + +def test_inspection_checks_live_recovery_process_identity(assembled, monkeypatch): + _api, packet, _, _ = assembled + release.execute(packet) + directory = release.attempt(packet) + recorded = {"pid": 987654321, "start_ticks": "fixture-ticks"} + release.write(directory / "recovery-process-1000.json", recorded) + monkeypatch.setattr(release, "process_identity", lambda pid: recorded if pid == recorded["pid"] else None) + assert release.inspect(packet)["status"] == "attempt_process_running" + monkeypatch.setattr(release, "process_identity", lambda pid: None) + assert release.inspect(packet)["last_phase"] == "completed" + + +def test_live_capture_lock_prevents_effects_and_retains_exclusive_attempt(assembled): + api, packet, _, _ = assembled + import fcntl + fd = os.open(api.lock_path, os.O_CREAT | os.O_RDWR, 0o600) + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + before = copy.deepcopy(api.objects) + try: + with pytest.raises(BlockingIOError): release.execute(packet) + assert api.objects == before + directory = release.attempt(packet) + assert (directory / "capsule.json").exists() + with pytest.raises(FileExistsError): release.execute(packet) + finally: + os.close(fd) + assert release.execute(packet, recover=True)["status"] == "configuration_recovered_verified" + + +@pytest.mark.parametrize("problem", ["truncated", "private", "earlier_success"]) +def test_foundation_journal_requires_private_complete_final_success(tmp_path, problem): + path = tmp_path / "foundation.jsonl" + data = json.dumps(successful_receipt()) + "\n" + if problem == "truncated": data += '{"status":' + if problem == "earlier_success": data += json.dumps({"status": "requires_inspection"}) + "\n" + path.write_text(data) + path.chmod(0o644 if problem == "private" else 0o600) + with pytest.raises(Refused): launcher.read_foundation_receipt(path) + + +def test_current_owner_wait_blocks_sealing_before_remote_call(tmp_path, monkeypatch): + tmp_path.chmod(0o700) + def waiting(base): + raise Refused("verified_credential_owner_return_required") + monkeypatch.setattr(release, "owner_return", waiting) + monkeypatch.setattr(launcher, "remote", lambda request: pytest.fail("no node call before owner return")) + with pytest.raises(Refused, match="owner_return"): + launcher.main(["--prepare", "--packet", str(tmp_path / "packet.json"), + "--foundation-receipt", str(tmp_path / "absent.jsonl")]) + assert not (tmp_path / "packet.json").exists() + + +def test_wrong_confirmation_cannot_reach_remote_mutation(assembled, monkeypatch): + api, packet, _, _ = assembled + path = api.root / "packet.json" + release.write(path, packet) + monkeypatch.setattr(launcher.sys.stdin, "isatty", lambda: True) + monkeypatch.setattr("builtins.input", lambda _: "DEPLOY sha256:wrong") + monkeypatch.setattr(launcher, "remote", lambda request: pytest.fail("wrong confirmation executed")) + with pytest.raises(Refused, match="not_confirmed"): + launcher.main(["--execute", "--packet", str(path), "--expect-digest", packet["proposal_digest"]]) + + +def test_bootstrap_checks_source_bytes_before_executing_them(tmp_path): + sentinel = tmp_path / "must-not-exist" + sources = {"sender_release.py": "from pathlib import Path\nPath(" + repr(str(sentinel)) + ").touch()\n"} + request = {"sources": sources, "source_hashes": {"sender_release.py": "0" * 64}} + result = subprocess.run([sys.executable, "-I", "-c", launcher.BOOTSTRAP], + input=json.dumps(request).encode(), capture_output=True, check=False, timeout=15) + assert result.returncode != 0 and not sentinel.exists() + + +def test_remote_timeout_is_unconfirmed_not_stopped_or_retried(monkeypatch): + calls = [] + def timeout(*args, **kwargs): + calls.append(args) + raise subprocess.TimeoutExpired("fixture-ssh", 900) + monkeypatch.setattr(launcher.subprocess, "run", timeout) + result, code = launcher.remote({"operation": "execute"}) + assert code == 1 and result["status"] == "remote_outcome_unconfirmed" + assert not result["retry_automatically"] and len(calls) == 1 + + +def test_process_identity_follows_real_process_lifetime(): + child = subprocess.Popen([sys.executable, "-c", "import time;time.sleep(30)"]) + try: + first = release.process_identity(child.pid) + assert first["pid"] == child.pid and first["start_ticks"].isdigit() + assert release.process_identity(child.pid) == first + finally: + child.terminate() + child.wait(timeout=10) + assert release.process_identity(child.pid) is None diff --git a/workplans/EMAIL-WP-0010-platform-sender-transition.md b/workplans/EMAIL-WP-0010-platform-sender-transition.md index d27a668..f33147a 100644 --- a/workplans/EMAIL-WP-0010-platform-sender-transition.md +++ b/workplans/EMAIL-WP-0010-platform-sender-transition.md @@ -51,9 +51,10 @@ archive-fingerprint compatibility and private-output refusal. Owner suite: ```task id: EMAIL-WP-0010-T02 -status: progress +status: wait +blocking_reason: "Executor and assembly tests complete; sealing the live proposal requires verified RPF-WP-0068-T04 and its successful foundation receipt." priority: high -depends_on: [EMAIL-WP-0010-T01] +depends_on: [EMAIL-WP-0010-T01, RPF-WP-0068-T04] state_hub_task_id: "acb6f995-9177-5a3c-909a-feb152d6bfed" ``` @@ -101,11 +102,26 @@ disposable container was removed. Thirty new adapter cases pass; full owner suit the source hashes and scope of each proof. Review: `docs/sender-native-adapter-2026-10-31.md`. -T02 remains progress: the final source-bound capsule, attended entry point, -assembled forward/recovery proof and exact execution/recovery packet are still -required. Native dry-runs and isolated snapshot integration do not establish a -production transition. Existing backup and foundation source packets remain -unchanged; no live grant, sender change, capture or mail send occurred. +2026-10-31 assembled execution follow-through: `scripts/release-sender.py` and +`scripts/sender_release.py` bind eight source files, native preimages, the exact +foundation receipt and verified owner handoff into an exclusive private capsule. +The launcher requires a human terminal and exact digest confirmation for deploy +or recovery; the node retains source capsule/journal/process identities and uses +the actual adapter/protocol. Twenty-one new cases prove the assembled forward and +recovery paths against modeled Kubernetes with real SQLite and WSGI probes, +including a lost configuration response, subsequent accepted/unknown sends, +capture-lock contention, source/receipt/owner drift and unconfirmed SSH outcomes. +All four new projection UIDs are sealed; replacement refuses deployment while +configuration recovery remains available. Owner suite: 270 passed / 21 existing +optional harness skips; Ruff clean. + +T02 is now wait on RPF-WP-0068-T04. A real preparation invocation refused with +`verified_credential_owner_return_required` before any node call or capsule write: +the owner task is still todo. Once that exact verified receipt exists, seal the +fresh native proposal and publish its exact execution/recovery review packet to +finish T02. Review: `docs/sender-attended-entry-2026-10-31.md`. No live grant, +sender change, production capture or mail send occurred; existing backup and +foundation source packets remain unchanged. T03/T04 retain live execution/return. ## T03 — Execute the admitted sender switch