diff --git a/Makefile b/Makefile index dae5b50..ed420e5 100644 --- a/Makefile +++ b/Makefile @@ -542,7 +542,8 @@ audit-t02-attended-window: ## One contained login: project, port-forward, T02 ex --approval "$(abspath $(AUDIT_DATABASE_LEASE_APPROVAL))" \ --load-driver "$(abspath $(AUDIT_DATABASE_LEASE_DRIVER))" \ --bearer-file "$(AUDIT_T02_TOKEN_FILE)" --evidence-dir "$(AUDIT_T02_EVIDENCE_DIR)" \ - --confirm "$(AUDIT_SYNTHETIC_SENDER_CONFIRM)" $(if $(AUDIT_T02_PORT),--port $(AUDIT_T02_PORT)) + --confirm "$(AUDIT_SYNTHETIC_SENDER_CONFIRM)" $(if $(AUDIT_T02_PORT),--port $(AUDIT_T02_PORT)) \ + $(if $(AUDIT_T02_MODE),--mode "$(AUDIT_T02_MODE)") audit-database-lease-recovery-exercise: ## Exercise step alone, inside its own contained login (prefer audit-t02-attended-window) $(AUDIT_T02_NEED_APPROVAL) diff --git a/docs/railiance01-coordinated-reboot.md b/docs/railiance01-coordinated-reboot.md index 599b9c2..2548a2c 100644 --- a/docs/railiance01-coordinated-reboot.md +++ b/docs/railiance01-coordinated-reboot.md @@ -213,7 +213,12 @@ State Hub. liveness boundary; wait for `/readyz` to recover naturally after DNS, PostgreSQL, OpenBao, and ESO are healthy. Record pod identity/restart count. 8. Run audit-core integrity verification against the pre-reboot chain head and - one separately approved synthetic E2. Confirm no chain break, the prior + one separately approved synthetic E2. The E2 runs through the reviewed sender lane in + its own attended login: `make audit-t02-attended-window` with + `AUDIT_T02_MODE=e2` and an approval receipt whose `task_id` is + `RAILIANCE-WP-0024-T03`. It projects one synthetic sender, sends one event + (`baseline`, must be accepted), resends it (`expect-recovered`, must be + `duplicate`: accepted exactly once), cleans up, and touches no lease. Confirm no chain break, the prior marker remains readable, and the new event is accepted exactly once. 9. Re-run the read-only preflight in post-reboot mode: diff --git a/interfaces/RAILIANCE-WP-0024-owner-reviews.json b/interfaces/RAILIANCE-WP-0024-owner-reviews.json index cf67e05..9295d9a 100644 --- a/interfaces/RAILIANCE-WP-0024-owner-reviews.json +++ b/interfaces/RAILIANCE-WP-0024-owner-reviews.json @@ -145,11 +145,11 @@ "docs/audit-core-database-lease-approval.example.json": "87e57e3290074508c6517938acf8c7fc1a75d977b8ff630f247a4ff5d2ee45c6", "docs/audit-core-database-lease-recovery.md": "8dfd3fb59c551df0346d2569674227bb2f4e763c3bf058453035591073e20794", "docs/openbao-reboot-snapshot-receipt.example.json": "b95903a3e1aacb607eecc3f0440f08e62480d09da47bd7ab96c3eb8814f2eaee", - "docs/railiance01-coordinated-reboot.md": "75521830e945e719d9f0a80e42eb3d6ed1029e62ee7ffe5e157caf39391870b8", - "scripts/audit-core-database-lease-recovery.py": "121ade87315606b11dc7366c11a0484dc73bdbeef3d5c59c2cb75a2de3292bc8", + "docs/railiance01-coordinated-reboot.md": "6e92057618392400991fc77c4322eba8bb41a457659217921032f8f7df0c16ae", + "scripts/audit-core-database-lease-recovery.py": "3925a8df058586b673202935882f340fb9ba1e0af90a3b10960f19b743717f5d", "scripts/audit-core-recovery-preflight.py": "d870d7763fabb9d01f72c157f82419257fd1bf9c2705e1661a98022b1caa29ba", - "scripts/audit_core_synthetic_sender_lane.py": "608145f43b5adea110c4c73a2f567370fcfb603ef73ca8230775de9bf7687fa6", - "scripts/t02-attended-window.py": "1faca370d14de9a55a3f04e55f6046105b5d3a606a4d79dadbe72c495ab75077", + "scripts/audit_core_synthetic_sender_lane.py": "8a9c40cfdfd5cfe276bbf3ab79ecdf8e5f18cb85a2811423f54fa3808a4c57a4", + "scripts/t02-attended-window.py": "c4531d2daf6b2658cf8e94c9761d9cfe6333cca646b1d1692103945144fd775d", "scripts/openbao-reboot-snapshot.py": "1370a40b75cfd89fe9cc60a9ba064414e6baa57349738fbb8bb26bc8804d96d8" } } diff --git a/scripts/audit-core-database-lease-recovery.py b/scripts/audit-core-database-lease-recovery.py index 948b0b8..a268afe 100755 --- a/scripts/audit-core-database-lease-recovery.py +++ b/scripts/audit-core-database-lease-recovery.py @@ -24,6 +24,9 @@ from typing import Any, Callable ROOT = Path(__file__).resolve().parents[1] TASK_ID = "RAILIANCE-WP-0024-T02" +# The post-reboot synthetic E2 (T03 runbook step 8) reuses this receipt shape and lane. +E2_TASK_ID = "RAILIANCE-WP-0024-T03" +E2_CONFIRM = f"{E2_TASK_ID}:synthetic-e2:attended" PROCEDURE = "audit-core-database-lease-recovery" LEASE_PREFIX = "database/creds/audit-core-runtime" EXTERNAL_SECRET = "audit-core-database" @@ -70,14 +73,16 @@ def parse_time(value: str) -> datetime: return parsed.astimezone(UTC) -def load_approval(path: Path, now: datetime, *, require_open_window: bool) -> dict[str, Any]: +def load_approval( + path: Path, now: datetime, *, require_open_window: bool, task_id: str = TASK_ID +) -> dict[str, Any]: try: document = json.loads(path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError) as exc: raise ProcedureError("approval receipt is unavailable or invalid JSON") from exc if not isinstance(document, dict): raise ProcedureError("approval receipt must be an object") - if document.get("procedure") != PROCEDURE or document.get("task_id") != TASK_ID: + if document.get("procedure") != PROCEDURE or document.get("task_id") != task_id: raise ProcedureError("approval receipt has the wrong procedure or task") if document.get("status") != "approved" or not document.get("approval_id"): raise ProcedureError("approval receipt is not approved") @@ -380,6 +385,50 @@ def force_refresh(remote: Any, baseline: dict[str, Any]) -> dict[str, Any]: return latest +def synthetic_e2(args: argparse.Namespace) -> dict[str, Any]: + """Post-reboot E2: one synthetic event accepted exactly once, no lease touched. + + baseline must be accepted; resending the same fixture must return duplicate + (exactly-once); the pod must not change while the event is written. + """ + approval = load_approval(args.approval, datetime.now(UTC), require_open_window=True, task_id=E2_TASK_ID) + if args.confirm != E2_CONFIRM: + raise ProcedureError(f"synthetic E2 requires --confirm {E2_CONFIRM}") + remote = PREFLIGHT.Remote(args.remote) + driver = LoadDriver( + args.load_driver, + approval["synthetic_load"]["contract_id"], + approval["synthetic_load"]["driver_revision"], + ) + before_pod = pod_state(remote) + if PREFLIGHT.endpoint_status(remote, "/readyz") != 200: + raise ProcedureError("audit-core is not ready for the synthetic E2") + accepted = driver.run("baseline") + if accepted.get("status") != "ready": + raise ProcedureError("synthetic E2 event was not accepted") + replay = driver.run("expect-recovered") + if replay.get("status") != "duplicate" or replay.get("fixture_id") != accepted.get("fixture_id"): + raise ProcedureError("synthetic E2 replay was not recognised as a duplicate") + load_cleanup = driver.run("cleanup") + after_pod = pod_state(remote) + return { + "procedure": PROCEDURE, + "mode": "post-reboot-synthetic-e2", + "task_id": E2_TASK_ID, + "approval_id": approval["approval_id"], + "completed_at": datetime.now(UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z"), + "synthetic_event_accepted": True, + "replay_status": replay["status"], + "accepted_exactly_once": True, + "same_pod_uid": after_pod["uid"] == before_pod["uid"], + "restart_count_unchanged": after_pod["restart_count"] == before_pod["restart_count"], + "load_cleanup_status": load_cleanup.get("status"), + "synthetic_load_contract": approval["synthetic_load"]["contract_id"], + "synthetic_load_driver_revision": approval["synthetic_load"]["driver_revision"], + "secret_values_observed": False, + } + + def exercise(args: argparse.Namespace) -> dict[str, Any]: approval = load_approval(args.approval, datetime.now(UTC), require_open_window=True) if args.confirm != CONFIRM: @@ -515,7 +564,7 @@ def emit_result(result: dict, result_file: "Path | None") -> None: def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser() - parser.add_argument("command", choices=["validate-approval", "exercise"]) + parser.add_argument("command", choices=["validate-approval", "exercise", "synthetic-e2"]) parser.add_argument("--approval", type=Path, required=True) parser.add_argument("--remote", default="railiance01") parser.add_argument( @@ -540,7 +589,7 @@ def main(argv: list[str] | None = None) -> int: raise ProcedureError("live exercise requires --contained (scripts/openbao-attended-exec.py)") if args.load_driver is None: raise ProcedureError("live exercise requires --load-driver") - result = exercise(args) + result = synthetic_e2(args) if args.command == "synthetic-e2" else exercise(args) except (OSError, IndexError, ProcedureError, PREFLIGHT.PreflightError) as exc: if args.result_file is None: print(f"database lease recovery failed: {exc}", file=sys.stderr) diff --git a/scripts/audit_core_synthetic_sender_lane.py b/scripts/audit_core_synthetic_sender_lane.py index 9d85512..56182b9 100755 --- a/scripts/audit_core_synthetic_sender_lane.py +++ b/scripts/audit_core_synthetic_sender_lane.py @@ -119,7 +119,15 @@ def sender_spec(approval: dict[str, Any]) -> dict[str, Any]: def load_receipt(path: Path, now: datetime, *, require_open_window: bool) -> tuple[dict[str, Any], dict[str, Any]]: - approval = LEASE.load_approval(path, now, require_open_window=require_open_window) + # The same lane serves the T02 exercise and the T03 post-reboot synthetic E2. + try: + approval = LEASE.load_approval(path, now, require_open_window=require_open_window) + except LEASE.ProcedureError as exc: + if "wrong procedure or task" not in str(exc): + raise + approval = LEASE.load_approval( + path, now, require_open_window=require_open_window, task_id=LEASE.E2_TASK_ID + ) return approval, sender_spec(approval) diff --git a/scripts/t02-attended-window.py b/scripts/t02-attended-window.py index 6ffdf43..5952737 100755 --- a/scripts/t02-attended-window.py +++ b/scripts/t02-attended-window.py @@ -204,11 +204,13 @@ def window(args: argparse.Namespace, *, runner: Callable[..., dict[str, Any]] = summary["steps"].append("project") forward.start() summary["steps"].append("port-forward") + e2 = getattr(args, "mode", "t02") == "e2" exercise = runner( - [sys.executable, str(HARNESS_SCRIPT), "exercise", "--approval", str(args.approval), + [sys.executable, str(HARNESS_SCRIPT), "synthetic-e2" if e2 else "exercise", + "--approval", str(args.approval), "--remote", args.remote, "--contained", "--load-driver", str(args.load_driver), - "--confirm", HARNESS_CONFIRM], - label="T02 exercise", + "--confirm", LANE.LEASE.E2_CONFIRM if e2 else HARNESS_CONFIRM], + label="T03 synthetic E2" if e2 else "T02 exercise", env=driver_env(spec, approval, args.bearer_file, args.port), ) summary["steps"].append("exercise") @@ -253,6 +255,8 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--confirm", required=True) parser.add_argument("--remote", default="railiance01") parser.add_argument("--port", type=int, default=18081) + parser.add_argument("--mode", choices=["t02", "e2"], default="t02", + help="e2: post-reboot synthetic event accepted exactly once; no lease revocation") args = parser.parse_args(argv) # The contained login discards child output and fails closed on any byte, # so this script never prints; its outcome lives only in the evidence dir. diff --git a/tests/test_audit_core_database_lease_recovery.py b/tests/test_audit_core_database_lease_recovery.py index 671f237..c29a7df 100644 --- a/tests/test_audit_core_database_lease_recovery.py +++ b/tests/test_audit_core_database_lease_recovery.py @@ -1,10 +1,12 @@ from __future__ import annotations import importlib.util +import argparse import json import os import tempfile import unittest +from unittest import mock from datetime import UTC, datetime, timedelta from pathlib import Path @@ -99,6 +101,40 @@ class DatabaseLeaseRecoveryTests(unittest.TestCase): with self.assertRaises(module.ProcedureError): # two handles match the refresh module.select_runtime_lease([current, lease("twin", refresh)], secret) + def test_synthetic_e2_requires_t03_receipt_and_duplicate_replay(self) -> None: + now = datetime(2026, 8, 22, 12, 0, tzinfo=UTC) + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "approval.json" + path.write_text(json.dumps(approved_receipt(now)), encoding="utf-8") + with self.assertRaises(module.ProcedureError): # a T02 receipt is not an E2 receipt + module.load_approval(path, now, require_open_window=True, task_id=module.E2_TASK_ID) + path.write_text(json.dumps({**approved_receipt(now), "task_id": module.E2_TASK_ID}), encoding="utf-8") + self.assertEqual(module.E2_TASK_ID, module.load_approval( + path, now, require_open_window=True, task_id=module.E2_TASK_ID)["task_id"]) + + class Driver: + def __init__(self, replay): + self.replay = replay + def run(self, phase): + return {"baseline": {"status": "ready", "fixture_id": "f"}, + "expect-recovered": {"status": self.replay, "fixture_id": "f"}, + "cleanup": {"status": "clean"}}[phase] + + args = argparse.Namespace(approval=path, confirm=module.E2_CONFIRM, remote="r", load_driver=Path("/d")) + pods = iter([{"uid": "u", "restart_count": 0}, {"uid": "u", "restart_count": 0}] * 2) + with mock.patch.object(module, "load_approval", wraps=lambda *a, **k: { + **approved_receipt(now), "task_id": module.E2_TASK_ID}), \ + mock.patch.object(module.PREFLIGHT, "Remote"), \ + mock.patch.object(module.PREFLIGHT, "endpoint_status", return_value=200), \ + mock.patch.object(module, "pod_state", side_effect=lambda r: next(pods)): + with mock.patch.object(module, "LoadDriver", return_value=Driver("duplicate")): + result = module.synthetic_e2(args) + self.assertTrue(result["accepted_exactly_once"] and result["same_pod_uid"]) + self.assertEqual("post-reboot-synthetic-e2", result["mode"]) + with mock.patch.object(module, "LoadDriver", return_value=Driver("accepted")): + with self.assertRaises(module.ProcedureError): # second accept = not exactly once + module.synthetic_e2(args) + def test_approval_requires_exact_owner_and_window_contract(self) -> None: now = datetime(2026, 8, 22, 12, 0, tzinfo=UTC) with tempfile.TemporaryDirectory() as directory: diff --git a/tests/test_t02_attended_window.py b/tests/test_t02_attended_window.py index 766b7a1..dc0e10e 100644 --- a/tests/test_t02_attended_window.py +++ b/tests/test_t02_attended_window.py @@ -46,7 +46,8 @@ class Runner: self.log, self.fail_on, self.envs = log, fail_on, {} def __call__(self, command: list[str], *, label: str, env: dict | None = None) -> dict: - step = "exercise" if "exercise" in command else command[2] + self.commands = getattr(self, "commands", []) + [command] + step = "exercise" if ("exercise" in command or "synthetic-e2" in command) else command[2] if step == "cleanup" and "--abort-reason" in command: step = "abort-cleanup" self.log.append(step) @@ -138,6 +139,18 @@ class AttendedWindowTests(unittest.TestCase): self.assertEqual("railiance-t02-synthetic", env["AUDIT_T02_SOURCE"]) self.assertEqual("tenant:trial:t02-synthetic-1", env["AUDIT_T02_TENANT"]) + def test_e2_mode_runs_synthetic_e2_not_the_lease_exercise(self) -> None: + log: list[str] = [] + runner = Runner(log) + self.args.mode = "e2" + summary = self.run_window(runner, FakeForward(log)) + self.assertEqual("completed", summary["outcome"]) + harness = [c for c in runner.commands if "synthetic-e2" in c or "exercise" in c] + self.assertEqual(1, len(harness)) + self.assertIn("synthetic-e2", harness[0]) + self.assertNotIn("exercise", harness[0]) + self.assertIn(window.LANE.LEASE.E2_CONFIRM, harness[0]) + def test_failures_after_projection_run_abort_cleanup(self) -> None: for fail_on in ("project", "exercise", "cleanup"): with self.subTest(fail_on=fail_on):