diff --git a/docs/audit-core-database-lease-recovery.md b/docs/audit-core-database-lease-recovery.md index 6852814..661335a 100644 --- a/docs/audit-core-database-lease-recovery.md +++ b/docs/audit-core-database-lease-recovery.md @@ -111,6 +111,13 @@ driver as `AUDIT_T02_SOURCE`/`AUDIT_T02_TENANT`; the confirm string is If cleanup is missed, audit-core's `expires_at` enforcement makes the identity inert at window end; the attended cleanup is still required to remove it. +If `project` fails after the registry write may already have reached OpenBao (for +example a transport error on the write itself), treat the identity as possibly +present: run `cleanup --abort-reason ""` before leaving the window. +The lane's `--token-file` is the attended operator token for the registry write; +it must come from the contained ops-warden attended login accepted under RPF-WP-0017 +(isolated 0600 helper state, removed afterwards), not from a long-lived token file. + ## Attended sequence Keep one terminal attached from baseline through recovery. A value-handling diff --git a/docs/railiance01-coordinated-reboot.md b/docs/railiance01-coordinated-reboot.md index 39b43b6..c744410 100644 --- a/docs/railiance01-coordinated-reboot.md +++ b/docs/railiance01-coordinated-reboot.md @@ -73,9 +73,15 @@ until supplied: denied by the preflight; - the RPF-WP-0017 re-acceptance receipt `reviews/RPF-WP-0017-ops-warden-reacceptance-2026-10-14.json` recording - acceptance of the exact ops-warden revision used for the attended login. The - 2026-10-14 revision `11b2a9d…` was reviewed as `reject`, so this gate is - currently closed; + acceptance of the exact ops-warden revision used for the attended login + (`11b2a9d…` was rejected; `6e14300…` accepted on 2026-10-14). The accepted + revision must be pushed to ops-warden `origin/main` and installed with + `uv tool install --reinstall /home/worsch/ops-warden` from that clean checkout + before the window. The preflight compares every installed `warden` module + byte-for-byte with the accepted revision and closes this gate on any mismatch; + rerun it at the hold point immediately before the attended login and abort if + it no longer matches (the 2026-09-01 WARDEN-WP-0036-T02 operation ran on + unaccepted code — do not repeat that); - a named attended driver/abort operator and the `railiance-infra` independent abort owner acknowledgement (`--infra-abort-owner-ack`); - independent provider-console access, not SSH alone: a value-safe diff --git a/interfaces/RAILIANCE-WP-0024-owner-reviews.json b/interfaces/RAILIANCE-WP-0024-owner-reviews.json index c9b3db1..2cdb404 100644 --- a/interfaces/RAILIANCE-WP-0024-owner-reviews.json +++ b/interfaces/RAILIANCE-WP-0024-owner-reviews.json @@ -139,11 +139,11 @@ }, "artifact_sha256": { "docs/audit-core-database-lease-approval.example.json": "87e57e3290074508c6517938acf8c7fc1a75d977b8ff630f247a4ff5d2ee45c6", - "docs/audit-core-database-lease-recovery.md": "630e9324d61fddc50c2c364649064a316a7b406b45cc6b4799cf0c4f8319b237", + "docs/audit-core-database-lease-recovery.md": "1c7643be1b97b56f5b5efb2c21803fbb0a8d3863953b61ebbfb85792932f7a4a", "docs/openbao-reboot-snapshot-receipt.example.json": "b95903a3e1aacb607eecc3f0440f08e62480d09da47bd7ab96c3eb8814f2eaee", - "docs/railiance01-coordinated-reboot.md": "1cbe079ca34252c1638879f0c03cf5b96d9a0078de5047ecf0cb5b3857a7121d", + "docs/railiance01-coordinated-reboot.md": "6fd6301f808449ad6519764d994b05aee41a96be879b917ee1d18d8a65d5f6fd", "scripts/audit-core-database-lease-recovery.py": "1de02f13dcd4e87e65de1c97ae14567f2b8e4a51aa0423e3fca78a224cf374b3", - "scripts/audit-core-recovery-preflight.py": "41ee3889db008ed90d51b855fc3470244051e7e3d7ba6c1c9256c3d115bbb23d", + "scripts/audit-core-recovery-preflight.py": "05e89ae9d0b9a636b67d9a8d0c203ac746c97208e32061cb23f1ca785eda8e5c", "scripts/audit_core_synthetic_sender_lane.py": "e71b3b3d56472a1cc91fc093d327a72116d89d07c1676f44bc24a8da36154bd4" } } diff --git a/scripts/audit-core-recovery-preflight.py b/scripts/audit-core-recovery-preflight.py index 3e9e38d..42e2296 100755 --- a/scripts/audit-core-recovery-preflight.py +++ b/scripts/audit-core-recovery-preflight.py @@ -638,6 +638,49 @@ def validate_ops_warden_receipt(path: Path) -> dict[str, Any]: return {"path": str(path), "accepted_revision": revision, "verified": True} +OPS_WARDEN_REPO = Path("/home/worsch/ops-warden") + + +def installed_warden_package(tool_python: Path) -> Path: + """Package directory of the warden actually on PATH (uv tool environment).""" + completed = subprocess.run( + [str(tool_python), "-I", "-c", "import os, warden; print(os.path.dirname(warden.__file__))"], + text=True, capture_output=True, check=False, + ) + if completed.returncode != 0 or not completed.stdout.strip(): + raise PreflightError("installed warden package could not be located") + return Path(completed.stdout.strip()) + + +def verify_installed_warden(package: Path, repo: Path, revision: str) -> dict[str, Any]: + """Prove the installed warden equals the accepted, published ops-warden revision. + + The uv tool is a non-editable directory install that records no revision, so + compare every installed module byte-for-byte with that revision's tree. + """ + def git(*args: str) -> subprocess.CompletedProcess[bytes]: + return subprocess.run(["git", "-C", str(repo), *args], capture_output=True, check=False) + + if git("merge-base", "--is-ancestor", revision, "origin/main").returncode != 0: + raise PreflightError("accepted ops-warden revision is not published on origin/main") + listed = git("ls-tree", "-r", "--name-only", revision, "src/warden/") + if listed.returncode != 0: + raise PreflightError("accepted ops-warden revision is not available locally") + expected = { + name.removeprefix("src/warden/") + for name in listed.stdout.decode().splitlines() + if name.endswith(".py") + } + installed = {path.relative_to(package).as_posix() for path in package.rglob("*.py")} + if installed != expected: + raise PreflightError("installed warden module set differs from the accepted revision") + for name in sorted(expected): + blob = git("show", f"{revision}:src/warden/{name}") + if blob.returncode != 0 or blob.stdout != (package / name).read_bytes(): + raise PreflightError("installed warden differs from the accepted revision") + return {"installed_revision_matches": True, "modules_compared": len(expected)} + + CONSOLE_RECEIPT_ID = re.compile(r"^RAIL01-CONSOLE-(\d{8})T\d{6}Z$") @@ -830,6 +873,13 @@ def reboot_preflight(remote: Remote, args: argparse.Namespace) -> dict[str, Any] warden_receipt = None try: warden_receipt = validate_ops_warden_receipt(Path(args.ops_warden_acceptance_receipt)) + warden_receipt.update( + verify_installed_warden( + installed_warden_package(Path(args.warden_tool_python)), + OPS_WARDEN_REPO, + warden_receipt["accepted_revision"], + ) + ) except PreflightError as exc: receipt_errors["ops_warden_acceptance_receipt"] = str(exc) console_receipt = None @@ -918,6 +968,11 @@ def main() -> int: parser.add_argument("--scenario-id", help="new scenario id; consumed drill ids are denied") parser.add_argument("--human-decision-id", help="fresh human GO decision id") parser.add_argument("--ops-warden-acceptance-receipt", default=OPS_WARDEN_RECEIPT) + parser.add_argument( + "--warden-tool-python", + default=str(Path.home() / ".local/share/uv/tools/ops-warden/bin/python"), + help="Python of the uv tool environment that provides `warden` on PATH", + ) parser.add_argument( "--accept-terminating-pvc", action="append", help="namespace/name of a pre-existing Terminating PVC accepted for the window", diff --git a/scripts/wp0024-owner-review.py b/scripts/wp0024-owner-review.py index 7e1f8b7..d74dd5c 100644 --- a/scripts/wp0024-owner-review.py +++ b/scripts/wp0024-owner-review.py @@ -152,7 +152,15 @@ def check_command(check_id: str) -> list[str]: if check_id == "database-lease-preflight": return [sys.executable, "scripts/audit-core-recovery-preflight.py", "database-lease"] if check_id == "node-reboot-preflight": - return [sys.executable, "scripts/audit-core-recovery-preflight.py", "node-reboot"] + # Procedure review lists the known stuck drill PVC explicitly; resolving it + # stays a rapp-postgres pre-window gate in the reboot runbook. + return [ + sys.executable, + "scripts/audit-core-recovery-preflight.py", + "node-reboot", + "--accept-terminating-pvc", + "platform-pg-drill/platform-pg-drill-1", + ] raise ReviewError(f"unknown check id: {check_id}") diff --git a/tests/test_audit_core_recovery_preflight.py b/tests/test_audit_core_recovery_preflight.py index 787ed92..3b5779a 100644 --- a/tests/test_audit_core_recovery_preflight.py +++ b/tests/test_audit_core_recovery_preflight.py @@ -19,6 +19,33 @@ SPEC.loader.exec_module(module) class RecoveryPreflightTests(unittest.TestCase): + def test_installed_warden_must_equal_published_accepted_revision(self) -> None: + import subprocess + + with tempfile.TemporaryDirectory() as tmp: + repo, package = Path(tmp) / "ops-warden", Path(tmp) / "site" / "warden" + (repo / "src" / "warden").mkdir(parents=True) + (repo / "src" / "warden" / "proxy.py").write_text("accepted\n", encoding="utf-8") + git = lambda *a: subprocess.run(["git", "-C", str(repo), *a], check=True, capture_output=True) + git("init", "-q") + git("-c", "user.email=t@t", "-c", "user.name=t", "commit", "-q", "--allow-empty", "-m", "base") + git("add", ".") + git("-c", "user.email=t@t", "-c", "user.name=t", "commit", "-q", "-m", "accepted") + revision = git("rev-parse", "HEAD").stdout.decode().strip() + package.mkdir(parents=True) + (package / "proxy.py").write_text("accepted\n", encoding="utf-8") + with self.assertRaises(module.PreflightError): # not published yet + module.verify_installed_warden(package, repo, revision) + git("update-ref", "refs/remotes/origin/main", revision) + self.assertTrue(module.verify_installed_warden(package, repo, revision)["installed_revision_matches"]) + (package / "proxy.py").write_text("relaxed\n", encoding="utf-8") + with self.assertRaises(module.PreflightError): + module.verify_installed_warden(package, repo, revision) + (package / "proxy.py").write_text("accepted\n", encoding="utf-8") + (package / "extra.py").write_text("x\n", encoding="utf-8") + with self.assertRaises(module.PreflightError): + module.verify_installed_warden(package, repo, revision) + def test_ops_warden_receipt_reads_current_decision_of_review_history(self) -> None: revision = "a" * 40 history = {