RPF-WP-0015: verify installed ops-warden against accepted published revision; list drill PVC in review check
Round-2 owner review follow-ups (railiance-infra, railiance-cluster, railiance-platform). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Assistant: claude-code Assistant-Model: opus Assistant-Process: 680842@bnt-lap001 Assistant-Session: 54eb9e8e-7d6b-4f0c-b7a6-a6db62075883
This commit is contained in:
codex
parent
ce8286dff9
commit
6152aa9e86
6 changed files with 110 additions and 7 deletions
|
|
@ -111,6 +111,13 @@ driver as `AUDIT_T02_SOURCE`/`AUDIT_T02_TENANT`; the confirm string is
|
|||
If cleanup is missed, audit-core's `expires_at` enforcement makes the identity
|
||||
inert at window end; the attended cleanup is still required to remove it.
|
||||
|
||||
If `project` fails after the registry write may already have reached OpenBao (for
|
||||
example a transport error on the write itself), treat the identity as possibly
|
||||
present: run `cleanup --abort-reason "<reason>"` before leaving the window.
|
||||
The lane's `--token-file` is the attended operator token for the registry write;
|
||||
it must come from the contained ops-warden attended login accepted under RPF-WP-0017
|
||||
(isolated 0600 helper state, removed afterwards), not from a long-lived token file.
|
||||
|
||||
## Attended sequence
|
||||
|
||||
Keep one terminal attached from baseline through recovery. A value-handling
|
||||
|
|
|
|||
|
|
@ -73,9 +73,15 @@ until supplied:
|
|||
denied by the preflight;
|
||||
- the RPF-WP-0017 re-acceptance receipt
|
||||
`reviews/RPF-WP-0017-ops-warden-reacceptance-2026-10-14.json` recording
|
||||
acceptance of the exact ops-warden revision used for the attended login. The
|
||||
2026-10-14 revision `11b2a9d…` was reviewed as `reject`, so this gate is
|
||||
currently closed;
|
||||
acceptance of the exact ops-warden revision used for the attended login
|
||||
(`11b2a9d…` was rejected; `6e14300…` accepted on 2026-10-14). The accepted
|
||||
revision must be pushed to ops-warden `origin/main` and installed with
|
||||
`uv tool install --reinstall /home/worsch/ops-warden` from that clean checkout
|
||||
before the window. The preflight compares every installed `warden` module
|
||||
byte-for-byte with the accepted revision and closes this gate on any mismatch;
|
||||
rerun it at the hold point immediately before the attended login and abort if
|
||||
it no longer matches (the 2026-09-01 WARDEN-WP-0036-T02 operation ran on
|
||||
unaccepted code — do not repeat that);
|
||||
- a named attended driver/abort operator and the `railiance-infra`
|
||||
independent abort owner acknowledgement (`--infra-abort-owner-ack`);
|
||||
- independent provider-console access, not SSH alone: a value-safe
|
||||
|
|
|
|||
|
|
@ -139,11 +139,11 @@
|
|||
},
|
||||
"artifact_sha256": {
|
||||
"docs/audit-core-database-lease-approval.example.json": "87e57e3290074508c6517938acf8c7fc1a75d977b8ff630f247a4ff5d2ee45c6",
|
||||
"docs/audit-core-database-lease-recovery.md": "630e9324d61fddc50c2c364649064a316a7b406b45cc6b4799cf0c4f8319b237",
|
||||
"docs/audit-core-database-lease-recovery.md": "1c7643be1b97b56f5b5efb2c21803fbb0a8d3863953b61ebbfb85792932f7a4a",
|
||||
"docs/openbao-reboot-snapshot-receipt.example.json": "b95903a3e1aacb607eecc3f0440f08e62480d09da47bd7ab96c3eb8814f2eaee",
|
||||
"docs/railiance01-coordinated-reboot.md": "1cbe079ca34252c1638879f0c03cf5b96d9a0078de5047ecf0cb5b3857a7121d",
|
||||
"docs/railiance01-coordinated-reboot.md": "6fd6301f808449ad6519764d994b05aee41a96be879b917ee1d18d8a65d5f6fd",
|
||||
"scripts/audit-core-database-lease-recovery.py": "1de02f13dcd4e87e65de1c97ae14567f2b8e4a51aa0423e3fca78a224cf374b3",
|
||||
"scripts/audit-core-recovery-preflight.py": "41ee3889db008ed90d51b855fc3470244051e7e3d7ba6c1c9256c3d115bbb23d",
|
||||
"scripts/audit-core-recovery-preflight.py": "05e89ae9d0b9a636b67d9a8d0c203ac746c97208e32061cb23f1ca785eda8e5c",
|
||||
"scripts/audit_core_synthetic_sender_lane.py": "e71b3b3d56472a1cc91fc093d327a72116d89d07c1676f44bc24a8da36154bd4"
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -638,6 +638,49 @@ def validate_ops_warden_receipt(path: Path) -> dict[str, Any]:
|
|||
return {"path": str(path), "accepted_revision": revision, "verified": True}
|
||||
|
||||
|
||||
OPS_WARDEN_REPO = Path("/home/worsch/ops-warden")
|
||||
|
||||
|
||||
def installed_warden_package(tool_python: Path) -> Path:
|
||||
"""Package directory of the warden actually on PATH (uv tool environment)."""
|
||||
completed = subprocess.run(
|
||||
[str(tool_python), "-I", "-c", "import os, warden; print(os.path.dirname(warden.__file__))"],
|
||||
text=True, capture_output=True, check=False,
|
||||
)
|
||||
if completed.returncode != 0 or not completed.stdout.strip():
|
||||
raise PreflightError("installed warden package could not be located")
|
||||
return Path(completed.stdout.strip())
|
||||
|
||||
|
||||
def verify_installed_warden(package: Path, repo: Path, revision: str) -> dict[str, Any]:
|
||||
"""Prove the installed warden equals the accepted, published ops-warden revision.
|
||||
|
||||
The uv tool is a non-editable directory install that records no revision, so
|
||||
compare every installed module byte-for-byte with that revision's tree.
|
||||
"""
|
||||
def git(*args: str) -> subprocess.CompletedProcess[bytes]:
|
||||
return subprocess.run(["git", "-C", str(repo), *args], capture_output=True, check=False)
|
||||
|
||||
if git("merge-base", "--is-ancestor", revision, "origin/main").returncode != 0:
|
||||
raise PreflightError("accepted ops-warden revision is not published on origin/main")
|
||||
listed = git("ls-tree", "-r", "--name-only", revision, "src/warden/")
|
||||
if listed.returncode != 0:
|
||||
raise PreflightError("accepted ops-warden revision is not available locally")
|
||||
expected = {
|
||||
name.removeprefix("src/warden/")
|
||||
for name in listed.stdout.decode().splitlines()
|
||||
if name.endswith(".py")
|
||||
}
|
||||
installed = {path.relative_to(package).as_posix() for path in package.rglob("*.py")}
|
||||
if installed != expected:
|
||||
raise PreflightError("installed warden module set differs from the accepted revision")
|
||||
for name in sorted(expected):
|
||||
blob = git("show", f"{revision}:src/warden/{name}")
|
||||
if blob.returncode != 0 or blob.stdout != (package / name).read_bytes():
|
||||
raise PreflightError("installed warden differs from the accepted revision")
|
||||
return {"installed_revision_matches": True, "modules_compared": len(expected)}
|
||||
|
||||
|
||||
CONSOLE_RECEIPT_ID = re.compile(r"^RAIL01-CONSOLE-(\d{8})T\d{6}Z$")
|
||||
|
||||
|
||||
|
|
@ -830,6 +873,13 @@ def reboot_preflight(remote: Remote, args: argparse.Namespace) -> dict[str, Any]
|
|||
warden_receipt = None
|
||||
try:
|
||||
warden_receipt = validate_ops_warden_receipt(Path(args.ops_warden_acceptance_receipt))
|
||||
warden_receipt.update(
|
||||
verify_installed_warden(
|
||||
installed_warden_package(Path(args.warden_tool_python)),
|
||||
OPS_WARDEN_REPO,
|
||||
warden_receipt["accepted_revision"],
|
||||
)
|
||||
)
|
||||
except PreflightError as exc:
|
||||
receipt_errors["ops_warden_acceptance_receipt"] = str(exc)
|
||||
console_receipt = None
|
||||
|
|
@ -918,6 +968,11 @@ def main() -> int:
|
|||
parser.add_argument("--scenario-id", help="new scenario id; consumed drill ids are denied")
|
||||
parser.add_argument("--human-decision-id", help="fresh human GO decision id")
|
||||
parser.add_argument("--ops-warden-acceptance-receipt", default=OPS_WARDEN_RECEIPT)
|
||||
parser.add_argument(
|
||||
"--warden-tool-python",
|
||||
default=str(Path.home() / ".local/share/uv/tools/ops-warden/bin/python"),
|
||||
help="Python of the uv tool environment that provides `warden` on PATH",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--accept-terminating-pvc", action="append",
|
||||
help="namespace/name of a pre-existing Terminating PVC accepted for the window",
|
||||
|
|
|
|||
|
|
@ -152,7 +152,15 @@ def check_command(check_id: str) -> list[str]:
|
|||
if check_id == "database-lease-preflight":
|
||||
return [sys.executable, "scripts/audit-core-recovery-preflight.py", "database-lease"]
|
||||
if check_id == "node-reboot-preflight":
|
||||
return [sys.executable, "scripts/audit-core-recovery-preflight.py", "node-reboot"]
|
||||
# Procedure review lists the known stuck drill PVC explicitly; resolving it
|
||||
# stays a rapp-postgres pre-window gate in the reboot runbook.
|
||||
return [
|
||||
sys.executable,
|
||||
"scripts/audit-core-recovery-preflight.py",
|
||||
"node-reboot",
|
||||
"--accept-terminating-pvc",
|
||||
"platform-pg-drill/platform-pg-drill-1",
|
||||
]
|
||||
raise ReviewError(f"unknown check id: {check_id}")
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -19,6 +19,33 @@ SPEC.loader.exec_module(module)
|
|||
|
||||
|
||||
class RecoveryPreflightTests(unittest.TestCase):
|
||||
def test_installed_warden_must_equal_published_accepted_revision(self) -> None:
|
||||
import subprocess
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
repo, package = Path(tmp) / "ops-warden", Path(tmp) / "site" / "warden"
|
||||
(repo / "src" / "warden").mkdir(parents=True)
|
||||
(repo / "src" / "warden" / "proxy.py").write_text("accepted\n", encoding="utf-8")
|
||||
git = lambda *a: subprocess.run(["git", "-C", str(repo), *a], check=True, capture_output=True)
|
||||
git("init", "-q")
|
||||
git("-c", "user.email=t@t", "-c", "user.name=t", "commit", "-q", "--allow-empty", "-m", "base")
|
||||
git("add", ".")
|
||||
git("-c", "user.email=t@t", "-c", "user.name=t", "commit", "-q", "-m", "accepted")
|
||||
revision = git("rev-parse", "HEAD").stdout.decode().strip()
|
||||
package.mkdir(parents=True)
|
||||
(package / "proxy.py").write_text("accepted\n", encoding="utf-8")
|
||||
with self.assertRaises(module.PreflightError): # not published yet
|
||||
module.verify_installed_warden(package, repo, revision)
|
||||
git("update-ref", "refs/remotes/origin/main", revision)
|
||||
self.assertTrue(module.verify_installed_warden(package, repo, revision)["installed_revision_matches"])
|
||||
(package / "proxy.py").write_text("relaxed\n", encoding="utf-8")
|
||||
with self.assertRaises(module.PreflightError):
|
||||
module.verify_installed_warden(package, repo, revision)
|
||||
(package / "proxy.py").write_text("accepted\n", encoding="utf-8")
|
||||
(package / "extra.py").write_text("x\n", encoding="utf-8")
|
||||
with self.assertRaises(module.PreflightError):
|
||||
module.verify_installed_warden(package, repo, revision)
|
||||
|
||||
def test_ops_warden_receipt_reads_current_decision_of_review_history(self) -> None:
|
||||
revision = "a" * 40
|
||||
history = {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue