From 17fdc27465417974a21777d3b29f5482012a43af Mon Sep 17 00:00:00 2001 From: codex Date: Tue, 3 Nov 2026 15:05:45 +0100 Subject: [PATCH] Let the audit-core recovery scripts use the attended admin kubeconfig. Since RCLUSTER-WP-0019 the node account cannot read the k3s admin kubeconfig, so ssh railiance01 kubectl fails at read node state. With AUDIT_KUBECONFIG set, cluster reads and the port-forward run locally against that file; host reads stay on ssh. Unset keeps the earlier path. The E2 receipt script takes owner approvals for the current contract digest from the hub. Co-Authored-By: Claude Opus 5.5 Assistant: claude-code Assistant-Model: opus Assistant-Process: 236506@bnt-lap001 Assistant-Session: 09087018-91e1-48c5-ab2f-acfbb62c92e4 --- docs/railiance01-coordinated-reboot.md | 9 +- docs/rpf-wp-0015-attended-windows.md | 10 ++ scripts/audit-core-recovery-preflight.py | 38 ++++++- scripts/rpf-wp-0015-e2-receipt.py | 121 +++++++++++++++-------- scripts/t02-attended-window.py | 11 ++- tests/test_audit_attended_kubeconfig.py | 72 ++++++++++++++ tests/test_rpf_wp_0015_e2_receipt.py | 20 +++- 7 files changed, 232 insertions(+), 49 deletions(-) create mode 100644 tests/test_audit_attended_kubeconfig.py diff --git a/docs/railiance01-coordinated-reboot.md b/docs/railiance01-coordinated-reboot.md index dc3f5c5..8b978cb 100644 --- a/docs/railiance01-coordinated-reboot.md +++ b/docs/railiance01-coordinated-reboot.md @@ -216,7 +216,14 @@ State Hub. one separately approved synthetic E2. The E2 runs through the reviewed sender lane in its own attended login: `make audit-t02-attended-window` with `AUDIT_T02_MODE=e2` and an approval receipt whose `task_id` is - `RAILIANCE-WP-0024-T03`. It projects one synthetic sender and sends one event + `RAILIANCE-WP-0024-T03`. + Since RCLUSTER-WP-0019 (2026-10-20) the node account cannot read the k3s admin + kubeconfig, so `ssh railiance01 kubectl` no longer reaches the cluster. Set + `AUDIT_KUBECONFIG=$HOME/.kube/config-railiance01` (the attended admin route, + ops-warden `kubernetes-railiance01-admin`) for that one command. Cluster reads + and the port-forward then run locally against that file; host reads stay on + ssh. `scripts/rpf-wp-0015-e2-receipt.py` writes the receipt from the current + owner approvals. It projects one synthetic sender and sends one event twice through the driver's `expect-recovered` phase: the first answer must be a fresh `accepted` (HTTP 200/202), the second `duplicate` for the same fixture (accepted exactly once); it touches no lease. The harness refuses unless both diff --git a/docs/rpf-wp-0015-attended-windows.md b/docs/rpf-wp-0015-attended-windows.md index 50cf533..85701ad 100644 --- a/docs/rpf-wp-0015-attended-windows.md +++ b/docs/rpf-wp-0015-attended-windows.md @@ -51,3 +51,13 @@ Runbook: `docs/railiance01-coordinated-reboot.md`. Additional gates before the w Stop immediately, self-revoke and never retry on any unexpected credential output from the attended login. + +## Cluster access since 2026-10-20 + +RCLUSTER-WP-0019 removed node-account access to the k3s admin kubeconfig. The +preflight, sender lane and window scripts therefore take +`AUDIT_KUBECONFIG=$HOME/.kube/config-railiance01` for one attended command. +Cluster reads and the port-forward then run locally against that file; host +reads such as uptime and boot id stay on `ssh railiance01`. Without the +variable the scripts keep the earlier ssh path, which now fails at +`read node state`. diff --git a/scripts/audit-core-recovery-preflight.py b/scripts/audit-core-recovery-preflight.py index ba752e0..8268f94 100755 --- a/scripts/audit-core-recovery-preflight.py +++ b/scripts/audit-core-recovery-preflight.py @@ -10,6 +10,7 @@ from __future__ import annotations import argparse import json +import os import re import shlex import subprocess @@ -42,9 +43,31 @@ def serving_pods(pod_list: dict[str, Any]) -> list[dict[str, Any]]: ] +KUBECONFIG_ENV = "AUDIT_KUBECONFIG" + + +def attended_kubeconfig(environ: dict[str, str] | None = None) -> str | None: + """Absolute path of the attended admin kubeconfig, or None. + + Since RCLUSTER-WP-0019 (2026-10-20) the node account can no longer read the + k3s admin kubeconfig. Attended cluster access uses the operator's own file + (ops-warden route ``kubernetes-railiance01-admin``), named explicitly for one + command. The file is passed to kubectl and never read here. Unset keeps + the earlier ``ssh kubectl`` path. + """ + value = (os.environ if environ is None else environ).get(KUBECONFIG_ENV) + if not value: + return None + path = Path(value) + if not path.is_absolute() or not path.is_file(): + raise PreflightError(f"{KUBECONFIG_ENV} must be an absolute path to an existing file") + return str(path) + + class Remote: - def __init__(self, host: str) -> None: + def __init__(self, host: str, kubeconfig: str | None = None) -> None: self.host = host + self.kubeconfig = kubeconfig if kubeconfig is not None else attended_kubeconfig() def run(self, command: list[str], *, label: str) -> str: remote_command = " ".join(shlex.quote(part) for part in command) @@ -61,7 +84,18 @@ class Remote: return completed.stdout.strip() def kubectl(self, args: list[str], *, label: str) -> str: - return self.run(["kubectl", *args], label=label) + if self.kubeconfig is None: + return self.run(["kubectl", *args], label=label) + completed = subprocess.run( + ["kubectl", "--kubeconfig", self.kubeconfig, *args], + text=True, + capture_output=True, + check=False, + ) + if completed.returncode != 0: + # Same rule as the ssh path: report only the failed step. + raise PreflightError(f"{label} failed (exit {completed.returncode})") + return completed.stdout.strip() def kubectl_json(self, args: list[str], *, label: str) -> dict[str, Any]: raw = self.kubectl([*args, "-o", "json"], label=label) diff --git a/scripts/rpf-wp-0015-e2-receipt.py b/scripts/rpf-wp-0015-e2-receipt.py index cf3594e..ba03dc7 100755 --- a/scripts/rpf-wp-0015-e2-receipt.py +++ b/scripts/rpf-wp-0015-e2-receipt.py @@ -5,52 +5,89 @@ Usage: scripts/rpf-wp-0015-e2-receipt.py OUTPUT.json [--start-in-min N] The window is 15 minutes and starts now, or N minutes from now. Writing the receipt and running the window is the founder GO for that window; D5 -(2fd5849c) authorizes the exercise class only. The owner acknowledgements are -the round-9 approvals of review contract 014627d6 (audit-core 0d0da0d2, -rapp-postgres 6d047a08, railiance-platform c4a80a60). They stay valid only -while the pinned artifacts in interfaces/RAILIANCE-WP-0024-owner-reviews.json -are unchanged; the script refuses otherwise. The receipt holds no credential -values. It refuses to overwrite an existing file. +(2fd5849c) authorizes the exercise class only. + +The owner acknowledgements are the owners' latest State Hub `approve` receipts +for the *current* review contract +(interfaces/RAILIANCE-WP-0024-owner-reviews.json). They are found by subject +`WP0024-OWNER-RECEIPT/v1//approve/`. The script +refuses when any pinned artifact differs from the contract, or when a +required owner has no approval for this digest. The receipt holds no +credential values, and the script never overwrites an existing file. """ -import hashlib +import importlib.util import json +import os import sys +import urllib.parse +import urllib.request from datetime import datetime, timedelta, timezone from pathlib import Path ROOT = Path(__file__).resolve().parents[1] -pins = json.loads((ROOT / "interfaces/RAILIANCE-WP-0024-owner-reviews.json").read_text())["artifact_sha256"] -drift = [p for p, h in pins.items() if hashlib.sha256((ROOT / p).read_bytes()).hexdigest() != h] -if drift: - sys.exit("owner-reviewed artifacts changed since contract 014627d6; new owner approvals needed: " + ", ".join(drift)) -if len(sys.argv) < 2: - sys.exit(__doc__) -out = sys.argv[1] -delay = int(sys.argv[sys.argv.index('--start-in-min') + 1]) if '--start-in-min' in sys.argv else 0 -start = datetime.now(timezone.utc).replace(second=0, microsecond=0) + timedelta(minutes=delay) -end = start + timedelta(minutes=15) -tag = start.strftime('%Y%m%dt%H%M') -z = lambda t: t.strftime('%Y-%m-%dT%H:%M:%SZ') -receipt = { - "procedure": "audit-core-database-lease-recovery", - "task_id": "RAILIANCE-WP-0024-T03", - "status": "approved", - "approval_id": f"rpf-wp-0015-t03-e2-{tag}z-founder-go", - "window": {"start": z(start), "end": z(end)}, - "abort_operator": "founder (platform-root), attended", - "owners": { - "audit-core": {"acknowledged": True, "message_id": "0d0da0d2-36a1-4ffc-b7bf-7896fad691f6"}, - "rapp-postgres": {"acknowledged": True, "message_id": "6d047a08-e9c8-4dbf-95b9-dca039791329"}, - "railiance-platform": {"acknowledged": True, "message_id": "c4a80a60-a415-4f2b-8928-751407473641"}, - }, - "synthetic_load": {"contract_id": "RAILIANCE-WP-0024-T02-SYNTHETIC-01", - "driver_revision": "sha256:941ba251f638869626b06e9cbf430c70188c0bdf4715e3eff9c7610366f68662"}, - "synthetic_sender": {"identity": f"t03-e2-synthetic-sender-{start:%Y%m%d}", - "source": f"t03-e2-synthetic-{start:%Y%m%d}", - "tenant": f"tenant:trial:t03-e2-synthetic-{start:%Y%m%d}", - "expires_at": z(end)}, -} -with open(out, 'x') as f: - json.dump(receipt, f, indent=2); f.write('\n') -print(json.dumps({"approval_id": receipt["approval_id"], "window": receipt["window"], - "confirm": f"audit-core-synthetic-sender:{receipt['approval_id']}:attended"})) +OWNERS = ("audit-core", "rapp-postgres", "railiance-platform") + + +def review_module(): + spec = importlib.util.spec_from_file_location("wp0024_review", ROOT / "scripts/wp0024-owner-review.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def approvals(review, contract, api_base): + digest = review.contract_digest(contract) + query = urllib.parse.urlencode({"to_agent": "railiance-platform", "limit": 1000}) + with urllib.request.urlopen(f"{api_base}/messages/?{query}", timeout=20) as response: + messages = json.load(response) + found = {} + for owner in OWNERS: + subject = f"{review.RECEIPT_PREFIX}/v1/{owner}/approve/{digest}" + hits = sorted((m for m in messages if m.get("subject") == subject and m.get("from_agent") == owner), + key=lambda m: m["created_at"]) + if not hits: + sys.exit(f"no {owner} approval for contract {digest[:8]}; run the owner round first") + found[owner] = {"acknowledged": True, "message_id": hits[-1]["id"]} + return digest, found + + +def main(argv): + if len(argv) < 2: + sys.exit(__doc__) + review = review_module() + contract = review.load_contract() + drift = [p for p, h in contract["artifact_sha256"].items() if review.file_sha256(ROOT / p) != h] + if drift: + sys.exit("reviewed artifacts differ from the owner-review contract: " + ", ".join(drift)) + digest, owners = approvals(review, contract, os.environ.get("STATE_HUB_URL", "http://127.0.0.1:8000")) + out = argv[1] + delay = int(argv[argv.index("--start-in-min") + 1]) if "--start-in-min" in argv else 0 + start = datetime.now(timezone.utc).replace(second=0, microsecond=0) + timedelta(minutes=delay) + end = start + timedelta(minutes=15) + tag = start.strftime("%Y%m%dt%H%M") + z = lambda t: t.strftime("%Y-%m-%dT%H:%M:%SZ") # noqa: E731 + receipt = { + "procedure": "audit-core-database-lease-recovery", + "task_id": "RAILIANCE-WP-0024-T03", + "status": "approved", + "approval_id": f"rpf-wp-0015-t03-e2-{tag}z-founder-go", + "window": {"start": z(start), "end": z(end)}, + "abort_operator": "founder (platform-root), attended", + "owners": owners, + "synthetic_load": {"contract_id": "RAILIANCE-WP-0024-T02-SYNTHETIC-01", + "driver_revision": "sha256:941ba251f638869626b06e9cbf430c70188c0bdf4715e3eff9c7610366f68662"}, + "synthetic_sender": {"identity": f"t03-e2-synthetic-sender-{start:%Y%m%d}", + "source": f"t03-e2-synthetic-{start:%Y%m%d}", + "tenant": f"tenant:trial:t03-e2-synthetic-{start:%Y%m%d}", + "expires_at": z(end)}, + } + with open(out, "x") as handle: + json.dump(receipt, handle, indent=2) + handle.write("\n") + print(json.dumps({"approval_id": receipt["approval_id"], "window": receipt["window"], + "contract_digest": digest, + "confirm": f"audit-core-synthetic-sender:{receipt['approval_id']}:attended"})) + + +if __name__ == "__main__": + main(sys.argv) diff --git a/scripts/t02-attended-window.py b/scripts/t02-attended-window.py index 5952737..7f509bb 100755 --- a/scripts/t02-attended-window.py +++ b/scripts/t02-attended-window.py @@ -123,12 +123,21 @@ class PortForward: the port-forward uses the same path so the window needs no local kubeconfig. """ - def __init__(self, remote: str, port: int) -> None: + def __init__(self, remote: str, port: int, kubeconfig: str | None = None) -> None: self.remote = remote self.port = port + # With AUDIT_KUBECONFIG set, forward locally against the attended admin + # kubeconfig (the node account has no cluster access since RCLUSTER-WP-0019). + self.kubeconfig = kubeconfig if kubeconfig is not None else LANE.PREFLIGHT.attended_kubeconfig() self.process: subprocess.Popen[bytes] | None = None def command(self) -> list[str]: + if self.kubeconfig is not None: + return [ + "timeout", str(REMOTE_PF_TIMEOUT_SECONDS), "kubectl", "--kubeconfig", self.kubeconfig, + "-n", "audit-core", "port-forward", "--address", "127.0.0.1", + "deploy/audit-core", f"{self.port}:{CONTAINER_PORT}", + ] remote_cmd = ( f"timeout {REMOTE_PF_TIMEOUT_SECONDS} kubectl -n audit-core port-forward " f"--address 127.0.0.1 deploy/audit-core {self.port}:{CONTAINER_PORT}" diff --git a/tests/test_audit_attended_kubeconfig.py b/tests/test_audit_attended_kubeconfig.py new file mode 100644 index 0000000..d37ffb7 --- /dev/null +++ b/tests/test_audit_attended_kubeconfig.py @@ -0,0 +1,72 @@ +"""AUDIT_KUBECONFIG routes cluster reads and the port-forward locally. + +Since RCLUSTER-WP-0019 the node account cannot read the k3s admin kubeconfig. +Attended runs name the operator's own kubeconfig for one command. Unset keeps +the earlier ssh path unchanged. +""" +import importlib.util +import sys +from pathlib import Path +from unittest.mock import patch + +import pytest + +ROOT = Path(__file__).resolve().parents[1] + + +def load(name, rel): + sys.path.insert(0, str(ROOT / "scripts")) + spec = importlib.util.spec_from_file_location(name, ROOT / rel) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +PRE = load("pre_kc", "scripts/audit-core-recovery-preflight.py") +WIN = load("win_kc", "scripts/t02-attended-window.py") + + +class Done: + def __init__(self, rc=0, out="{}"): + self.returncode, self.stdout, self.stderr = rc, out, "secret-looking stderr" + + +def test_unset_keeps_ssh_path(monkeypatch): + monkeypatch.delenv("AUDIT_KUBECONFIG", raising=False) + remote = PRE.Remote("railiance01") + with patch.object(PRE.subprocess, "run", return_value=Done()) as run: + remote.kubectl(["get", "nodes"], label="read node state") + assert run.call_args.args[0][:3] == ["ssh", "-o", "BatchMode=yes"] + assert WIN.PortForward("railiance01", 18080).command()[0] == "ssh" + + +def test_set_runs_kubectl_locally_with_that_file(monkeypatch, tmp_path): + kc = tmp_path / "config"; kc.write_text("not read") + monkeypatch.setenv("AUDIT_KUBECONFIG", str(kc)) + remote = PRE.Remote("railiance01") + with patch.object(PRE.subprocess, "run", return_value=Done()) as run: + remote.kubectl(["get", "nodes", "-o", "json"], label="read node state") + assert run.call_args.args[0] == ["kubectl", "--kubeconfig", str(kc), "get", "nodes", "-o", "json"] + # Host-level reads still use ssh to the node. + with patch.object(PRE.subprocess, "run", return_value=Done(out="12.0 1.0")) as run: + remote.run(["cat", "/proc/uptime"], label="read host uptime") + assert run.call_args.args[0][0] == "ssh" + command = WIN.PortForward("railiance01", 18080).command() + assert command[:5] == ["timeout", str(WIN.REMOTE_PF_TIMEOUT_SECONDS), "kubectl", "--kubeconfig", str(kc)] + assert "ssh" not in command and "127.0.0.1" in command + + +def test_failure_reports_only_the_step(monkeypatch, tmp_path): + kc = tmp_path / "config"; kc.write_text("") + monkeypatch.setenv("AUDIT_KUBECONFIG", str(kc)) + with patch.object(PRE.subprocess, "run", return_value=Done(rc=1)): + with pytest.raises(PRE.PreflightError) as err: + PRE.Remote("railiance01").kubectl(["get", "nodes"], label="read node state") + assert str(err.value) == "read node state failed (exit 1)" + + +@pytest.mark.parametrize("value", ["relative/config", "/nonexistent/config"]) +def test_bad_path_is_refused(monkeypatch, value): + monkeypatch.setenv("AUDIT_KUBECONFIG", value) + with pytest.raises(PRE.PreflightError): + PRE.Remote("railiance01") diff --git a/tests/test_rpf_wp_0015_e2_receipt.py b/tests/test_rpf_wp_0015_e2_receipt.py index 8dd2f23..f872fc5 100644 --- a/tests/test_rpf_wp_0015_e2_receipt.py +++ b/tests/test_rpf_wp_0015_e2_receipt.py @@ -17,10 +17,24 @@ def lane(): return module -def test_receipt_is_accepted_by_the_lane_as_t03(tmp_path): +def generator(): + spec = importlib.util.spec_from_file_location("e2_receipt", ROOT / "scripts/rpf-wp-0015-e2-receipt.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_receipt_is_accepted_by_the_lane_as_t03(tmp_path, monkeypatch, capsys): + gen = generator() + owners = {o: {"acknowledged": True, "message_id": f"{o}-msg"} for o in gen.OWNERS} + monkeypatch.setattr(gen, "approvals", lambda review, contract, base: ("d" * 64, owners)) + review = gen.review_module() + pins = review.load_contract()["artifact_sha256"] + monkeypatch.setattr(review, "file_sha256", lambda path: pins[str(path.relative_to(ROOT))]) + monkeypatch.setattr(gen, "review_module", lambda: review) out = tmp_path / "approval.json" - printed = json.loads(subprocess.run([sys.executable, str(ROOT / "scripts/rpf-wp-0015-e2-receipt.py"), str(out)], - check=True, capture_output=True, text=True).stdout) + gen.main(["x", str(out)]) + printed = json.loads(capsys.readouterr().out) m = lane() approval, spec = m.load_receipt(out, datetime.now(UTC), require_open_window=True) assert approval["task_id"] == "RAILIANCE-WP-0024-T03"