Let the audit-core recovery scripts use the attended admin kubeconfig.

Since RCLUSTER-WP-0019 the node account cannot read the k3s admin kubeconfig,
so ssh railiance01 kubectl fails at read node state. With AUDIT_KUBECONFIG
set, cluster reads and the port-forward run locally against that file; host
reads stay on ssh. Unset keeps the earlier path. The E2 receipt script takes
owner approvals for the current contract digest from the hub.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

Assistant: claude-code
Assistant-Model: opus
Assistant-Process: 236506@bnt-lap001
Assistant-Session: 09087018-91e1-48c5-ab2f-acfbb62c92e4

docs/railiance01-coordinated-reboot.md

@@ -216,7 +216,14 @@ State Hub.
    one separately approved synthetic E2. The E2 runs through the reviewed sender lane in
    its own attended login: `make audit-t02-attended-window` with
    `AUDIT_T02_MODE=e2` and an approval receipt whose `task_id` is
-   `RAILIANCE-WP-0024-T03`. It projects one synthetic sender and sends one event
+   `RAILIANCE-WP-0024-T03`.
+   Since RCLUSTER-WP-0019 (2026-10-20) the node account cannot read the k3s admin
+   kubeconfig, so `ssh railiance01 kubectl` no longer reaches the cluster. Set
+   `AUDIT_KUBECONFIG=$HOME/.kube/config-railiance01` (the attended admin route,
+   ops-warden `kubernetes-railiance01-admin`) for that one command. Cluster reads
+   and the port-forward then run locally against that file; host reads stay on
+   ssh. `scripts/rpf-wp-0015-e2-receipt.py` writes the receipt from the current
+   owner approvals. It projects one synthetic sender and sends one event
    twice through the driver's `expect-recovered` phase: the first answer must be a
    fresh `accepted` (HTTP 200/202), the second `duplicate` for the same fixture
    (accepted exactly once); it touches no lease. The harness refuses unless both

docs/rpf-wp-0015-attended-windows.md

@@ -51,3 +51,13 @@ Runbook: `docs/railiance01-coordinated-reboot.md`. Additional gates before the w
 
 Stop immediately, self-revoke and never retry on any unexpected credential output
 from the attended login.
+
+## Cluster access since 2026-10-20
+
+RCLUSTER-WP-0019 removed node-account access to the k3s admin kubeconfig. The
+preflight, sender lane and window scripts therefore take
+`AUDIT_KUBECONFIG=$HOME/.kube/config-railiance01` for one attended command.
+Cluster reads and the port-forward then run locally against that file; host
+reads such as uptime and boot id stay on `ssh railiance01`. Without the
+variable the scripts keep the earlier ssh path, which now fails at
+`read node state`.

scripts/audit-core-recovery-preflight.py

@@ -10,6 +10,7 @@ from __future__ import annotations
 
 import argparse
 import json
+import os
 import re
 import shlex
 import subprocess
@@ -42,9 +43,31 @@ def serving_pods(pod_list: dict[str, Any]) -> list[dict[str, Any]]:
     ]
 
 
+KUBECONFIG_ENV = "AUDIT_KUBECONFIG"
+
+
+def attended_kubeconfig(environ: dict[str, str] | None = None) -> str | None:
+    """Absolute path of the attended admin kubeconfig, or None.
+
+    Since RCLUSTER-WP-0019 (2026-10-20) the node account can no longer read the
+    k3s admin kubeconfig. Attended cluster access uses the operator's own file
+    (ops-warden route ``kubernetes-railiance01-admin``), named explicitly for one
+    command. The file is passed to kubectl and never read here. Unset keeps
+    the earlier ``ssh <host> kubectl`` path.
+    """
+    value = (os.environ if environ is None else environ).get(KUBECONFIG_ENV)
+    if not value:
+        return None
+    path = Path(value)
+    if not path.is_absolute() or not path.is_file():
+        raise PreflightError(f"{KUBECONFIG_ENV} must be an absolute path to an existing file")
+    return str(path)
+
+
 class Remote:
-    def __init__(self, host: str) -> None:
+    def __init__(self, host: str, kubeconfig: str | None = None) -> None:
         self.host = host
+        self.kubeconfig = kubeconfig if kubeconfig is not None else attended_kubeconfig()
 
     def run(self, command: list[str], *, label: str) -> str:
         remote_command = " ".join(shlex.quote(part) for part in command)
@@ -61,7 +84,18 @@ class Remote:
         return completed.stdout.strip()
 
     def kubectl(self, args: list[str], *, label: str) -> str:
-        return self.run(["kubectl", *args], label=label)
+        if self.kubeconfig is None:
+            return self.run(["kubectl", *args], label=label)
+        completed = subprocess.run(
+            ["kubectl", "--kubeconfig", self.kubeconfig, *args],
+            text=True,
+            capture_output=True,
+            check=False,
+        )
+        if completed.returncode != 0:
+            # Same rule as the ssh path: report only the failed step.
+            raise PreflightError(f"{label} failed (exit {completed.returncode})")
+        return completed.stdout.strip()
 
     def kubectl_json(self, args: list[str], *, label: str) -> dict[str, Any]:
         raw = self.kubectl([*args, "-o", "json"], label=label)

scripts/rpf-wp-0015-e2-receipt.py

@@ -5,52 +5,89 @@ Usage: scripts/rpf-wp-0015-e2-receipt.py OUTPUT.json [--start-in-min N]
 
 The window is 15 minutes and starts now, or N minutes from now. Writing the
 receipt and running the window is the founder GO for that window; D5
-(2fd5849c) authorizes the exercise class only. The owner acknowledgements are
-the round-9 approvals of review contract 014627d6 (audit-core 0d0da0d2,
-rapp-postgres 6d047a08, railiance-platform c4a80a60). They stay valid only
-while the pinned artifacts in interfaces/RAILIANCE-WP-0024-owner-reviews.json
-are unchanged; the script refuses otherwise. The receipt holds no credential
-values. It refuses to overwrite an existing file.
+(2fd5849c) authorizes the exercise class only.
+
+The owner acknowledgements are the owners' latest State Hub `approve` receipts
+for the *current* review contract
+(interfaces/RAILIANCE-WP-0024-owner-reviews.json). They are found by subject
+`WP0024-OWNER-RECEIPT/v1/<owner>/approve/<contract digest>`. The script
+refuses when any pinned artifact differs from the contract, or when a
+required owner has no approval for this digest. The receipt holds no
+credential values, and the script never overwrites an existing file.
 """
-import hashlib
+import importlib.util
 import json
+import os
 import sys
+import urllib.parse
+import urllib.request
 from datetime import datetime, timedelta, timezone
 from pathlib import Path
 
 ROOT = Path(__file__).resolve().parents[1]
-pins = json.loads((ROOT / "interfaces/RAILIANCE-WP-0024-owner-reviews.json").read_text())["artifact_sha256"]
-drift = [p for p, h in pins.items() if hashlib.sha256((ROOT / p).read_bytes()).hexdigest() != h]
-if drift:
-    sys.exit("owner-reviewed artifacts changed since contract 014627d6; new owner approvals needed: " + ", ".join(drift))
-if len(sys.argv) < 2:
-    sys.exit(__doc__)
-out = sys.argv[1]
-delay = int(sys.argv[sys.argv.index('--start-in-min') + 1]) if '--start-in-min' in sys.argv else 0
-start = datetime.now(timezone.utc).replace(second=0, microsecond=0) + timedelta(minutes=delay)
-end = start + timedelta(minutes=15)
-tag = start.strftime('%Y%m%dt%H%M')
-z = lambda t: t.strftime('%Y-%m-%dT%H:%M:%SZ')
-receipt = {
-    "procedure": "audit-core-database-lease-recovery",
-    "task_id": "RAILIANCE-WP-0024-T03",
-    "status": "approved",
-    "approval_id": f"rpf-wp-0015-t03-e2-{tag}z-founder-go",
-    "window": {"start": z(start), "end": z(end)},
-    "abort_operator": "founder (platform-root), attended",
-    "owners": {
-        "audit-core": {"acknowledged": True, "message_id": "0d0da0d2-36a1-4ffc-b7bf-7896fad691f6"},
-        "rapp-postgres": {"acknowledged": True, "message_id": "6d047a08-e9c8-4dbf-95b9-dca039791329"},
-        "railiance-platform": {"acknowledged": True, "message_id": "c4a80a60-a415-4f2b-8928-751407473641"},
-    },
-    "synthetic_load": {"contract_id": "RAILIANCE-WP-0024-T02-SYNTHETIC-01",
-                       "driver_revision": "sha256:941ba251f638869626b06e9cbf430c70188c0bdf4715e3eff9c7610366f68662"},
-    "synthetic_sender": {"identity": f"t03-e2-synthetic-sender-{start:%Y%m%d}",
-                         "source": f"t03-e2-synthetic-{start:%Y%m%d}",
-                         "tenant": f"tenant:trial:t03-e2-synthetic-{start:%Y%m%d}",
-                         "expires_at": z(end)},
-}
-with open(out, 'x') as f:
-    json.dump(receipt, f, indent=2); f.write('\n')
-print(json.dumps({"approval_id": receipt["approval_id"], "window": receipt["window"],
-                  "confirm": f"audit-core-synthetic-sender:{receipt['approval_id']}:attended"}))
+OWNERS = ("audit-core", "rapp-postgres", "railiance-platform")
+
+
+def review_module():
+    spec = importlib.util.spec_from_file_location("wp0024_review", ROOT / "scripts/wp0024-owner-review.py")
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+def approvals(review, contract, api_base):
+    digest = review.contract_digest(contract)
+    query = urllib.parse.urlencode({"to_agent": "railiance-platform", "limit": 1000})
+    with urllib.request.urlopen(f"{api_base}/messages/?{query}", timeout=20) as response:
+        messages = json.load(response)
+    found = {}
+    for owner in OWNERS:
+        subject = f"{review.RECEIPT_PREFIX}/v1/{owner}/approve/{digest}"
+        hits = sorted((m for m in messages if m.get("subject") == subject and m.get("from_agent") == owner),
+                      key=lambda m: m["created_at"])
+        if not hits:
+            sys.exit(f"no {owner} approval for contract {digest[:8]}; run the owner round first")
+        found[owner] = {"acknowledged": True, "message_id": hits[-1]["id"]}
+    return digest, found
+
+
+def main(argv):
+    if len(argv) < 2:
+        sys.exit(__doc__)
+    review = review_module()
+    contract = review.load_contract()
+    drift = [p for p, h in contract["artifact_sha256"].items() if review.file_sha256(ROOT / p) != h]
+    if drift:
+        sys.exit("reviewed artifacts differ from the owner-review contract: " + ", ".join(drift))
+    digest, owners = approvals(review, contract, os.environ.get("STATE_HUB_URL", "http://127.0.0.1:8000"))
+    out = argv[1]
+    delay = int(argv[argv.index("--start-in-min") + 1]) if "--start-in-min" in argv else 0
+    start = datetime.now(timezone.utc).replace(second=0, microsecond=0) + timedelta(minutes=delay)
+    end = start + timedelta(minutes=15)
+    tag = start.strftime("%Y%m%dt%H%M")
+    z = lambda t: t.strftime("%Y-%m-%dT%H:%M:%SZ")  # noqa: E731
+    receipt = {
+        "procedure": "audit-core-database-lease-recovery",
+        "task_id": "RAILIANCE-WP-0024-T03",
+        "status": "approved",
+        "approval_id": f"rpf-wp-0015-t03-e2-{tag}z-founder-go",
+        "window": {"start": z(start), "end": z(end)},
+        "abort_operator": "founder (platform-root), attended",
+        "owners": owners,
+        "synthetic_load": {"contract_id": "RAILIANCE-WP-0024-T02-SYNTHETIC-01",
+                           "driver_revision": "sha256:941ba251f638869626b06e9cbf430c70188c0bdf4715e3eff9c7610366f68662"},
+        "synthetic_sender": {"identity": f"t03-e2-synthetic-sender-{start:%Y%m%d}",
+                             "source": f"t03-e2-synthetic-{start:%Y%m%d}",
+                             "tenant": f"tenant:trial:t03-e2-synthetic-{start:%Y%m%d}",
+                             "expires_at": z(end)},
+    }
+    with open(out, "x") as handle:
+        json.dump(receipt, handle, indent=2)
+        handle.write("\n")
+    print(json.dumps({"approval_id": receipt["approval_id"], "window": receipt["window"],
+                      "contract_digest": digest,
+                      "confirm": f"audit-core-synthetic-sender:{receipt['approval_id']}:attended"}))
+
+
+if __name__ == "__main__":
+    main(sys.argv)

scripts/t02-attended-window.py

@@ -123,12 +123,21 @@ class PortForward:
     the port-forward uses the same path so the window needs no local kubeconfig.
     """
 
-    def __init__(self, remote: str, port: int) -> None:
+    def __init__(self, remote: str, port: int, kubeconfig: str | None = None) -> None:
         self.remote = remote
         self.port = port
+        # With AUDIT_KUBECONFIG set, forward locally against the attended admin
+        # kubeconfig (the node account has no cluster access since RCLUSTER-WP-0019).
+        self.kubeconfig = kubeconfig if kubeconfig is not None else LANE.PREFLIGHT.attended_kubeconfig()
         self.process: subprocess.Popen[bytes] | None = None
 
     def command(self) -> list[str]:
+        if self.kubeconfig is not None:
+            return [
+                "timeout", str(REMOTE_PF_TIMEOUT_SECONDS), "kubectl", "--kubeconfig", self.kubeconfig,
+                "-n", "audit-core", "port-forward", "--address", "127.0.0.1",
+                "deploy/audit-core", f"{self.port}:{CONTAINER_PORT}",
+            ]
         remote_cmd = (
             f"timeout {REMOTE_PF_TIMEOUT_SECONDS} kubectl -n audit-core port-forward "
             f"--address 127.0.0.1 deploy/audit-core {self.port}:{CONTAINER_PORT}"

tests/test_audit_attended_kubeconfig.py

@@ -0,0 +1,72 @@
+"""AUDIT_KUBECONFIG routes cluster reads and the port-forward locally.
+
+Since RCLUSTER-WP-0019 the node account cannot read the k3s admin kubeconfig.
+Attended runs name the operator's own kubeconfig for one command. Unset keeps
+the earlier ssh path unchanged.
+"""
+import importlib.util
+import sys
+from pathlib import Path
+from unittest.mock import patch
+
+import pytest
+
+ROOT = Path(__file__).resolve().parents[1]
+
+
+def load(name, rel):
+    sys.path.insert(0, str(ROOT / "scripts"))
+    spec = importlib.util.spec_from_file_location(name, ROOT / rel)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+PRE = load("pre_kc", "scripts/audit-core-recovery-preflight.py")
+WIN = load("win_kc", "scripts/t02-attended-window.py")
+
+
+class Done:
+    def __init__(self, rc=0, out="{}"):
+        self.returncode, self.stdout, self.stderr = rc, out, "secret-looking stderr"
+
+
+def test_unset_keeps_ssh_path(monkeypatch):
+    monkeypatch.delenv("AUDIT_KUBECONFIG", raising=False)
+    remote = PRE.Remote("railiance01")
+    with patch.object(PRE.subprocess, "run", return_value=Done()) as run:
+        remote.kubectl(["get", "nodes"], label="read node state")
+    assert run.call_args.args[0][:3] == ["ssh", "-o", "BatchMode=yes"]
+    assert WIN.PortForward("railiance01", 18080).command()[0] == "ssh"
+
+
+def test_set_runs_kubectl_locally_with_that_file(monkeypatch, tmp_path):
+    kc = tmp_path / "config"; kc.write_text("not read")
+    monkeypatch.setenv("AUDIT_KUBECONFIG", str(kc))
+    remote = PRE.Remote("railiance01")
+    with patch.object(PRE.subprocess, "run", return_value=Done()) as run:
+        remote.kubectl(["get", "nodes", "-o", "json"], label="read node state")
+    assert run.call_args.args[0] == ["kubectl", "--kubeconfig", str(kc), "get", "nodes", "-o", "json"]
+    # Host-level reads still use ssh to the node.
+    with patch.object(PRE.subprocess, "run", return_value=Done(out="12.0 1.0")) as run:
+        remote.run(["cat", "/proc/uptime"], label="read host uptime")
+    assert run.call_args.args[0][0] == "ssh"
+    command = WIN.PortForward("railiance01", 18080).command()
+    assert command[:5] == ["timeout", str(WIN.REMOTE_PF_TIMEOUT_SECONDS), "kubectl", "--kubeconfig", str(kc)]
+    assert "ssh" not in command and "127.0.0.1" in command
+
+
+def test_failure_reports_only_the_step(monkeypatch, tmp_path):
+    kc = tmp_path / "config"; kc.write_text("")
+    monkeypatch.setenv("AUDIT_KUBECONFIG", str(kc))
+    with patch.object(PRE.subprocess, "run", return_value=Done(rc=1)):
+        with pytest.raises(PRE.PreflightError) as err:
+            PRE.Remote("railiance01").kubectl(["get", "nodes"], label="read node state")
+    assert str(err.value) == "read node state failed (exit 1)"
+
+
+@pytest.mark.parametrize("value", ["relative/config", "/nonexistent/config"])
+def test_bad_path_is_refused(monkeypatch, value):
+    monkeypatch.setenv("AUDIT_KUBECONFIG", value)
+    with pytest.raises(PRE.PreflightError):
+        PRE.Remote("railiance01")

tests/test_rpf_wp_0015_e2_receipt.py

@@ -17,10 +17,24 @@ def lane():
     return module
 
 
-def test_receipt_is_accepted_by_the_lane_as_t03(tmp_path):
+def generator():
+    spec = importlib.util.spec_from_file_location("e2_receipt", ROOT / "scripts/rpf-wp-0015-e2-receipt.py")
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+def test_receipt_is_accepted_by_the_lane_as_t03(tmp_path, monkeypatch, capsys):
+    gen = generator()
+    owners = {o: {"acknowledged": True, "message_id": f"{o}-msg"} for o in gen.OWNERS}
+    monkeypatch.setattr(gen, "approvals", lambda review, contract, base: ("d" * 64, owners))
+    review = gen.review_module()
+    pins = review.load_contract()["artifact_sha256"]
+    monkeypatch.setattr(review, "file_sha256", lambda path: pins[str(path.relative_to(ROOT))])
+    monkeypatch.setattr(gen, "review_module", lambda: review)
     out = tmp_path / "approval.json"
-    printed = json.loads(subprocess.run([sys.executable, str(ROOT / "scripts/rpf-wp-0015-e2-receipt.py"), str(out)],
-                                        check=True, capture_output=True, text=True).stdout)
+    gen.main(["x", str(out)])
+    printed = json.loads(capsys.readouterr().out)
     m = lane()
     approval, spec = m.load_receipt(out, datetime.now(UTC), require_open_window=True)
     assert approval["task_id"] == "RAILIANCE-WP-0024-T03"