From a2da4085e022e983043d91c31ace7cb9b30b8bab Mon Sep 17 00:00:00 2001 From: codex Date: Wed, 28 Oct 2026 00:02:10 +0100 Subject: [PATCH] RPF-WP-0064-T10: break-glass drill mode for the isolated restore adapter Adds --mode break-glass-drill with its own signed admission (drill: "break-glass", attempt id RPF-WP-0064-T10-*, snapshot_label). After the phase A restore and unseal it checks the held root token with lookup-self only, runs a fresh generate-root ceremony with re-entered shares, does one read-only sys/auth action, revokes the generated token (204 then 403) and drills a cancelled attempt. Phase A is unchanged. The verifier types the receipt extension and rejects token, accessor, OTP, nonce, encoded-token and share shapes. Offline 34 and native 16 pass. Founder runbooks for the drill and the read-only production SSO-down walk are in the T10 section. Co-Authored-By: Claude Opus 5.5 Assistant: claude-code Assistant-Model: opus Assistant-Process: 4515@bnt-lap001 Assistant-Session: 0bf96127-6e04-41be-bfda-d11717c3f354 --- WORK-RECORDS.md | 2 +- docs/openbao.md | 4 +- scripts/openbao-isolated-restore/preflight.py | 3 +- .../restore_adapter.py | 233 +++++++++-- .../run-isolated-restore.sh | 25 +- .../verify-restore-receipt.py | 105 ++++- tests/openbao_restore_rehearsal.py | 16 +- tests/test_openbao_break_glass_drill.py | 372 ++++++++++++++++++ tests/test_openbao_isolated_restore.py | 4 +- tests/test_openbao_isolated_restore_native.py | 34 ++ ...-0064-isolated-original-openbao-restore.md | 118 +++++- 11 files changed, 867 insertions(+), 49 deletions(-) create mode 100644 tests/test_openbao_break_glass_drill.py diff --git a/WORK-RECORDS.md b/WORK-RECORDS.md index 7fe3b66..9dc28cc 100644 --- a/WORK-RECORDS.md +++ b/WORK-RECORDS.md @@ -204,7 +204,7 @@ | task | RPF-WP-0064-T07 | done | — | workplans/RPF-WP-0064-isolated-original-openbao-restore.md | | task | RPF-WP-0064-T08 | done | — | workplans/RPF-WP-0064-isolated-original-openbao-restore.md | | task | RPF-WP-0064-T09 | todo | — | workplans/RPF-WP-0064-isolated-original-openbao-restore.md | -| task | RPF-WP-0064-T10 | todo | — | workplans/RPF-WP-0064-isolated-original-openbao-restore.md | +| task | RPF-WP-0064-T10 | progress | — | workplans/RPF-WP-0064-isolated-original-openbao-restore.md | | task | RPF-WP-0065-T01 | done | — | workplans/RPF-WP-0065-netkingdom-role-claim-bindings.md | | task | RPF-WP-0065-T02 | done | — | workplans/RPF-WP-0065-netkingdom-role-claim-bindings.md | | task | RPF-WP-0065-T03 | done | — | workplans/RPF-WP-0065-netkingdom-role-claim-bindings.md | diff --git a/docs/openbao.md b/docs/openbao.md index 3df1d6b..107d04c 100644 --- a/docs/openbao.md +++ b/docs/openbao.md @@ -197,7 +197,9 @@ break-glass material with the same handling as unseal shares. Use this when KeyCape or Authelia is down and an OpenBao action can't wait. It belongs to NetKingdom NK-WP-0055-T13 and is drilled against an isolated restored -OpenBao only (RPF-WP-0064-T10). Never drill it on production. +OpenBao only (RPF-WP-0064-T10). Never drill it on production. The drill runs as +`scripts/openbao-isolated-restore/run-isolated-restore.sh --break-glass-drill` with its own signed +admission; the founder runbook is in the RPF-WP-0064 workplan, T10. **What fails with SSO down.** Operator OIDC logins through `auth/netkingdom` fail, and so does the warden lane. ops-warden SSH certificates fail too, diff --git a/scripts/openbao-isolated-restore/preflight.py b/scripts/openbao-isolated-restore/preflight.py index f17a758..98b125d 100644 --- a/scripts/openbao-isolated-restore/preflight.py +++ b/scripts/openbao-isolated-restore/preflight.py @@ -47,10 +47,11 @@ def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__.split("\n", 1)[0]) for name in ("admission", "admission-sig", "admission-sha256", "ciphertext", "receipt"): parser.add_argument("--" + name, required=True) + parser.add_argument("--mode", choices=(adapter.PHASE_A, adapter.DRILL_MODE), default=adapter.PHASE_A) opts = parser.parse_args(argv) try: adm, _raw, _signer, _protected = adapter.load_admission( - Path(opts.admission), Path(opts.admission_sig), opts.admission_sha256, adapter.anchor_path()) + Path(opts.admission), Path(opts.admission_sig), opts.admission_sha256, adapter.anchor_path(), opts.mode) require(adm["material"] == "original", "admission_refused") adapter.check_window(adm) adapter.check_source_closure(adm, adapter.ROOT) diff --git a/scripts/openbao-isolated-restore/restore_adapter.py b/scripts/openbao-isolated-restore/restore_adapter.py index ce5c9ca..815addf 100644 --- a/scripts/openbao-isolated-restore/restore_adapter.py +++ b/scripts/openbao-isolated-restore/restore_adapter.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Isolated original-OpenBao restore adapter (RPF-WP-0064-T02, phase A only). +"""Isolated original-OpenBao restore adapter (RPF-WP-0064-T02 phase A; T10 break-glass drill). Restores one age-encrypted Raft snapshot into a disposable, networkless Docker container on the founder workstation, unseals it with the original Shamir @@ -22,6 +22,15 @@ Outcomes: ``restored`` | ``refused`` (required object missing or not matching its source; never repaired) | ``aborted`` (any other failed check, timeout, operator abort). Only the launcher ``run-isolated-restore.sh`` is the attended entry point; ``Session`` is importable for the synthetic rehearsal tests. + +``--mode break-glass-drill`` (RPF-WP-0064-T10, NK-WP-0055-T13) needs its own +signed admission (``drill: "break-glass"``, attempt id ``RPF-WP-0064-T10-*``). +It runs the same restore and unseal, forgets the unseal shares, then: checks a +held root token with ``lookup-self`` only (never revoked or otherwise used); +runs a fresh generate-root ceremony with a server OTP and two re-entered +shares; does one read-only action (``sys/auth``); revokes the generated token +(204) and confirms ``lookup-self`` is refused (403); starts and cancels one +more attempt. Outcomes: ``drilled`` | ``aborted``. No object readback. """ from __future__ import annotations @@ -127,13 +136,27 @@ ALLOWED_READS = frozenset({"sys/seal-status", "sys/leader", "auth/token/lookup-s *(o["path"] for o in OBJECTS)}) ALLOWED_WRITES = frozenset({"sys/unseal", "sys/generate-root/attempt", "sys/generate-root/update", "auth/token/revoke-self"}) +# --------------------------------------------------------------------------- T10 break-glass drill +PHASE_A, DRILL_MODE = "phase-a", "break-glass-drill" +DRILL = "break-glass" +GENERATE_ROOT_ATTEMPT = "sys/generate-root/attempt" +MINIMAL_ACTION_PATH = "sys/auth" +# T10 extensions: generate-root status (GET) and cancel (DELETE); sys/auth and +# auth/token/lookup-self are already allowed reads, sys/generate-root/update an allowed write. +ALLOWED_READS = ALLOWED_READS | {GENERATE_ROOT_ATTEMPT} +ALLOWED_DELETES = frozenset({GENERATE_ROOT_ATTEMPT}) +DRILL_READBACK = "generate-root-from-reentered-shares" +DRILL_CLAIMS_NOT_MADE = ("production_root_token_validity", "production_generate_root_availability", + "sso_down_access_path") +DRILL_ABORT_REASONS = frozenset({"held_token_refused", "generate_root_failed", "minimal_action_failed", + "revocation_failed", "cancel_drill_failed"}) ABORT_REASONS = frozenset({ "admission_refused", "preflight_refused", "custody_identity_refused", "ciphertext_mismatch", "decrypt_failed", "plaintext_mismatch", "target_start_failed", "network_containment_failed", "throwaway_init_failed", "restore_failed", "unseal_refused", "leader_not_elected", "cluster_mismatch", "applied_index_behind", "readback_credential_failed", "readback_failed", "deadline_exceeded", "operator_abort", "parent_lost", "internal_error", -}) +}) | DRILL_ABORT_REASONS # Production runs the chart with setNodeId, so the snapshot's voter is the pod # name. The target takes the same id; a native rehearsal showed a different id # still elects a leader after a forced restore, so this is alignment, not the @@ -167,6 +190,12 @@ AGE_IDENTITY_RE = re.compile(r"^AGE-SECRET-KEY-1[0-9A-Z]{58}$") AGE_RECIPIENT_RE = re.compile(r"^age1[023456789acdefghjklmnpqrstuvwxyz]{58}$") SHARE_RE = re.compile(r"^(?:[A-Za-z0-9+/]{40,100}={0,2}|[a-f0-9]{60,130})$") ATTEMPT_RE = re.compile(r"^RPF-WP-0064-(?:T07|T04)-[A-Za-z0-9-]{3,64}$") +DRILL_ATTEMPT_RE = re.compile(r"^RPF-WP-0064-T10-[A-Za-z0-9-]{3,64}$") +# Snapshot name for the claim limit, e.g. openbao-raft-20261014T1532Z.snap.age. +SNAPSHOT_LABEL_RE = re.compile(r"^(?=.*20\d{6})[a-z0-9][a-z0-9._-]{2,95}$") +HELD_TOKEN_RE = re.compile(r"^[A-Za-z0-9._-]{16,256}$") +POLICY_NAME_RE = re.compile(r"^[a-z0-9][a-z0-9_.-]{0,63}$") +HTTP_CODE_RE = re.compile(rb"Code: (\d{3})\.") COMMIT_RE = re.compile(r"^[a-f0-9]{40}$") DURATION_RE = re.compile(r"^(\d+)([smh]?)$") @@ -229,14 +258,23 @@ SNAPSHOT_FIELDS = {"ciphertext_sha256", "ciphertext_bytes", "plaintext_sha256", "source_cluster_id", "source_applied_index", "source_receipt_sha256"} -def validate_admission(adm: Any) -> dict[str, Any]: - """Shape-only validation of the signed admission (no I/O).""" - require(isinstance(adm, dict) and set(adm) == ADMISSION_FIELDS, "admission_refused") +def validate_admission(adm: Any, mode: str = PHASE_A) -> dict[str, Any]: + """Shape-only validation of the signed admission (no I/O); phase-A and drill admissions are disjoint.""" + require(mode in (PHASE_A, DRILL_MODE), "admission_refused") + drill = mode == DRILL_MODE + fields = ADMISSION_FIELDS | {"drill", "snapshot_label"} if drill else ADMISSION_FIELDS + require(isinstance(adm, dict) and set(adm) == fields, "admission_refused") require(adm["schema"] == ADMISSION_SCHEMA and adm["namespace"] == NAMESPACE, "admission_refused") - require(isinstance(adm["attempt_id"], str) and ATTEMPT_RE.fullmatch(adm["attempt_id"]), "admission_refused") + attempt_re = DRILL_ATTEMPT_RE if drill else ATTEMPT_RE + require(isinstance(adm["attempt_id"], str) and attempt_re.fullmatch(adm["attempt_id"]), "admission_refused") require(adm["phase"] == "A" and adm["phase_b_admitted"] is False, "admission_refused") require(adm["material"] in ("original", "synthetic"), "admission_refused") - require(adm["readback_credential"] == "generate-root-from-entered-shares", "admission_refused") + require(adm["readback_credential"] == (DRILL_READBACK if drill else "generate-root-from-entered-shares"), + "admission_refused") + if drill: + require(adm["drill"] == DRILL, "admission_refused") + require(isinstance(adm["snapshot_label"], str) and SNAPSHOT_LABEL_RE.fullmatch(adm["snapshot_label"]), + "admission_refused") nb, na = parse_iso(adm["not_before"]), parse_iso(adm["not_after"]) require(nb < na and (na - nb).total_seconds() <= 24 * 3600, "admission_refused") dur = adm["max_duration_seconds"] @@ -338,11 +376,11 @@ def read_owner_file(path: Path, limit: int) -> bytes: return raw -def load_admission(admission: Path, signature: Path, admission_sha256: str, - signers: Path) -> tuple[dict[str, Any], bytes, dict[str, str], bool]: +def load_admission(admission: Path, signature: Path, admission_sha256: str, signers: Path, + mode: str = PHASE_A) -> tuple[dict[str, Any], bytes, dict[str, str], bool]: raw = read_owner_file(admission, 64 * 1024) require(DIGEST_RE.fullmatch(admission_sha256) and sha256_bytes(raw) == admission_sha256, "admission_refused") - adm = validate_admission(strict_json(raw)) + adm = validate_admission(strict_json(raw), mode) signer = verify_signature(raw, read_owner_file(signature, 8192), signers) protected = anchor_protected(signers) if adm["material"] == "original": @@ -401,6 +439,18 @@ def wipe(buf: bytearray | None) -> None: buf[i] = 0 +def http_code(stderr: bytes) -> int | None: + """HTTP status from the bao CLI error text (``Code: 403.``); None if absent.""" + match = HTTP_CODE_RE.search(stderr) + return int(match.group(1)) if match else None + + +def policy_names(value: Any) -> list[str]: + """Policy names only, sorted; anything not shaped like a policy name is dropped.""" + return sorted(p for p in (value if isinstance(value, list) else []) + if isinstance(p, str) and POLICY_NAME_RE.fullmatch(p))[:32] + + def tty_prompt(label: str) -> bytearray: """Hidden prompt on the controlling terminal; the value never touches argv/env/disk.""" require(sys.stdin.isatty(), "preflight_refused") @@ -415,7 +465,10 @@ class Session: def __init__(self, *, admission: dict[str, Any], admission_sha256: str, signer: dict[str, str], anchor_is_protected: bool, ciphertext: Path, receipt_path: Path, root: Path = ROOT, prompt: Callable[[str], bytearray] = tty_prompt, docker: str = "docker", - say: Callable[[str], None] = lambda m: print(m, flush=True), start_reaper: bool = True): + say: Callable[[str], None] = lambda m: print(m, flush=True), start_reaper: bool = True, + mode: str = PHASE_A): + require(mode in (PHASE_A, DRILL_MODE) and (mode == DRILL_MODE) == ("drill" in admission), "admission_refused") + self.mode = mode self.adm, self.root, self.prompt, self.docker, self.say = admission, root, prompt, docker, say self.ciphertext, self.receipt_path, self.start_reaper = ciphertext, receipt_path, start_reaper self.name = "rpf64-restore-" + secrets.token_hex(6) @@ -458,6 +511,22 @@ class Session: "kv_data_read": False, "no_secret_material_recorded": True, "claims_not_made": list(CLAIMS_NOT_MADE), } + if mode == DRILL_MODE: + self.receipt["claims_not_made"] += list(DRILL_CLAIMS_NOT_MADE) + self.receipt.update({ + "drill": DRILL, + "held_root_token": {"valid": None, "policies": [], "ttl_zero": None, "expire_time_null": None, + "orphan": None}, + "generate_root": {"started": False, "shares_entered": 0, "complete": False, "otp_decode_ok": False, + "policies": [], "minimal_action_path": MINIMAL_ACTION_PATH, + "minimal_action_ok": False}, + "revocation": {"revoke_status": None, "post_revoke_lookup": None}, + "cancel_drill": {"started_then_cancelled": False}, + # The held-token result is valid only as of this snapshot, never for production now. + "claim_limits": {"production_root_token_validity": { + "as_of_snapshot": admission["snapshot_label"], + "snapshot_plaintext_sha256": snap["plaintext_sha256"]}}, + }) # ---- infrastructure def remaining(self) -> float: @@ -501,6 +570,10 @@ class Session: body = json.loads(done.stdout or b"{}") return body.get("data", body) if isinstance(body, dict) and isinstance(body.get("data"), dict) else body + def delete(self, path: str, reason: str = "internal_error") -> None: + require(path in ALLOWED_DELETES, "internal_error") + self.bao(["delete", path], reason=reason) + def tell_reaper(self, line: str) -> None: if self.reaper and self.reaper.stdin: try: @@ -778,6 +851,109 @@ class Session: self.receipt["objects"] = rows return rows + # ---- T10 break-glass drill steps (after restore + unseal) + def forget_unseal_shares(self) -> None: + """The drill re-enters shares for generate-root; the unseal entries are not reused.""" + for share in self.shares: + wipe(share) + self.shares.clear() + + def drill_held_token(self) -> None: + self.say("Drill 1/5: held root-token check (hidden prompt; lookup-self only, never revoked).") + held = self.prompt("Held initial root token") + try: + require(HELD_TOKEN_RE.fullmatch(held.decode("ascii", "replace")), "held_token_refused") + done = self.bao(["read", "-format=json", "auth/token/lookup-self"], credential=held.decode(), + check=False, reason="held_token_refused") + finally: + wipe(held) + record = self.receipt["held_root_token"] + if done.returncode == 0: + body = json.loads(done.stdout) + data = body.get("data") if isinstance(body, dict) and isinstance(body.get("data"), dict) else {} + record.update(valid=True, policies=policy_names(data.get("policies")), + ttl_zero=type(data.get("ttl")) is int and data["ttl"] == 0, + expire_time_null="expire_time" in data and data["expire_time"] is None, + orphan=data["orphan"] if type(data.get("orphan")) is bool else None) + body.clear() + data = {} + elif http_code(done.stderr) == 403: + record.update(valid=False) + else: + raise Refused("held_token_refused") + + def drill_generate_root(self) -> None: + self.say("Drill 2/5: fresh generate-root ceremony; re-enter two shares (hidden prompts).") + record = self.receipt["generate_root"] + status = self.read(GENERATE_ROOT_ATTEMPT, reason="generate_root_failed") or {} + require(status.get("started") is False, "generate_root_failed") + attempt = self.write(GENERATE_ROOT_ATTEMPT, {}, reason="generate_root_failed") + otp, nonce, encoded, update = attempt.get("otp"), attempt.get("nonce"), None, {} + attempt.clear() + require(isinstance(otp, str) and otp and isinstance(nonce, str) and nonce, "generate_root_failed") + record["started"] = True + for index in (1, 2): + share = self.prompt(f"Re-enter unseal share {index} of 2 for generate-root") + try: + require(SHARE_RE.fullmatch(share.decode("ascii", "replace")), "generate_root_failed") + update = self.write("sys/generate-root/update", {"key": share.decode(), "nonce": nonce}, + reason="generate_root_failed") + finally: + wipe(share) + record["shares_entered"] = index + encoded = update.get("encoded_token") or update.get("encoded_root_token") or encoded + record["complete"] = update.get("complete") is True and bool(encoded) + require(record["complete"], "generate_root_failed") + try: + raw = base64.b64decode(encoded + "=" * (-len(encoded) % 4)) + require(len(raw) == len(otp), "generate_root_failed") + self.credential = bytes(a ^ b for a, b in zip(raw, otp.encode())).decode("ascii") + except (ValueError, UnicodeDecodeError) as exc: + raise Refused("generate_root_failed") from exc + finally: + otp = nonce = encoded = None + update.clear() + record["otp_decode_ok"] = True + lookup = self.read("auth/token/lookup-self", self.credential, reason="generate_root_failed") or {} + record["policies"] = policy_names(lookup.get("policies")) + lookup.clear() + require(record["policies"] == ["root"], "generate_root_failed") + + def drill_minimal_action(self) -> None: + self.say("Drill 3/5: one read-only action (sys/auth metadata).") + mounts = self.read(MINIMAL_ACTION_PATH, self.credential, reason="minimal_action_failed") + ok = isinstance(mounts, dict) and isinstance(mounts.get("token/"), dict) + self.receipt["generate_root"]["minimal_action_ok"] = ok + require(ok, "minimal_action_failed") + + def drill_revoke(self) -> None: + self.say("Drill 4/5: revoke-self the generated token, then confirm lookup-self is refused.") + record = self.receipt["revocation"] + require("auth/token/revoke-self" in ALLOWED_WRITES, "internal_error") + done = self.bao(["write", "-format=json", "auth/token/revoke-self", "-"], credential=self.credential, + body=b"{}", check=False, reason="revocation_failed") + # The CLI prints nothing for 204 No Content; any response body means another status. + record["revoke_status"] = (204 if not done.stdout.strip() else 200) if done.returncode == 0 \ + else http_code(done.stderr) + check = self.bao(["read", "-format=json", "auth/token/lookup-self"], credential=self.credential, + check=False, reason="revocation_failed") + record["post_revoke_lookup"] = 200 if check.returncode == 0 else http_code(check.stderr) + revoked = record["revoke_status"] == 204 and record["post_revoke_lookup"] == 403 + self.receipt["teardown"]["readback_credential_revoked"] = revoked + require(revoked, "revocation_failed") + self.credential = None + + def drill_cancel(self) -> None: + self.say("Drill 5/5: start one more generate-root attempt and cancel it.") + attempt = self.write(GENERATE_ROOT_ATTEMPT, {}, reason="cancel_drill_failed") + started = attempt.get("started") is True + attempt.clear() + self.delete(GENERATE_ROOT_ATTEMPT, reason="cancel_drill_failed") + status = self.read(GENERATE_ROOT_ATTEMPT, reason="cancel_drill_failed") or {} + ok = started and status.get("started") is False and status.get("progress") == 0 + self.receipt["cancel_drill"]["started_then_cancelled"] = ok + require(ok, "cancel_drill_failed") + def revoke(self) -> None: if not self.credential or not self.container_started: return @@ -844,16 +1020,25 @@ class Session: self.start_target() self.restore() self.unseal_original() - self.generate_credential() - rows = self.readback() - missing = [r for r in rows if not r["present"]] - mismatch = [r for r in rows if r["present"] and not r["source_match"]] - if missing: - result, reason = "refused", "object_missing" - elif mismatch: - result, reason = "refused", "object_mismatch" + if self.mode == DRILL_MODE: + self.forget_unseal_shares() + self.drill_held_token() + self.drill_generate_root() + self.drill_minimal_action() + self.drill_revoke() + self.drill_cancel() + result, reason = "drilled", "break_glass_drill_complete" else: - result, reason = "restored", "all_required_objects_match" + self.generate_credential() + rows = self.readback() + missing = [r for r in rows if not r["present"]] + mismatch = [r for r in rows if r["present"] and not r["source_match"]] + if missing: + result, reason = "refused", "object_missing" + elif mismatch: + result, reason = "refused", "object_mismatch" + else: + result, reason = "restored", "all_required_objects_match" except Refused as exc: result, reason = "aborted", exc.reason if exc.reason in ABORT_REASONS else "internal_error" except Exception: @@ -953,11 +1138,12 @@ def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__.split("\n", 1)[0]) for name in ("admission", "admission-sig", "admission-sha256", "ciphertext", "receipt"): parser.add_argument("--" + name, required=True) + parser.add_argument("--mode", choices=(PHASE_A, DRILL_MODE), default=PHASE_A) opts = parser.parse_args(args) os.umask(0o077) try: adm, _raw, signer, protected = load_admission(Path(opts.admission), Path(opts.admission_sig), - opts.admission_sha256, anchor_path()) + opts.admission_sha256, anchor_path(), opts.mode) require(adm["material"] == "original", "admission_refused") check_source_closure(adm, ROOT) receipt = Path(opts.receipt) @@ -966,12 +1152,13 @@ def main(argv: list[str] | None = None) -> int: print(f"ERROR: {exc.reason}; no custody input requested.", file=sys.stderr) return 1 session = Session(admission=adm, admission_sha256=opts.admission_sha256, signer=signer, - anchor_is_protected=protected, ciphertext=Path(opts.ciphertext), receipt_path=receipt) + anchor_is_protected=protected, ciphertext=Path(opts.ciphertext), receipt_path=receipt, + mode=opts.mode) result = session.execute() verify = subprocess.run([sys.executable, str(ROOT / HERE / "verify-restore-receipt.py"), "--receipt", str(receipt), "--source-root", str(ROOT)], capture_output=True, timeout=60) print("Independent verifier: " + ("PASS" if verify.returncode == 0 else "FAIL")) - return 0 if result["result"] in ("restored", "refused") and verify.returncode == 0 else 2 + return 0 if result["result"] in ("restored", "refused", "drilled") and verify.returncode == 0 else 2 if __name__ == "__main__": diff --git a/scripts/openbao-isolated-restore/run-isolated-restore.sh b/scripts/openbao-isolated-restore/run-isolated-restore.sh index 81159d3..e1c715d 100644 --- a/scripts/openbao-isolated-restore/run-isolated-restore.sh +++ b/scripts/openbao-isolated-restore/run-isolated-restore.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Attended launcher: isolated original-OpenBao restore, phase A (RPF-WP-0064). +# Attended launcher: isolated original-OpenBao restore, phase A or T10 break-glass drill (RPF-WP-0064). # Founder workstation only. Never railiance01, never a kubeconfig, no network. set +x set -euo pipefail @@ -7,7 +7,9 @@ ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd)" cd -- "$ROOT" umask 077 unset KUBECONFIG BAO_TOKEN VAULT_TOKEN BAO_ADDR VAULT_ADDR -usage='Usage: run-isolated-restore.sh ADMISSION ADMISSION_SIG ADMISSION_SHA256 CIPHERTEXT RECEIPT_DIR' +usage='Usage: run-isolated-restore.sh [--break-glass-drill] ADMISSION ADMISSION_SIG ADMISSION_SHA256 CIPHERTEXT RECEIPT_DIR' +MODE=phase-a +if [[ "${1:-}" == --break-glass-drill ]]; then MODE=break-glass-drill; shift; fi [[ $# == 5 ]] || { printf '%s\n' "$usage"; exit 1; } [[ "$3" =~ ^sha256:[a-f0-9]{64}$ ]] || { printf '%s\n' 'ERROR: exact sha256: admission digest required.'; exit 1; } for index in 1 2 4; do @@ -19,13 +21,20 @@ done for tool in python3 age age-keygen docker ssh-keygen git; do command -v "$tool" >/dev/null || { printf 'ERROR: required tool unavailable: %s\n' "$tool"; exit 1; } done -RECEIPT="$5/rpf64-restore-$(date -u +%Y%m%dT%H%M%SZ).json" -python3 scripts/openbao-isolated-restore/preflight.py --admission "$1" --admission-sig "$2" \ +RECEIPT="$5/rpf64-${MODE}-$(date -u +%Y%m%dT%H%M%SZ).json" +python3 scripts/openbao-isolated-restore/preflight.py --mode "$MODE" --admission "$1" --admission-sig "$2" \ --admission-sha256 "$3" --ciphertext "$4" --receipt "$RECEIPT" -printf '%s\n' 'Phase A only: networkless restore and metadata readback. No consumer continuity,' -printf '%s\n' 'live JWT login, phase B network or owner acceptance is claimed.' -printf '%s\n' 'You will be asked for the backup age identity, then two original unseal shares.' +if [[ "$MODE" == break-glass-drill ]]; then + printf '%s\n' 'Break-glass drill (RPF-WP-0064-T10) on an isolated, networkless restore. Never production.' + printf '%s\n' 'Prompts: backup age identity, two unseal shares, the held initial root token (lookup-self only),' + printf '%s\n' 'then two shares AGAIN for a fresh generate-root ceremony. Production root-token validity,' + printf '%s\n' 'production generate-root availability and the SSO-down access path are not claimed.' +else + printf '%s\n' 'Phase A only: networkless restore and metadata readback. No consumer continuity,' + printf '%s\n' 'live JWT login, phase B network or owner acceptance is claimed.' + printf '%s\n' 'You will be asked for the backup age identity, then two original unseal shares.' +fi printf '%s\n' 'Press Ctrl-C at any time to abort; the container is removed and an aborted receipt written.' -python3 scripts/openbao-isolated-restore/restore_adapter.py --admission "$1" --admission-sig "$2" \ +python3 scripts/openbao-isolated-restore/restore_adapter.py --mode "$MODE" --admission "$1" --admission-sig "$2" \ --admission-sha256 "$3" --ciphertext "$4" --receipt "$RECEIPT" printf 'Safe receipt: %s\n' "$RECEIPT" diff --git a/scripts/openbao-isolated-restore/verify-restore-receipt.py b/scripts/openbao-isolated-restore/verify-restore-receipt.py index fd145b6..16210f5 100644 --- a/scripts/openbao-isolated-restore/verify-restore-receipt.py +++ b/scripts/openbao-isolated-restore/verify-restore-receipt.py @@ -6,6 +6,12 @@ consistency, deadline, required teardown evidence, claims not made, refuses placeholders and anything shaped like a credential, share, key or KV value, and compares ``source_sha256`` against a clean checkout at ``adapter_commit``. +RPF-WP-0064-T10 break-glass drill receipts (``drill: "break-glass"``) carry a +typed extension: held-token facts, generate-root ceremony facts, revocation +status codes, the cancel drill and the snapshot the held-token result is +limited to. Every extension value is an enumerated, typed fact; anything else +(token, accessor, OTP, nonce, encoded token, share) is rejected. + Exit 0 = receipt accepted. Prints only PASS/FAIL and a reason code. """ from __future__ import annotations @@ -52,6 +58,17 @@ REQUIRED_OBJECTS = ( "secrets_engine_jwt_role", "netkingdom_oidc_config", "platform_admin_role", "policy_secrets_engine_login_self", "kv_registration_metadata", ) +DRILL = "break-glass" +DRILL_TOP = {"drill", "held_root_token", "generate_root", "revocation", "cancel_drill", "claim_limits"} +HELD = {"valid", "policies", "ttl_zero", "expire_time_null", "orphan"} +GENERATE_ROOT = {"started", "shares_entered", "complete", "otp_decode_ok", "policies", "minimal_action_path", + "minimal_action_ok"} +REVOCATION = {"revoke_status", "post_revoke_lookup"} +DRILL_CLAIMS = {"production_root_token_validity", "production_generate_root_availability", "sso_down_access_path"} +DRILL_ABORT = {"held_token_refused", "generate_root_failed", "minimal_action_failed", "revocation_failed", + "cancel_drill_failed"} +POLICY_NAME = re.compile(r"^[a-z0-9][a-z0-9_.-]{0,63}$") +SNAPSHOT_LABEL = re.compile(r"^(?=.*20\d{6})[a-z0-9][a-z0-9._-]{2,95}$") CLAIMS = {"consumer_continuity", "live_jwt_login", "phase_b_network", "owner_acceptance", "recurring_restore_assurance", "production_redirect"} ABORT = {"admission_refused", "preflight_refused", "custody_identity_refused", "ciphertext_mismatch", @@ -72,6 +89,9 @@ SECRET_SHAPES = ( re.compile(r"(? bool: """Random base64 (shares, keys) mixes cases and digits; repo paths do not.""" return any(re.search(r"[A-Z]", m) and re.search(r"[a-z]", m) and re.search(r"[0-9+]", m) for m in BASE64_BLOB.findall(text)) + + +def mixed_run(text: str) -> bool: + return any(re.search(r"[A-Z]", m) and re.search(r"[a-z]", m) and re.search(r"[0-9]", m) + for m in MIXED_RUN.findall(text)) PLACEHOLDERS = re.compile(r"(?i)replace-with|placeholder|todo|changeme|<[^>]*>|example\.invalid|YYYY") FORBIDDEN_KEYS = re.compile(r"(?i)^(?:data|value|values|secret|secrets|token|root_token|unseal_keys.*|keys?|" - r"otp|nonce|encoded_.*|password|plaintext)$") + r"otp|nonce|encoded_.*|password|plaintext|accessor|client_token|shares?|id)$") class Reject(Exception): @@ -137,6 +162,8 @@ def check_content(receipt: dict) -> None: for shape in SECRET_SHAPES: need(not shape.search(stripped), "secret_shaped_value") need(not base64_blob(stripped), "secret_shaped_value") + if path != "$.signer_fingerprint": # SHA256: is regex-checked above + need(not mixed_run(stripped), "secret_shaped_value") def check_sources(receipt: dict, root: Path) -> None: @@ -154,14 +181,58 @@ def check_sources(receipt: dict, root: Path) -> None: need("sha256:" + hashlib.sha256(path.read_bytes()).hexdigest() == expected, "source_hash_mismatch") +def check_drill(receipt: dict, result: str) -> None: + """Typed T10 extension; every value is an enumerated fact, never a credential.""" + need(receipt["drill"] == DRILL, "drill") + held = receipt["held_root_token"] + need(isinstance(held, dict) and set(held) == HELD, "drill_fields") + need(all(held[k] in (True, False, None) for k in ("valid", "ttl_zero", "expire_time_null", "orphan")), + "drill_fields") + need(isinstance(held["policies"], list) and len(held["policies"]) <= 32 + and all(isinstance(p, str) and POLICY_NAME.fullmatch(p) for p in held["policies"]), "drill_fields") + if held["valid"] is not True: + need(held["policies"] == [] and held["ttl_zero"] is None and held["expire_time_null"] is None + and held["orphan"] is None, "drill_inconsistent") + gen = receipt["generate_root"] + need(isinstance(gen, dict) and set(gen) == GENERATE_ROOT, "drill_fields") + need(all(type(gen[k]) is bool for k in ("started", "complete", "otp_decode_ok", "minimal_action_ok")), + "drill_fields") + need(type(gen["shares_entered"]) is int and 0 <= gen["shares_entered"] <= 2, "drill_fields") + need(isinstance(gen["policies"], list) and gen["policies"] in ([], ["root"]), "drill_fields") + need(gen["minimal_action_path"] == "sys/auth", "drill_fields") + rev = receipt["revocation"] + need(isinstance(rev, dict) and set(rev) == REVOCATION, "drill_fields") + need(all(rev[k] is None or (type(rev[k]) is int and 100 <= rev[k] <= 599) for k in REVOCATION), "drill_fields") + cancel = receipt["cancel_drill"] + need(isinstance(cancel, dict) and set(cancel) == {"started_then_cancelled"} + and type(cancel["started_then_cancelled"]) is bool, "drill_fields") + limits = receipt["claim_limits"] + need(isinstance(limits, dict) and set(limits) == {"production_root_token_validity"}, "claim_limits") + basis = limits["production_root_token_validity"] + need(isinstance(basis, dict) and set(basis) == {"as_of_snapshot", "snapshot_plaintext_sha256"} + and isinstance(basis["as_of_snapshot"], str) and SNAPSHOT_LABEL.fullmatch(basis["as_of_snapshot"]) + and basis["snapshot_plaintext_sha256"] == receipt["snapshot"]["plaintext_sha256"], "claim_limits") + need(receipt["objects"] == [], "objects") + if result == "drilled": + need(held["valid"] in (True, False), "drill_inconsistent") + need(gen["started"] and gen["shares_entered"] == 2 and gen["complete"] and gen["otp_decode_ok"] + and gen["policies"] == ["root"] and gen["minimal_action_ok"], "drill_inconsistent") + need(rev["revoke_status"] == 204 and rev["post_revoke_lookup"] == 403, "drill_inconsistent") + need(cancel["started_then_cancelled"] is True, "drill_inconsistent") + need(receipt["teardown"]["readback_credential_revoked"] is True, "drill_inconsistent") + elif gen["policies"]: + need(gen["otp_decode_ok"], "drill_inconsistent") + + def verify(receipt: object, *, source_root: Path, not_after: datetime | None = None, now: datetime | None = None) -> str: - need(isinstance(receipt, dict) and set(receipt) == TOP, "schema_fields") + drill = isinstance(receipt, dict) and "drill" in receipt + need(isinstance(receipt, dict) and set(receipt) == (TOP | DRILL_TOP if drill else TOP), "schema_fields") need(receipt["schema"] == SCHEMA and receipt["admission_namespace"] == NAMESPACE, "schema") need(receipt["extends"] == "docs/openbao-restore-drill-evidence.example.json", "schema") need(receipt["phase"] == "A" and receipt["material"] in ("original", "synthetic"), "phase") - need(isinstance(receipt["attempt_id"], str) and re.fullmatch(r"RPF-WP-0064-T0[47]-[A-Za-z0-9-]{3,64}", - receipt["attempt_id"]), "attempt_id") + attempt = r"RPF-WP-0064-T10-[A-Za-z0-9-]{3,64}" if drill else r"RPF-WP-0064-T0[47]-[A-Za-z0-9-]{3,64}" + need(isinstance(receipt["attempt_id"], str) and re.fullmatch(attempt, receipt["attempt_id"]), "attempt_id") need(digest_ok(receipt["admission_sha256"]), "admission_digest") need(isinstance(receipt["signer_principal"], str) and re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._@-]{0,127}", receipt["signer_principal"]), "signer") @@ -175,8 +246,9 @@ def verify(receipt: object, *, source_root: Path, not_after: datetime | None = N and digest_ok(image.split("@", 1)[1]), "image") need(receipt["image_id"] is None or digest_ok(receipt["image_id"]), "image") need(receipt["no_secret_material_recorded"] is True and receipt["kv_data_read"] is False, "secret_policy") - need(isinstance(receipt["claims_not_made"], list) and set(receipt["claims_not_made"]) == CLAIMS - and len(receipt["claims_not_made"]) == len(CLAIMS), "claims_not_made") + claims = CLAIMS | DRILL_CLAIMS if drill else CLAIMS + need(isinstance(receipt["claims_not_made"], list) and set(receipt["claims_not_made"]) == claims + and len(receipt["claims_not_made"]) == len(claims), "claims_not_made") started, finished = parse_time(receipt["started_at"]), parse_time(receipt["finished_at"]) deadline = receipt["deadline_seconds"] @@ -237,7 +309,22 @@ def verify(receipt: object, *, source_root: Path, not_after: datetime | None = N names = [row["object"] for row in objects] result, reason = receipt["result"], receipt["reason"] - if result in ("restored", "refused"): + if drill: + check_drill(receipt, result) + if drill and result == "drilled": + need(reason == "break_glass_drill_complete", "result_inconsistent") + need(snap["ciphertext_verified"] and snap["plaintext_verified"], "restore_inconsistent") + need(restored["sealed_after_restore"] is True and restored["unsealed_with_original_shares"] is True + and restored["cluster_id_match"] is True and restored["applied_index_not_behind"] is True + and restored["throwaway_init_invalidated"] is True + and restored["cluster_id"] == snap["source_cluster_id"] + and restored["applied_index"] >= snap["source_applied_index"], "restore_inconsistent") + need(network["network_mode"] == "none" and network["docker_networks"] == ["none"] + and network["interfaces"] == ["lo"] and network["published_ports"] == [] and network["mounts"] == [] + and network["egress_refused"] is True, "network_not_contained") + need(teardown["performed_by"] == "session", "teardown_missing") + need(digest_ok(receipt["image_id"]), "image") + elif result in ("restored", "refused") and not drill: need(tuple(names) == REQUIRED_OBJECTS, "objects_incomplete") need(snap["ciphertext_verified"] and snap["plaintext_verified"], "restore_inconsistent") need(restored["sealed_after_restore"] is True and restored["unsealed_with_original_shares"] is True @@ -258,7 +345,7 @@ def verify(receipt: object, *, source_root: Path, not_after: datetime | None = N missing = any(not r["present"] for r in objects) need(not all_ok and reason == ("object_missing" if missing else "object_mismatch"), "result_inconsistent") elif result == "aborted": - need(reason in ABORT, "result_inconsistent") + need(reason in (ABORT | DRILL_ABORT if drill else ABORT), "result_inconsistent") need(not objects or len(objects) == len(REQUIRED_OBJECTS), "objects") if network["docker_networks"]: need(network["docker_networks"] == ["none"] and network["published_ports"] == [], "network_not_contained") @@ -274,7 +361,7 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--receipt", required=True) parser.add_argument("--source-root", required=True, help="clean checkout at the receipt's adapter_commit") parser.add_argument("--not-after", help="refuse after this UTC time (YYYY-MM-DDTHH:MM:SSZ)") - parser.add_argument("--expect-result", choices=("restored", "refused", "aborted")) + parser.add_argument("--expect-result", choices=("restored", "refused", "aborted", "drilled")) opts = parser.parse_args(argv) try: path = Path(opts.receipt) diff --git a/tests/openbao_restore_rehearsal.py b/tests/openbao_restore_rehearsal.py index 5c9eac6..8586452 100644 --- a/tests/openbao_restore_rehearsal.py +++ b/tests/openbao_restore_rehearsal.py @@ -208,7 +208,7 @@ class Rehearsal: bao.seed(kv_registration=kv_registration, jwt_role=jwt_role) cluster_id, index = bao.facts() plain = bao.snapshot() - shares = list(bao.shares) + shares, root_credential = list(bao.shares), bao.credential self.secrets_seen += [bao.credential, *shares, getattr(bao, "kv_value", "")] cipher = sh(["age", "-r", self.age_recipient], input=plain).stdout path = self.work / ("snapshot-" + secrets.token_hex(3) + ".age") @@ -216,7 +216,7 @@ class Rehearsal: with os.fdopen(fd, "wb") as out: out.write(cipher) return {"path": path, "cipher": cipher, "plain_sha": adapter.sha256_bytes(plain), "cluster_id": cluster_id, - "index": index, "shares": shares} + "index": index, "shares": shares, "root_credential": root_credential} def admission(self, snap: dict, source_root: Path, commit: str, **override) -> dict: now = adapter.now_utc() @@ -256,9 +256,19 @@ class Rehearsal: os.chmod(sig, 0o600) return path, sig, adapter.sha256_bytes(raw) + def drill_admission(self, snap: dict, source_root: Path, commit: str, **override) -> dict: + """RPF-WP-0064-T10 drill admission: same snapshot facts, disjoint attempt id and fields.""" + adm = self.admission(snap, source_root, commit, **override) + adm.update({"attempt_id": "RPF-WP-0064-T10-rehearsal-" + secrets.token_hex(3), "drill": "break-glass", + "snapshot_label": "openbao-raft-20261014t000000z-synthetic.snap.age", + "readback_credential": adapter.DRILL_READBACK}) + return adm + def session(self, adm: dict, snap: dict, source_root: Path, answers: list, **kw): path, sig, digest = self.signed(adm) - loaded, _raw, signer, protected = adapter.load_admission(path, sig, digest, Path(os.environ[adapter.ANCHOR_ENV])) + mode = kw.get("mode", adapter.PHASE_A) + loaded, _raw, signer, protected = adapter.load_admission(path, sig, digest, Path(os.environ[adapter.ANCHOR_ENV]), + mode) queue = list(answers) def prompt(_label: str) -> bytearray: diff --git a/tests/test_openbao_break_glass_drill.py b/tests/test_openbao_break_glass_drill.py new file mode 100644 index 0000000..84f205b --- /dev/null +++ b/tests/test_openbao_break_glass_drill.py @@ -0,0 +1,372 @@ +"""RPF-WP-0064-T10 break-glass drill: offline tests with a fake OpenBao (no Docker). + +The fake answers the bao CLI calls the drill makes, keeps generated tokens, +OTPs and nonces in memory only, and records which credential reached which +path, so the tests can show the held token is used for ``lookup-self`` only. +""" +from __future__ import annotations + +import base64 +import copy +import json +import secrets +import subprocess +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import test_openbao_isolated_restore as U # noqa: E402 + +A, V, R = U.A, U.V, U.R +LABEL = "openbao-raft-20261014t153200z.snap.age" + + +def b64share() -> str: + return base64.b64encode(secrets.token_bytes(33)).decode() + + +class FakeBao: + """Just enough OpenBao for the drill steps; failure switches per test.""" + + def __init__(self, held_valid=True): + self.held = "s." + secrets.token_urlsafe(18)[:24] + self.held_valid = held_valid + self.generated: str | None = None + self.revoked = False + self.attempt: dict | None = None + self.calls: list[tuple[str, ...]] = [] + self.held_paths: list[str] = [] + self.generated_policies = ["root"] + self.revoke_body = b"" + self.cancel_works = True + + @staticmethod + def ok(body=b""): + return subprocess.CompletedProcess([], 0, body if isinstance(body, bytes) else json.dumps(body).encode(), b"") + + @staticmethod + def code(status): + return subprocess.CompletedProcess([], 2, b"", f"Error: Error making API request.\n\nCode: {status}. " + "Errors:\n\n* permission denied\n".encode()) + + def __call__(self, args, *, credential=None, body=b"", limit=30, check=True, reason="internal_error"): + verb, path = args[0], args[-2] if args[-1] == "-" else args[-1] + self.calls.append((verb, path)) + if credential is not None and credential == self.held: + self.held_paths.append(f"{verb} {path}") + done = self.answer(verb, path, credential, body) + if check and done.returncode != 0: + raise A.Refused(reason) + return done + + def status(self): + att = self.attempt or {} + return {"started": bool(self.attempt), "progress": att.get("progress", 0), "required": 2, + "complete": False, "otp": "", "nonce": "", "encoded_token": "", "otp_length": 26} + + def answer(self, verb, path, credential, body): + if path == "auth/token/lookup-self" and verb == "read": + if credential == self.held and self.held_valid: + return self.ok({"data": {"policies": ["root"], "ttl": 0, "expire_time": None, "orphan": True, + "id": self.held, "accessor": "Ab3" * 8}}) + if credential and credential == self.generated and not self.revoked: + return self.ok({"data": {"policies": self.generated_policies, "ttl": 0, "expire_time": None, + "id": self.generated}}) + return self.code(403) + if path == "sys/generate-root/attempt" and verb == "read": + return self.ok({"data": self.status()}) + if path == "sys/generate-root/attempt" and verb == "write": + self.attempt = {"otp": secrets.token_urlsafe(30)[:26].replace("-", "a").replace("_", "b"), + "nonce": "0b0e" + secrets.token_hex(14), "progress": 0} + return self.ok({"data": {**self.status(), "otp": self.attempt["otp"], "nonce": self.attempt["nonce"]}}) + if path == "sys/generate-root/attempt" and verb == "delete": + if self.cancel_works: + self.attempt = None + return self.ok(b"Success! Data deleted\n") + if path == "sys/generate-root/update": + payload = json.loads(body) + assert payload["nonce"] == self.attempt["nonce"] + self.attempt["progress"] += 1 + if self.attempt["progress"] < 2: + return self.ok({"data": {**self.status(), "progress": 1}}) + self.generated = "s." + secrets.token_urlsafe(18)[:24] + otp = self.attempt["otp"].encode() + encoded = base64.b64encode(bytes(a ^ b for a, b in zip(self.generated.encode(), otp))).decode() + self.attempt = None + return self.ok({"data": {"complete": True, "encoded_token": encoded, "progress": 2}}) + if path == "sys/auth" and credential == self.generated and not self.revoked: + return self.ok({"data": {"token/": {"type": "token"}, "netkingdom/": {"type": "oidc"}}}) + if path == "auth/token/revoke-self" and credential == self.generated: + self.revoked = True + return self.ok(self.revoke_body) + return self.code(403) + + +class DrillBase(U.Base): + def drill_admission(self, **override): + adm = self.admission(attempt_id="RPF-WP-0064-T10-unit-01", drill="break-glass", snapshot_label=LABEL, + readback_credential=A.DRILL_READBACK) + adm.update(override) + return adm + + def session(self, fake, answers, **adm_override): + queue = list(answers) + self.prompts: list[str] = [] + + def prompt(label): + self.prompts.append(label) + return bytearray(queue.pop(0).encode()) + adm = self.drill_admission(**adm_override) + session = A.Session(admission=adm, admission_sha256=U.digest("adm"), + signer={"principal": "founder@rehearsal", "fingerprint": "SHA256:" + "AbCd" * 10 + "AbC"}, + anchor_is_protected=False, ciphertext=self.work / "none", receipt_path=self.work / "r", + root=self.root, prompt=prompt, say=lambda _m: None, start_reaper=False, + mode=A.DRILL_MODE) + session.bao = fake + return session + + def run_drill(self, fake, answers=None): + answers = answers if answers is not None else [fake.held, b64share(), b64share()] + session = self.session(fake, answers) + session.drill_held_token() + session.drill_generate_root() + session.drill_minimal_action() + session.drill_revoke() + session.drill_cancel() + return session + + def drill_receipt(self, session=None): + """A complete ``drilled`` receipt from the adapter's own drill record plus a valid restore.""" + session = session or self.run_drill(FakeBao()) + base = self.receipt() + rc = copy.deepcopy(session.receipt) + for key in ("started_at", "finished_at", "elapsed_seconds", "image_id", "network", "restored", "snapshot"): + rc[key] = base[key] + rc["claim_limits"]["production_root_token_validity"]["snapshot_plaintext_sha256"] = \ + rc["snapshot"]["plaintext_sha256"] + rc["image"] = base["image"] + rc["teardown"].update(container_removed=True, container_absent_verified=True) + rc.update(result="drilled", reason="break_glass_drill_complete") + return rc + + +class DrillAdmissionTests(DrillBase): + def test_valid_drill_admission(self): + A.validate_admission(self.drill_admission(), A.DRILL_MODE) + + def test_phase_a_and_drill_admissions_are_disjoint(self): + with self.assertRaises(A.Refused): + A.validate_admission(self.drill_admission(), A.PHASE_A) + with self.assertRaises(A.Refused): + A.validate_admission(self.admission(), A.DRILL_MODE) + with self.assertRaises(A.Refused): + A.Session(admission=self.admission(), admission_sha256=U.digest("a"), signer={}, anchor_is_protected=False, + ciphertext=self.work, receipt_path=self.work / "x", mode=A.DRILL_MODE, start_reaper=False) + + def test_drill_refusals(self): + cases = [{"drill": "phase-a"}, {"attempt_id": "RPF-WP-0064-T07-01"}, {"snapshot_label": "latest"}, + {"snapshot_label": "Openbao-20261014"}, {"snapshot_label": "openbao-raft-snapshot"}, + {"readback_credential": "generate-root-from-entered-shares"}, {"phase_b_admitted": True}, + {"material": "original", "max_duration_seconds": 600}] + for override in cases: + with self.subTest(override=override), self.assertRaises(A.Refused): + A.validate_admission(self.drill_admission(**override), A.DRILL_MODE) + + def test_allowed_surfaces(self): + self.assertIn("sys/generate-root/attempt", A.ALLOWED_READS) + self.assertIn("sys/generate-root/attempt", A.ALLOWED_WRITES) + self.assertIn("sys/generate-root/update", A.ALLOWED_WRITES) + self.assertEqual(A.ALLOWED_DELETES, frozenset({"sys/generate-root/attempt"})) + self.assertIn("auth/token/lookup-self", A.ALLOWED_READS) + self.assertIn("sys/auth", A.ALLOWED_READS) + self.assertFalse(any("/data/" in p for p in A.ALLOWED_READS)) + self.assertEqual(A.MAX_DURATION_SECONDS, 2700) + listener = json.loads(A.SERVER_CONFIG)["listener"]["tcp"] + self.assertEqual(listener["address"], "127.0.0.1:8200") + self.assertIs(listener["disable_unauthed_generate_root_endpoints"], False) + with self.assertRaises(A.Refused): + self.session(FakeBao(), []).delete("sys/unseal") + + +class DrillStepTests(DrillBase): + def test_full_drill_records_only_facts(self): + fake = FakeBao() + shares = [b64share(), b64share()] + session = self.run_drill(fake, [fake.held, *shares]) + rc = session.receipt + self.assertEqual(self.prompts, ["Held initial root token", "Re-enter unseal share 1 of 2 for generate-root", + "Re-enter unseal share 2 of 2 for generate-root"]) + self.assertEqual(rc["held_root_token"], {"valid": True, "policies": ["root"], "ttl_zero": True, + "expire_time_null": True, "orphan": True}) + self.assertEqual(rc["generate_root"], {"started": True, "shares_entered": 2, "complete": True, + "otp_decode_ok": True, "policies": ["root"], + "minimal_action_path": "sys/auth", "minimal_action_ok": True}) + self.assertEqual(rc["revocation"], {"revoke_status": 204, "post_revoke_lookup": 403}) + self.assertEqual(rc["cancel_drill"], {"started_then_cancelled": True}) + self.assertIsNone(session.credential) + self.assertTrue(rc["teardown"]["readback_credential_revoked"]) + # The held token reached lookup-self only: never revoked, never used for anything else. + self.assertEqual(fake.held_paths, ["read auth/token/lookup-self"]) + self.assertIn(("delete", "sys/generate-root/attempt"), fake.calls) + text = json.dumps(rc) + for value in (fake.held, fake.generated, *shares): + self.assertNotIn(value, text) + self.assertEqual(rc["claim_limits"]["production_root_token_validity"], + {"as_of_snapshot": LABEL, "snapshot_plaintext_sha256": rc["snapshot"]["plaintext_sha256"]}) + self.assertTrue(set(A.DRILL_CLAIMS_NOT_MADE) <= set(rc["claims_not_made"])) + + def test_invalid_held_token_is_a_finding_not_an_abort(self): + fake = FakeBao(held_valid=False) + rc = self.run_drill(fake).receipt + self.assertEqual(rc["held_root_token"], {"valid": False, "policies": [], "ttl_zero": None, + "expire_time_null": None, "orphan": None}) + self.assertEqual(self.verify(self.drill_receipt(self.run_drill(FakeBao(held_valid=False)))), "drilled") + + def failing(self, fake, step, reason, answers=None): + session = self.session(fake, answers if answers is not None else [fake.held, b64share(), b64share()]) + steps = [session.drill_held_token, session.drill_generate_root, session.drill_minimal_action, + session.drill_revoke, session.drill_cancel] + with self.assertRaises(A.Refused) as ctx: + for fn in steps[:step + 1]: + fn() + self.assertEqual(ctx.exception.reason, reason) + return session + + def test_failures(self): + self.failing(FakeBao(), 0, "held_token_refused", ["not a token!", b64share(), b64share()]) + fake = FakeBao() + fake.generated_policies = ["default"] + self.failing(fake, 1, "generate_root_failed") + self.failing(FakeBao(), 1, "generate_root_failed", [FakeBao().held, "short", b64share()]) + fake = FakeBao() + fake.revoke_body = b'{"data": {}}' + session = self.failing(fake, 3, "revocation_failed") + self.assertFalse(session.receipt["teardown"]["readback_credential_revoked"]) + fake = FakeBao() + fake.cancel_works = False + self.failing(fake, 4, "cancel_drill_failed") + + def test_stale_attempt_refused(self): + fake = FakeBao() + fake.attempt = {"otp": "x", "nonce": "y", "progress": 1} + self.failing(fake, 1, "generate_root_failed") + + +class DrillVerifierTests(DrillBase): + def test_drilled_receipt_passes(self): + self.assertEqual(self.verify(self.drill_receipt()), "drilled") + + def test_aborted_drill_receipt_passes(self): + session = self.session(FakeBao(), []) + rc = self.drill_receipt(session) + rc.update(result="aborted", reason="generate_root_failed") + self.assertEqual(self.verify(rc), "aborted") + + def test_drill_and_phase_a_shapes_do_not_mix(self): + rc = self.drill_receipt() + rc.update(result="restored", reason="all_required_objects_match") + with self.assertRaises(V.Reject): + self.verify(rc) + phase_a = self.receipt() + phase_a.update(result="drilled", reason="break_glass_drill_complete") + with self.assertRaises(V.Reject): + self.verify(phase_a) + + def test_adversarial_fixtures(self): + token = "s." + "Kq3x9ZbT7mW2pL8vR4nY6cJd" + hvs = "hvs.CAESI" + "Kq3x9ZbT7mW2pL8vR4nY6cJd" * 3 + accessor = "Kq3x9ZbT7mW2pL8vR4nY6cJd" + otp = "Zq8T1mVb4Xw9Kc2Lp7Rn3Hs6Yd" + nonce = "0b0e6c2d-9f3a-4e1b-8c7d-5a6b7c8d9e0f" + encoded = base64.b64encode(bytes(range(40, 66))).decode() + share = base64.b64encode(bytes(range(1, 34))).decode() + hex_share = "ab01" * 17 + gen = lambda **kw: (lambda r: r["generate_root"].update(**kw)) # noqa: E731 + held = lambda **kw: (lambda r: r["held_root_token"].update(**kw)) # noqa: E731 + cases = { + "token_value_policies": held(policies=[token]), + "token_in_path": lambda r: r.update(attempt_id="RPF-WP-0064-T10-" + token.replace(".", "-")), + "hvs_in_label": lambda r: r["claim_limits"]["production_root_token_validity"].update(as_of_snapshot=hvs), + "token_key": lambda r: r["held_root_token"].update(token=token), + "root_token_key": lambda r: r.update(root_token=token), + "accessor_key": held(accessor=accessor), + "accessor_value": gen(minimal_action_path=accessor), + "accessor_in_network": lambda r: r["network"].update(network_mode=accessor), + "otp_key": gen(otp=otp), + "otp_value": gen(policies=[otp]), + "otp_in_last_error": lambda r: r["restored"]["leader_wait"].update(last_error=otp) if + r["restored"]["leader_wait"] else r["network"].update(mounts=[otp]), + "nonce_key": gen(nonce=nonce), + "nonce_value": gen(minimal_action_path=nonce), + "encoded_token_key": gen(encoded_token=encoded), + "encoded_root_token_key": gen(encoded_root_token=encoded), + "encoded_value": lambda r: r["network"].update(mounts=[encoded]), + "share_key": gen(shares=[share]), + "share_value": gen(policies=[share]), + "share_in_cancel": lambda r: r.update(cancel_drill={"started_then_cancelled": share}), + "hex_share": lambda r: r["network"].update(mounts=[hex_share]), + "unseal_keys": lambda r: r.update(unseal_keys_b64=[share]), + "client_token": lambda r: r["revocation"].update(client_token=token), + } + for name, mutate in cases.items(): + with self.subTest(name): + rc = self.drill_receipt() + mutate(rc) + with self.assertRaises(V.Reject): + self.verify(rc) + + def test_inconsistent_drill_receipts(self): + cases = { + "revoke_200": (lambda r: r["revocation"].update(revoke_status=200), "drill_inconsistent"), + "lookup_after_revoke_ok": (lambda r: r["revocation"].update(post_revoke_lookup=200), + "drill_inconsistent"), + "one_share": (lambda r: r["generate_root"].update(shares_entered=1), "drill_inconsistent"), + "not_root": (lambda r: r["generate_root"].update(policies=["default"]), "drill_fields"), + "no_cancel": (lambda r: r["cancel_drill"].update(started_then_cancelled=False), "drill_inconsistent"), + "minimal_path": (lambda r: r["generate_root"].update(minimal_action_path="sys/mounts"), "drill_fields"), + "held_invalid_with_facts": (lambda r: r["held_root_token"].update(valid=False), "drill_inconsistent"), + "held_unchecked": (lambda r: r["held_root_token"].update(valid=None, policies=[], ttl_zero=None, + expire_time_null=None, orphan=None), + "drill_inconsistent"), + "claims_missing": (lambda r: r["claims_not_made"].remove("sso_down_access_path"), "claims_not_made"), + "snapshot_unnamed": (lambda r: r["claim_limits"]["production_root_token_validity"].update( + as_of_snapshot="latest"), "claim_limits"), + "snapshot_other_sha": (lambda r: r["claim_limits"]["production_root_token_validity"].update( + snapshot_plaintext_sha256=U.digest("other")), "claim_limits"), + "objects_read": (lambda r: r.update(objects=self.receipt()["objects"]), "objects"), + "phase_a_attempt": (lambda r: r.update(attempt_id="RPF-WP-0064-T07-01x"), "attempt_id"), + "unrevoked_teardown": (lambda r: r["teardown"].update(readback_credential_revoked=None), + "drill_inconsistent"), + "kv_data_read": (lambda r: r.update(kv_data_read=True), "secret_policy"), + "drill_reason_on_phase_a": (lambda r: r.update(result="aborted", reason="nope"), "result_inconsistent"), + } + for name, (mutate, code) in cases.items(): + with self.subTest(name): + rc = self.drill_receipt() + mutate(rc) + with self.assertRaises(V.Reject) as ctx: + self.verify(rc) + self.assertEqual(str(ctx.exception), code) + + def test_cli_accepts_drilled(self): + path = self.work / "drill.json" + path.write_text(json.dumps(self.drill_receipt())) + done = subprocess.run([sys.executable, str(R.VERIFIER), "--receipt", str(path), "--source-root", + str(self.root), "--expect-result", "drilled"], capture_output=True) + self.assertEqual(done.returncode, 0, done.stdout) + + +class DrillLauncherTests(DrillBase): + def test_launcher_mode_flag_refuses_before_prompt(self): + launcher = R.ROOT / "scripts/openbao-isolated-restore/run-isolated-restore.sh" + f = self.work / "f" + f.write_text("x") + out = subprocess.run(["bash", str(launcher), "--break-glass-drill", str(f), str(f), U.digest("x"), str(f), + str(self.work)], capture_output=True, stdin=subprocess.DEVNULL).stdout + self.assertIn(b"interactive terminal", out) + self.assertIn(b"--break-glass-drill", subprocess.run(["bash", str(launcher)], capture_output=True).stdout) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_openbao_isolated_restore.py b/tests/test_openbao_isolated_restore.py index 19c9a39..9ebc8e5 100644 --- a/tests/test_openbao_isolated_restore.py +++ b/tests/test_openbao_isolated_restore.py @@ -196,7 +196,9 @@ class VerifierTests(Base): self.assertEqual(tuple(o["object"] for o in rc["objects"]), V.REQUIRED_OBJECTS) self.assertEqual(tuple(o["object"] for o in A.OBJECTS), V.REQUIRED_OBJECTS) self.assertEqual(set(A.CLAIMS_NOT_MADE), V.CLAIMS) - self.assertEqual(A.ABORT_REASONS, V.ABORT) + self.assertEqual(A.ABORT_REASONS, V.ABORT | V.DRILL_ABORT) + self.assertEqual(A.DRILL_ABORT_REASONS, V.DRILL_ABORT) + self.assertEqual(set(A.DRILL_CLAIMS_NOT_MADE), V.DRILL_CLAIMS) with self.assertRaises(V.Reject): self.verify(rc) # placeholders diff --git a/tests/test_openbao_isolated_restore_native.py b/tests/test_openbao_isolated_restore_native.py index a2b23fc..77b3f10 100644 --- a/tests/test_openbao_isolated_restore_native.py +++ b/tests/test_openbao_isolated_restore_native.py @@ -171,6 +171,40 @@ class NativeRehearsal(unittest.TestCase): self.assertEqual(out["reason"], "operator_abort") self.assertTrue(out["teardown"]["container_absent_verified"]) + # ---- RPF-WP-0064-T10 break-glass drill (generated material; the held token is synthetic) + def run_drill(self, held: str, reentered: list[str]): + adm = self.h.drill_admission(self.full, self.root, self.commit) + answers = [self.h.age_identity, self.full["shares"][0], self.full["shares"][2], held, *reentered] + return self.h.session(adm, self.full, self.root, answers, mode=R.adapter.DRILL_MODE) + + def test_break_glass_drill_with_valid_held_token(self): + # The synthetic source's initial root token stands in for the founder's held token. + session, receipt = self.run_drill(self.full["root_credential"], [self.full["shares"][1], self.full["shares"][0]]) + out = self.finish(session, receipt, expect="drilled") + self.assertEqual(out["held_root_token"], {"valid": True, "policies": ["root"], "ttl_zero": True, + "expire_time_null": True, "orphan": True}) + self.assertEqual(out["generate_root"]["policies"], ["root"]) + self.assertTrue(out["generate_root"]["minimal_action_ok"]) + self.assertEqual(out["revocation"], {"revoke_status": 204, "post_revoke_lookup": 403}) + self.assertTrue(out["cancel_drill"]["started_then_cancelled"]) + self.assertEqual(out["objects"], []) + self.assertEqual(out["claim_limits"]["production_root_token_validity"]["snapshot_plaintext_sha256"], + self.full["plain_sha"]) + + def test_break_glass_drill_with_fake_held_token(self): + fake = "s." + base64.b32encode(os.urandom(15)).decode().lower()[:24] + self.h.secrets_seen.append(fake) + session, receipt = self.run_drill(fake, [self.full["shares"][0], self.full["shares"][1]]) + out = self.finish(session, receipt, expect="drilled") + self.assertIs(out["held_root_token"]["valid"], False) + + def test_break_glass_drill_wrong_reentered_share_aborts(self): + wrong = base64.b64encode(os.urandom(33)).decode() + session, receipt = self.run_drill(self.full["root_credential"], [self.full["shares"][0], wrong]) + out = self.finish(session, receipt) + self.assertEqual(out["reason"], "generate_root_failed") + self.assertTrue(out["teardown"]["container_absent_verified"]) + def reaper_case(self, *, kill_parent: bool, deadline_in: float) -> dict: session, receipt = self.run_session() parent = subprocess.Popen(["sleep", "300"]) diff --git a/workplans/RPF-WP-0064-isolated-original-openbao-restore.md b/workplans/RPF-WP-0064-isolated-original-openbao-restore.md index 8892041..d07c189 100644 --- a/workplans/RPF-WP-0064-isolated-original-openbao-restore.md +++ b/workplans/RPF-WP-0064-isolated-original-openbao-restore.md @@ -8,7 +8,7 @@ status: active flavor: implementation owner: claude created: "2026-10-23" -updated: "2026-10-23" +updated: "2026-10-28" related: - SECRETS-WP-0015 - NK-WP-0049 @@ -588,7 +588,7 @@ packet or of the other three services. ```task id: RPF-WP-0064-T10 -status: todo +status: progress priority: high depends_on: [RPF-WP-0064-T08] state_hub_task_id: "5facf182-91fb-58bc-a2dd-c6724049e70f" @@ -673,3 +673,117 @@ the ceremony drill succeeds, or record why it is kept. - NetKingdom has a value-free post-event review record and the rapp-openbao listener decision No live run is part of the source preparation. + +### Implementation (2026-10-28, source only; no live action) + +`--mode break-glass-drill` is in `scripts/openbao-isolated-restore/restore_adapter.py`, with `--mode` in +`preflight.py` and `--break-glass-drill` in `run-isolated-restore.sh`. The phase A path is unchanged. + +- **Admission.** The v1 admission plus `drill: "break-glass"`, `snapshot_label` (lowercase object name + containing its date) and `readback_credential: "generate-root-from-reentered-shares"`. The attempt id is + `RPF-WP-0064-T10-*`. Phase A and drill admissions are disjoint and refuse each other's mode. +- **Allowed surfaces.** One added read: `GET sys/generate-root/attempt`. One added delete: + `ALLOWED_DELETES = {sys/generate-root/attempt}`. The writes `sys/generate-root/attempt` and + `sys/generate-root/update` were already allowed, as were `auth/token/lookup-self` and `sys/auth`. These are + unchanged: `--network none`, the loopback listener, 2700 s, teardown, the reaper and no `/data/` reads. +- **Order.** Phase A restore and unseal. The unseal shares are then wiped. Next come the held-token + `lookup-self`, the generate-root attempt (refused if one is already in progress), the two re-entered shares, + the OTP decode in memory, `lookup-self` = `["root"]`, `GET sys/auth`, `revoke-self` (204: the CLI prints no + body) and `lookup-self` (403). Last, an attempt is started, deleted, and the status shows `started: false`. +- **Held-token outcome.** An invalid held token (403) is a finding, `valid: false`, not an abort. +- **Results.** The result is `drilled` (`break_glass_drill_complete`) or `aborted`. The new abort reasons are + `held_token_refused`, `generate_root_failed`, `minimal_action_failed`, `revocation_failed` and + `cancel_drill_failed`. +- **Receipt.** The receipt adds `drill`, `held_root_token`, `generate_root`, `revocation`, `cancel_drill` and + `claim_limits.production_root_token_validity {as_of_snapshot, snapshot_plaintext_sha256}`. It also adds three + `claims_not_made` entries. `objects` is `[]`. +- **Verifier.** The verifier types every extension field and binds `snapshot_plaintext_sha256` to + `snapshot.plaintext_sha256`. It rejects these keys: `token`, `accessor`, `client_token`, `otp`, `nonce`, + `encoded_*`, `share(s)` and `unseal_keys*`. It also rejects any 20+ character mixed-case alphanumeric run, + which covers tokens, accessors, OTPs and encoded tokens. +- **Tests.** Offline (`python3 -m unittest tests.test_openbao_isolated_restore + tests.test_openbao_break_glass_drill`): 34 pass, 15 of them new in `tests/test_openbao_break_glass_drill.py`. They use a fake OpenBao and adversarial token, accessor, OTP, + nonce, encoded-token and share fixtures. Native (`make openbao-isolated-restore-rehearsal`): 16 pass, 3 of + them new. The new native cases are a synthetic held root token (`drilled`, valid, `["root"]`, ttl 0), a + fake held token (`drilled`, `valid: false`) and a wrong re-entered share (`aborted`, `generate_root_failed`). + The native rehearsal uses local Docker with `--network none`, the pinned 2.5.4 image and generated material. + +### Founder runbook: drill (isolated restore on the workstation, never production) + +Run each block in your own terminal. Nothing below needs the cluster. Snapshot facts are reused from the +signed T07 `-03` admission (the 2026-10-14 snapshot, plaintext `a6312444…590b`). If a newer RPF-WP-0066 +scheduled snapshot is used instead, put its facts in the admission and its name in `snapshot_label`. + +```bash +# 0. Clean checkout at the reviewed commit; offline tests pass +cd ~/railiance-platform && git pull --ff-only && git status --porcelain # must print nothing +python3 -m unittest tests.test_openbao_isolated_restore tests.test_openbao_break_glass_drill +COMMIT=$(git rev-parse HEAD); echo "$COMMIT" + +# 1. Build the drill admission from the T07 -03 admission (same snapshot facts) +W=/dev/shm/rpf64-t10; mkdir -m 700 -p "$W" +T07_ADM=/path/to/RPF-WP-0064-T07-20261024-03.json # your signed -03 admission +CIPHERTEXT=/path/to/the-2026-10-14-snapshot.age # same file as T07 (8069449c…8c41, 375618 bytes) +LABEL=openbao-raft-20261014... # its object name in railiance-backups/openbao/, lowercase +ID=RPF-WP-0064-T10-$(date -u +%Y%m%d)-01 +NB=$(date -u +%Y-%m-%dT%H:%M:00Z); NA=$(date -u -d '+6 hours' +%Y-%m-%dT%H:%M:00Z) +SRC=$(python3 -c 'import importlib.util as u,json;s=u.spec_from_file_location("a","scripts/openbao-isolated-restore/restore_adapter.py");m=u.module_from_spec(s);s.loader.exec_module(m);print(json.dumps(m.source_digests(m.ROOT)))') +jq --arg id "$ID" --arg nb "$NB" --arg na "$NA" --arg c "$COMMIT" --arg label "$LABEL" --argjson src "$SRC" \ + '.attempt_id=$id | .not_before=$nb | .not_after=$na | .adapter_commit=$c | .source_sha256=$src + | .drill="break-glass" | .snapshot_label=$label | .readback_credential="generate-root-from-reentered-shares"' \ + "$T07_ADM" > "$W/$ID.json" +chmod 600 "$W/$ID.json"; jq '{attempt_id, drill, snapshot_label, not_before, not_after, adapter_commit}' "$W/$ID.json" + +# 2. Sign it (founder key in /etc/railiance-openbao-restore/allowed_signers; fixed namespace) +ssh-keygen -Y sign -f ~/.ssh/ -n railiance-openbao-restore-attempt "$W/$ID.json" +chmod 600 "$W/$ID.json.sig" +ADM_SHA=sha256:$(sha256sum "$W/$ID.json" | cut -d' ' -f1); echo "$ADM_SHA" + +# 3. Run the drill (hidden prompts, in this order: backup age identity; unseal share 1; unseal share 2; +# held initial root token; share 1 AGAIN; share 2 AGAIN). Clear the clipboard after each paste. +mkdir -m 700 -p ~/rpf64-receipts +bash scripts/openbao-isolated-restore/run-isolated-restore.sh --break-glass-drill \ + "$W/$ID.json" "$W/$ID.json.sig" "$ADM_SHA" "$CIPHERTEXT" ~/rpf64-receipts + +# 4. Verify independently from a clean clone at the admitted commit, then show the value-free facts +RECEIPT=$(ls -t ~/rpf64-receipts/rpf64-break-glass-drill-*.json | head -1); chmod 400 "$RECEIPT" +rm -rf /dev/shm/rpf64-verify && git clone -q ~/railiance-platform /dev/shm/rpf64-verify \ + && git -C /dev/shm/rpf64-verify checkout -q "$COMMIT" +python3 /dev/shm/rpf64-verify/scripts/openbao-isolated-restore/verify-restore-receipt.py \ + --receipt "$RECEIPT" --source-root /dev/shm/rpf64-verify --expect-result drilled +jq '{attempt_id, result, reason, elapsed_seconds, restored: {cluster_id_match: .restored.cluster_id_match, + applied_index: .restored.applied_index}, held_root_token, generate_root, revocation, cancel_drill, + claim_limits, claims_not_made, teardown}' "$RECEIPT" +rm -rf "$W" /dev/shm/rpf64-verify +``` + +**Paste back:** + +- the admission sha256 (`$ADM_SHA`) and the commit; +- the verifier line, which is `PASS: drilled receipt sha256:…` or `FAIL: `; +- the `jq` output from step 4. + +Never paste the receipt path contents beyond that, nor any token, share or OTP. A `FAIL` or `aborted` is still +evidence: paste the same lines. The receipt stays outside Git (mode 0400). + +### Founder runbook: production SSO-down walk (read-only) + +The walk uses the static-key `k3s-api-railiance01` tunnel and the admin kubeconfig. Local port 18200 is taken by +a tunnel, so the port-forward uses 18210. There is no unseal, no generate-root attempt and no token on +production. + +```bash +bridge status k3s-api-railiance01 # up; restore with: bridge up k3s-api-railiance01 +export KUBECONFIG=~/.kube/config-railiance01 +kubectl -n openbao get pod openbao-0 -o wide +kubectl -n openbao exec openbao-0 -- bao status -format=json \ + | jq '{initialized, sealed, t, n, version, ha_enabled}' # exit 2 only means sealed +kubectl -n openbao port-forward pod/openbao-0 18210:8200 >/dev/null 2>&1 & PF=$!; sleep 3 +curl -sS http://127.0.0.1:18210/v1/sys/seal-status | jq '{initialized, sealed, t, n}' +curl -sS -o /dev/null -w 'generate-root status GET: %{http_code}\n' \ + http://127.0.0.1:18210/v1/sys/generate-root/attempt # 405 = unauthenticated generate-root disabled +kill "$PF"; unset KUBECONFIG +``` + +**Paste back** the pod line, the two `jq` outputs and the `generate-root status GET` code. Record them in the +NK-WP-0055-T13 post-event review record. The GET only reads status and starts nothing.