a2da4085e0
Adds --mode break-glass-drill with its own signed admission (drill: "break-glass", attempt id RPF-WP-0064-T10-*, snapshot_label). After the phase A restore and unseal it checks the held root token with lookup-self only, runs a fresh generate-root ceremony with re-entered shares, does one read-only sys/auth action, revokes the generated token (204 then 403) and drills a cancelled attempt. Phase A is unchanged. The verifier types the receipt extension and rejects token, accessor, OTP, nonce, encoded-token and share shapes. Offline 34 and native 16 pass. Founder runbooks for the drill and the read-only production SSO-down walk are in the T10 section. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Assistant: claude-code Assistant-Model: opus Assistant-Process: 4515@bnt-lap001 Assistant-Session: 0bf96127-6e04-41be-bfda-d11717c3f354
291 lines
15 KiB
Python
291 lines
15 KiB
Python
"""Synthetic-material rehearsal fixture for RPF-WP-0064-T04.
|
|
|
|
Everything here is generated per run: SSH admission key, age identity, Shamir
|
|
shares, root credentials and the KV value. Source containers run the pinned
|
|
OpenBao image with ``--network none`` and tmpfs storage only and are removed.
|
|
No real snapshot, key, share or cluster is touched.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import importlib.util
|
|
import json
|
|
import os
|
|
import secrets
|
|
import shutil
|
|
import subprocess
|
|
import tempfile
|
|
import time
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
REHEARSAL_IMAGE = "quay.io/openbao/openbao:2.5.4"
|
|
|
|
|
|
def load(name: str, rel: str):
|
|
spec = importlib.util.spec_from_file_location(name, ROOT / rel)
|
|
module = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
adapter = load("restore_adapter", "scripts/openbao-isolated-restore/restore_adapter.py")
|
|
VERIFIER = ROOT / "scripts/openbao-isolated-restore/verify-restore-receipt.py"
|
|
|
|
|
|
def sh(argv, **kw) -> subprocess.CompletedProcess:
|
|
kw.setdefault("capture_output", True)
|
|
kw.setdefault("timeout", 120)
|
|
return subprocess.run(argv, **kw)
|
|
|
|
|
|
def docker_status() -> tuple[bool, str]:
|
|
if not shutil.which("docker") or not shutil.which("age") or not shutil.which("ssh-keygen"):
|
|
return False, "docker/age/ssh-keygen unavailable"
|
|
if sh(["docker", "version", "--format", "{{.Server.Version}}"], timeout=20).returncode != 0:
|
|
return False, "docker daemon unavailable"
|
|
inspect = sh(["docker", "image", "inspect", REHEARSAL_IMAGE, "--format", "{{json .RepoDigests}}"], timeout=20)
|
|
if inspect.returncode != 0:
|
|
return False, f"{REHEARSAL_IMAGE} not pulled locally (make openbao-isolated-restore-image)"
|
|
digests = [d for d in json.loads(inspect.stdout) if d.startswith(adapter.IMAGE_REPOSITORY + "@")]
|
|
return (True, digests[0]) if digests else (False, "image has no repository digest")
|
|
|
|
|
|
def make_source_root(base: Path, mutate: dict[str, bytes] | None = None) -> tuple[Path, str]:
|
|
"""A clean git checkout holding exactly the adapter source closure."""
|
|
root = base / ("src-" + secrets.token_hex(3))
|
|
for name in adapter.SOURCE_PATHS:
|
|
target = root / name
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
target.write_bytes((mutate or {}).get(name, (ROOT / name).read_bytes()))
|
|
env = {**os.environ, "GIT_AUTHOR_NAME": "rehearsal", "GIT_AUTHOR_EMAIL": "rehearsal@invalid",
|
|
"GIT_COMMITTER_NAME": "rehearsal", "GIT_COMMITTER_EMAIL": "rehearsal@invalid"}
|
|
for argv in (["git", "init", "-q"], ["git", "add", "-A"],
|
|
["git", "-c", "core.hooksPath=/dev/null", "commit", "-q", "-m", "rehearsal closure"]):
|
|
done = sh(argv, cwd=root, env=env)
|
|
assert done.returncode == 0, done.stderr
|
|
commit = sh(["git", "rev-parse", "HEAD"], cwd=root).stdout.decode().strip()
|
|
return root, commit
|
|
|
|
|
|
# The source mirrors production (rapp-openbao chart setNodeId: the pod name),
|
|
# built independently of the adapter's target config. A rehearsal with a target
|
|
# node id differing from this voter still restored and elected a leader, so the
|
|
# node id alone did not explain T07 attempt -01; it stays aligned with production.
|
|
PRODUCTION_NODE_ID = "openbao-0"
|
|
# It also declares production's file audit device (rapp-openbao values), so the
|
|
# snapshot's audit table re-enables it on the target after unseal.
|
|
PRODUCTION_AUDIT_PATH = "/openbao/audit/openbao-audit.log"
|
|
SOURCE_CONFIG = json.dumps({**json.loads(adapter.SERVER_CONFIG),
|
|
"storage": {"raft": {"path": "/openbao/data", "node_id": PRODUCTION_NODE_ID}},
|
|
"audit": {"file": {"file": {"description": "rehearsal mirror of production",
|
|
"options": {"file_path": PRODUCTION_AUDIT_PATH}}}}},
|
|
separators=(",", ":"))
|
|
|
|
|
|
class SyntheticBao:
|
|
"""A generated-material OpenBao in a networkless container (source side)."""
|
|
|
|
def __init__(self, image: str):
|
|
self.image = image
|
|
self.name = "rpf64-rehearsal-src-" + secrets.token_hex(4)
|
|
self.credential = ""
|
|
self.shares: list[str] = []
|
|
|
|
def __enter__(self):
|
|
argv = ["docker", "run", "--detach", "--pull=never", "--name", self.name, "--network", "none",
|
|
"--user", "100:1000", "--read-only", "--cap-drop", "ALL", "--security-opt", "no-new-privileges"]
|
|
for target, options in adapter.TMPFS.items():
|
|
argv += ["--tmpfs", f"{target}:{options}"]
|
|
argv += ["-e", "BAO_LOCAL_CONFIG=" + SOURCE_CONFIG, "-e", "BAO_ADDR=http://127.0.0.1:8200",
|
|
self.image, "server"]
|
|
assert sh(argv).returncode == 0
|
|
try:
|
|
return self._start()
|
|
except BaseException:
|
|
sh(["docker", "rm", "-f", "-v", self.name])
|
|
raise
|
|
|
|
def _start(self):
|
|
for _ in range(60):
|
|
if self.bao(["status", "-format=json"]).returncode in (0, 2):
|
|
break
|
|
time.sleep(0.5)
|
|
init = json.loads(self.bao(["operator", "init", "-key-shares=3", "-key-threshold=2",
|
|
"-format=json"], check=True).stdout)
|
|
self.shares, self.credential = init["unseal_keys_b64"], init["root_token"]
|
|
for share in self.shares[:2]:
|
|
self.bao(["write", "sys/unseal", "-"], body=json.dumps({"key": share}).encode(), check=True)
|
|
for _ in range(60):
|
|
leader = self.bao(["read", "-format=json", "sys/leader"])
|
|
if leader.returncode == 0 and json.loads(leader.stdout)["data"].get("is_self"):
|
|
break
|
|
time.sleep(0.5)
|
|
audits = json.loads(self.bao(["audit", "list", "-format=json"], auth=True, check=True).stdout)
|
|
assert any(a.get("type") == "file" and a.get("options", {}).get("file_path") == PRODUCTION_AUDIT_PATH
|
|
for a in audits.values()), "source must carry production's file audit device"
|
|
return self
|
|
|
|
def __exit__(self, *exc):
|
|
sh(["docker", "rm", "-f", "-v", self.name])
|
|
|
|
def bao(self, args, *, body: bytes = b"", auth: bool = False, check: bool = False):
|
|
script = 'IFS= read -r t; BAO_TOKEN="$t"; export BAO_TOKEN; exec "$@"' if auth else 'exec "$@"'
|
|
data = (self.credential.encode() + b"\n" + body) if auth else body
|
|
done = sh(["docker", "exec", "-i", self.name, "sh", "-c", script, "sh", "bao", *args], input=data)
|
|
if check:
|
|
assert done.returncode == 0, done.stderr.decode()[-400:]
|
|
return done
|
|
|
|
def put(self, path: str, payload: dict):
|
|
self.bao(["write", path, "-"], body=json.dumps(payload).encode(), auth=True, check=True)
|
|
|
|
def seed(self, *, kv_registration: bool = True, jwt_role: bool = True) -> None:
|
|
"""Mirror the real source shapes (T01) with generated values only."""
|
|
src = lambda rel: json.loads((ROOT / rel).read_bytes()) # noqa: E731
|
|
self.bao(["auth", "enable", "-path=keycape-services", "jwt"], auth=True, check=True)
|
|
self.bao(["auth", "enable", "-path=netkingdom", "oidc"], auth=True, check=True)
|
|
self.bao(["secrets", "enable", "-path=platform", "kv-v2"], auth=True, check=True)
|
|
self.put("auth/keycape-services/config",
|
|
{**src("openbao/auth/keycape-services-config.json"), "skip_jwks_validation": True})
|
|
if jwt_role:
|
|
self.put("auth/keycape-services/role/secrets-engine-openbao",
|
|
src("openbao/auth/secrets-engine-jwt-role.json"))
|
|
oidc = {k: v for k, v in src("openbao/auth/netkingdom-oidc-config.json").items() if k != "_note"}
|
|
self.put("auth/netkingdom/config", {**oidc, "oidc_client_secret": "synthetic-" + secrets.token_hex(8),
|
|
"skip_jwks_validation": True})
|
|
self.put("auth/netkingdom/role/platform-admin", src("openbao/auth/netkingdom-platform-admin-role.json"))
|
|
policy = (ROOT / "openbao/policies/secrets-engine-login-self.hcl").read_bytes()
|
|
self.bao(["policy", "write", "secrets-engine-login-self", "-"], body=policy, auth=True, check=True)
|
|
if kv_registration:
|
|
self.kv_value = "synthetic-kv-" + secrets.token_urlsafe(24)
|
|
self.put("platform/data/workloads/keycape/secrets-engine-openbao",
|
|
{"data": {"client_credential": self.kv_value}})
|
|
|
|
def facts(self) -> tuple[str, int]:
|
|
status = json.loads(self.bao(["read", "-format=json", "sys/seal-status"], check=True).stdout)["data"]
|
|
leader = json.loads(self.bao(["read", "-format=json", "sys/leader"], check=True).stdout)["data"]
|
|
return status["cluster_id"], leader["raft_applied_index"]
|
|
|
|
def snapshot(self) -> bytes:
|
|
done = self.bao(["operator", "raft", "snapshot", "save", "/dev/stdout"], auth=True, check=True)
|
|
assert done.stdout[:2] == b"\x1f\x8b"
|
|
return done.stdout
|
|
|
|
|
|
class Rehearsal:
|
|
"""Generated custody, admission signing and session plumbing."""
|
|
|
|
def __init__(self, image: str):
|
|
self.image = image
|
|
self.work = Path(tempfile.mkdtemp(prefix="rpf64-rehearsal-", dir="/dev/shm"))
|
|
os.chmod(self.work, 0o700)
|
|
self.anchor = self.work / "anchor"
|
|
self.anchor.mkdir(mode=0o700)
|
|
key = self.work / "founder_ed25519"
|
|
assert sh(["ssh-keygen", "-q", "-t", "ed25519", "-N", "", "-C", "founder@rehearsal", "-f",
|
|
str(key)]).returncode == 0
|
|
self.ssh_key = key
|
|
pub = (self.work / "founder_ed25519.pub").read_text().strip()
|
|
(self.anchor / "allowed_signers").write_text(
|
|
f'founder@rehearsal namespaces="{adapter.NAMESPACE}" {pub}\n')
|
|
os.environ[adapter.ANCHOR_ENV] = str(self.anchor / "allowed_signers")
|
|
self.age_identity, self.age_recipient = self.age_pair()
|
|
self.secrets_seen: list[str] = [self.age_identity]
|
|
|
|
@staticmethod
|
|
def age_pair() -> tuple[str, str]:
|
|
out = sh(["age-keygen"]).stdout.decode().splitlines()
|
|
identity = next(line for line in out if line.startswith("AGE-SECRET-KEY-1"))
|
|
recipient = sh(["age-keygen", "-y"], input=identity.encode()).stdout.decode().strip()
|
|
return identity, recipient
|
|
|
|
def close(self) -> None:
|
|
os.environ.pop(adapter.ANCHOR_ENV, None)
|
|
shutil.rmtree(self.work, ignore_errors=True)
|
|
|
|
def snapshot_set(self, *, kv_registration=True, jwt_role=True) -> dict:
|
|
with SyntheticBao(self.image) as bao:
|
|
bao.seed(kv_registration=kv_registration, jwt_role=jwt_role)
|
|
cluster_id, index = bao.facts()
|
|
plain = bao.snapshot()
|
|
shares, root_credential = list(bao.shares), bao.credential
|
|
self.secrets_seen += [bao.credential, *shares, getattr(bao, "kv_value", "")]
|
|
cipher = sh(["age", "-r", self.age_recipient], input=plain).stdout
|
|
path = self.work / ("snapshot-" + secrets.token_hex(3) + ".age")
|
|
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
|
|
with os.fdopen(fd, "wb") as out:
|
|
out.write(cipher)
|
|
return {"path": path, "cipher": cipher, "plain_sha": adapter.sha256_bytes(plain), "cluster_id": cluster_id,
|
|
"index": index, "shares": shares, "root_credential": root_credential}
|
|
|
|
def admission(self, snap: dict, source_root: Path, commit: str, **override) -> dict:
|
|
now = adapter.now_utc()
|
|
adm = {
|
|
"schema": adapter.ADMISSION_SCHEMA, "namespace": adapter.NAMESPACE,
|
|
"attempt_id": "RPF-WP-0064-T04-rehearsal-" + secrets.token_hex(3), "phase": "A",
|
|
"phase_b_admitted": False, "material": "synthetic",
|
|
"not_before": adapter.iso(now.replace(second=0) - __import__("datetime").timedelta(minutes=1)),
|
|
"not_after": adapter.iso(now + __import__("datetime").timedelta(hours=1)),
|
|
"max_duration_seconds": adapter.MAX_DURATION_SECONDS, "abort_operator": "founder",
|
|
"image": self.image, "adapter_commit": commit,
|
|
"snapshot": {"ciphertext_sha256": adapter.sha256_bytes(snap["cipher"]),
|
|
"ciphertext_bytes": len(snap["cipher"]), "plaintext_sha256": snap["plain_sha"],
|
|
"age_recipient": self.age_recipient, "source_cluster_id": snap["cluster_id"],
|
|
"source_applied_index": snap["index"],
|
|
"source_receipt_sha256": adapter.sha256_bytes(b"synthetic-source-receipt" + os.urandom(8))},
|
|
"unseal": {"key_threshold": 2, "key_shares": 3},
|
|
"readback_credential": "generate-root-from-entered-shares",
|
|
"source_sha256": adapter.source_digests(source_root),
|
|
}
|
|
for key, value in override.items():
|
|
if key in adm["snapshot"]:
|
|
adm["snapshot"][key] = value
|
|
else:
|
|
adm[key] = value
|
|
return adm
|
|
|
|
def signed(self, adm: dict) -> tuple[Path, Path, str]:
|
|
raw = json.dumps(adm, indent=2, sort_keys=True).encode()
|
|
path = self.work / (adm["attempt_id"] + ".json")
|
|
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
|
|
with os.fdopen(fd, "wb") as out:
|
|
out.write(raw)
|
|
done = sh(["ssh-keygen", "-q", "-Y", "sign", "-f", str(self.ssh_key), "-n", adapter.NAMESPACE, str(path)])
|
|
assert done.returncode == 0, done.stderr
|
|
sig = Path(str(path) + ".sig")
|
|
os.chmod(sig, 0o600)
|
|
return path, sig, adapter.sha256_bytes(raw)
|
|
|
|
def drill_admission(self, snap: dict, source_root: Path, commit: str, **override) -> dict:
|
|
"""RPF-WP-0064-T10 drill admission: same snapshot facts, disjoint attempt id and fields."""
|
|
adm = self.admission(snap, source_root, commit, **override)
|
|
adm.update({"attempt_id": "RPF-WP-0064-T10-rehearsal-" + secrets.token_hex(3), "drill": "break-glass",
|
|
"snapshot_label": "openbao-raft-20261014t000000z-synthetic.snap.age",
|
|
"readback_credential": adapter.DRILL_READBACK})
|
|
return adm
|
|
|
|
def session(self, adm: dict, snap: dict, source_root: Path, answers: list, **kw):
|
|
path, sig, digest = self.signed(adm)
|
|
mode = kw.get("mode", adapter.PHASE_A)
|
|
loaded, _raw, signer, protected = adapter.load_admission(path, sig, digest, Path(os.environ[adapter.ANCHOR_ENV]),
|
|
mode)
|
|
queue = list(answers)
|
|
|
|
def prompt(_label: str) -> bytearray:
|
|
item = queue.pop(0)
|
|
value = item() if callable(item) else item
|
|
return bytearray(value.encode())
|
|
receipt = self.work / (adm["attempt_id"] + "-receipt.json")
|
|
session = adapter.Session(admission=loaded, admission_sha256=digest, signer=signer,
|
|
anchor_is_protected=protected, ciphertext=snap["path"], receipt_path=receipt,
|
|
root=source_root, prompt=prompt, say=lambda _m: None, **kw)
|
|
return session, receipt
|
|
|
|
@staticmethod
|
|
def verify(receipt: Path, source_root: Path, *extra: str) -> subprocess.CompletedProcess:
|
|
return sh(["python3", str(VERIFIER), "--receipt", str(receipt), "--source-root", str(source_root), *extra])
|
|
|
|
|
|
def leftover_containers() -> list[str]:
|
|
done = sh(["docker", "ps", "-a", "--format", "{{.Names}}", "--filter", "name=rpf64-"])
|
|
return [n for n in done.stdout.decode().split() if n]
|