railiance-platform/scripts/openbao-isolated-restore/restore_adapter.py
codex 1701a3fa2a
All checks were successful
CCR intake gate / gate (push) Successful in 1m11s
CI Smoke / host-smoke (push) Successful in 0s
CI Smoke / container-smoke (push) Successful in 1s
RPF-WP-0064-T06: pin the live production OpenBao image digest in the restore adapter
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

Assistant: claude-code
Assistant-Model: opus
Assistant-Process: 4384@bnt-lap001
Assistant-Session: 80d1f3da-7a91-4519-9737-9ba128b4da64
2026-10-24 10:17:11 +02:00

917 lines
47 KiB
Python

#!/usr/bin/env python3
"""Isolated original-OpenBao restore adapter (RPF-WP-0064-T02, phase A only).
Restores one age-encrypted Raft snapshot into a disposable, networkless Docker
container on the founder workstation, unseals it with the original Shamir
shares, reads back configuration metadata, and tears everything down. It never
runs on railiance01, never uses a kubeconfig, never reads KV ``data`` and never
prints or records a secret value.
Custody inputs (founder-attended, hidden prompts only, never argv/env/files):
the backup age identity (held in a sealed memfd for ``age -d``) and two
original unseal shares (process memory only). The decrypted snapshot lives in
a sealed memfd and is streamed to the container's stdin; the container's Raft
directory is tmpfs. A throwaway 1-of-1 init makes the empty target writable;
the restore invalidates it. Readback uses a root credential generated in the
restored target from the same two shares, revoked before teardown.
A reaper process watches the session: on parent loss or the hard deadline it
kills the session, removes the container and writes an ``aborted`` receipt.
Outcomes: ``restored`` | ``refused`` (required object missing or not matching
its source; never repaired) | ``aborted`` (any other failed check, timeout,
operator abort). Only the launcher ``run-isolated-restore.sh`` is the attended
entry point; ``Session`` is importable for the synthetic rehearsal tests.
"""
from __future__ import annotations
import base64
import fcntl
import getpass
import hashlib
import json
import os
import re
import secrets
import select
import signal
import stat
import subprocess
import sys
import time
from datetime import UTC, datetime
from pathlib import Path
from typing import Any, Callable
ROOT = Path(__file__).resolve().parents[2]
HERE = "scripts/openbao-isolated-restore/"
NAMESPACE = "railiance-openbao-restore-attempt"
ADMISSION_SCHEMA = "railiance-openbao-restore-admission/v1"
RECEIPT_SCHEMA = "railiance-openbao-restore-receipt/v2"
IMAGE_REPOSITORY = "quay.io/openbao/openbao"
# RPF-WP-0064-T06: production digest, observed 2026-10-24 as the imageID of the live
# openbao-0 container (spec image quay.io/openbao/openbao:2.5.4); equal to the T04
# rehearsal pull. Original-material runs refuse any other image.
PINNED_IMAGE_DIGEST: str | None = "sha256:436eaf9778cad75507ff70ea26ace30dcbe15606e619ac3823495663d7f7c115"
MAX_DURATION_SECONDS = 2700
REAPER_GRACE_SECONDS = 60
DEFAULT_ANCHOR = Path("/etc/railiance-openbao-restore/allowed_signers")
ANCHOR_ENV = "RAILIANCE_OPENBAO_RESTORE_ANCHOR"
SSH_ENV = {"PATH": "/usr/bin:/bin", "LC_ALL": "C"}
CLAIMS_NOT_MADE = (
"consumer_continuity", "live_jwt_login", "phase_b_network",
"owner_acceptance", "recurring_restore_assurance", "production_redirect",
)
ADAPTER_PATHS = tuple(HERE + n for n in (
"restore_adapter.py", "preflight.py", "run-isolated-restore.sh", "verify-restore-receipt.py"))
OBJECT_SOURCE_PATHS = (
"openbao/auth/keycape-services-config.json",
"openbao/auth/secrets-engine-jwt-role.json",
"openbao/auth/netkingdom-oidc-config.json",
"openbao/auth/netkingdom-platform-admin-role.json",
"openbao/policies/secrets-engine-login-self.hcl",
"docs/contracts/secrets-engine-openbao-jwt-login.yaml",
)
SOURCE_PATHS = tuple(sorted(ADAPTER_PATHS + OBJECT_SOURCE_PATHS))
KV_METADATA_PATH = "platform/metadata/workloads/keycape/secrets-engine-openbao"
# Required objects. ``source`` objects compare the declared fields of the
# source file (minus ``ignore``) with the restored readback after normalising.
OBJECTS: tuple[dict[str, Any], ...] = (
{"object": "auth_mount_keycape_services", "kind": "mount", "path": "sys/auth",
"mount": "keycape-services/", "types": ("jwt",)},
{"object": "auth_mount_netkingdom", "kind": "mount", "path": "sys/auth",
"mount": "netkingdom/", "types": ("oidc", "jwt")},
{"object": "kv_mount_platform", "kind": "mount", "path": "sys/mounts",
"mount": "platform/", "types": ("kv",), "kv_version": "2"},
{"object": "keycape_services_config", "kind": "source", "path": "auth/keycape-services/config",
"source": "openbao/auth/keycape-services-config.json", "ignore": ()},
{"object": "secrets_engine_jwt_role", "kind": "source",
"path": "auth/keycape-services/role/secrets-engine-openbao",
"source": "openbao/auth/secrets-engine-jwt-role.json", "ignore": ()},
{"object": "netkingdom_oidc_config", "kind": "source", "path": "auth/netkingdom/config",
"source": "openbao/auth/netkingdom-oidc-config.json", "ignore": ("_note",)},
{"object": "platform_admin_role", "kind": "source", "path": "auth/netkingdom/role/platform-admin",
"source": "openbao/auth/netkingdom-platform-admin-role.json", "ignore": ()},
{"object": "policy_secrets_engine_login_self", "kind": "policy",
"path": "sys/policies/acl/secrets-engine-login-self",
"source": "openbao/policies/secrets-engine-login-self.hcl"},
{"object": "kv_registration_metadata", "kind": "kv_metadata", "path": KV_METADATA_PATH},
)
ALLOWED_READS = frozenset({"sys/seal-status", "sys/leader", "auth/token/lookup-self",
*(o["path"] for o in OBJECTS)})
ALLOWED_WRITES = frozenset({"sys/unseal", "sys/generate-root/attempt", "sys/generate-root/update",
"auth/token/revoke-self"})
ABORT_REASONS = frozenset({
"admission_refused", "preflight_refused", "custody_identity_refused", "ciphertext_mismatch",
"decrypt_failed", "plaintext_mismatch", "target_start_failed", "network_containment_failed",
"throwaway_init_failed", "restore_failed", "unseal_refused", "cluster_mismatch",
"applied_index_behind", "readback_credential_failed", "readback_failed", "deadline_exceeded",
"operator_abort", "parent_lost", "internal_error",
})
SERVER_CONFIG = json.dumps({
"storage": {"raft": {"path": "/openbao/data", "node_id": "isolated-restore"}},
# OpenBao 2.x disables unauthenticated generate-root by default; the
# readback credential needs it, on a loopback-only listener without a network.
"listener": {"tcp": {"address": "127.0.0.1:8200", "tls_disable": True,
"disable_unauthed_generate_root_endpoints": False}},
"api_addr": "http://127.0.0.1:8200", "cluster_addr": "http://127.0.0.1:8201",
"disable_mlock": True, "ui": False,
}, separators=(",", ":"))
TMPFS = {
"/openbao/data": "rw,nosuid,nodev,noexec,size=536870912,uid=100,gid=1000,mode=0700",
"/openbao/config": "rw,nosuid,nodev,noexec,size=1048576,uid=100,gid=1000,mode=0700",
"/tmp": "rw,nosuid,nodev,noexec,size=16777216,uid=100,gid=1000,mode=0700",
# The image declares VOLUMEs here; tmpfs overlays stop anonymous volumes.
"/openbao/file": "rw,nosuid,nodev,noexec,size=1048576,uid=100,gid=1000,mode=0700",
"/openbao/logs": "rw,nosuid,nodev,noexec,size=1048576,uid=100,gid=1000,mode=0700",
}
DIGEST_RE = re.compile(r"^sha256:[a-f0-9]{64}$")
IMAGE_RE = re.compile(r"^" + re.escape(IMAGE_REPOSITORY) + r"@(sha256:[a-f0-9]{64})$")
AGE_IDENTITY_RE = re.compile(r"^AGE-SECRET-KEY-1[0-9A-Z]{58}$")
AGE_RECIPIENT_RE = re.compile(r"^age1[023456789acdefghjklmnpqrstuvwxyz]{58}$")
SHARE_RE = re.compile(r"^(?:[A-Za-z0-9+/]{40,100}={0,2}|[a-f0-9]{60,130})$")
ATTEMPT_RE = re.compile(r"^RPF-WP-0064-(?:T07|T04)-[A-Za-z0-9-]{3,64}$")
COMMIT_RE = re.compile(r"^[a-f0-9]{40}$")
DURATION_RE = re.compile(r"^(\d+)([smh]?)$")
class Refused(Exception):
"""A fail-closed check; ``reason`` is an ABORT_REASONS code or ``object_*``."""
def __init__(self, reason: str):
super().__init__(reason)
self.reason = reason
def require(condition: Any, reason: str) -> None:
if not condition:
raise Refused(reason)
def now_utc() -> datetime:
return datetime.now(UTC).replace(microsecond=0)
def iso(value: datetime) -> str:
return value.astimezone(UTC).strftime("%Y-%m-%dT%H:%M:%SZ")
def parse_iso(value: Any) -> datetime:
require(isinstance(value, str) and re.fullmatch(r"\d{4}-\d\d-\d\dT\d\d:\d\d:\d\dZ", value), "admission_refused")
return datetime.strptime(value, "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=UTC)
def sha256_bytes(data: bytes) -> str:
return "sha256:" + hashlib.sha256(data).hexdigest()
def canonical(value: Any) -> bytes:
return json.dumps(value, sort_keys=True, separators=(",", ":")).encode()
def strict_json(raw: bytes) -> Any:
def unique(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
out: dict[str, Any] = {}
for key, value in pairs:
require(key not in out, "admission_refused")
out[key] = value
return out
return json.loads(raw, object_pairs_hook=unique)
def placeholder_digest(value: str) -> bool:
hexpart = value.split(":", 1)[-1]
return len(set(hexpart)) <= 2
# --------------------------------------------------------------------------- admission
ADMISSION_FIELDS = {"schema", "namespace", "attempt_id", "phase", "phase_b_admitted", "material",
"not_before", "not_after", "max_duration_seconds", "abort_operator", "image",
"adapter_commit", "snapshot", "unseal", "readback_credential", "source_sha256"}
SNAPSHOT_FIELDS = {"ciphertext_sha256", "ciphertext_bytes", "plaintext_sha256", "age_recipient",
"source_cluster_id", "source_applied_index", "source_receipt_sha256"}
def validate_admission(adm: Any) -> dict[str, Any]:
"""Shape-only validation of the signed admission (no I/O)."""
require(isinstance(adm, dict) and set(adm) == ADMISSION_FIELDS, "admission_refused")
require(adm["schema"] == ADMISSION_SCHEMA and adm["namespace"] == NAMESPACE, "admission_refused")
require(isinstance(adm["attempt_id"], str) and ATTEMPT_RE.fullmatch(adm["attempt_id"]), "admission_refused")
require(adm["phase"] == "A" and adm["phase_b_admitted"] is False, "admission_refused")
require(adm["material"] in ("original", "synthetic"), "admission_refused")
require(adm["readback_credential"] == "generate-root-from-entered-shares", "admission_refused")
nb, na = parse_iso(adm["not_before"]), parse_iso(adm["not_after"])
require(nb < na and (na - nb).total_seconds() <= 24 * 3600, "admission_refused")
dur = adm["max_duration_seconds"]
require(type(dur) is int and 60 <= dur <= MAX_DURATION_SECONDS, "admission_refused")
if adm["material"] == "original":
require(dur == MAX_DURATION_SECONDS, "admission_refused")
require(isinstance(adm["abort_operator"], str) and re.fullmatch(r"[a-z][a-z0-9-]{1,63}", adm["abort_operator"]),
"admission_refused")
match = IMAGE_RE.fullmatch(adm["image"]) if isinstance(adm["image"], str) else None
require(match and not placeholder_digest(match.group(1)), "admission_refused")
require(isinstance(adm["adapter_commit"], str) and COMMIT_RE.fullmatch(adm["adapter_commit"]), "admission_refused")
snap = adm["snapshot"]
require(isinstance(snap, dict) and set(snap) == SNAPSHOT_FIELDS, "admission_refused")
for field in ("ciphertext_sha256", "plaintext_sha256", "source_receipt_sha256"):
require(isinstance(snap[field], str) and DIGEST_RE.fullmatch(snap[field])
and not placeholder_digest(snap[field]), "admission_refused")
require(type(snap["ciphertext_bytes"]) is int and 0 < snap["ciphertext_bytes"] <= 512 * 1024 * 1024,
"admission_refused")
require(isinstance(snap["age_recipient"], str) and AGE_RECIPIENT_RE.fullmatch(snap["age_recipient"]),
"admission_refused")
require(isinstance(snap["source_cluster_id"], str)
and re.fullmatch(r"[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}", snap["source_cluster_id"]),
"admission_refused")
require(type(snap["source_applied_index"]) is int and snap["source_applied_index"] > 0, "admission_refused")
require(adm["unseal"] == {"key_threshold": 2, "key_shares": 3}, "admission_refused")
sources = adm["source_sha256"]
require(isinstance(sources, dict) and set(sources) == set(SOURCE_PATHS), "admission_refused")
for value in sources.values():
require(isinstance(value, str) and DIGEST_RE.fullmatch(value), "admission_refused")
return adm
def source_digests(root: Path) -> dict[str, str]:
out = {}
for name in SOURCE_PATHS:
path = root / name
require(not any(p.is_symlink() for p in (path, *path.parents)) and path.is_file(), "preflight_refused")
out[name] = sha256_bytes(path.read_bytes())
return out
def check_source_closure(adm: dict[str, Any], root: Path) -> None:
"""Admission names the exact committed adapter and source files on disk."""
require(source_digests(root) == adm["source_sha256"], "preflight_refused")
head = subprocess.run(["git", "-C", str(root), "rev-parse", "HEAD"], capture_output=True, timeout=10)
require(head.returncode == 0 and head.stdout.decode().strip() == adm["adapter_commit"], "preflight_refused")
dirty = subprocess.run(["git", "-C", str(root), "status", "--porcelain", "--", *SOURCE_PATHS],
capture_output=True, timeout=10)
require(dirty.returncode == 0 and not dirty.stdout.strip(), "preflight_refused")
def anchor_path() -> Path:
return Path(os.environ.get(ANCHOR_ENV) or DEFAULT_ANCHOR)
def anchor_protected(signers: Path) -> bool:
"""True if the running account cannot rewrite the trust anchor."""
euid = os.geteuid()
for path in (signers.parent, signers):
info = os.stat(path, follow_symlinks=False)
if info.st_mode & 0o022 or (info.st_uid == euid and euid != 0):
return False
return True
def verify_signature(data: bytes, signature: bytes, signers: Path) -> dict[str, str]:
"""``ssh-keygen -Y`` over exact admission bytes with the fixed namespace."""
import tempfile
require(signers.is_absolute() and not signers.is_symlink() and not signers.parent.is_symlink()
and signers.is_file(), "admission_refused")
require(signature.startswith(b"-----BEGIN SSH SIGNATURE-----") and len(signature) <= 8192, "admission_refused")
with tempfile.TemporaryDirectory(prefix="rpf64-adm-") as tmp:
sig = Path(tmp) / "admission.sig"
sig.write_bytes(signature)
found = subprocess.run(["ssh-keygen", "-Y", "find-principals", "-f", str(signers), "-n", NAMESPACE,
"-s", str(sig)], capture_output=True, env=SSH_ENV, timeout=10)
principals = found.stdout.decode("utf-8", "replace").split()
require(found.returncode == 0 and len(principals) == 1
and re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._@-]{0,127}", principals[0]), "admission_refused")
checked = subprocess.run(["ssh-keygen", "-Y", "verify", "-f", str(signers), "-I", principals[0],
"-n", NAMESPACE, "-s", str(sig)], input=data, capture_output=True,
env=SSH_ENV, timeout=10)
match = re.search(r"key (SHA256:[A-Za-z0-9+/=]+)", checked.stdout.decode("utf-8", "replace"))
require(checked.returncode == 0 and match, "admission_refused")
return {"principal": principals[0], "fingerprint": match.group(1)}
def read_owner_file(path: Path, limit: int) -> bytes:
"""Regular, owner-only, single-link file read without following symlinks."""
path = Path(os.path.abspath(path))
require(not any(p.is_symlink() for p in (path, *path.parents)), "preflight_refused")
fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK)
with os.fdopen(fd, "rb") as handle:
info = os.fstat(handle.fileno())
require(stat.S_ISREG(info.st_mode) and info.st_uid == os.getuid() and info.st_nlink == 1
and not info.st_mode & 0o077 and 0 < info.st_size <= limit, "preflight_refused")
raw = handle.read(limit + 1)
require(0 < len(raw) <= limit, "preflight_refused")
return raw
def load_admission(admission: Path, signature: Path, admission_sha256: str,
signers: Path) -> tuple[dict[str, Any], bytes, dict[str, str], bool]:
raw = read_owner_file(admission, 64 * 1024)
require(DIGEST_RE.fullmatch(admission_sha256) and sha256_bytes(raw) == admission_sha256, "admission_refused")
adm = validate_admission(strict_json(raw))
signer = verify_signature(raw, read_owner_file(signature, 8192), signers)
protected = anchor_protected(signers)
if adm["material"] == "original":
require(protected, "admission_refused")
require(PINNED_IMAGE_DIGEST and adm["image"] == f"{IMAGE_REPOSITORY}@{PINNED_IMAGE_DIGEST}",
"admission_refused")
return adm, raw, signer, protected
def check_window(adm: dict[str, Any], at: datetime | None = None) -> None:
at = at or now_utc()
require(parse_iso(adm["not_before"]) <= at < parse_iso(adm["not_after"]), "admission_refused")
# --------------------------------------------------------------------------- normalisation
def _seconds(value: Any) -> Any:
if isinstance(value, str):
match = DURATION_RE.fullmatch(value)
if match:
return int(match.group(1)) * {"": 1, "s": 1, "m": 60, "h": 3600}[match.group(2)]
return value
def normalise(value: Any) -> Any:
if isinstance(value, dict):
return {k: normalise(v) for k, v in sorted(value.items())}
if isinstance(value, list):
items = [normalise(v) for v in value]
return sorted(items) if all(isinstance(v, str) for v in items) else items
return _seconds(value)
def project(source: dict[str, Any], observed: dict[str, Any], ignore: tuple[str, ...]) -> tuple[bool, dict]:
fields = sorted(k for k in source if k not in ignore)
projected = {k: normalise(observed.get(k)) for k in fields}
expected = {k: normalise(source[k]) for k in fields}
return projected == expected, projected
# --------------------------------------------------------------------------- memfd custody
def sealed_memfd(name: str, data: bytes | bytearray) -> int:
fd = os.memfd_create(name, os.MFD_CLOEXEC | os.MFD_ALLOW_SEALING)
view = memoryview(data)
while view:
view = view[os.write(fd, view):]
fcntl.fcntl(fd, fcntl.F_ADD_SEALS, fcntl.F_SEAL_SHRINK | fcntl.F_SEAL_GROW | fcntl.F_SEAL_WRITE | fcntl.F_SEAL_SEAL)
os.lseek(fd, 0, os.SEEK_SET)
return fd
def wipe(buf: bytearray | None) -> None:
if buf is not None:
for i in range(len(buf)):
buf[i] = 0
def tty_prompt(label: str) -> bytearray:
"""Hidden prompt on the controlling terminal; the value never touches argv/env/disk."""
require(sys.stdin.isatty(), "preflight_refused")
return bytearray(getpass.getpass(f"{label} (hidden): ").strip().encode())
# --------------------------------------------------------------------------- session
class Session:
"""One phase-A restore attempt. Inject ``prompt``/``docker``/``say`` in tests only."""
def __init__(self, *, admission: dict[str, Any], admission_sha256: str, signer: dict[str, str],
anchor_is_protected: bool, ciphertext: Path, receipt_path: Path, root: Path = ROOT,
prompt: Callable[[str], bytearray] = tty_prompt, docker: str = "docker",
say: Callable[[str], None] = lambda m: print(m, flush=True), start_reaper: bool = True):
self.adm, self.root, self.prompt, self.docker, self.say = admission, root, prompt, docker, say
self.ciphertext, self.receipt_path, self.start_reaper = ciphertext, receipt_path, start_reaper
self.name = "rpf64-restore-" + secrets.token_hex(6)
self.deadline = time.monotonic() + admission["max_duration_seconds"]
self.not_after = parse_iso(admission["not_after"])
self.started = now_utc()
self.container_started = False
self.identity_fd: int | None = None
self.plaintext_fd: int | None = None
self.shares: list[bytearray] = []
self.credential: str | None = None
self.reaper: subprocess.Popen | None = None
snap = admission["snapshot"]
self.receipt: dict[str, Any] = {
"schema": RECEIPT_SCHEMA,
"extends": "docs/openbao-restore-drill-evidence.example.json",
"attempt_id": admission["attempt_id"], "material": admission["material"], "phase": "A",
"admission_sha256": admission_sha256, "admission_namespace": NAMESPACE,
"signer_principal": signer["principal"], "signer_fingerprint": signer["fingerprint"],
"anchor_protected": anchor_is_protected, "abort_operator": admission["abort_operator"],
"adapter_commit": admission["adapter_commit"], "source_sha256": dict(admission["source_sha256"]),
"started_at": iso(self.started), "finished_at": None, "not_after": admission["not_after"],
"deadline_seconds": admission["max_duration_seconds"], "elapsed_seconds": None,
"image": admission["image"], "image_id": None,
"snapshot": {"ciphertext_sha256": snap["ciphertext_sha256"], "ciphertext_bytes": snap["ciphertext_bytes"],
"plaintext_sha256": snap["plaintext_sha256"], "source_cluster_id": snap["source_cluster_id"],
"source_applied_index": snap["source_applied_index"],
"source_receipt_sha256": snap["source_receipt_sha256"],
"ciphertext_verified": False, "plaintext_verified": False},
"restored": {"cluster_id": None, "applied_index": None, "sealed_after_restore": None,
"unsealed_with_original_shares": False, "cluster_id_match": False,
"applied_index_not_behind": False, "throwaway_init_invalidated": False},
"network": {"network_mode": None, "docker_networks": [], "interfaces": [], "published_ports": [],
"mounts": [], "egress_refused": False},
"objects": [], "result": None, "reason": None,
"teardown": {"container_removed": False, "container_absent_verified": False,
"plaintext_memfd_closed": True, "custody_memfd_closed": True,
"readback_credential_revoked": None, "docker_volumes_created": 0, "performed_by": "session"},
"kv_data_read": False, "no_secret_material_recorded": True,
"claims_not_made": list(CLAIMS_NOT_MADE),
}
# ---- infrastructure
def remaining(self) -> float:
left = min(self.deadline - time.monotonic(), (self.not_after - now_utc()).total_seconds())
require(left > 0, "deadline_exceeded")
return left
def run(self, argv: list[str], *, data: bytes = b"", limit: float = 60, check: bool = True,
reason: str = "internal_error") -> subprocess.CompletedProcess:
try:
done = subprocess.run(argv, input=data, capture_output=True, timeout=min(limit, self.remaining()))
except subprocess.TimeoutExpired as exc:
raise Refused("deadline_exceeded" if self.remaining_or_zero() <= 1 else reason) from exc
require(not check or done.returncode == 0, reason)
return done
def remaining_or_zero(self) -> float:
return max(0.0, min(self.deadline - time.monotonic(), (self.not_after - now_utc()).total_seconds()))
def bao(self, args: list[str], *, credential: str | None = None, body: bytes = b"", limit: float = 30,
check: bool = True, reason: str = "internal_error") -> subprocess.CompletedProcess:
script = 'IFS= read -r t; BAO_TOKEN="$t"; export BAO_TOKEN; exec "$@"' if credential else 'exec "$@"'
data = (credential.encode() + b"\n" + body) if credential else body
return self.run([self.docker, "exec", "-i", self.name, "sh", "-c", script, "sh", "bao", *args],
data=data, limit=limit, check=check, reason=reason)
def read(self, path: str, credential: str | None = None, reason: str = "readback_failed") -> dict | None:
require(path in ALLOWED_READS and "/data/" not in path, "internal_error")
done = self.bao(["read", "-format=json", path], credential=credential, check=False, reason=reason)
if done.returncode != 0:
if b"No value found" in done.stderr or done.returncode == 2:
return None
raise Refused(reason)
body = json.loads(done.stdout)
return body.get("data", body) if isinstance(body, dict) else None
def write(self, path: str, payload: dict, credential: str | None = None, reason: str = "internal_error") -> dict:
require(path in ALLOWED_WRITES, "internal_error")
done = self.bao(["write", "-format=json", path, "-"], credential=credential, body=json.dumps(payload).encode(),
reason=reason)
body = json.loads(done.stdout or b"{}")
return body.get("data", body) if isinstance(body, dict) and isinstance(body.get("data"), dict) else body
def tell_reaper(self, line: str) -> None:
if self.reaper and self.reaper.stdin:
try:
self.reaper.stdin.write((line + "\n").encode())
self.reaper.stdin.flush()
except OSError:
pass
def spawn_reaper(self) -> None:
deadline_epoch = time.time() + self.remaining_or_zero() + REAPER_GRACE_SECONDS
header = {"parent_pid": os.getpid(), "deadline_epoch": deadline_epoch, "docker": self.docker,
"receipt_path": str(self.receipt_path), "base_receipt": self.receipt}
self.reaper = subprocess.Popen([sys.executable, str(Path(__file__).resolve()), "--reaper"],
stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
start_new_session=True, close_fds=True)
self.tell_reaper(json.dumps(header))
# ---- steps
def custody_identity(self) -> None:
self.say("Step 1/7: custody identity (hidden prompt).")
identity = self.prompt("Backup age identity")
try:
require(AGE_IDENTITY_RE.fullmatch(identity.decode("ascii", "replace")), "custody_identity_refused")
self.identity_fd = sealed_memfd("rpf64-age-identity", identity + b"\n")
self.receipt["teardown"]["custody_memfd_closed"] = False
finally:
wipe(identity)
done = subprocess.run(["age-keygen", "-y", f"/dev/fd/{self.identity_fd}"], pass_fds=(self.identity_fd,),
capture_output=True, timeout=min(10, self.remaining()))
require(done.returncode == 0 and done.stdout.decode().strip() == self.adm["snapshot"]["age_recipient"],
"custody_identity_refused")
def decrypt(self) -> None:
self.say("Step 2/7: verify ciphertext and decrypt into a sealed memfd.")
snap = self.adm["snapshot"]
raw = self.ciphertext.read_bytes()
require(len(raw) == snap["ciphertext_bytes"] and sha256_bytes(raw) == snap["ciphertext_sha256"],
"ciphertext_mismatch")
self.receipt["snapshot"]["ciphertext_verified"] = True
os.lseek(self.identity_fd, 0, os.SEEK_SET)
try:
done = subprocess.run(["age", "-d", "-i", f"/dev/fd/{self.identity_fd}"], input=raw,
pass_fds=(self.identity_fd,), capture_output=True,
timeout=min(300, self.remaining()))
except subprocess.TimeoutExpired as exc:
raise Refused("deadline_exceeded") from exc
require(done.returncode == 0 and done.stdout, "decrypt_failed")
plain = bytearray(done.stdout)
del done
try:
require(sha256_bytes(bytes(plain)) == snap["plaintext_sha256"], "plaintext_mismatch")
self.plaintext_fd = sealed_memfd("rpf64-snapshot", plain)
self.receipt["teardown"]["plaintext_memfd_closed"] = False
finally:
wipe(plain)
self.receipt["snapshot"]["plaintext_verified"] = True
self.close_identity()
def start_target(self) -> None:
self.say("Step 3/7: start networkless disposable target.")
image = self.adm["image"]
inspect = self.run([self.docker, "image", "inspect", image, "--format", "{{json .RepoDigests}}|{{.Id}}"],
reason="target_start_failed")
digests, image_id = inspect.stdout.decode().strip().split("|", 1)
require(image in json.loads(digests) and DIGEST_RE.fullmatch(image_id), "target_start_failed")
self.receipt["image_id"] = image_id
argv = [self.docker, "run", "--detach", "--pull=never", "--name", self.name, "--network", "none",
"--user", "100:1000", "--read-only", "--cap-drop", "ALL", "--security-opt", "no-new-privileges",
"--pids-limit", "256", "--memory", "1g", "--label", "railiance.restore.attempt=" + self.adm["attempt_id"]]
for target, options in TMPFS.items():
argv += ["--tmpfs", f"{target}:{options}"]
argv += ["-e", "BAO_LOCAL_CONFIG=" + SERVER_CONFIG, "-e", "BAO_ADDR=http://127.0.0.1:8200",
"-e", "SKIP_CHOWN=1", image, "server"]
self.tell_reaper("C " + self.name)
self.container_started = True
self.run(argv, reason="target_start_failed")
self.check_network()
for _ in range(60):
status = self.read("sys/seal-status", reason="target_start_failed")
if status and status.get("initialized") is False:
return
time.sleep(0.5)
self.remaining()
raise Refused("target_start_failed")
def check_network(self) -> None:
done = self.run([self.docker, "container", "inspect", self.name], reason="network_containment_failed")
info = json.loads(done.stdout)[0]
host, net = info["HostConfig"], info["NetworkSettings"]
record = self.receipt["network"]
record["network_mode"] = host.get("NetworkMode")
record["docker_networks"] = sorted((net.get("Networks") or {}).keys())
record["published_ports"] = sorted((net.get("Ports") or {}).keys())
record["mounts"] = sorted(m.get("Type", "?") for m in info.get("Mounts") or [])
require(record["network_mode"] == "none" and record["docker_networks"] == ["none"]
and not record["published_ports"] and not record["mounts"] and not host.get("PortBindings")
and not host.get("Binds") and host.get("Privileged") is False and host.get("ReadonlyRootfs") is True
and sorted((host.get("Tmpfs") or {}).keys()) == sorted(TMPFS), "network_containment_failed")
dev = self.run([self.docker, "exec", self.name, "cat", "/proc/net/dev"], reason="network_containment_failed")
names = sorted(line.split(":", 1)[0].strip() for line in dev.stdout.decode().splitlines()[2:] if ":" in line)
record["interfaces"] = names
require(names == ["lo"], "network_containment_failed")
probe = self.run([self.docker, "exec", self.name, "sh", "-c",
"wget -q -T 3 -O /dev/null http://192.0.2.1/ || exit 7; "
"wget -q -T 3 -O /dev/null https://kc.coulomb.social/ || exit 7; exit 0"],
check=False, limit=20, reason="network_containment_failed")
record["egress_refused"] = probe.returncode == 7
require(record["egress_refused"], "network_containment_failed")
def restore(self) -> None:
self.say("Step 4/7: throwaway init, then forced Raft restore from the sealed memfd.")
init = self.bao(["operator", "init", "-key-shares=1", "-key-threshold=1", "-format=json"],
reason="throwaway_init_failed")
throwaway = json.loads(init.stdout)
del init
unseal = self.write("sys/unseal", {"key": throwaway["unseal_keys_b64"][0]}, reason="throwaway_init_failed")
require(unseal.get("sealed") is False, "throwaway_init_failed")
self.wait_active("throwaway_init_failed")
throwaway_credential = throwaway["root_token"]
throwaway.clear()
script = 'IFS= read -r t; BAO_TOKEN="$t"; export BAO_TOKEN; exec bao operator raft snapshot restore -force /dev/stdin'
proc = subprocess.Popen([self.docker, "exec", "-i", self.name, "sh", "-c", script], stdin=subprocess.PIPE,
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
try:
proc.stdin.write(throwaway_credential.encode() + b"\n")
os.lseek(self.plaintext_fd, 0, os.SEEK_SET)
while chunk := os.read(self.plaintext_fd, 1 << 20):
proc.stdin.write(chunk)
proc.stdin.close()
code = proc.wait(timeout=min(600, self.remaining()))
except subprocess.TimeoutExpired as exc:
proc.kill()
raise Refused("deadline_exceeded" if self.remaining_or_zero() <= 1 else "restore_failed") from exc
except BrokenPipeError as exc:
raise Refused("restore_failed") from exc
require(code == 0, "restore_failed")
self.close_plaintext()
self.throwaway_credential = throwaway_credential
def unseal_original(self) -> None:
self.say("Step 5/7: unseal with two original shares (hidden prompts).")
sealed = None
for _ in range(60):
status = self.read("sys/seal-status", reason="unseal_refused")
sealed = bool(status and status.get("sealed"))
if sealed:
break
time.sleep(0.5)
self.remaining()
self.receipt["restored"]["sealed_after_restore"] = sealed
require(sealed, "unseal_refused")
for index in (1, 2):
share = self.prompt(f"Original unseal share {index} of 2")
if not SHARE_RE.fullmatch(share.decode("ascii", "replace")):
wipe(share)
raise Refused("unseal_refused")
self.shares.append(share)
result = self.write("sys/unseal", {"key": share.decode()}, reason="unseal_refused")
require(result.get("sealed") is False, "unseal_refused")
self.receipt["restored"]["unsealed_with_original_shares"] = True
leader = self.wait_active("unseal_refused")
status = self.read("sys/seal-status", reason="unseal_refused") or {}
snap, restored = self.adm["snapshot"], self.receipt["restored"]
restored["cluster_id"] = status.get("cluster_id")
restored["applied_index"] = leader.get("raft_applied_index")
restored["cluster_id_match"] = restored["cluster_id"] == snap["source_cluster_id"]
restored["applied_index_not_behind"] = (isinstance(restored["applied_index"], int)
and restored["applied_index"] >= snap["source_applied_index"])
lookup = self.bao(["read", "-format=json", "auth/token/lookup-self"], credential=self.throwaway_credential,
check=False)
restored["throwaway_init_invalidated"] = lookup.returncode != 0
self.throwaway_credential = None
require(restored["throwaway_init_invalidated"], "restore_failed")
require(restored["cluster_id_match"], "cluster_mismatch")
require(restored["applied_index_not_behind"], "applied_index_behind")
def wait_active(self, reason: str) -> dict:
for _ in range(120):
leader = self.read("sys/leader", reason=reason) or {}
if leader.get("is_self") and isinstance(leader.get("raft_applied_index"), int) \
and leader["raft_applied_index"] > 0:
return leader
time.sleep(0.5)
self.remaining()
raise Refused(reason)
def generate_credential(self) -> None:
self.say("Step 6/7: generate an in-memory readback credential from the entered shares.")
attempt = self.write("sys/generate-root/attempt", {}, reason="readback_credential_failed")
otp, nonce, encoded = attempt.get("otp"), attempt.get("nonce"), None
require(otp and nonce, "readback_credential_failed")
for share in self.shares:
update = self.write("sys/generate-root/update", {"key": share.decode(), "nonce": nonce},
reason="readback_credential_failed")
encoded = update.get("encoded_token") or update.get("encoded_root_token") or encoded
require(encoded, "readback_credential_failed")
raw = base64.b64decode(encoded + "=" * (-len(encoded) % 4))
require(len(raw) == len(otp), "readback_credential_failed")
self.credential = bytes(a ^ b for a, b in zip(raw, otp.encode())).decode()
for share in self.shares:
wipe(share)
self.shares.clear()
def readback(self) -> list[dict[str, Any]]:
self.say("Step 7/7: metadata-only readback against the committed sources.")
rows = []
cache: dict[str, Any] = {}
for spec in OBJECTS:
row = {"object": spec["object"], "path": spec["path"], "present": False, "source_match": False,
"source_path": spec.get("source"), "observed_sha256": None}
if spec["kind"] == "mount":
if spec["path"] not in cache:
cache[spec["path"]] = self.read(spec["path"], self.credential) or {}
mount = cache[spec["path"]].get(spec["mount"])
if isinstance(mount, dict):
row["present"] = True
observed = {"type": mount.get("type"),
"kv_version": (mount.get("options") or {}).get("version")}
row["source_match"] = observed["type"] in spec["types"] and \
("kv_version" not in spec or observed["kv_version"] == spec["kv_version"])
row["observed_sha256"] = sha256_bytes(canonical(observed))
elif spec["kind"] == "source":
observed = self.read(spec["path"], self.credential)
if isinstance(observed, dict):
row["present"] = True
source = json.loads((self.root / spec["source"]).read_bytes())
row["source_match"], projected = project(source, observed, spec["ignore"])
row["observed_sha256"] = sha256_bytes(canonical(projected))
elif spec["kind"] == "policy":
observed = self.read(spec["path"], self.credential)
text = (observed or {}).get("policy")
if isinstance(text, str):
row["present"] = True
row["source_match"] = text.strip() == (self.root / spec["source"]).read_text().strip()
row["observed_sha256"] = sha256_bytes(text.strip().encode())
elif spec["kind"] == "kv_metadata":
meta = self.read(spec["path"], self.credential)
if isinstance(meta, dict):
current = meta.get("current_version")
version = (meta.get("versions") or {}).get(str(current)) or {}
observed = {"current_version": current, "versions": len(meta.get("versions") or {}),
"created_time": meta.get("created_time"), "updated_time": meta.get("updated_time"),
"current_deleted": bool(version.get("deletion_time")),
"current_destroyed": bool(version.get("destroyed"))}
row["present"] = isinstance(current, int) and current > 0
row["source_match"] = row["present"] and not observed["current_deleted"] \
and not observed["current_destroyed"]
row["observed_sha256"] = sha256_bytes(canonical(observed))
rows.append(row)
self.receipt["objects"] = rows
return rows
def revoke(self) -> None:
if not self.credential or not self.container_started:
return
try:
self.write("auth/token/revoke-self", {}, credential=self.credential)
check = self.bao(["read", "-format=json", "auth/token/lookup-self"], credential=self.credential,
check=False)
self.receipt["teardown"]["readback_credential_revoked"] = check.returncode != 0
except Exception:
self.receipt["teardown"]["readback_credential_revoked"] = False
finally:
self.credential = None
# ---- teardown
def close_identity(self) -> None:
if self.identity_fd is not None:
os.close(self.identity_fd)
self.identity_fd = None
self.receipt["teardown"]["custody_memfd_closed"] = True
def close_plaintext(self) -> None:
if self.plaintext_fd is not None:
os.close(self.plaintext_fd)
self.plaintext_fd = None
self.receipt["teardown"]["plaintext_memfd_closed"] = True
def teardown(self) -> None:
self.revoke()
self.close_identity()
self.close_plaintext()
for share in self.shares:
wipe(share)
self.shares.clear()
self.throwaway_credential = None
td = self.receipt["teardown"]
subprocess.run([self.docker, "rm", "-f", "-v", self.name], capture_output=True, timeout=60)
absent = subprocess.run([self.docker, "container", "inspect", self.name], capture_output=True, timeout=30)
td["container_removed"] = td["container_absent_verified"] = absent.returncode != 0
def write_receipt(self) -> None:
finished = now_utc()
self.receipt["finished_at"] = iso(finished)
self.receipt["elapsed_seconds"] = int((finished - self.started).total_seconds())
fd = os.open(self.receipt_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600)
with os.fdopen(fd, "w") as out:
json.dump(self.receipt, out, indent=2, sort_keys=True)
out.write("\n")
def execute(self) -> dict[str, Any]:
self.throwaway_credential = None
handlers = {}
def interrupted(signum, _frame):
raise Refused("operator_abort")
for sig in (signal.SIGINT, signal.SIGTERM, signal.SIGHUP):
handlers[sig] = signal.signal(sig, interrupted)
result, reason = "aborted", "internal_error"
try:
check_window(self.adm)
if self.start_reaper:
self.spawn_reaper()
self.custody_identity()
self.decrypt()
self.start_target()
self.restore()
self.unseal_original()
self.generate_credential()
rows = self.readback()
missing = [r for r in rows if not r["present"]]
mismatch = [r for r in rows if r["present"] and not r["source_match"]]
if missing:
result, reason = "refused", "object_missing"
elif mismatch:
result, reason = "refused", "object_mismatch"
else:
result, reason = "restored", "all_required_objects_match"
except Refused as exc:
result, reason = "aborted", exc.reason if exc.reason in ABORT_REASONS else "internal_error"
except Exception:
result, reason = "aborted", "internal_error"
finally:
try:
self.teardown()
finally:
for sig, handler in handlers.items():
signal.signal(sig, handler)
self.receipt["result"], self.receipt["reason"] = result, reason
self.write_receipt()
self.tell_reaper("DONE")
if self.reaper:
try:
self.reaper.stdin.close()
self.reaper.wait(timeout=10)
except Exception:
pass
self.say(f"Result: {result} ({reason}). Receipt written; verify it independently.")
return self.receipt
# --------------------------------------------------------------------------- reaper
def reaper_main() -> int:
"""Backstop: on parent loss or hard deadline remove containers, write ``aborted``."""
stream = sys.stdin.buffer
buffer = b""
while b"\n" not in buffer:
chunk = os.read(stream.fileno(), 65536)
if not chunk:
return 1
buffer += chunk
first, buffer = buffer.split(b"\n", 1)
header = json.loads(first)
parent, deadline = header["parent_pid"], header["deadline_epoch"]
try:
parent_fd = os.pidfd_open(parent)
except OSError:
parent_fd = None
names: list[str] = []
reason = "parent_lost"
while True:
while b"\n" in buffer:
line, buffer = buffer.split(b"\n", 1)
text = line.decode()
if text == "DONE":
return 0
if text.startswith("C ") and re.fullmatch(r"rpf64-restore-[a-f0-9]{12}", text[2:]):
names.append(text[2:])
wait = deadline - time.time()
if wait <= 0:
reason = "deadline_exceeded"
break
ready, _, _ = select.select([stream], [], [], min(wait, 5))
if not ready:
continue
chunk = os.read(stream.fileno(), 4096)
if not chunk:
break
buffer += chunk
if parent_fd is not None:
try:
signal.pidfd_send_signal(parent_fd, signal.SIGKILL)
except OSError:
pass
select.select([parent_fd], [], [], 10)
os.close(parent_fd)
removed = True
for name in names:
subprocess.run([header["docker"], "rm", "-f", "-v", name], capture_output=True, timeout=60)
absent = subprocess.run([header["docker"], "container", "inspect", name], capture_output=True, timeout=30)
removed = removed and absent.returncode != 0
receipt = header["base_receipt"]
finished = now_utc()
receipt.update({"result": "aborted", "reason": reason, "finished_at": iso(finished),
"elapsed_seconds": int((finished - parse_iso(receipt["started_at"])).total_seconds())})
receipt["teardown"].update({"container_removed": removed, "container_absent_verified": removed,
"plaintext_memfd_closed": True, "custody_memfd_closed": True,
"readback_credential_revoked": None, "performed_by": "reaper"})
try:
fd = os.open(header["receipt_path"], os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600)
except FileExistsError:
return 0
with os.fdopen(fd, "w") as out:
json.dump(receipt, out, indent=2, sort_keys=True)
out.write("\n")
return 0
def main(argv: list[str] | None = None) -> int:
import argparse
args = argv if argv is not None else sys.argv[1:]
if args == ["--reaper"]:
return reaper_main()
parser = argparse.ArgumentParser(description=__doc__.split("\n", 1)[0])
for name in ("admission", "admission-sig", "admission-sha256", "ciphertext", "receipt"):
parser.add_argument("--" + name, required=True)
opts = parser.parse_args(args)
os.umask(0o077)
try:
adm, _raw, signer, protected = load_admission(Path(opts.admission), Path(opts.admission_sig),
opts.admission_sha256, anchor_path())
require(adm["material"] == "original", "admission_refused")
check_source_closure(adm, ROOT)
receipt = Path(opts.receipt)
require(receipt.is_absolute() and not receipt.exists() and receipt.parent.is_dir(), "preflight_refused")
except Refused as exc:
print(f"ERROR: {exc.reason}; no custody input requested.", file=sys.stderr)
return 1
session = Session(admission=adm, admission_sha256=opts.admission_sha256, signer=signer,
anchor_is_protected=protected, ciphertext=Path(opts.ciphertext), receipt_path=receipt)
result = session.execute()
verify = subprocess.run([sys.executable, str(ROOT / HERE / "verify-restore-receipt.py"), "--receipt",
str(receipt), "--source-root", str(ROOT)], capture_output=True, timeout=60)
print("Independent verifier: " + ("PASS" if verify.returncode == 0 else "FAIL"))
return 0 if result["result"] in ("restored", "refused") and verify.returncode == 0 else 2
if __name__ == "__main__":
raise SystemExit(main())