a2da4085e0
Adds --mode break-glass-drill with its own signed admission (drill: "break-glass", attempt id RPF-WP-0064-T10-*, snapshot_label). After the phase A restore and unseal it checks the held root token with lookup-self only, runs a fresh generate-root ceremony with re-entered shares, does one read-only sys/auth action, revokes the generated token (204 then 403) and drills a cancelled attempt. Phase A is unchanged. The verifier types the receipt extension and rejects token, accessor, OTP, nonce, encoded-token and share shapes. Offline 34 and native 16 pass. Founder runbooks for the drill and the read-only production SSO-down walk are in the T10 section. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Assistant: claude-code Assistant-Model: opus Assistant-Process: 4515@bnt-lap001 Assistant-Session: 0bf96127-6e04-41be-bfda-d11717c3f354
1165 lines
62 KiB
Python
1165 lines
62 KiB
Python
#!/usr/bin/env python3
|
|
"""Isolated original-OpenBao restore adapter (RPF-WP-0064-T02 phase A; T10 break-glass drill).
|
|
|
|
Restores one age-encrypted Raft snapshot into a disposable, networkless Docker
|
|
container on the founder workstation, unseals it with the original Shamir
|
|
shares, reads back configuration metadata, and tears everything down. It never
|
|
runs on railiance01, never uses a kubeconfig, never reads KV ``data`` and never
|
|
prints or records a secret value.
|
|
|
|
Custody inputs (founder-attended, hidden prompts only, never argv/env/files):
|
|
the backup age identity (held in a sealed memfd for ``age -d``) and two
|
|
original unseal shares (process memory only). The decrypted snapshot lives in
|
|
a sealed memfd and is streamed to the container's stdin; the container's Raft
|
|
directory is tmpfs. A throwaway 1-of-1 init makes the empty target writable;
|
|
the restore invalidates it. Readback uses a root credential generated in the
|
|
restored target from the same two shares, revoked before teardown.
|
|
|
|
A reaper process watches the session: on parent loss or the hard deadline it
|
|
kills the session, removes the container and writes an ``aborted`` receipt.
|
|
|
|
Outcomes: ``restored`` | ``refused`` (required object missing or not matching
|
|
its source; never repaired) | ``aborted`` (any other failed check, timeout,
|
|
operator abort). Only the launcher ``run-isolated-restore.sh`` is the attended
|
|
entry point; ``Session`` is importable for the synthetic rehearsal tests.
|
|
|
|
``--mode break-glass-drill`` (RPF-WP-0064-T10, NK-WP-0055-T13) needs its own
|
|
signed admission (``drill: "break-glass"``, attempt id ``RPF-WP-0064-T10-*``).
|
|
It runs the same restore and unseal, forgets the unseal shares, then: checks a
|
|
held root token with ``lookup-self`` only (never revoked or otherwise used);
|
|
runs a fresh generate-root ceremony with a server OTP and two re-entered
|
|
shares; does one read-only action (``sys/auth``); revokes the generated token
|
|
(204) and confirms ``lookup-self`` is refused (403); starts and cancels one
|
|
more attempt. Outcomes: ``drilled`` | ``aborted``. No object readback.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import base64
|
|
import fcntl
|
|
import getpass
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import re
|
|
import secrets
|
|
import select
|
|
import signal
|
|
import stat
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
from typing import Any, Callable
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
HERE = "scripts/openbao-isolated-restore/"
|
|
NAMESPACE = "railiance-openbao-restore-attempt"
|
|
ADMISSION_SCHEMA = "railiance-openbao-restore-admission/v1"
|
|
RECEIPT_SCHEMA = "railiance-openbao-restore-receipt/v2"
|
|
IMAGE_REPOSITORY = "quay.io/openbao/openbao"
|
|
# RPF-WP-0064-T06: production digest, observed 2026-10-24 as the imageID of the live
|
|
# openbao-0 container (spec image quay.io/openbao/openbao:2.5.4); equal to the T04
|
|
# rehearsal pull. Original-material runs refuse any other image.
|
|
PINNED_IMAGE_DIGEST: str | None = "sha256:436eaf9778cad75507ff70ea26ace30dcbe15606e619ac3823495663d7f7c115"
|
|
MAX_DURATION_SECONDS = 2700
|
|
LEADER_WAIT_SECONDS = 180
|
|
LEADER_FIELDS = {"ha_enabled": bool, "is_self": bool, "leader_address": str, "leader_cluster_address": str,
|
|
"raft_committed_index": int, "raft_applied_index": int, "active_time": str}
|
|
SEAL_FIELDS = {"sealed": bool, "initialized": bool, "storage_type": str, "version": str}
|
|
_SECRETISH = re.compile(r"\b(?:hvs|hvb|hvr|s|b|r)\.[A-Za-z0-9_-]{8,}|[A-Za-z0-9+/=_-]{32,}")
|
|
|
|
|
|
def typed(value, kind):
|
|
"""Keep a status value only if it has the expected type; bound strings and redact token-like text."""
|
|
if kind is int:
|
|
return value if type(value) is int else None
|
|
if kind is bool:
|
|
return value if type(value) is bool else None
|
|
if isinstance(value, str):
|
|
return _SECRETISH.sub("[redacted]", value)[:200]
|
|
return None
|
|
|
|
|
|
def sanitize_status_line(raw: bytes) -> str | None:
|
|
"""First non-empty stderr line, bounded, with token-like text redacted."""
|
|
for line in raw.decode("utf-8", "replace").splitlines():
|
|
line = line.strip()
|
|
if line:
|
|
return _SECRETISH.sub("[redacted]", line)[:200]
|
|
return None
|
|
REAPER_GRACE_SECONDS = 60
|
|
DEFAULT_ANCHOR = Path("/etc/railiance-openbao-restore/allowed_signers")
|
|
ANCHOR_ENV = "RAILIANCE_OPENBAO_RESTORE_ANCHOR"
|
|
SSH_ENV = {"PATH": "/usr/bin:/bin", "LC_ALL": "C"}
|
|
CLAIMS_NOT_MADE = (
|
|
"consumer_continuity", "live_jwt_login", "phase_b_network",
|
|
"owner_acceptance", "recurring_restore_assurance", "production_redirect",
|
|
)
|
|
ADAPTER_PATHS = tuple(HERE + n for n in (
|
|
"restore_adapter.py", "preflight.py", "run-isolated-restore.sh", "verify-restore-receipt.py"))
|
|
OBJECT_SOURCE_PATHS = (
|
|
"openbao/auth/keycape-services-config.json",
|
|
"openbao/auth/secrets-engine-jwt-role.json",
|
|
"openbao/auth/netkingdom-oidc-config.json",
|
|
"openbao/auth/netkingdom-platform-admin-role.json",
|
|
"openbao/policies/secrets-engine-login-self.hcl",
|
|
"docs/contracts/secrets-engine-openbao-jwt-login.yaml",
|
|
)
|
|
SOURCE_PATHS = tuple(sorted(ADAPTER_PATHS + OBJECT_SOURCE_PATHS))
|
|
KV_METADATA_PATH = "platform/metadata/workloads/keycape/secrets-engine-openbao"
|
|
|
|
# Required objects. ``source`` objects compare the declared fields of the
|
|
# source file (minus ``ignore``) with the restored readback after normalising.
|
|
OBJECTS: tuple[dict[str, Any], ...] = (
|
|
{"object": "auth_mount_keycape_services", "kind": "mount", "path": "sys/auth",
|
|
"mount": "keycape-services/", "types": ("jwt",)},
|
|
{"object": "auth_mount_netkingdom", "kind": "mount", "path": "sys/auth",
|
|
"mount": "netkingdom/", "types": ("oidc", "jwt")},
|
|
{"object": "kv_mount_platform", "kind": "mount", "path": "sys/mounts",
|
|
"mount": "platform/", "types": ("kv",), "kv_version": "2"},
|
|
{"object": "keycape_services_config", "kind": "source", "path": "auth/keycape-services/config",
|
|
"source": "openbao/auth/keycape-services-config.json", "ignore": ()},
|
|
{"object": "secrets_engine_jwt_role", "kind": "source",
|
|
"path": "auth/keycape-services/role/secrets-engine-openbao",
|
|
"source": "openbao/auth/secrets-engine-jwt-role.json", "ignore": ()},
|
|
{"object": "netkingdom_oidc_config", "kind": "source", "path": "auth/netkingdom/config",
|
|
"source": "openbao/auth/netkingdom-oidc-config.json", "ignore": ("_note",)},
|
|
{"object": "platform_admin_role", "kind": "source", "path": "auth/netkingdom/role/platform-admin",
|
|
"source": "openbao/auth/netkingdom-platform-admin-role.json", "ignore": ()},
|
|
{"object": "policy_secrets_engine_login_self", "kind": "policy",
|
|
"path": "sys/policies/acl/secrets-engine-login-self",
|
|
"source": "openbao/policies/secrets-engine-login-self.hcl"},
|
|
{"object": "kv_registration_metadata", "kind": "kv_metadata", "path": KV_METADATA_PATH},
|
|
)
|
|
ALLOWED_READS = frozenset({"sys/seal-status", "sys/leader", "auth/token/lookup-self",
|
|
*(o["path"] for o in OBJECTS)})
|
|
ALLOWED_WRITES = frozenset({"sys/unseal", "sys/generate-root/attempt", "sys/generate-root/update",
|
|
"auth/token/revoke-self"})
|
|
# --------------------------------------------------------------------------- T10 break-glass drill
|
|
PHASE_A, DRILL_MODE = "phase-a", "break-glass-drill"
|
|
DRILL = "break-glass"
|
|
GENERATE_ROOT_ATTEMPT = "sys/generate-root/attempt"
|
|
MINIMAL_ACTION_PATH = "sys/auth"
|
|
# T10 extensions: generate-root status (GET) and cancel (DELETE); sys/auth and
|
|
# auth/token/lookup-self are already allowed reads, sys/generate-root/update an allowed write.
|
|
ALLOWED_READS = ALLOWED_READS | {GENERATE_ROOT_ATTEMPT}
|
|
ALLOWED_DELETES = frozenset({GENERATE_ROOT_ATTEMPT})
|
|
DRILL_READBACK = "generate-root-from-reentered-shares"
|
|
DRILL_CLAIMS_NOT_MADE = ("production_root_token_validity", "production_generate_root_availability",
|
|
"sso_down_access_path")
|
|
DRILL_ABORT_REASONS = frozenset({"held_token_refused", "generate_root_failed", "minimal_action_failed",
|
|
"revocation_failed", "cancel_drill_failed"})
|
|
ABORT_REASONS = frozenset({
|
|
"admission_refused", "preflight_refused", "custody_identity_refused", "ciphertext_mismatch",
|
|
"decrypt_failed", "plaintext_mismatch", "target_start_failed", "network_containment_failed",
|
|
"throwaway_init_failed", "restore_failed", "unseal_refused", "leader_not_elected", "cluster_mismatch",
|
|
"applied_index_behind", "readback_credential_failed", "readback_failed", "deadline_exceeded",
|
|
"operator_abort", "parent_lost", "internal_error",
|
|
}) | DRILL_ABORT_REASONS
|
|
# Production runs the chart with setNodeId, so the snapshot's voter is the pod
|
|
# name. The target takes the same id; a native rehearsal showed a different id
|
|
# still elects a leader after a forced restore, so this is alignment, not the
|
|
# T07 attempt -01 cause.
|
|
RESTORE_NODE_ID = "openbao-0"
|
|
SERVER_CONFIG = json.dumps({
|
|
"storage": {"raft": {"path": "/openbao/data", "node_id": RESTORE_NODE_ID}},
|
|
# OpenBao 2.x disables unauthenticated generate-root by default; the
|
|
# readback credential needs it, on a loopback-only listener without a network.
|
|
"listener": {"tcp": {"address": "127.0.0.1:8200", "tls_disable": True,
|
|
"disable_unauthed_generate_root_endpoints": False}},
|
|
"api_addr": "http://127.0.0.1:8200", "cluster_addr": "http://127.0.0.1:8201",
|
|
"disable_mlock": True, "ui": False,
|
|
}, separators=(",", ":"))
|
|
TMPFS = {
|
|
"/openbao/data": "rw,nosuid,nodev,noexec,size=536870912,uid=100,gid=1000,mode=0700",
|
|
"/openbao/config": "rw,nosuid,nodev,noexec,size=1048576,uid=100,gid=1000,mode=0700",
|
|
"/tmp": "rw,nosuid,nodev,noexec,size=16777216,uid=100,gid=1000,mode=0700",
|
|
# The image declares VOLUMEs here; tmpfs overlays stop anonymous volumes.
|
|
"/openbao/file": "rw,nosuid,nodev,noexec,size=1048576,uid=100,gid=1000,mode=0700",
|
|
"/openbao/logs": "rw,nosuid,nodev,noexec,size=1048576,uid=100,gid=1000,mode=0700",
|
|
# Production declares a file audit device at /openbao/audit/openbao-audit.log
|
|
# (rapp-openbao values). The restored audit table re-enables it on unseal; if
|
|
# the path is unwritable OpenBao seals itself again (T07 attempts -01/-02).
|
|
# The log holds HMAC'd entries only and dies with the container.
|
|
"/openbao/audit": "rw,nosuid,nodev,noexec,size=16777216,uid=100,gid=1000,mode=0700",
|
|
}
|
|
DIGEST_RE = re.compile(r"^sha256:[a-f0-9]{64}$")
|
|
IMAGE_RE = re.compile(r"^" + re.escape(IMAGE_REPOSITORY) + r"@(sha256:[a-f0-9]{64})$")
|
|
AGE_IDENTITY_RE = re.compile(r"^AGE-SECRET-KEY-1[0-9A-Z]{58}$")
|
|
AGE_RECIPIENT_RE = re.compile(r"^age1[023456789acdefghjklmnpqrstuvwxyz]{58}$")
|
|
SHARE_RE = re.compile(r"^(?:[A-Za-z0-9+/]{40,100}={0,2}|[a-f0-9]{60,130})$")
|
|
ATTEMPT_RE = re.compile(r"^RPF-WP-0064-(?:T07|T04)-[A-Za-z0-9-]{3,64}$")
|
|
DRILL_ATTEMPT_RE = re.compile(r"^RPF-WP-0064-T10-[A-Za-z0-9-]{3,64}$")
|
|
# Snapshot name for the claim limit, e.g. openbao-raft-20261014T1532Z.snap.age.
|
|
SNAPSHOT_LABEL_RE = re.compile(r"^(?=.*20\d{6})[a-z0-9][a-z0-9._-]{2,95}$")
|
|
HELD_TOKEN_RE = re.compile(r"^[A-Za-z0-9._-]{16,256}$")
|
|
POLICY_NAME_RE = re.compile(r"^[a-z0-9][a-z0-9_.-]{0,63}$")
|
|
HTTP_CODE_RE = re.compile(rb"Code: (\d{3})\.")
|
|
COMMIT_RE = re.compile(r"^[a-f0-9]{40}$")
|
|
DURATION_RE = re.compile(r"^(\d+)([smh]?)$")
|
|
|
|
|
|
class Refused(Exception):
|
|
"""A fail-closed check; ``reason`` is an ABORT_REASONS code or ``object_*``."""
|
|
|
|
def __init__(self, reason: str):
|
|
super().__init__(reason)
|
|
self.reason = reason
|
|
|
|
|
|
def require(condition: Any, reason: str) -> None:
|
|
if not condition:
|
|
raise Refused(reason)
|
|
|
|
|
|
def now_utc() -> datetime:
|
|
return datetime.now(UTC).replace(microsecond=0)
|
|
|
|
|
|
def iso(value: datetime) -> str:
|
|
return value.astimezone(UTC).strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
|
|
|
|
def parse_iso(value: Any) -> datetime:
|
|
require(isinstance(value, str) and re.fullmatch(r"\d{4}-\d\d-\d\dT\d\d:\d\d:\d\dZ", value), "admission_refused")
|
|
return datetime.strptime(value, "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=UTC)
|
|
|
|
|
|
def sha256_bytes(data: bytes) -> str:
|
|
return "sha256:" + hashlib.sha256(data).hexdigest()
|
|
|
|
|
|
def canonical(value: Any) -> bytes:
|
|
return json.dumps(value, sort_keys=True, separators=(",", ":")).encode()
|
|
|
|
|
|
def strict_json(raw: bytes) -> Any:
|
|
def unique(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
|
|
out: dict[str, Any] = {}
|
|
for key, value in pairs:
|
|
require(key not in out, "admission_refused")
|
|
out[key] = value
|
|
return out
|
|
return json.loads(raw, object_pairs_hook=unique)
|
|
|
|
|
|
def placeholder_digest(value: str) -> bool:
|
|
hexpart = value.split(":", 1)[-1]
|
|
return len(set(hexpart)) <= 2
|
|
|
|
|
|
# --------------------------------------------------------------------------- admission
|
|
|
|
ADMISSION_FIELDS = {"schema", "namespace", "attempt_id", "phase", "phase_b_admitted", "material",
|
|
"not_before", "not_after", "max_duration_seconds", "abort_operator", "image",
|
|
"adapter_commit", "snapshot", "unseal", "readback_credential", "source_sha256"}
|
|
SNAPSHOT_FIELDS = {"ciphertext_sha256", "ciphertext_bytes", "plaintext_sha256", "age_recipient",
|
|
"source_cluster_id", "source_applied_index", "source_receipt_sha256"}
|
|
|
|
|
|
def validate_admission(adm: Any, mode: str = PHASE_A) -> dict[str, Any]:
|
|
"""Shape-only validation of the signed admission (no I/O); phase-A and drill admissions are disjoint."""
|
|
require(mode in (PHASE_A, DRILL_MODE), "admission_refused")
|
|
drill = mode == DRILL_MODE
|
|
fields = ADMISSION_FIELDS | {"drill", "snapshot_label"} if drill else ADMISSION_FIELDS
|
|
require(isinstance(adm, dict) and set(adm) == fields, "admission_refused")
|
|
require(adm["schema"] == ADMISSION_SCHEMA and adm["namespace"] == NAMESPACE, "admission_refused")
|
|
attempt_re = DRILL_ATTEMPT_RE if drill else ATTEMPT_RE
|
|
require(isinstance(adm["attempt_id"], str) and attempt_re.fullmatch(adm["attempt_id"]), "admission_refused")
|
|
require(adm["phase"] == "A" and adm["phase_b_admitted"] is False, "admission_refused")
|
|
require(adm["material"] in ("original", "synthetic"), "admission_refused")
|
|
require(adm["readback_credential"] == (DRILL_READBACK if drill else "generate-root-from-entered-shares"),
|
|
"admission_refused")
|
|
if drill:
|
|
require(adm["drill"] == DRILL, "admission_refused")
|
|
require(isinstance(adm["snapshot_label"], str) and SNAPSHOT_LABEL_RE.fullmatch(adm["snapshot_label"]),
|
|
"admission_refused")
|
|
nb, na = parse_iso(adm["not_before"]), parse_iso(adm["not_after"])
|
|
require(nb < na and (na - nb).total_seconds() <= 24 * 3600, "admission_refused")
|
|
dur = adm["max_duration_seconds"]
|
|
require(type(dur) is int and 60 <= dur <= MAX_DURATION_SECONDS, "admission_refused")
|
|
if adm["material"] == "original":
|
|
require(dur == MAX_DURATION_SECONDS, "admission_refused")
|
|
require(isinstance(adm["abort_operator"], str) and re.fullmatch(r"[a-z][a-z0-9-]{1,63}", adm["abort_operator"]),
|
|
"admission_refused")
|
|
match = IMAGE_RE.fullmatch(adm["image"]) if isinstance(adm["image"], str) else None
|
|
require(match and not placeholder_digest(match.group(1)), "admission_refused")
|
|
require(isinstance(adm["adapter_commit"], str) and COMMIT_RE.fullmatch(adm["adapter_commit"]), "admission_refused")
|
|
snap = adm["snapshot"]
|
|
require(isinstance(snap, dict) and set(snap) == SNAPSHOT_FIELDS, "admission_refused")
|
|
for field in ("ciphertext_sha256", "plaintext_sha256", "source_receipt_sha256"):
|
|
require(isinstance(snap[field], str) and DIGEST_RE.fullmatch(snap[field])
|
|
and not placeholder_digest(snap[field]), "admission_refused")
|
|
require(type(snap["ciphertext_bytes"]) is int and 0 < snap["ciphertext_bytes"] <= 512 * 1024 * 1024,
|
|
"admission_refused")
|
|
require(isinstance(snap["age_recipient"], str) and AGE_RECIPIENT_RE.fullmatch(snap["age_recipient"]),
|
|
"admission_refused")
|
|
require(isinstance(snap["source_cluster_id"], str)
|
|
and re.fullmatch(r"[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}", snap["source_cluster_id"]),
|
|
"admission_refused")
|
|
require(type(snap["source_applied_index"]) is int and snap["source_applied_index"] > 0, "admission_refused")
|
|
require(adm["unseal"] == {"key_threshold": 2, "key_shares": 3}, "admission_refused")
|
|
sources = adm["source_sha256"]
|
|
require(isinstance(sources, dict) and set(sources) == set(SOURCE_PATHS), "admission_refused")
|
|
for value in sources.values():
|
|
require(isinstance(value, str) and DIGEST_RE.fullmatch(value), "admission_refused")
|
|
return adm
|
|
|
|
|
|
def source_digests(root: Path) -> dict[str, str]:
|
|
out = {}
|
|
for name in SOURCE_PATHS:
|
|
path = root / name
|
|
require(not any(p.is_symlink() for p in (path, *path.parents)) and path.is_file(), "preflight_refused")
|
|
out[name] = sha256_bytes(path.read_bytes())
|
|
return out
|
|
|
|
|
|
def check_source_closure(adm: dict[str, Any], root: Path) -> None:
|
|
"""Admission names the exact committed adapter and source files on disk."""
|
|
require(source_digests(root) == adm["source_sha256"], "preflight_refused")
|
|
head = subprocess.run(["git", "-C", str(root), "rev-parse", "HEAD"], capture_output=True, timeout=10)
|
|
require(head.returncode == 0 and head.stdout.decode().strip() == adm["adapter_commit"], "preflight_refused")
|
|
dirty = subprocess.run(["git", "-C", str(root), "status", "--porcelain", "--", *SOURCE_PATHS],
|
|
capture_output=True, timeout=10)
|
|
require(dirty.returncode == 0 and not dirty.stdout.strip(), "preflight_refused")
|
|
|
|
|
|
def anchor_path() -> Path:
|
|
return Path(os.environ.get(ANCHOR_ENV) or DEFAULT_ANCHOR)
|
|
|
|
|
|
def anchor_protected(signers: Path) -> bool:
|
|
"""True if the running account cannot rewrite the trust anchor."""
|
|
euid = os.geteuid()
|
|
for path in (signers.parent, signers):
|
|
info = os.stat(path, follow_symlinks=False)
|
|
if info.st_mode & 0o022 or (info.st_uid == euid and euid != 0):
|
|
return False
|
|
return True
|
|
|
|
|
|
def verify_signature(data: bytes, signature: bytes, signers: Path) -> dict[str, str]:
|
|
"""``ssh-keygen -Y`` over exact admission bytes with the fixed namespace."""
|
|
import tempfile
|
|
require(signers.is_absolute() and not signers.is_symlink() and not signers.parent.is_symlink()
|
|
and signers.is_file(), "admission_refused")
|
|
require(signature.startswith(b"-----BEGIN SSH SIGNATURE-----") and len(signature) <= 8192, "admission_refused")
|
|
with tempfile.TemporaryDirectory(prefix="rpf64-adm-") as tmp:
|
|
sig = Path(tmp) / "admission.sig"
|
|
sig.write_bytes(signature)
|
|
found = subprocess.run(["ssh-keygen", "-Y", "find-principals", "-f", str(signers), "-n", NAMESPACE,
|
|
"-s", str(sig)], capture_output=True, env=SSH_ENV, timeout=10)
|
|
principals = found.stdout.decode("utf-8", "replace").split()
|
|
require(found.returncode == 0 and len(principals) == 1
|
|
and re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._@-]{0,127}", principals[0]), "admission_refused")
|
|
checked = subprocess.run(["ssh-keygen", "-Y", "verify", "-f", str(signers), "-I", principals[0],
|
|
"-n", NAMESPACE, "-s", str(sig)], input=data, capture_output=True,
|
|
env=SSH_ENV, timeout=10)
|
|
match = re.search(r"key (SHA256:[A-Za-z0-9+/=]+)", checked.stdout.decode("utf-8", "replace"))
|
|
require(checked.returncode == 0 and match, "admission_refused")
|
|
return {"principal": principals[0], "fingerprint": match.group(1)}
|
|
|
|
|
|
def read_owner_file(path: Path, limit: int) -> bytes:
|
|
"""Regular, owner-only, single-link file read without following symlinks."""
|
|
path = Path(os.path.abspath(path))
|
|
require(not any(p.is_symlink() for p in (path, *path.parents)), "preflight_refused")
|
|
fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK)
|
|
with os.fdopen(fd, "rb") as handle:
|
|
info = os.fstat(handle.fileno())
|
|
require(stat.S_ISREG(info.st_mode) and info.st_uid == os.getuid() and info.st_nlink == 1
|
|
and not info.st_mode & 0o077 and 0 < info.st_size <= limit, "preflight_refused")
|
|
raw = handle.read(limit + 1)
|
|
require(0 < len(raw) <= limit, "preflight_refused")
|
|
return raw
|
|
|
|
|
|
def load_admission(admission: Path, signature: Path, admission_sha256: str, signers: Path,
|
|
mode: str = PHASE_A) -> tuple[dict[str, Any], bytes, dict[str, str], bool]:
|
|
raw = read_owner_file(admission, 64 * 1024)
|
|
require(DIGEST_RE.fullmatch(admission_sha256) and sha256_bytes(raw) == admission_sha256, "admission_refused")
|
|
adm = validate_admission(strict_json(raw), mode)
|
|
signer = verify_signature(raw, read_owner_file(signature, 8192), signers)
|
|
protected = anchor_protected(signers)
|
|
if adm["material"] == "original":
|
|
require(protected, "admission_refused")
|
|
require(PINNED_IMAGE_DIGEST and adm["image"] == f"{IMAGE_REPOSITORY}@{PINNED_IMAGE_DIGEST}",
|
|
"admission_refused")
|
|
return adm, raw, signer, protected
|
|
|
|
|
|
def check_window(adm: dict[str, Any], at: datetime | None = None) -> None:
|
|
at = at or now_utc()
|
|
require(parse_iso(adm["not_before"]) <= at < parse_iso(adm["not_after"]), "admission_refused")
|
|
|
|
|
|
# --------------------------------------------------------------------------- normalisation
|
|
|
|
def _seconds(value: Any) -> Any:
|
|
if isinstance(value, str):
|
|
match = DURATION_RE.fullmatch(value)
|
|
if match:
|
|
return int(match.group(1)) * {"": 1, "s": 1, "m": 60, "h": 3600}[match.group(2)]
|
|
return value
|
|
|
|
|
|
def normalise(value: Any) -> Any:
|
|
if isinstance(value, dict):
|
|
return {k: normalise(v) for k, v in sorted(value.items())}
|
|
if isinstance(value, list):
|
|
items = [normalise(v) for v in value]
|
|
return sorted(items) if all(isinstance(v, str) for v in items) else items
|
|
return _seconds(value)
|
|
|
|
|
|
def project(source: dict[str, Any], observed: dict[str, Any], ignore: tuple[str, ...]) -> tuple[bool, dict]:
|
|
fields = sorted(k for k in source if k not in ignore)
|
|
projected = {k: normalise(observed.get(k)) for k in fields}
|
|
expected = {k: normalise(source[k]) for k in fields}
|
|
return projected == expected, projected
|
|
|
|
|
|
# --------------------------------------------------------------------------- memfd custody
|
|
|
|
def sealed_memfd(name: str, data: bytes | bytearray) -> int:
|
|
fd = os.memfd_create(name, os.MFD_CLOEXEC | os.MFD_ALLOW_SEALING)
|
|
view = memoryview(data)
|
|
while view:
|
|
view = view[os.write(fd, view):]
|
|
fcntl.fcntl(fd, fcntl.F_ADD_SEALS, fcntl.F_SEAL_SHRINK | fcntl.F_SEAL_GROW | fcntl.F_SEAL_WRITE | fcntl.F_SEAL_SEAL)
|
|
os.lseek(fd, 0, os.SEEK_SET)
|
|
return fd
|
|
|
|
|
|
def wipe(buf: bytearray | None) -> None:
|
|
if buf is not None:
|
|
for i in range(len(buf)):
|
|
buf[i] = 0
|
|
|
|
|
|
def http_code(stderr: bytes) -> int | None:
|
|
"""HTTP status from the bao CLI error text (``Code: 403.``); None if absent."""
|
|
match = HTTP_CODE_RE.search(stderr)
|
|
return int(match.group(1)) if match else None
|
|
|
|
|
|
def policy_names(value: Any) -> list[str]:
|
|
"""Policy names only, sorted; anything not shaped like a policy name is dropped."""
|
|
return sorted(p for p in (value if isinstance(value, list) else [])
|
|
if isinstance(p, str) and POLICY_NAME_RE.fullmatch(p))[:32]
|
|
|
|
|
|
def tty_prompt(label: str) -> bytearray:
|
|
"""Hidden prompt on the controlling terminal; the value never touches argv/env/disk."""
|
|
require(sys.stdin.isatty(), "preflight_refused")
|
|
return bytearray(getpass.getpass(f"{label} (hidden): ").strip().encode())
|
|
|
|
|
|
# --------------------------------------------------------------------------- session
|
|
|
|
class Session:
|
|
"""One phase-A restore attempt. Inject ``prompt``/``docker``/``say`` in tests only."""
|
|
|
|
def __init__(self, *, admission: dict[str, Any], admission_sha256: str, signer: dict[str, str],
|
|
anchor_is_protected: bool, ciphertext: Path, receipt_path: Path, root: Path = ROOT,
|
|
prompt: Callable[[str], bytearray] = tty_prompt, docker: str = "docker",
|
|
say: Callable[[str], None] = lambda m: print(m, flush=True), start_reaper: bool = True,
|
|
mode: str = PHASE_A):
|
|
require(mode in (PHASE_A, DRILL_MODE) and (mode == DRILL_MODE) == ("drill" in admission), "admission_refused")
|
|
self.mode = mode
|
|
self.adm, self.root, self.prompt, self.docker, self.say = admission, root, prompt, docker, say
|
|
self.ciphertext, self.receipt_path, self.start_reaper = ciphertext, receipt_path, start_reaper
|
|
self.name = "rpf64-restore-" + secrets.token_hex(6)
|
|
self.deadline = time.monotonic() + admission["max_duration_seconds"]
|
|
self.not_after = parse_iso(admission["not_after"])
|
|
self.started = now_utc()
|
|
self.container_started = False
|
|
self.identity_fd: int | None = None
|
|
self.plaintext_fd: int | None = None
|
|
self.shares: list[bytearray] = []
|
|
self.credential: str | None = None
|
|
self.reaper: subprocess.Popen | None = None
|
|
snap = admission["snapshot"]
|
|
self.receipt: dict[str, Any] = {
|
|
"schema": RECEIPT_SCHEMA,
|
|
"extends": "docs/openbao-restore-drill-evidence.example.json",
|
|
"attempt_id": admission["attempt_id"], "material": admission["material"], "phase": "A",
|
|
"admission_sha256": admission_sha256, "admission_namespace": NAMESPACE,
|
|
"signer_principal": signer["principal"], "signer_fingerprint": signer["fingerprint"],
|
|
"anchor_protected": anchor_is_protected, "abort_operator": admission["abort_operator"],
|
|
"adapter_commit": admission["adapter_commit"], "source_sha256": dict(admission["source_sha256"]),
|
|
"started_at": iso(self.started), "finished_at": None, "not_after": admission["not_after"],
|
|
"deadline_seconds": admission["max_duration_seconds"], "elapsed_seconds": None,
|
|
"image": admission["image"], "image_id": None,
|
|
"snapshot": {"ciphertext_sha256": snap["ciphertext_sha256"], "ciphertext_bytes": snap["ciphertext_bytes"],
|
|
"plaintext_sha256": snap["plaintext_sha256"], "source_cluster_id": snap["source_cluster_id"],
|
|
"source_applied_index": snap["source_applied_index"],
|
|
"source_receipt_sha256": snap["source_receipt_sha256"],
|
|
"ciphertext_verified": False, "plaintext_verified": False},
|
|
"restored": {"cluster_id": None, "applied_index": None, "sealed_after_restore": None,
|
|
"unsealed_with_original_shares": False, "cluster_id_match": False,
|
|
"applied_index_not_behind": False, "throwaway_init_invalidated": False,
|
|
"leader_wait": None},
|
|
"network": {"network_mode": None, "docker_networks": [], "interfaces": [], "published_ports": [],
|
|
"mounts": [], "egress_refused": False},
|
|
"objects": [], "result": None, "reason": None,
|
|
"teardown": {"container_removed": False, "container_absent_verified": False,
|
|
"plaintext_memfd_closed": True, "custody_memfd_closed": True,
|
|
"readback_credential_revoked": None, "docker_volumes_created": 0, "performed_by": "session"},
|
|
"kv_data_read": False, "no_secret_material_recorded": True,
|
|
"claims_not_made": list(CLAIMS_NOT_MADE),
|
|
}
|
|
if mode == DRILL_MODE:
|
|
self.receipt["claims_not_made"] += list(DRILL_CLAIMS_NOT_MADE)
|
|
self.receipt.update({
|
|
"drill": DRILL,
|
|
"held_root_token": {"valid": None, "policies": [], "ttl_zero": None, "expire_time_null": None,
|
|
"orphan": None},
|
|
"generate_root": {"started": False, "shares_entered": 0, "complete": False, "otp_decode_ok": False,
|
|
"policies": [], "minimal_action_path": MINIMAL_ACTION_PATH,
|
|
"minimal_action_ok": False},
|
|
"revocation": {"revoke_status": None, "post_revoke_lookup": None},
|
|
"cancel_drill": {"started_then_cancelled": False},
|
|
# The held-token result is valid only as of this snapshot, never for production now.
|
|
"claim_limits": {"production_root_token_validity": {
|
|
"as_of_snapshot": admission["snapshot_label"],
|
|
"snapshot_plaintext_sha256": snap["plaintext_sha256"]}},
|
|
})
|
|
|
|
# ---- infrastructure
|
|
def remaining(self) -> float:
|
|
left = min(self.deadline - time.monotonic(), (self.not_after - now_utc()).total_seconds())
|
|
require(left > 0, "deadline_exceeded")
|
|
return left
|
|
|
|
def run(self, argv: list[str], *, data: bytes = b"", limit: float = 60, check: bool = True,
|
|
reason: str = "internal_error") -> subprocess.CompletedProcess:
|
|
try:
|
|
done = subprocess.run(argv, input=data, capture_output=True, timeout=min(limit, self.remaining()))
|
|
except subprocess.TimeoutExpired as exc:
|
|
raise Refused("deadline_exceeded" if self.remaining_or_zero() <= 1 else reason) from exc
|
|
require(not check or done.returncode == 0, reason)
|
|
return done
|
|
|
|
def remaining_or_zero(self) -> float:
|
|
return max(0.0, min(self.deadline - time.monotonic(), (self.not_after - now_utc()).total_seconds()))
|
|
|
|
def bao(self, args: list[str], *, credential: str | None = None, body: bytes = b"", limit: float = 30,
|
|
check: bool = True, reason: str = "internal_error") -> subprocess.CompletedProcess:
|
|
script = 'IFS= read -r t; BAO_TOKEN="$t"; export BAO_TOKEN; exec "$@"' if credential else 'exec "$@"'
|
|
data = (credential.encode() + b"\n" + body) if credential else body
|
|
return self.run([self.docker, "exec", "-i", self.name, "sh", "-c", script, "sh", "bao", *args],
|
|
data=data, limit=limit, check=check, reason=reason)
|
|
|
|
def read(self, path: str, credential: str | None = None, reason: str = "readback_failed") -> dict | None:
|
|
require(path in ALLOWED_READS and "/data/" not in path, "internal_error")
|
|
done = self.bao(["read", "-format=json", path], credential=credential, check=False, reason=reason)
|
|
if done.returncode != 0:
|
|
if b"No value found" in done.stderr or done.returncode == 2:
|
|
return None
|
|
raise Refused(reason)
|
|
body = json.loads(done.stdout)
|
|
return body.get("data", body) if isinstance(body, dict) else None
|
|
|
|
def write(self, path: str, payload: dict, credential: str | None = None, reason: str = "internal_error") -> dict:
|
|
require(path in ALLOWED_WRITES, "internal_error")
|
|
done = self.bao(["write", "-format=json", path, "-"], credential=credential, body=json.dumps(payload).encode(),
|
|
reason=reason)
|
|
body = json.loads(done.stdout or b"{}")
|
|
return body.get("data", body) if isinstance(body, dict) and isinstance(body.get("data"), dict) else body
|
|
|
|
def delete(self, path: str, reason: str = "internal_error") -> None:
|
|
require(path in ALLOWED_DELETES, "internal_error")
|
|
self.bao(["delete", path], reason=reason)
|
|
|
|
def tell_reaper(self, line: str) -> None:
|
|
if self.reaper and self.reaper.stdin:
|
|
try:
|
|
self.reaper.stdin.write((line + "\n").encode())
|
|
self.reaper.stdin.flush()
|
|
except OSError:
|
|
pass
|
|
|
|
def spawn_reaper(self) -> None:
|
|
deadline_epoch = time.time() + self.remaining_or_zero() + REAPER_GRACE_SECONDS
|
|
header = {"parent_pid": os.getpid(), "deadline_epoch": deadline_epoch, "docker": self.docker,
|
|
"receipt_path": str(self.receipt_path), "base_receipt": self.receipt}
|
|
self.reaper = subprocess.Popen([sys.executable, str(Path(__file__).resolve()), "--reaper"],
|
|
stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
|
|
start_new_session=True, close_fds=True)
|
|
self.tell_reaper(json.dumps(header))
|
|
|
|
# ---- steps
|
|
def custody_identity(self) -> None:
|
|
self.say("Step 1/7: custody identity (hidden prompt).")
|
|
identity = self.prompt("Backup age identity")
|
|
try:
|
|
require(AGE_IDENTITY_RE.fullmatch(identity.decode("ascii", "replace")), "custody_identity_refused")
|
|
self.identity_fd = sealed_memfd("rpf64-age-identity", identity + b"\n")
|
|
self.receipt["teardown"]["custody_memfd_closed"] = False
|
|
finally:
|
|
wipe(identity)
|
|
done = subprocess.run(["age-keygen", "-y", f"/dev/fd/{self.identity_fd}"], pass_fds=(self.identity_fd,),
|
|
capture_output=True, timeout=min(10, self.remaining()))
|
|
require(done.returncode == 0 and done.stdout.decode().strip() == self.adm["snapshot"]["age_recipient"],
|
|
"custody_identity_refused")
|
|
|
|
def decrypt(self) -> None:
|
|
self.say("Step 2/7: verify ciphertext and decrypt into a sealed memfd.")
|
|
snap = self.adm["snapshot"]
|
|
raw = self.ciphertext.read_bytes()
|
|
require(len(raw) == snap["ciphertext_bytes"] and sha256_bytes(raw) == snap["ciphertext_sha256"],
|
|
"ciphertext_mismatch")
|
|
self.receipt["snapshot"]["ciphertext_verified"] = True
|
|
os.lseek(self.identity_fd, 0, os.SEEK_SET)
|
|
try:
|
|
done = subprocess.run(["age", "-d", "-i", f"/dev/fd/{self.identity_fd}"], input=raw,
|
|
pass_fds=(self.identity_fd,), capture_output=True,
|
|
timeout=min(300, self.remaining()))
|
|
except subprocess.TimeoutExpired as exc:
|
|
raise Refused("deadline_exceeded") from exc
|
|
require(done.returncode == 0 and done.stdout, "decrypt_failed")
|
|
plain = bytearray(done.stdout)
|
|
del done
|
|
try:
|
|
require(sha256_bytes(bytes(plain)) == snap["plaintext_sha256"], "plaintext_mismatch")
|
|
self.plaintext_fd = sealed_memfd("rpf64-snapshot", plain)
|
|
self.receipt["teardown"]["plaintext_memfd_closed"] = False
|
|
finally:
|
|
wipe(plain)
|
|
self.receipt["snapshot"]["plaintext_verified"] = True
|
|
self.close_identity()
|
|
|
|
def start_target(self) -> None:
|
|
self.say("Step 3/7: start networkless disposable target.")
|
|
image = self.adm["image"]
|
|
inspect = self.run([self.docker, "image", "inspect", image, "--format", "{{json .RepoDigests}}|{{.Id}}"],
|
|
reason="target_start_failed")
|
|
digests, image_id = inspect.stdout.decode().strip().split("|", 1)
|
|
require(image in json.loads(digests) and DIGEST_RE.fullmatch(image_id), "target_start_failed")
|
|
self.receipt["image_id"] = image_id
|
|
argv = [self.docker, "run", "--detach", "--pull=never", "--name", self.name, "--network", "none",
|
|
"--user", "100:1000", "--read-only", "--cap-drop", "ALL", "--security-opt", "no-new-privileges",
|
|
"--pids-limit", "256", "--memory", "1g", "--label", "railiance.restore.attempt=" + self.adm["attempt_id"]]
|
|
for target, options in TMPFS.items():
|
|
argv += ["--tmpfs", f"{target}:{options}"]
|
|
argv += ["-e", "BAO_LOCAL_CONFIG=" + SERVER_CONFIG, "-e", "BAO_ADDR=http://127.0.0.1:8200",
|
|
"-e", "SKIP_CHOWN=1", image, "server"]
|
|
self.tell_reaper("C " + self.name)
|
|
self.container_started = True
|
|
self.run(argv, reason="target_start_failed")
|
|
self.check_network()
|
|
for _ in range(60):
|
|
status = self.read("sys/seal-status", reason="target_start_failed")
|
|
if status and status.get("initialized") is False:
|
|
return
|
|
time.sleep(0.5)
|
|
self.remaining()
|
|
raise Refused("target_start_failed")
|
|
|
|
def check_network(self) -> None:
|
|
done = self.run([self.docker, "container", "inspect", self.name], reason="network_containment_failed")
|
|
info = json.loads(done.stdout)[0]
|
|
host, net = info["HostConfig"], info["NetworkSettings"]
|
|
record = self.receipt["network"]
|
|
record["network_mode"] = host.get("NetworkMode")
|
|
record["docker_networks"] = sorted((net.get("Networks") or {}).keys())
|
|
record["published_ports"] = sorted((net.get("Ports") or {}).keys())
|
|
record["mounts"] = sorted(m.get("Type", "?") for m in info.get("Mounts") or [])
|
|
require(record["network_mode"] == "none" and record["docker_networks"] == ["none"]
|
|
and not record["published_ports"] and not record["mounts"] and not host.get("PortBindings")
|
|
and not host.get("Binds") and host.get("Privileged") is False and host.get("ReadonlyRootfs") is True
|
|
and sorted((host.get("Tmpfs") or {}).keys()) == sorted(TMPFS), "network_containment_failed")
|
|
dev = self.run([self.docker, "exec", self.name, "cat", "/proc/net/dev"], reason="network_containment_failed")
|
|
names = sorted(line.split(":", 1)[0].strip() for line in dev.stdout.decode().splitlines()[2:] if ":" in line)
|
|
record["interfaces"] = names
|
|
require(names == ["lo"], "network_containment_failed")
|
|
probe = self.run([self.docker, "exec", self.name, "sh", "-c",
|
|
"wget -q -T 3 -O /dev/null http://192.0.2.1/ || exit 7; "
|
|
"wget -q -T 3 -O /dev/null https://kc.coulomb.social/ || exit 7; exit 0"],
|
|
check=False, limit=20, reason="network_containment_failed")
|
|
record["egress_refused"] = probe.returncode == 7
|
|
require(record["egress_refused"], "network_containment_failed")
|
|
|
|
def restore(self) -> None:
|
|
self.say("Step 4/7: throwaway init, then forced Raft restore from the sealed memfd.")
|
|
init = self.bao(["operator", "init", "-key-shares=1", "-key-threshold=1", "-format=json"],
|
|
reason="throwaway_init_failed")
|
|
throwaway = json.loads(init.stdout)
|
|
del init
|
|
unseal = self.write("sys/unseal", {"key": throwaway["unseal_keys_b64"][0]}, reason="throwaway_init_failed")
|
|
require(unseal.get("sealed") is False, "throwaway_init_failed")
|
|
self.wait_active("throwaway_init_failed")
|
|
throwaway_credential = throwaway["root_token"]
|
|
throwaway.clear()
|
|
script = 'IFS= read -r t; BAO_TOKEN="$t"; export BAO_TOKEN; exec bao operator raft snapshot restore -force /dev/stdin'
|
|
proc = subprocess.Popen([self.docker, "exec", "-i", self.name, "sh", "-c", script], stdin=subprocess.PIPE,
|
|
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
|
try:
|
|
proc.stdin.write(throwaway_credential.encode() + b"\n")
|
|
os.lseek(self.plaintext_fd, 0, os.SEEK_SET)
|
|
while chunk := os.read(self.plaintext_fd, 1 << 20):
|
|
proc.stdin.write(chunk)
|
|
proc.stdin.close()
|
|
code = proc.wait(timeout=min(600, self.remaining()))
|
|
except subprocess.TimeoutExpired as exc:
|
|
proc.kill()
|
|
raise Refused("deadline_exceeded" if self.remaining_or_zero() <= 1 else "restore_failed") from exc
|
|
except BrokenPipeError as exc:
|
|
raise Refused("restore_failed") from exc
|
|
require(code == 0, "restore_failed")
|
|
self.close_plaintext()
|
|
self.throwaway_credential = throwaway_credential
|
|
|
|
def unseal_original(self) -> None:
|
|
self.say("Step 5/7: unseal with two original shares (hidden prompts).")
|
|
sealed = None
|
|
for _ in range(60):
|
|
status = self.read("sys/seal-status", reason="unseal_refused")
|
|
sealed = bool(status and status.get("sealed"))
|
|
if sealed:
|
|
break
|
|
time.sleep(0.5)
|
|
self.remaining()
|
|
self.receipt["restored"]["sealed_after_restore"] = sealed
|
|
require(sealed, "unseal_refused")
|
|
for index in (1, 2):
|
|
share = self.prompt(f"Original unseal share {index} of 2")
|
|
if not SHARE_RE.fullmatch(share.decode("ascii", "replace")):
|
|
wipe(share)
|
|
raise Refused("unseal_refused")
|
|
self.shares.append(share)
|
|
result = self.write("sys/unseal", {"key": share.decode()}, reason="unseal_refused")
|
|
require(result.get("sealed") is False, "unseal_refused")
|
|
self.receipt["restored"]["unsealed_with_original_shares"] = True
|
|
leader = self.wait_active("leader_not_elected")
|
|
status = self.read("sys/seal-status", reason="unseal_refused") or {}
|
|
snap, restored = self.adm["snapshot"], self.receipt["restored"]
|
|
restored["cluster_id"] = status.get("cluster_id")
|
|
restored["applied_index"] = leader.get("raft_applied_index")
|
|
restored["cluster_id_match"] = restored["cluster_id"] == snap["source_cluster_id"]
|
|
restored["applied_index_not_behind"] = (isinstance(restored["applied_index"], int)
|
|
and restored["applied_index"] >= snap["source_applied_index"])
|
|
lookup = self.bao(["read", "-format=json", "auth/token/lookup-self"], credential=self.throwaway_credential,
|
|
check=False)
|
|
restored["throwaway_init_invalidated"] = lookup.returncode != 0
|
|
self.throwaway_credential = None
|
|
require(restored["throwaway_init_invalidated"], "restore_failed")
|
|
require(restored["cluster_id_match"], "cluster_mismatch")
|
|
require(restored["applied_index_not_behind"], "applied_index_behind")
|
|
|
|
def wait_active(self, reason: str) -> dict:
|
|
# Bounded, non-secret diagnostics: T07 attempt -01 timed out here with no
|
|
# trace. Only typed status fields and a sanitized first error line are kept.
|
|
started, polls, leader, seal, error = time.monotonic(), 0, {}, {}, None
|
|
wait = self.receipt["restored"]
|
|
try:
|
|
while time.monotonic() - started < LEADER_WAIT_SECONDS:
|
|
polls += 1
|
|
done = self.bao(["read", "-format=json", "sys/leader"], check=False, reason=reason)
|
|
if done.returncode == 0:
|
|
body = json.loads(done.stdout)
|
|
leader = body.get("data", body) if isinstance(body, dict) else {}
|
|
error = None
|
|
if leader.get("is_self") and isinstance(leader.get("raft_applied_index"), int) \
|
|
and leader["raft_applied_index"] > 0:
|
|
return leader
|
|
else:
|
|
error = sanitize_status_line(done.stderr)
|
|
time.sleep(1)
|
|
self.remaining()
|
|
status = self.bao(["read", "-format=json", "sys/seal-status"], check=False, reason=reason)
|
|
if status.returncode == 0:
|
|
body = json.loads(status.stdout)
|
|
seal = body.get("data", body) if isinstance(body, dict) else {}
|
|
raise Refused(reason)
|
|
finally:
|
|
wait["leader_wait"] = {
|
|
"waited_seconds": int(time.monotonic() - started), "polls": polls,
|
|
"elected": bool(leader.get("is_self")) and isinstance(leader.get("raft_applied_index"), int)
|
|
and leader.get("raft_applied_index", 0) > 0,
|
|
"last_leader": {k: typed(leader.get(k), t) for k, t in LEADER_FIELDS.items()},
|
|
"last_seal": {k: typed(seal.get(k), t) for k, t in SEAL_FIELDS.items()},
|
|
"last_error": error,
|
|
}
|
|
|
|
def generate_credential(self) -> None:
|
|
self.say("Step 6/7: generate an in-memory readback credential from the entered shares.")
|
|
attempt = self.write("sys/generate-root/attempt", {}, reason="readback_credential_failed")
|
|
otp, nonce, encoded = attempt.get("otp"), attempt.get("nonce"), None
|
|
require(otp and nonce, "readback_credential_failed")
|
|
for share in self.shares:
|
|
update = self.write("sys/generate-root/update", {"key": share.decode(), "nonce": nonce},
|
|
reason="readback_credential_failed")
|
|
encoded = update.get("encoded_token") or update.get("encoded_root_token") or encoded
|
|
require(encoded, "readback_credential_failed")
|
|
raw = base64.b64decode(encoded + "=" * (-len(encoded) % 4))
|
|
require(len(raw) == len(otp), "readback_credential_failed")
|
|
self.credential = bytes(a ^ b for a, b in zip(raw, otp.encode())).decode()
|
|
for share in self.shares:
|
|
wipe(share)
|
|
self.shares.clear()
|
|
|
|
def readback(self) -> list[dict[str, Any]]:
|
|
self.say("Step 7/7: metadata-only readback against the committed sources.")
|
|
rows = []
|
|
cache: dict[str, Any] = {}
|
|
for spec in OBJECTS:
|
|
row = {"object": spec["object"], "path": spec["path"], "present": False, "source_match": False,
|
|
"source_path": spec.get("source"), "observed_sha256": None}
|
|
if spec["kind"] == "mount":
|
|
if spec["path"] not in cache:
|
|
cache[spec["path"]] = self.read(spec["path"], self.credential) or {}
|
|
mount = cache[spec["path"]].get(spec["mount"])
|
|
if isinstance(mount, dict):
|
|
row["present"] = True
|
|
observed = {"type": mount.get("type"),
|
|
"kv_version": (mount.get("options") or {}).get("version")}
|
|
row["source_match"] = observed["type"] in spec["types"] and \
|
|
("kv_version" not in spec or observed["kv_version"] == spec["kv_version"])
|
|
row["observed_sha256"] = sha256_bytes(canonical(observed))
|
|
elif spec["kind"] == "source":
|
|
observed = self.read(spec["path"], self.credential)
|
|
if isinstance(observed, dict):
|
|
row["present"] = True
|
|
source = json.loads((self.root / spec["source"]).read_bytes())
|
|
row["source_match"], projected = project(source, observed, spec["ignore"])
|
|
row["observed_sha256"] = sha256_bytes(canonical(projected))
|
|
elif spec["kind"] == "policy":
|
|
observed = self.read(spec["path"], self.credential)
|
|
text = (observed or {}).get("policy")
|
|
if isinstance(text, str):
|
|
row["present"] = True
|
|
row["source_match"] = text.strip() == (self.root / spec["source"]).read_text().strip()
|
|
row["observed_sha256"] = sha256_bytes(text.strip().encode())
|
|
elif spec["kind"] == "kv_metadata":
|
|
meta = self.read(spec["path"], self.credential)
|
|
if isinstance(meta, dict):
|
|
current = meta.get("current_version")
|
|
version = (meta.get("versions") or {}).get(str(current)) or {}
|
|
observed = {"current_version": current, "versions": len(meta.get("versions") or {}),
|
|
"created_time": meta.get("created_time"), "updated_time": meta.get("updated_time"),
|
|
"current_deleted": bool(version.get("deletion_time")),
|
|
"current_destroyed": bool(version.get("destroyed"))}
|
|
row["present"] = isinstance(current, int) and current > 0
|
|
row["source_match"] = row["present"] and not observed["current_deleted"] \
|
|
and not observed["current_destroyed"]
|
|
row["observed_sha256"] = sha256_bytes(canonical(observed))
|
|
rows.append(row)
|
|
self.receipt["objects"] = rows
|
|
return rows
|
|
|
|
# ---- T10 break-glass drill steps (after restore + unseal)
|
|
def forget_unseal_shares(self) -> None:
|
|
"""The drill re-enters shares for generate-root; the unseal entries are not reused."""
|
|
for share in self.shares:
|
|
wipe(share)
|
|
self.shares.clear()
|
|
|
|
def drill_held_token(self) -> None:
|
|
self.say("Drill 1/5: held root-token check (hidden prompt; lookup-self only, never revoked).")
|
|
held = self.prompt("Held initial root token")
|
|
try:
|
|
require(HELD_TOKEN_RE.fullmatch(held.decode("ascii", "replace")), "held_token_refused")
|
|
done = self.bao(["read", "-format=json", "auth/token/lookup-self"], credential=held.decode(),
|
|
check=False, reason="held_token_refused")
|
|
finally:
|
|
wipe(held)
|
|
record = self.receipt["held_root_token"]
|
|
if done.returncode == 0:
|
|
body = json.loads(done.stdout)
|
|
data = body.get("data") if isinstance(body, dict) and isinstance(body.get("data"), dict) else {}
|
|
record.update(valid=True, policies=policy_names(data.get("policies")),
|
|
ttl_zero=type(data.get("ttl")) is int and data["ttl"] == 0,
|
|
expire_time_null="expire_time" in data and data["expire_time"] is None,
|
|
orphan=data["orphan"] if type(data.get("orphan")) is bool else None)
|
|
body.clear()
|
|
data = {}
|
|
elif http_code(done.stderr) == 403:
|
|
record.update(valid=False)
|
|
else:
|
|
raise Refused("held_token_refused")
|
|
|
|
def drill_generate_root(self) -> None:
|
|
self.say("Drill 2/5: fresh generate-root ceremony; re-enter two shares (hidden prompts).")
|
|
record = self.receipt["generate_root"]
|
|
status = self.read(GENERATE_ROOT_ATTEMPT, reason="generate_root_failed") or {}
|
|
require(status.get("started") is False, "generate_root_failed")
|
|
attempt = self.write(GENERATE_ROOT_ATTEMPT, {}, reason="generate_root_failed")
|
|
otp, nonce, encoded, update = attempt.get("otp"), attempt.get("nonce"), None, {}
|
|
attempt.clear()
|
|
require(isinstance(otp, str) and otp and isinstance(nonce, str) and nonce, "generate_root_failed")
|
|
record["started"] = True
|
|
for index in (1, 2):
|
|
share = self.prompt(f"Re-enter unseal share {index} of 2 for generate-root")
|
|
try:
|
|
require(SHARE_RE.fullmatch(share.decode("ascii", "replace")), "generate_root_failed")
|
|
update = self.write("sys/generate-root/update", {"key": share.decode(), "nonce": nonce},
|
|
reason="generate_root_failed")
|
|
finally:
|
|
wipe(share)
|
|
record["shares_entered"] = index
|
|
encoded = update.get("encoded_token") or update.get("encoded_root_token") or encoded
|
|
record["complete"] = update.get("complete") is True and bool(encoded)
|
|
require(record["complete"], "generate_root_failed")
|
|
try:
|
|
raw = base64.b64decode(encoded + "=" * (-len(encoded) % 4))
|
|
require(len(raw) == len(otp), "generate_root_failed")
|
|
self.credential = bytes(a ^ b for a, b in zip(raw, otp.encode())).decode("ascii")
|
|
except (ValueError, UnicodeDecodeError) as exc:
|
|
raise Refused("generate_root_failed") from exc
|
|
finally:
|
|
otp = nonce = encoded = None
|
|
update.clear()
|
|
record["otp_decode_ok"] = True
|
|
lookup = self.read("auth/token/lookup-self", self.credential, reason="generate_root_failed") or {}
|
|
record["policies"] = policy_names(lookup.get("policies"))
|
|
lookup.clear()
|
|
require(record["policies"] == ["root"], "generate_root_failed")
|
|
|
|
def drill_minimal_action(self) -> None:
|
|
self.say("Drill 3/5: one read-only action (sys/auth metadata).")
|
|
mounts = self.read(MINIMAL_ACTION_PATH, self.credential, reason="minimal_action_failed")
|
|
ok = isinstance(mounts, dict) and isinstance(mounts.get("token/"), dict)
|
|
self.receipt["generate_root"]["minimal_action_ok"] = ok
|
|
require(ok, "minimal_action_failed")
|
|
|
|
def drill_revoke(self) -> None:
|
|
self.say("Drill 4/5: revoke-self the generated token, then confirm lookup-self is refused.")
|
|
record = self.receipt["revocation"]
|
|
require("auth/token/revoke-self" in ALLOWED_WRITES, "internal_error")
|
|
done = self.bao(["write", "-format=json", "auth/token/revoke-self", "-"], credential=self.credential,
|
|
body=b"{}", check=False, reason="revocation_failed")
|
|
# The CLI prints nothing for 204 No Content; any response body means another status.
|
|
record["revoke_status"] = (204 if not done.stdout.strip() else 200) if done.returncode == 0 \
|
|
else http_code(done.stderr)
|
|
check = self.bao(["read", "-format=json", "auth/token/lookup-self"], credential=self.credential,
|
|
check=False, reason="revocation_failed")
|
|
record["post_revoke_lookup"] = 200 if check.returncode == 0 else http_code(check.stderr)
|
|
revoked = record["revoke_status"] == 204 and record["post_revoke_lookup"] == 403
|
|
self.receipt["teardown"]["readback_credential_revoked"] = revoked
|
|
require(revoked, "revocation_failed")
|
|
self.credential = None
|
|
|
|
def drill_cancel(self) -> None:
|
|
self.say("Drill 5/5: start one more generate-root attempt and cancel it.")
|
|
attempt = self.write(GENERATE_ROOT_ATTEMPT, {}, reason="cancel_drill_failed")
|
|
started = attempt.get("started") is True
|
|
attempt.clear()
|
|
self.delete(GENERATE_ROOT_ATTEMPT, reason="cancel_drill_failed")
|
|
status = self.read(GENERATE_ROOT_ATTEMPT, reason="cancel_drill_failed") or {}
|
|
ok = started and status.get("started") is False and status.get("progress") == 0
|
|
self.receipt["cancel_drill"]["started_then_cancelled"] = ok
|
|
require(ok, "cancel_drill_failed")
|
|
|
|
def revoke(self) -> None:
|
|
if not self.credential or not self.container_started:
|
|
return
|
|
try:
|
|
self.write("auth/token/revoke-self", {}, credential=self.credential)
|
|
check = self.bao(["read", "-format=json", "auth/token/lookup-self"], credential=self.credential,
|
|
check=False)
|
|
self.receipt["teardown"]["readback_credential_revoked"] = check.returncode != 0
|
|
except Exception:
|
|
self.receipt["teardown"]["readback_credential_revoked"] = False
|
|
finally:
|
|
self.credential = None
|
|
|
|
# ---- teardown
|
|
def close_identity(self) -> None:
|
|
if self.identity_fd is not None:
|
|
os.close(self.identity_fd)
|
|
self.identity_fd = None
|
|
self.receipt["teardown"]["custody_memfd_closed"] = True
|
|
|
|
def close_plaintext(self) -> None:
|
|
if self.plaintext_fd is not None:
|
|
os.close(self.plaintext_fd)
|
|
self.plaintext_fd = None
|
|
self.receipt["teardown"]["plaintext_memfd_closed"] = True
|
|
|
|
def teardown(self) -> None:
|
|
self.revoke()
|
|
self.close_identity()
|
|
self.close_plaintext()
|
|
for share in self.shares:
|
|
wipe(share)
|
|
self.shares.clear()
|
|
self.throwaway_credential = None
|
|
td = self.receipt["teardown"]
|
|
subprocess.run([self.docker, "rm", "-f", "-v", self.name], capture_output=True, timeout=60)
|
|
absent = subprocess.run([self.docker, "container", "inspect", self.name], capture_output=True, timeout=30)
|
|
td["container_removed"] = td["container_absent_verified"] = absent.returncode != 0
|
|
|
|
def write_receipt(self) -> None:
|
|
finished = now_utc()
|
|
self.receipt["finished_at"] = iso(finished)
|
|
self.receipt["elapsed_seconds"] = int((finished - self.started).total_seconds())
|
|
fd = os.open(self.receipt_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600)
|
|
with os.fdopen(fd, "w") as out:
|
|
json.dump(self.receipt, out, indent=2, sort_keys=True)
|
|
out.write("\n")
|
|
|
|
def execute(self) -> dict[str, Any]:
|
|
self.throwaway_credential = None
|
|
handlers = {}
|
|
|
|
def interrupted(signum, _frame):
|
|
raise Refused("operator_abort")
|
|
for sig in (signal.SIGINT, signal.SIGTERM, signal.SIGHUP):
|
|
handlers[sig] = signal.signal(sig, interrupted)
|
|
result, reason = "aborted", "internal_error"
|
|
try:
|
|
check_window(self.adm)
|
|
if self.start_reaper:
|
|
self.spawn_reaper()
|
|
self.custody_identity()
|
|
self.decrypt()
|
|
self.start_target()
|
|
self.restore()
|
|
self.unseal_original()
|
|
if self.mode == DRILL_MODE:
|
|
self.forget_unseal_shares()
|
|
self.drill_held_token()
|
|
self.drill_generate_root()
|
|
self.drill_minimal_action()
|
|
self.drill_revoke()
|
|
self.drill_cancel()
|
|
result, reason = "drilled", "break_glass_drill_complete"
|
|
else:
|
|
self.generate_credential()
|
|
rows = self.readback()
|
|
missing = [r for r in rows if not r["present"]]
|
|
mismatch = [r for r in rows if r["present"] and not r["source_match"]]
|
|
if missing:
|
|
result, reason = "refused", "object_missing"
|
|
elif mismatch:
|
|
result, reason = "refused", "object_mismatch"
|
|
else:
|
|
result, reason = "restored", "all_required_objects_match"
|
|
except Refused as exc:
|
|
result, reason = "aborted", exc.reason if exc.reason in ABORT_REASONS else "internal_error"
|
|
except Exception:
|
|
result, reason = "aborted", "internal_error"
|
|
finally:
|
|
try:
|
|
self.teardown()
|
|
finally:
|
|
for sig, handler in handlers.items():
|
|
signal.signal(sig, handler)
|
|
self.receipt["result"], self.receipt["reason"] = result, reason
|
|
self.write_receipt()
|
|
self.tell_reaper("DONE")
|
|
if self.reaper:
|
|
try:
|
|
self.reaper.stdin.close()
|
|
self.reaper.wait(timeout=10)
|
|
except Exception:
|
|
pass
|
|
self.say(f"Result: {result} ({reason}). Receipt written; verify it independently.")
|
|
return self.receipt
|
|
|
|
|
|
# --------------------------------------------------------------------------- reaper
|
|
|
|
def reaper_main() -> int:
|
|
"""Backstop: on parent loss or hard deadline remove containers, write ``aborted``."""
|
|
stream = sys.stdin.buffer
|
|
buffer = b""
|
|
while b"\n" not in buffer:
|
|
chunk = os.read(stream.fileno(), 65536)
|
|
if not chunk:
|
|
return 1
|
|
buffer += chunk
|
|
first, buffer = buffer.split(b"\n", 1)
|
|
header = json.loads(first)
|
|
parent, deadline = header["parent_pid"], header["deadline_epoch"]
|
|
try:
|
|
parent_fd = os.pidfd_open(parent)
|
|
except OSError:
|
|
parent_fd = None
|
|
names: list[str] = []
|
|
reason = "parent_lost"
|
|
while True:
|
|
while b"\n" in buffer:
|
|
line, buffer = buffer.split(b"\n", 1)
|
|
text = line.decode()
|
|
if text == "DONE":
|
|
return 0
|
|
if text.startswith("C ") and re.fullmatch(r"rpf64-restore-[a-f0-9]{12}", text[2:]):
|
|
names.append(text[2:])
|
|
wait = deadline - time.time()
|
|
if wait <= 0:
|
|
reason = "deadline_exceeded"
|
|
break
|
|
ready, _, _ = select.select([stream], [], [], min(wait, 5))
|
|
if not ready:
|
|
continue
|
|
chunk = os.read(stream.fileno(), 4096)
|
|
if not chunk:
|
|
break
|
|
buffer += chunk
|
|
if parent_fd is not None:
|
|
try:
|
|
signal.pidfd_send_signal(parent_fd, signal.SIGKILL)
|
|
except OSError:
|
|
pass
|
|
select.select([parent_fd], [], [], 10)
|
|
os.close(parent_fd)
|
|
removed = True
|
|
for name in names:
|
|
subprocess.run([header["docker"], "rm", "-f", "-v", name], capture_output=True, timeout=60)
|
|
absent = subprocess.run([header["docker"], "container", "inspect", name], capture_output=True, timeout=30)
|
|
removed = removed and absent.returncode != 0
|
|
receipt = header["base_receipt"]
|
|
finished = now_utc()
|
|
receipt.update({"result": "aborted", "reason": reason, "finished_at": iso(finished),
|
|
"elapsed_seconds": int((finished - parse_iso(receipt["started_at"])).total_seconds())})
|
|
receipt["teardown"].update({"container_removed": removed, "container_absent_verified": removed,
|
|
"plaintext_memfd_closed": True, "custody_memfd_closed": True,
|
|
"readback_credential_revoked": None, "performed_by": "reaper"})
|
|
try:
|
|
fd = os.open(header["receipt_path"], os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600)
|
|
except FileExistsError:
|
|
return 0
|
|
with os.fdopen(fd, "w") as out:
|
|
json.dump(receipt, out, indent=2, sort_keys=True)
|
|
out.write("\n")
|
|
return 0
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
import argparse
|
|
args = argv if argv is not None else sys.argv[1:]
|
|
if args == ["--reaper"]:
|
|
return reaper_main()
|
|
parser = argparse.ArgumentParser(description=__doc__.split("\n", 1)[0])
|
|
for name in ("admission", "admission-sig", "admission-sha256", "ciphertext", "receipt"):
|
|
parser.add_argument("--" + name, required=True)
|
|
parser.add_argument("--mode", choices=(PHASE_A, DRILL_MODE), default=PHASE_A)
|
|
opts = parser.parse_args(args)
|
|
os.umask(0o077)
|
|
try:
|
|
adm, _raw, signer, protected = load_admission(Path(opts.admission), Path(opts.admission_sig),
|
|
opts.admission_sha256, anchor_path(), opts.mode)
|
|
require(adm["material"] == "original", "admission_refused")
|
|
check_source_closure(adm, ROOT)
|
|
receipt = Path(opts.receipt)
|
|
require(receipt.is_absolute() and not receipt.exists() and receipt.parent.is_dir(), "preflight_refused")
|
|
except Refused as exc:
|
|
print(f"ERROR: {exc.reason}; no custody input requested.", file=sys.stderr)
|
|
return 1
|
|
session = Session(admission=adm, admission_sha256=opts.admission_sha256, signer=signer,
|
|
anchor_is_protected=protected, ciphertext=Path(opts.ciphertext), receipt_path=receipt,
|
|
mode=opts.mode)
|
|
result = session.execute()
|
|
verify = subprocess.run([sys.executable, str(ROOT / HERE / "verify-restore-receipt.py"), "--receipt",
|
|
str(receipt), "--source-root", str(ROOT)], capture_output=True, timeout=60)
|
|
print("Independent verifier: " + ("PASS" if verify.returncode == 0 else "FAIL"))
|
|
return 0 if result["result"] in ("restored", "refused", "drilled") and verify.returncode == 0 else 2
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|