diff --git a/Makefile b/Makefile index 6e97af9..dae5b50 100644 --- a/Makefile +++ b/Makefile @@ -585,6 +585,22 @@ audit-synthetic-sender-cleanup: ## Idempotent cleanup inside a contained login ( railiance01-reboot-preflight: ## Read-only gate for the coordinated single-node reboot python3 scripts/audit-core-recovery-preflight.py node-reboot $(RAILIANCE01_REBOOT_PREFLIGHT_ARGS) +.PHONY: openbao-reboot-snapshot openbao-reboot-snapshot-test +openbao-reboot-snapshot: ## Attended Raft snapshot -> age -> verify -> offsite; silent child writes SNAPSHOT_RECEIPT + @test -n "$(SNAPSHOT_SCENARIO_ID)" || { echo 'set SNAPSHOT_SCENARIO_ID=' >&2; exit 2; } + @test -n "$(SNAPSHOT_RECEIPT)" || { echo 'set SNAPSHOT_RECEIPT=/abs/new/file outside Git (the contained child must stay silent)' >&2; exit 2; } + @test ! -e "$(SNAPSHOT_RECEIPT)" || { echo 'SNAPSHOT_RECEIPT already exists' >&2; exit 2; } + @test -n "$(SNAPSHOT_CUSTODIAN_ESCROW_REF)" || { echo 'set SNAPSHOT_CUSTODIAN_ESCROW_REF=' >&2; exit 2; } + env -u OPENBAO_TOKEN -u VAULT_TOKEN -u BAO_TOKEN \ + $(ATTENDED_EXEC) scripts/openbao-reboot-snapshot.py \ + --scenario-id "$(SNAPSHOT_SCENARIO_ID)" --receipt-file "$(abspath $(SNAPSHOT_RECEIPT))" \ + $(if $(SNAPSHOT_PRIVATE_DIR),--private-dir "$(abspath $(SNAPSHOT_PRIVATE_DIR))") \ + --custodian-escrow-ref "$(SNAPSHOT_CUSTODIAN_ESCROW_REF)" \ + $(if $(SNAPSHOT_CUSTODIAN_RECIPIENT),--custodian-recipient "$(SNAPSHOT_CUSTODIAN_RECIPIENT)") + +openbao-reboot-snapshot-test: ## Offline unit tests for the attended snapshot lane (no OpenBao, no network) + PYTHONDONTWRITEBYTECODE=1 python3 -m unittest tests.test_openbao_reboot_snapshot + wp0024-owner-show: ## Show one owner's hash-bound review contract (set OWNER) @test -n "$(OWNER)" || { echo 'set OWNER=audit-core|rapp-postgres|railiance-cluster|railiance-infra' >&2; exit 2; } @python3 scripts/wp0024-owner-review.py show --owner "$(OWNER)" diff --git a/docs/railiance01-coordinated-reboot.md b/docs/railiance01-coordinated-reboot.md index c744410..599b9c2 100644 --- a/docs/railiance01-coordinated-reboot.md +++ b/docs/railiance01-coordinated-reboot.md @@ -129,6 +129,44 @@ State Hub. Pass that receipt to the preflight with `--openbao-snapshot-evidence /path/to/receipt.json`; a merely existing, stale, placeholder, wrong-cluster, unencrypted, or on-host receipt is denied. + + The reviewed lane is `make openbao-reboot-snapshot` (offline tests: + `make openbao-reboot-snapshot-test`). It runs + `scripts/openbao-reboot-snapshot.py` as the single silent child of the + contained `openbao-platform-admin-login` (via + `scripts/openbao-attended-exec.py`): + + ```bash + make openbao-reboot-snapshot \ + SNAPSHOT_SCENARIO_ID=RPF-WP-0015-T03- \ + SNAPSHOT_RECEIPT=/abs/new/receipt.json \ + SNAPSHOT_CUSTODIAN_ESCROW_REF= \ + SNAPSHOT_CUSTODIAN_RECIPIENT=age1... # only if that offline key differs from the offsite recipient + SNAPSHOT_PRIVATE_DIR=/abs/0700/dir # optional; default is the session HOME (prefer tmpfs) + ``` + + Recoverability requires a decryption key held outside OpenBao: the escrow + reference is mandatory and the preflight rejects a receipt without one. If the + custodian's offline key is the offsite recipient, the reference alone is + recorded; otherwise the snapshot is also encrypted to + `SNAPSHOT_CUSTODIAN_RECIPIENT` (age: any one recipient key decrypts). The + receipt lists all recipients. Reading the offsite `AGE_PRIVATE_KEY` for the + decrypt-stream proof is a founder decision (in memory only), recorded in + `reviews/RPF-WP-0015-T03-founder-go-20261014.json#escrow_key_read` and cited by + the receipt. Snapshots upload to the founder-created `openbao/` folder. + + In order it reads `bao status` (initialized, unsealed, Raft), saves the + snapshot through the tunnel into a fresh 0700 temp dir outside Git, re-reads + status (same `cluster_id`, index not behind), age-encrypts to the offsite + lane recipient declared in `lib/railiance-backup-common.sh` (plus the custodian + recipient, if given), proves the + lane's escrow key derives that recipient and decrypt-streams the ciphertext + to the plaintext sha256, then uploads ciphertext only to the governed + Nextcloud lane with `If-None-Match: *`. The temp dir is always shredded. + The receipt (0600, `encrypted_location_ref` + `offhost-custody::`) is self-checked with the preflight's own + validator before it is written; on any failure the file holds a value-safe + `status: failed` record instead, which the preflight rejects. 7. Have both Shamir share custodians present. Prove provider-console access and SSH access (same-day `RAIL01-CONSOLE-*` receipt). Record the exact restart operator, abort operator, and the railiance-infra independent abort owner. diff --git a/interfaces/RAILIANCE-WP-0024-owner-reviews.json b/interfaces/RAILIANCE-WP-0024-owner-reviews.json index 8cc9ba0..3706c8f 100644 --- a/interfaces/RAILIANCE-WP-0024-owner-reviews.json +++ b/interfaces/RAILIANCE-WP-0024-owner-reviews.json @@ -125,7 +125,8 @@ "docs/audit-core-database-lease-approval.example.json", "docs/openbao-reboot-snapshot-receipt.example.json", "scripts/audit_core_synthetic_sender_lane.py", - "scripts/t02-attended-window.py" + "scripts/t02-attended-window.py", + "scripts/openbao-reboot-snapshot.py" ], "assertions": [ "T02 remains gated on a named window, abort operator, exact owner receipts and a revision-pinned synthetic-load driver; repository access alone cannot revoke a lease.", @@ -144,10 +145,11 @@ "docs/audit-core-database-lease-approval.example.json": "87e57e3290074508c6517938acf8c7fc1a75d977b8ff630f247a4ff5d2ee45c6", "docs/audit-core-database-lease-recovery.md": "8dfd3fb59c551df0346d2569674227bb2f4e763c3bf058453035591073e20794", "docs/openbao-reboot-snapshot-receipt.example.json": "b95903a3e1aacb607eecc3f0440f08e62480d09da47bd7ab96c3eb8814f2eaee", - "docs/railiance01-coordinated-reboot.md": "6fd6301f808449ad6519764d994b05aee41a96be879b917ee1d18d8a65d5f6fd", + "docs/railiance01-coordinated-reboot.md": "75521830e945e719d9f0a80e42eb3d6ed1029e62ee7ffe5e157caf39391870b8", "scripts/audit-core-database-lease-recovery.py": "121ade87315606b11dc7366c11a0484dc73bdbeef3d5c59c2cb75a2de3292bc8", - "scripts/audit-core-recovery-preflight.py": "05e89ae9d0b9a636b67d9a8d0c203ac746c97208e32061cb23f1ca785eda8e5c", + "scripts/audit-core-recovery-preflight.py": "d870d7763fabb9d01f72c157f82419257fd1bf9c2705e1661a98022b1caa29ba", "scripts/audit_core_synthetic_sender_lane.py": "608145f43b5adea110c4c73a2f567370fcfb603ef73ca8230775de9bf7687fa6", - "scripts/t02-attended-window.py": "1faca370d14de9a55a3f04e55f6046105b5d3a606a4d79dadbe72c495ab75077" + "scripts/t02-attended-window.py": "1faca370d14de9a55a3f04e55f6046105b5d3a606a4d79dadbe72c495ab75077", + "scripts/openbao-reboot-snapshot.py": "c29b403cfe64685346b5d08cfe8cc9836d39c5ad17ff021e7cfa5c1c14e0a59b" } } diff --git a/reviews/RPF-WP-0015-T03-founder-go-20261014.json b/reviews/RPF-WP-0015-T03-founder-go-20261014.json new file mode 100644 index 0000000..db4f37f --- /dev/null +++ b/reviews/RPF-WP-0015-T03-founder-go-20261014.json @@ -0,0 +1,33 @@ +{ + "decision_id": "rpf-wp-0015-t03-founder-go-20261014-01", + "scenario_id": "RPF-WP-0015-T03-REBOOT-20261014-01", + "approved_window_id": "rpf-wp-0015-t03-window-20261014-01", + "task_id": "RAILIANCE-WP-0024-T03", + "decided_by_role": "founder", + "decided_in": "attended Claude Code session 2026-10-14 (answer: 'Today, I have 2 unseal holders')", + "recorded_at": "2026-10-14T17:24:57Z", + "authority": "D5 2fd5849c (outage exercise authorized 2026-10-11) + this per-window founder GO", + "abort_operator": "founder (platform-root), attended", + "independent_abort_owner": "railiance-infra (procedure receipt 0300d069)", + "owner_procedure_approvals_contract": "ea3f53d664f7357ccfefa31800467a82e19576e92af20783e811de24e49a2b7a", + "owner_procedure_approvals": { + "audit-core": "b3c564a6-93f9-4c07-872b-7feb22cccb73", + "rapp-postgres": "08a84caf-5cf9-4f90-8dba-98f9f43677e2", + "railiance-platform": "0f8f0692-1549-4070-8428-0d9284e4d8d5", + "railiance-cluster": "4bb0d7af-b644-44f3-acc8-69c7321e9824", + "railiance-infra": "0300d069-90ad-47fa-992d-ead7b686a720" + }, + "conditions": "Window proceeds only after the hold-point preflight reports ready_for_live_execution true and a final founder go/no-go. Consumed decisions 449a697a/85724c0c and scenario WARDEN-WP-0027-T02-DRILL-20260822-01 are not reused.", + "secret_values_observed": false, + "escrow_key_read": { + "decision": "Allow, in memory only", + "question": "Snapshot lane verifies the encrypted copy by decrypting it in memory with the offsite escrow key (AGE_PRIVATE_KEY at platform/workloads/railiance/backup/offsite-lane, normally restore-drills only). Allow that read during the attended snapshot?", + "decided_by_role": "founder", + "decided_at": "2026-10-14 (attended Claude Code session, before the snapshot lane review)", + "scope": "RPF-WP-0015-T03 attended snapshot lane only; key held in process memory and a shredded 0700 temp dir, never logged or written to Git/hub" + }, + "upload_folder": { + "decision": "railiance-backups/openbao/ created by the founder on the offsite share", + "decided_by_role": "founder" + } +} diff --git a/scripts/audit-core-recovery-preflight.py b/scripts/audit-core-recovery-preflight.py index 42e2296..ba752e0 100755 --- a/scripts/audit-core-recovery-preflight.py +++ b/scripts/audit-core-recovery-preflight.py @@ -169,6 +169,10 @@ def validate_snapshot_receipt( for key in required_true: if receipt.get(key) is not True: raise PreflightError(f"OpenBao snapshot receipt requires {key}=true") + # A snapshot whose only decryption key lives inside OpenBao cannot restore OpenBao. + escrow = receipt.get("custodian_escrow_ref") + if not isinstance(escrow, str) or not escrow.strip() or "replace" in escrow.lower(): + raise PreflightError("OpenBao snapshot receipt names no decryption key held outside OpenBao") if ( receipt["source_cluster"] != "railiance01" or receipt["source_namespace"] != "openbao" diff --git a/scripts/openbao-reboot-snapshot.py b/scripts/openbao-reboot-snapshot.py new file mode 100755 index 0000000..1589f49 --- /dev/null +++ b/scripts/openbao-reboot-snapshot.py @@ -0,0 +1,440 @@ +#!/usr/bin/env python3 +"""Attended OpenBao Raft snapshot lane for the railiance01 reboot hold point. + +RPF-WP-0015-T03, docs/railiance01-coordinated-reboot.md step 6. Runs only as +the single, silent child of the contained ops-warden login:: + + python3 scripts/openbao-attended-exec.py -- scripts/openbao-reboot-snapshot.py \ + --scenario-id --receipt-file /abs/new/file --private-dir /abs/0700/dir + +Order: read-only ``bao status`` -> ``bao operator raft snapshot save`` into a +fresh 0700 temp dir -> ``bao status`` again (same cluster, index not behind) -> +age-encrypt to the declared offsite-lane recipient -> decrypt-stream verify +against the plaintext sha256 with the lane's escrow key (held only in memory) +-> upload ciphertext with ``If-None-Match: *`` to the governed Nextcloud lane +-> shred the temp dir (always) -> self-validate and write the receipt. + +The process prints nothing: Warden fails closed on any child output. The only +output is the 0600 receipt file (or a value-safe failure record) at +``--receipt-file``. Only booleans, hashes, ids and status codes are recorded. +""" +from __future__ import annotations + +import argparse +import hashlib +import hmac +import importlib.util +import json +import os +import re +import shutil +import signal +import stat +import subprocess +import sys +import tempfile +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +ROOT = Path(__file__).resolve().parents[1] +SCRIPTS = ROOT / "scripts" +if str(SCRIPTS) not in sys.path: + sys.path.insert(0, str(SCRIPTS)) + +RECIPIENT_SOURCE = ROOT / "lib" / "railiance-backup-common.sh" +LANE_PATH = "platform/workloads/railiance/backup/offsite-lane" +# Established offsite-lane folder used by every existing filesdrop upload. +UPLOAD_PREFIX = "openbao" # founder-created railiance-backups/openbao/ (2026-10-14) +SOURCE = {"source_cluster": "railiance01", "source_namespace": "openbao", "source_pod": "openbao-0"} +OPERATOR = "railiance-platform" +SCENARIO_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9-]{2,95}$") +AGE_RECIPIENT_RE = re.compile(r"^age1[023456789acdefghjklmnpqrstuvwxyz]{58}$") +AGE_HEADER = b"age-encryption.org/v1\n" +GZIP_MAGIC = b"\x1f\x8b" +CHUNK = 1024 * 1024 + + +def _load(name: str, filename: str) -> Any: + spec = importlib.util.spec_from_file_location(name, SCRIPTS / filename) + if not spec or not spec.loader: + raise RuntimeError(f"cannot load {filename}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +# Reuse, do not copy: contained login env + safe_run from the lease harness, +# the reviewed receipt validator, and the governed Nextcloud request helper. +LEASE = _load("audit_core_database_lease_recovery", "audit-core-database-lease-recovery.py") +PREFLIGHT = LEASE.PREFLIGHT +import migrate_nextcloud_backup_account as NEXTCLOUD # noqa: E402 + + +class SnapshotError(RuntimeError): + pass + + +def fail(message: str) -> None: + raise SnapshotError(message) + + +def stamp(value: datetime) -> str: + return value.astimezone(UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z") + + +def compact(value: datetime) -> str: + return value.astimezone(UTC).strftime("%Y%m%dT%H%M%SZ") + + +def utc_now() -> datetime: + return datetime.now(UTC) + + +def declared_recipient(path: Path = RECIPIENT_SOURCE) -> str: + """The offsite lane's public recipient, as declared in the backup helper.""" + try: + text = path.read_text(encoding="utf-8") + except OSError: + fail("offsite recipient declaration unavailable") + found = re.findall(r'RAILIANCE_BACKUP_AGE_PUBLIC_KEY:=(age1[0-9a-z]+)\}', text) + if len(found) != 1 or not AGE_RECIPIENT_RE.fullmatch(found[0]): + fail("offsite recipient declaration is not exactly one age recipient") + return found[0] + + +def inside_git_tree(path: Path) -> bool: + for candidate in (path, *path.parents): + if (candidate / ".git").exists(): + return True + return False + + +def private_dir(value: str | None, environ: dict[str, str]) -> Path: + raw = value or environ.get("HOME", "") + path = Path(raw) + if not raw or not path.is_absolute(): + fail("private dir must be an absolute path") + try: + info = path.lstat() + except OSError: + fail("private dir is missing") + if not stat.S_ISDIR(info.st_mode) or info.st_uid != os.getuid() or info.st_mode & 0o077: + fail("private dir must be an owner-only (0700) directory") + if inside_git_tree(path.resolve()): + fail("private dir must not be inside a Git tree") + return path + + +def open_receipt(value: str) -> int: + path = Path(value) + if not path.is_absolute() or inside_git_tree(path.parent.resolve()): + raise SnapshotError("receipt file must be an absolute path outside Git") + return os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(CHUNK), b""): + digest.update(chunk) + return digest.hexdigest() + + +def shred(path: Path) -> None: + """Overwrite then unlink every regular file below ``path``; remove the tree.""" + if not path.exists(): + return + for item in sorted(path.rglob("*"), reverse=True): + try: + if item.is_file() and not item.is_symlink(): + size = item.stat().st_size + with item.open("r+b") as handle: + remaining = size + while remaining > 0: + step = min(CHUNK, remaining) + handle.write(b"\0" * step) + remaining -= step + handle.flush() + os.fsync(handle.fileno()) + item.unlink() + elif item.is_dir() and not item.is_symlink(): + item.rmdir() + else: + item.unlink() + except OSError: + pass + shutil.rmtree(path, ignore_errors=True) + + +class Lane: + """All side effects of one attended run; tests replace the boundaries.""" + + def __init__(self, env: dict[str, str]) -> None: + self.env = env + self.tool_env = {"PATH": env.get("PATH", "/usr/bin:/bin"), "HOME": env.get("HOME", "/")} + + def bao(self, args: list[str], *, label: str) -> str: + return LEASE.safe_run(["bao", *args], label=label, env=self.env).stdout + + def status(self) -> dict[str, Any]: + try: + status = json.loads(self.bao(["status", "-format=json"], label="read OpenBao status")) + return { + "initialized": status["initialized"] is True, + "unsealed": status["sealed"] is False, + "storage_type": status.get("storage_type"), + "cluster_id": str(status["cluster_id"]), + "raft_applied_index": int(status["raft_applied_index"]), + } + except (json.JSONDecodeError, KeyError, TypeError, ValueError): + fail("OpenBao status is incomplete") + + def snapshot(self, target: Path) -> None: + self.bao(["operator", "raft", "snapshot", "save", str(target)], label="save OpenBao Raft snapshot") + + def lane_field(self, field: str) -> str: + value = self.bao(["kv", "get", f"-field={field}", LANE_PATH], label=f"read offsite lane {field}") + value = value.strip() + if not value or "\n" in value or "\r" in value: + fail(f"offsite lane {field} is unusable") + return value + + def tool(self, argv: list[str], *, label: str, payload: bytes = b"") -> bytes: + try: + result = subprocess.run(argv, input=payload, capture_output=True, env=self.tool_env, timeout=1800) + except (OSError, subprocess.TimeoutExpired): + fail(f"{label} unavailable") + if result.returncode != 0: + fail(f"{label} failed (exit {result.returncode})") + return result.stdout + + def encrypt(self, recipients: list[str], source: Path, target: Path) -> None: + argv = ["age"] + for recipient in recipients: + argv += ["-r", recipient] + self.tool([*argv, "-o", str(target), str(source)], label="age encrypt") + + def recipient_of(self, identity: bytes) -> str: + return self.tool(["age-keygen", "-y", "/dev/stdin"], label="derive escrow recipient", payload=identity).decode().strip() + + def decrypt_digest(self, identity: bytes, ciphertext: Path) -> str: + """Stream-decrypt into a hash; plaintext never touches disk again.""" + digest = hashlib.sha256() + try: + proc = subprocess.Popen( + ["age", "-d", "-i", "/dev/stdin", str(ciphertext)], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, env=self.tool_env, + ) + except OSError: + fail("age decrypt unavailable") + try: + assert proc.stdin and proc.stdout + proc.stdin.write(identity) + proc.stdin.close() + for chunk in iter(lambda: proc.stdout.read(CHUNK), b""): + digest.update(chunk) + code = proc.wait(timeout=1800) + except (OSError, subprocess.TimeoutExpired): + proc.kill() + proc.wait() + fail("age decrypt failed") + finally: + if proc.stdout: + proc.stdout.close() + if code != 0: + fail(f"age decrypt failed (exit {code})") + return digest.hexdigest() + + def upload(self, url: str, token: str, body: bytes) -> int: + status, _ = NEXTCLOUD.request( + url, "PUT", body, (token, ""), + {"Content-Type": "application/octet-stream", "If-None-Match": "*", + "Content-Length": str(len(body))}, + ) + return status + + +def run( + args: argparse.Namespace, lane: Lane, receipt: dict[str, Any], now=utc_now +) -> tuple[dict[str, Any], dict[str, Any]]: + recipient = declared_recipient() + base = private_dir(args.private_dir, lane.env) + stage = receipt.setdefault("stages", []) + + stage.append("status_before") + before = lane.status() + if not (before["initialized"] and before["unsealed"] and before["storage_type"] == "raft"): + fail("OpenBao source is not an initialized, unsealed Raft node") + if before["raft_applied_index"] <= 0: + fail("OpenBao Raft applied index is not positive") + + created = now() + work = Path(tempfile.mkdtemp(prefix="openbao-reboot-snapshot-", dir=base)) + try: + os.chmod(work, 0o700) + plain = work / "openbao.snap" + cipher = work / "openbao.snap.age" + stage.append("snapshot_save") + lane.snapshot(plain) + if not plain.is_file() or plain.stat().st_size == 0: + fail("snapshot file was not written") + os.chmod(plain, 0o600) + with plain.open("rb") as handle: + if handle.read(2) != GZIP_MAGIC: + fail("snapshot is not a Raft snapshot archive") + + stage.append("status_after") + after = lane.status() + if after["cluster_id"] != before["cluster_id"] or after["raft_applied_index"] < before["raft_applied_index"]: + fail("OpenBao identity or Raft index changed incoherently during the snapshot") + if not (after["initialized"] and after["unsealed"]): + fail("OpenBao source sealed during the snapshot") + plain_sha = sha256_file(plain) + + stage.append("encrypt") + recipients = [recipient, *([args.custodian_recipient] if args.custodian_recipient else [])] + lane.encrypt(recipients, plain, cipher) + os.chmod(cipher, 0o600) + with cipher.open("rb") as handle: + if handle.read(len(AGE_HEADER)) != AGE_HEADER: + fail("ciphertext is not an age file") + cipher_sha = sha256_file(cipher) + if cipher_sha == plain_sha: + fail("ciphertext equals plaintext") + + stage.append("verify") + url = lane.lane_field("NC_WEBDAV_URL") + token = lane.lane_field("NC_WEBDAV_TOKEN") + identity = bytearray((lane.lane_field("AGE_PRIVATE_KEY") + "\n").encode()) + try: + escrow_recipient = lane.recipient_of(bytes(identity)) + if not hmac.compare_digest(escrow_recipient, recipient): + fail("escrow key does not match the declared offsite recipient") + decrypted_sha = lane.decrypt_digest(bytes(identity), cipher) + finally: + identity[:] = b"\0" * len(identity) + if not hmac.compare_digest(decrypted_sha, plain_sha): + fail("decrypted snapshot hash does not match the plaintext") + + stage.append("upload") + if url not in ( + f"{NEXTCLOUD.HOST}/public.php/dav/filesdrop/{token}", + f"{NEXTCLOUD.HOST}/public.php/dav/files/{token}", + ): + fail("offsite lane URL does not match the governed provider contract") + uploaded = now() + name = f"openbao-raft-snapshot-{args.scenario_id}-{compact(uploaded)}.snap.age" + status = lane.upload(f"{url}/{UPLOAD_PREFIX}/{name}", token, cipher.read_bytes()) + if status != 201: + fail("offsite upload did not create a new object") + # Only a 201 on a no-overwrite PUT names a location; the drop cannot read back. + receipt["upload_http_status"] = status + receipt["encrypted_location_ref"] = f"offhost-custody:{args.scenario_id}:{compact(uploaded)}" + finally: + stage.append("shred") + shred(work) + receipt["temp_dir_removed"] = not work.exists() + + if not receipt["temp_dir_removed"]: + fail("private temp dir could not be removed") + return { + "receipt_version": 1, + "receipt_id": f"{args.scenario_id}-SNAPSHOT-{compact(created)}", + "created_at": stamp(created), + "operator": OPERATOR, + **SOURCE, + "cluster_id": before["cluster_id"], + # Lower bound: the snapshot contains at least the pre-save index. + "raft_applied_index": before["raft_applied_index"], + "snapshot_created": True, + "source_initialized": True, + "source_unsealed": True, + "snapshot_sha256": f"sha256:{plain_sha}", + "snapshot_encrypted": True, + "encrypted_snapshot_sha256": f"sha256:{cipher_sha}", + "encrypted_location_ref": f"offhost-custody:{args.scenario_id}:{compact(uploaded)}", + "encrypted_copy_off_host": True, + "off_host_evidence": "upload acknowledged (HTTP 201, If-None-Match: *); remote hash not read back", + "upload_folder": UPLOAD_PREFIX, + "recipients": [recipient, *([args.custodian_recipient] if args.custodian_recipient else [])], + "custodian_escrow_ref": args.custodian_escrow_ref, + "escrow_key_read_decision": "reviews/RPF-WP-0015-T03-founder-go-20261014.json#escrow_key_read", + "encryption_verified": True, + "hash_verified": True, + "no_secret_material_recorded": True, + }, after + + +class QuietParser(argparse.ArgumentParser): + def error(self, message: str) -> None: # never print usage + raise SnapshotError("invalid arguments") + + +def parser() -> argparse.ArgumentParser: + p = QuietParser(description=__doc__, add_help=False) + p.add_argument("--scenario-id", required=True) + p.add_argument("--receipt-file", required=True) + p.add_argument("--private-dir") + p.add_argument("--remote", default="railiance01", choices=["railiance01"]) + p.add_argument("--custodian-recipient", help="extra age recipient whose key is held outside OpenBao") + p.add_argument("--custodian-escrow-ref", required=True, + help="value-safe reference to where a decryption key is held outside OpenBao") + return p + + +def main(argv: list[str] | None = None, *, lane_factory=Lane, now=utc_now) -> int: + try: + args = parser().parse_args(argv) + if not SCENARIO_RE.fullmatch(args.scenario_id): + return 2 + # Always required: where a decryption key is held outside OpenBao (alone, when the + # offline key is the offsite recipient; with --custodian-recipient otherwise). + if not (args.custodian_escrow_ref or "").strip() or ( + args.custodian_recipient and not AGE_RECIPIENT_RE.fullmatch(args.custodian_recipient) + ): + return 2 + fd = open_receipt(args.receipt_file) + except (SnapshotError, OSError): + return 2 + record: dict[str, Any] = {"status": "failed", "scenario_id": args.scenario_id, "error": "internal_error"} + result: dict[str, Any] | None = None + def _abort(signum, _frame): # warden timeout/abort: still run every finally/shred + raise SnapshotError(f"interrupted by signal {signum}") + + signal.signal(signal.SIGTERM, _abort) + signal.signal(signal.SIGHUP, _abort) + try: + os.umask(0o077) + env = LEASE.contained_bao_env() + receipt, live = run(args, lane_factory(env), record, now=now) + # Self-check with the exact preflight validator before claiming success. + check = Path(tempfile.mkdtemp(prefix="openbao-receipt-check-", dir=private_dir(args.private_dir, env))) + try: + probe = check / "receipt.json" + probe.write_text(json.dumps(receipt), encoding="utf-8") + PREFLIGHT.validate_snapshot_receipt(probe, live_openbao=live, now=now(), max_age_hours=1.0) + finally: + shutil.rmtree(check, ignore_errors=True) + result = receipt + except (SnapshotError, LEASE.ProcedureError, PREFLIGHT.PreflightError) as exc: + record["error"] = str(exc) + except Exception: # noqa: BLE001 - value-safe: never record exception text + record["error"] = "internal_error" + finally: + with os.fdopen(fd, "w", encoding="utf-8") as handle: + json.dump(result if result is not None else record, handle, indent=2, sort_keys=False) + handle.write("\n") + return 0 if result is not None else 1 + + +def silence() -> None: + devnull = os.open(os.devnull, os.O_WRONLY) + os.dup2(devnull, 1) + os.dup2(devnull, 2) + os.close(devnull) + + +if __name__ == "__main__": + silence() + raise SystemExit(main()) diff --git a/tests/test_audit_core_recovery_preflight.py b/tests/test_audit_core_recovery_preflight.py index 3b5779a..ea6a0eb 100644 --- a/tests/test_audit_core_recovery_preflight.py +++ b/tests/test_audit_core_recovery_preflight.py @@ -188,6 +188,7 @@ host: 50 bytes "encryption_verified": True, "hash_verified": True, "no_secret_material_recorded": True, + "custodian_escrow_ref": "founder-offline:test", } with tempfile.TemporaryDirectory() as directory: path = Path(directory) / "receipt.json" @@ -225,6 +226,7 @@ host: 50 bytes "encryption_verified": True, "hash_verified": True, "no_secret_material_recorded": True, + "custodian_escrow_ref": "founder-offline:test", } with tempfile.TemporaryDirectory() as directory: path = Path(directory) / "receipt.json" diff --git a/tests/test_openbao_reboot_snapshot.py b/tests/test_openbao_reboot_snapshot.py new file mode 100644 index 0000000..8dde645 --- /dev/null +++ b/tests/test_openbao_reboot_snapshot.py @@ -0,0 +1,316 @@ +"""Offline tests for scripts/openbao-reboot-snapshot.py (no OpenBao, no network). + +``bao`` and the Nextcloud upload are faked; ``age`` runs for real against a +throwaway key when installed (otherwise those cases are skipped). +""" +from __future__ import annotations + +import gzip +import importlib.util +import io +import json +import os +import shutil +import subprocess +import sys +import tempfile +import unittest +from contextlib import redirect_stderr, redirect_stdout +from datetime import UTC, datetime +from pathlib import Path +from unittest import mock + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / "scripts" / "openbao-reboot-snapshot.py" +SPEC = importlib.util.spec_from_file_location("openbao_reboot_snapshot", SCRIPT) +assert SPEC and SPEC.loader +snap = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(snap) + +HAVE_AGE = bool(shutil.which("age") and shutil.which("age-keygen")) +CLUSTER = "fd28df5d-98ec-57dd-42ec-9b3e4f4e53bf" +SCENARIO = "RPF-WP-0015-T03-REBOOT-20261014-01" +TOKEN = "tok3nValue" +URL = f"{snap.NEXTCLOUD.HOST}/public.php/dav/filesdrop/{TOKEN}" + + +def throwaway_key() -> tuple[str, str]: + out = subprocess.run(["age-keygen"], capture_output=True, text=True, check=True).stdout + identity = next(line for line in out.splitlines() if line.startswith("AGE-SECRET-KEY-1")) + recipient = subprocess.run( + ["age-keygen", "-y"], input=identity + "\n", capture_output=True, text=True, check=True + ).stdout.strip() + return identity, recipient + + +class FakeLane(snap.Lane): + """Replaces only the bao CLI and the HTTP boundary.""" + + def __init__(self, env, *, identity, status_seq=None, upload_status=201, url=URL, payload=None): + super().__init__(env) + self.calls: list[str] = [] + self.identity = identity + self.status_seq = list(status_seq or [self.status_json(100), self.status_json(105)]) + self.upload_status = upload_status + self.url = url + self.payload = payload if payload is not None else gzip.compress(b"raft-state" * 1000) + self.uploads: list[tuple[str, bytes]] = [] + self.snapshot_path: Path | None = None + + @staticmethod + def status_json(index, sealed=False, initialized=True, cluster=CLUSTER): + return json.dumps({"initialized": initialized, "sealed": sealed, "storage_type": "raft", + "cluster_id": cluster, "raft_applied_index": index, "t": 2, "n": 3}) + + def bao(self, args, *, label): + assert self.env.get("WARDEN_ROUTING_CATALOG") and "BAO_TOKEN" not in self.env + if args[:1] == ["status"]: + self.calls.append("status") + return self.status_seq.pop(0) + if args[:4] == ["operator", "raft", "snapshot", "save"]: + self.calls.append("snapshot") + self.snapshot_path = Path(args[4]) + assert not self.snapshot_path.exists() + self.snapshot_path.write_bytes(self.payload) + return "" + if args[:2] == ["kv", "get"]: + field = args[2].removeprefix("-field=") + assert args[3] == snap.LANE_PATH + self.calls.append(f"kv:{field}") + return {"NC_WEBDAV_URL": self.url, "NC_WEBDAV_TOKEN": TOKEN, + "AGE_PRIVATE_KEY": self.identity}[field] + "\n" + raise AssertionError(f"unexpected bao call {args}") + + def upload(self, url, token, body): + self.calls.append("upload") + self.uploads.append((url, body)) + return self.upload_status + + +@unittest.skipUnless(HAVE_AGE, "age/age-keygen not installed") +class SnapshotLaneTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.identity, cls.recipient = throwaway_key() + + def setUp(self): + self.tmp = Path(tempfile.mkdtemp(prefix="snap-test-")) + self.home = self.tmp / "home" + self.home.mkdir(mode=0o700) + self.receipt = self.tmp / "receipt.json" + self.env = {"WARDEN_ROUTING_CATALOG": "/catalog.yaml", "BAO_ADDR": "http://127.0.0.1:18200", + "HOME": str(self.home), "PATH": os.environ.get("PATH", "/usr/bin:/bin")} + self.lanes: list[FakeLane] = [] + patcher = mock.patch.object(snap, "declared_recipient", return_value=self.recipient) + patcher.start() + self.addCleanup(patcher.stop) + self.addCleanup(shutil.rmtree, self.tmp, True) + + def factory(self, **kwargs): + def make(env): + lane = FakeLane(env, identity=kwargs.pop("identity", self.identity), **kwargs) + self.lanes.append(lane) + return lane + return make + + def run_main(self, env=None, extra=(), **kwargs): + out, err = io.StringIO(), io.StringIO() + escrow = () if "--custodian-escrow-ref" in extra else ("--custodian-escrow-ref", "founder-offline:test") + argv = ["--scenario-id", SCENARIO, "--receipt-file", str(self.receipt), *escrow, *extra] + with mock.patch.dict(os.environ, env or self.env, clear=True), redirect_stdout(out), redirect_stderr(err): + code = snap.main(argv, lane_factory=self.factory(**kwargs)) + self.assertEqual(out.getvalue() + err.getvalue(), "", "child must stay silent") + return code, json.loads(self.receipt.read_text()) + + def assert_clean(self): + self.assertEqual(list(self.home.iterdir()), [], "private temp dir must be shredded") + + def test_happy_path_order_and_receipt_validates(self): + code, receipt = self.run_main() + self.assertEqual(code, 0) + lane = self.lanes[0] + self.assertEqual(lane.calls, ["status", "snapshot", "status", "kv:NC_WEBDAV_URL", + "kv:NC_WEBDAV_TOKEN", "kv:AGE_PRIVATE_KEY", "upload"]) + self.assertEqual(self.receipt.stat().st_mode & 0o777, 0o600) + live = {"cluster_id": CLUSTER, "raft_applied_index": 105} + checked = snap.PREFLIGHT.validate_snapshot_receipt( + self.receipt, live_openbao=live, now=datetime.now(UTC), max_age_hours=24) + self.assertTrue(checked["verified"]) + self.assertEqual(receipt["raft_applied_index"], 100) + self.assertTrue(receipt["encrypted_location_ref"].startswith(f"offhost-custody:{SCENARIO}:")) + url, body = lane.uploads[0] + self.assertTrue(url.startswith(f"{URL}/openbao/openbao-raft-snapshot-{SCENARIO}-")) + self.assertTrue(body.startswith(snap.AGE_HEADER)) + self.assertNotIn(lane.payload, body) + text = self.receipt.read_text() + for value in (self.identity, TOKEN, "AGE-SECRET-KEY"): + self.assertNotIn(value, text) + self.assertFalse(lane.snapshot_path.exists()) + self.assert_clean() + + def test_upload_conflict_fails_and_cleans_up(self): + code, record = self.run_main(upload_status=412) + self.assertEqual(code, 1) + self.assertEqual(record["status"], "failed") + self.assertIn("did not create", record["error"]) + self.assertTrue(record["temp_dir_removed"]) + self.assertNotIn("receipt_version", record) + self.assertNotIn("encrypted_location_ref", record) # no location without a 201 + self.assertNotIn("upload_http_status", record) + self.assert_clean() + + def test_custodian_recipient_alone_decrypts_upload(self): + identity, recipient = throwaway_key() + code, receipt = self.run_main( + extra=("--custodian-recipient", recipient, "--custodian-escrow-ref", "founder-offline:test")) + self.assertEqual(code, 0) + self.assertEqual(receipt["recipients"][1], recipient) + self.assertEqual(receipt["custodian_escrow_ref"], "founder-offline:test") + self.assertEqual(receipt["upload_folder"], "openbao") + _, body = self.lanes[0].uploads[0] + with tempfile.TemporaryDirectory() as tmp: + key, ct = Path(tmp) / "k", Path(tmp) / "c" + key.write_bytes(identity if isinstance(identity, bytes) else identity.encode()) + ct.write_bytes(body) + out = subprocess.run(["age", "-d", "-i", str(key), str(ct)], capture_output=True, check=False) + self.assertEqual(out.returncode, 0) + self.assertEqual(out.stdout, self.lanes[0].payload) + + def test_escrow_ref_alone_suffices_and_is_cited(self): + code, receipt = self.run_main() + self.assertEqual(code, 0) + self.assertEqual(len(receipt["recipients"]), 1) + self.assertEqual(receipt["custodian_escrow_ref"], "founder-offline:test") + self.assertIn("escrow_key_read", receipt["escrow_key_read_decision"]) + + def test_preflight_rejects_receipt_without_escrow_ref(self): + code, receipt = self.run_main() + self.assertEqual(code, 0) + receipt["custodian_escrow_ref"] = None + bad = self.home / "bad.json" + bad.write_text(json.dumps(receipt)) + with self.assertRaises(snap.PREFLIGHT.PreflightError): + snap.PREFLIGHT.validate_snapshot_receipt( + bad, live_openbao={"cluster_id": CLUSTER, "raft_applied_index": 105}, + now=datetime.now(UTC), max_age_hours=24) + bad.unlink() + + def test_custodian_recipient_requires_escrow_ref(self): + _, recipient = throwaway_key() + argv = ["--scenario-id", SCENARIO, "--receipt-file", str(self.receipt), "--custodian-recipient", recipient] + with mock.patch.dict(os.environ, self.env, clear=True): + self.assertEqual(snap.main(argv, lane_factory=self.factory()), 2) + self.assertFalse(self.receipt.exists()) + + def test_escrow_key_mismatch_never_uploads(self): + other, _ = throwaway_key() + code, record = self.run_main(identity=other) + self.assertEqual(code, 1) + self.assertIn("escrow key does not match", record["error"]) + self.assertNotIn("upload", self.lanes[0].calls) + self.assertNotIn(other, json.dumps(record)) + self.assert_clean() + + def test_decrypt_hash_mismatch_never_uploads(self): + with mock.patch.object(snap.Lane, "decrypt_digest", return_value="0" * 64): + code, record = self.run_main() + self.assertEqual(code, 1) + self.assertIn("decrypted snapshot hash", record["error"]) + self.assertNotIn("upload", self.lanes[0].calls) + self.assert_clean() + + def test_sealed_source_never_snapshots(self): + code, record = self.run_main(status_seq=[FakeLane.status_json(100, sealed=True)]) + self.assertEqual(code, 1) + self.assertEqual(self.lanes[0].calls, ["status"]) + self.assert_clean() + + def test_cluster_change_during_snapshot_fails(self): + seq = [FakeLane.status_json(100), FakeLane.status_json(105, cluster="other")] + code, record = self.run_main(status_seq=seq) + self.assertEqual(code, 1) + self.assertNotIn("kv:AGE_PRIVATE_KEY", self.lanes[0].calls) + self.assert_clean() + + def test_non_snapshot_payload_refused(self): + code, record = self.run_main(payload=b"plain text, not a raft archive") + self.assertEqual(code, 1) + self.assertIn("not a Raft snapshot", record["error"]) + self.assert_clean() + + def test_unexpected_provider_url_never_uploads(self): + code, record = self.run_main(url="https://elsewhere.example/dav") + self.assertEqual(code, 1) + self.assertNotIn("upload", self.lanes[0].calls) + self.assert_clean() + + def test_internal_error_is_value_safe(self): + with mock.patch.object(snap, "sha256_file", side_effect=ValueError(TOKEN)): + code, record = self.run_main() + self.assertEqual(record["error"], "internal_error") + self.assertNotIn(TOKEN, json.dumps(record)) + self.assert_clean() + + def test_refused_outside_contained_login(self): + for env in ({"HOME": str(self.home)}, + {**self.env, "BAO_TOKEN": "x"}, + {**self.env, "BAO_ADDR": "https://openbao.example"}): + self.receipt.unlink(missing_ok=True) + self.lanes.clear() + code, record = self.run_main(env=env) + self.assertEqual(code, 1) + self.assertEqual(record["status"], "failed") + self.assertEqual(self.lanes, [], "no bao call may happen outside containment") + + def test_private_dir_must_be_owner_only_and_outside_git(self): + loose = self.tmp / "loose" + loose.mkdir(mode=0o755) + loose.chmod(0o755) + code, record = self.run_main(extra=["--private-dir", str(loose)]) + self.assertIn("owner-only", record["error"]) + self.receipt.unlink() + code, record = self.run_main(extra=["--private-dir", str(ROOT / "scripts")]) + self.assertEqual(code, 1) + self.assertEqual(self.lanes[-1].calls if self.lanes else [], []) + + def test_existing_receipt_or_bad_scenario_refused(self): + self.receipt.write_text("keep") + with mock.patch.dict(os.environ, self.env, clear=True): + self.assertEqual(snap.main(["--scenario-id", SCENARIO, "--receipt-file", str(self.receipt)]), 2) + self.assertEqual(snap.main(["--scenario-id", "../x", "--receipt-file", str(self.tmp / "n.json")]), 2) + self.assertEqual(snap.main(["--bogus"]), 2) + self.assertEqual(self.receipt.read_text(), "keep") + + +class BoundaryTest(unittest.TestCase): + def test_declared_recipient_comes_from_offsite_lane_helper(self): + recipient = snap.declared_recipient() + self.assertRegex(recipient, snap.AGE_RECIPIENT_RE) + self.assertIn(recipient, (ROOT / "lib" / "railiance-backup-common.sh").read_text()) + + def test_upload_uses_no_overwrite_governed_request(self): + lane = snap.Lane({"PATH": "/usr/bin"}) + with mock.patch.object(snap.NEXTCLOUD, "request", return_value=(201, b"")) as request: + self.assertEqual(lane.upload(URL + "/forgejo/x.age", TOKEN, b"cipher"), 201) + url, method, body, auth, headers = request.call_args.args + self.assertEqual((method, body, auth), ("PUT", b"cipher", (TOKEN, ""))) + self.assertEqual(headers["If-None-Match"], "*") + + def test_script_is_silent_and_refuses_as_process_outside_login(self): + with tempfile.TemporaryDirectory() as tmp: + receipt = Path(tmp) / "r.json" + env = {k: v for k, v in os.environ.items() + if k not in ("WARDEN_ROUTING_CATALOG", "BAO_TOKEN", "VAULT_TOKEN", "OPENBAO_TOKEN")} + env["HOME"] = tmp + result = subprocess.run( + [sys.executable, str(SCRIPT), "--scenario-id", SCENARIO, "--receipt-file", str(receipt), "--custodian-escrow-ref", "founder-offline:test"], + capture_output=True, env=env, timeout=60) + self.assertEqual((result.stdout, result.stderr), (b"", b"")) + self.assertNotEqual(result.returncode, 0) + self.assertIn("contained attended login", json.loads(receipt.read_text())["error"]) + bad = subprocess.run([sys.executable, str(SCRIPT), "--help"], capture_output=True, timeout=60) + self.assertEqual((bad.stdout, bad.stderr, bad.returncode), (b"", b"", 2)) + + +if __name__ == "__main__": + unittest.main()