diff --git a/Makefile b/Makefile index 140e8a4..8fe6aca 100644 --- a/Makefile +++ b/Makefile @@ -305,6 +305,23 @@ openbao-validate-emergency-evidence: ## Validate non-secret OpenBao emergency se OPENBAO_EMERGENCY_EVIDENCE='$(OPENBAO_EMERGENCY_EVIDENCE)' \ scripts/openbao-validate-emergency-drill-evidence.sh +OPENBAO_REHEARSAL_IMAGE ?= quay.io/openbao/openbao:2.5.4 + +openbao-isolated-restore-test: ## RPF-WP-0064 offline tests: admission, v2 receipt verifier, launcher (no Docker) + PYTHONDONTWRITEBYTECODE=1 python3 -m unittest tests.test_openbao_isolated_restore + +openbao-isolated-restore-image: ## Pull the public OpenBao image used by the synthetic rehearsal (no custody) + docker pull -q $(OPENBAO_REHEARSAL_IMAGE) + +openbao-isolated-restore-rehearsal: ## RPF-WP-0064-T04 native rehearsal: --network none containers, generated material only + PYTHONDONTWRITEBYTECODE=1 python3 -m unittest -v tests.test_openbao_isolated_restore_native + +openbao-verify-restore-receipt: ## Verify a v2 isolated-restore receipt (RECEIPT=/abs/path SOURCE_ROOT=clean checkout [NOT_AFTER=]) + python3 scripts/openbao-isolated-restore/verify-restore-receipt.py --receipt '$(RECEIPT)' \ + --source-root '$(or $(SOURCE_ROOT),$(CURDIR))' $(if $(NOT_AFTER),--not-after '$(NOT_AFTER)') + +.PHONY: openbao-isolated-restore-test openbao-isolated-restore-image openbao-isolated-restore-rehearsal openbao-verify-restore-receipt + ##@ Credential broker credential-grants-validate: ## Validate non-secret credential grant catalog diff --git a/WORK-RECORDS.md b/WORK-RECORDS.md index 38d4cc0..352947d 100644 --- a/WORK-RECORDS.md +++ b/WORK-RECORDS.md @@ -43,7 +43,7 @@ | workplan | RPF-WP-0061 | active | — | workplans/RPF-WP-0061-self-service-ccr-intake.md | | workplan | RPF-WP-0062 | finished | — | workplans/RPF-WP-0062-custodian-agent-observer-identity.md | | workplan | RPF-WP-0063 | finished | — | workplans/RPF-WP-0063-wave2-resource-right-sizing.md | -| workplan | RPF-WP-0064 | proposed | — | workplans/RPF-WP-0064-isolated-original-openbao-restore.md | +| workplan | RPF-WP-0064 | active | — | workplans/RPF-WP-0064-isolated-original-openbao-restore.md | | task | RPF-WP-ADHOC-2026-10-10-T01 | done | — | workplans/ADHOC-2026-10-10.md | | task | RPF-WP-ADHOC-2026-10-11-T01 | done | — | workplans/ADHOC-2026-10-11.md | | task | RPF-WP-ADHOC-2026-10-11-T02 | done | — | workplans/ADHOC-2026-10-11.md | @@ -194,9 +194,9 @@ | task | RPF-WP-0062-T03 | done | — | workplans/RPF-WP-0062-custodian-agent-observer-identity.md | | task | RPF-WP-0063-T01 | done | — | workplans/RPF-WP-0063-wave2-resource-right-sizing.md | | task | RPF-WP-0064-T01 | done | — | workplans/RPF-WP-0064-isolated-original-openbao-restore.md | -| task | RPF-WP-0064-T02 | todo | — | workplans/RPF-WP-0064-isolated-original-openbao-restore.md | -| task | RPF-WP-0064-T03 | todo | — | workplans/RPF-WP-0064-isolated-original-openbao-restore.md | -| task | RPF-WP-0064-T04 | todo | — | workplans/RPF-WP-0064-isolated-original-openbao-restore.md | +| task | RPF-WP-0064-T02 | done | — | workplans/RPF-WP-0064-isolated-original-openbao-restore.md | +| task | RPF-WP-0064-T03 | done | — | workplans/RPF-WP-0064-isolated-original-openbao-restore.md | +| task | RPF-WP-0064-T04 | done | — | workplans/RPF-WP-0064-isolated-original-openbao-restore.md | | task | RPF-WP-0064-T05 | wait | — | workplans/RPF-WP-0064-isolated-original-openbao-restore.md | | task | RPF-WP-0064-T06 | wait | — | workplans/RPF-WP-0064-isolated-original-openbao-restore.md | | task | RPF-WP-0064-T07 | wait | — | workplans/RPF-WP-0064-isolated-original-openbao-restore.md | diff --git a/tests/test_openbao_isolated_restore_native.py b/tests/test_openbao_isolated_restore_native.py new file mode 100644 index 0000000..d34d18a --- /dev/null +++ b/tests/test_openbao_isolated_restore_native.py @@ -0,0 +1,192 @@ +"""RPF-WP-0064-T04 native rehearsal: real OpenBao containers, generated material only. + +Runs the pinned OpenBao image with ``--network none``. Skips (and says so) when +Docker or the image is unavailable; a skip is not native coverage. The only +mocked element is the network-interface abort, where a wrapper reports an +extra network in ``docker container inspect`` because attaching a real network +is out of bounds for this rehearsal. +""" +from __future__ import annotations + +import base64 +import json +import os +import signal +import stat +import subprocess +import sys +import time +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import openbao_restore_rehearsal as R # noqa: E402 + +OK, IMAGE = R.docker_status() + + +@unittest.skipUnless(OK, f"native rehearsal skipped: {IMAGE}") +class NativeRehearsal(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.h = R.Rehearsal(IMAGE) + cls.root, cls.commit = R.make_source_root(cls.h.work) + cls.full = cls.h.snapshot_set() + cls.partial = cls.h.snapshot_set(kv_registration=False, jwt_role=False) + + @classmethod + def tearDownClass(cls): + cls.h.close() + + def tearDown(self): + self.assertEqual([n for n in R.leftover_containers() if n.startswith("rpf64-restore-")], []) + + def run_session(self, snap=None, answers=None, root=None, commit=None, adm_override=None, **kw): + snap = snap or self.full + root, commit = root or self.root, commit or self.commit + adm = self.h.admission(snap, root, commit, **(adm_override or {})) + if answers is None: + answers = [self.h.age_identity, snap["shares"][0], snap["shares"][2]] + session, receipt = self.h.session(adm, snap, root, answers, **kw) + return session, receipt + + def finish(self, session, receipt, root=None, expect="aborted"): + out = session.execute() + self.assertEqual(out["result"], expect, out["reason"]) + self.assertEqual(stat.S_IMODE(receipt.stat().st_mode), 0o600) + text = receipt.read_text() + for value in self.h.secrets_seen: + if value: + self.assertNotIn(value, text) + verified = R.Rehearsal.verify(receipt, root or self.root, "--expect-result", expect) + self.assertEqual(verified.returncode, 0, verified.stdout) + return out + + def test_restored_end_to_end(self): + session, receipt = self.run_session() + out = self.finish(session, receipt, expect="restored") + self.assertEqual(out["reason"], "all_required_objects_match") + self.assertTrue(all(r["present"] and r["source_match"] for r in out["objects"])) + self.assertEqual(out["restored"]["cluster_id"], self.full["cluster_id"]) + self.assertEqual(out["network"]["interfaces"], ["lo"]) + self.assertTrue(out["network"]["egress_refused"]) + self.assertTrue(out["teardown"]["readback_credential_revoked"]) + self.assertTrue(out["restored"]["throwaway_init_invalidated"]) + late = R.Rehearsal.verify(receipt, self.root, "--not-after", "2026-01-01T00:00:00Z") + self.assertNotEqual(late.returncode, 0) + # Independent source check: a checkout with one changed source refuses. + other, _ = R.make_source_root(self.h.work, {"openbao/auth/secrets-engine-jwt-role.json": b"{}\n"}) + self.assertNotEqual(R.Rehearsal.verify(receipt, other).returncode, 0) + + def test_wrong_age_identity_aborts_before_target(self): + other, _ = R.Rehearsal.age_pair() + session, receipt = self.run_session(answers=[other]) + out = self.finish(session, receipt) + self.assertEqual(out["reason"], "custody_identity_refused") + self.assertEqual(out["network"]["docker_networks"], []) + + def test_wrong_ciphertext_hash_aborts(self): + session, receipt = self.run_session(adm_override={"ciphertext_sha256": R.adapter.sha256_bytes(b"other")}) + self.assertEqual(self.finish(session, receipt)["reason"], "ciphertext_mismatch") + + def test_wrong_plaintext_hash_aborts(self): + session, receipt = self.run_session(adm_override={"plaintext_sha256": R.adapter.sha256_bytes(b"other")}) + self.assertEqual(self.finish(session, receipt)["reason"], "plaintext_mismatch") + + def test_wrong_share_aborts_and_removes_target(self): + wrong = base64.b64encode(os.urandom(33)).decode() + session, receipt = self.run_session(answers=[self.h.age_identity, self.full["shares"][0], wrong]) + out = self.finish(session, receipt) + self.assertEqual(out["reason"], "unseal_refused") + self.assertTrue(out["restored"]["sealed_after_restore"]) + self.assertTrue(out["teardown"]["container_absent_verified"]) + + def test_missing_objects_refused_not_repaired(self): + session, receipt = self.run_session(snap=self.partial) + out = self.finish(session, receipt, expect="refused") + self.assertEqual(out["reason"], "object_missing") + missing = sorted(r["object"] for r in out["objects"] if not r["present"]) + self.assertEqual(missing, ["kv_registration_metadata", "secrets_engine_jwt_role"]) + + def test_source_mismatch_refused(self): + role = json.loads((R.ROOT / "openbao/auth/secrets-engine-jwt-role.json").read_bytes()) + role["token_ttl"] = "10m" + root, commit = R.make_source_root(self.h.work, { + "openbao/auth/secrets-engine-jwt-role.json": json.dumps(role, indent=2).encode()}) + session, receipt = self.run_session(root=root, commit=commit) + out = self.finish(session, receipt, root=root, expect="refused") + self.assertEqual(out["reason"], "object_mismatch") + self.assertEqual([r["object"] for r in out["objects"] if not r["source_match"]], ["secrets_engine_jwt_role"]) + + def test_network_interface_present_aborts(self): + # Mocked element: real container, inspect output reports an extra network. + wrapper = self.h.work / "docker-extra-network" + wrapper.write_text( + "#!/usr/bin/env python3\nimport json,subprocess,sys\n" + "a=sys.argv[1:]\nr=subprocess.run(['docker',*a],capture_output=True)\n" + "out=r.stdout\n" + "if a[:2]==['container','inspect'] and r.returncode==0:\n" + " d=json.loads(out);d[0]['NetworkSettings']['Networks']['bridge']={};out=json.dumps(d).encode()\n" + "sys.stdout.buffer.write(out);sys.stderr.buffer.write(r.stderr);sys.exit(r.returncode)\n") + wrapper.chmod(0o700) + session, receipt = self.run_session(docker=str(wrapper)) + out = session.execute() + self.assertEqual((out["result"], out["reason"]), ("aborted", "network_containment_failed")) + self.assertEqual(out["network"]["docker_networks"], ["bridge", "none"]) + self.assertTrue(out["teardown"]["container_absent_verified"]) + # The verifier refuses a receipt claiming any non-"none" network. + self.assertNotEqual(R.Rehearsal.verify(receipt, self.root).returncode, 0) + + def test_timeout_aborts(self): + holder = {} + + def slow_share(): + holder["s"].deadline = time.monotonic() - 1 + return self.full["shares"][0] + session, receipt = self.run_session(answers=[self.h.age_identity, slow_share, self.full["shares"][2]]) + holder["s"] = session + out = self.finish(session, receipt) + self.assertEqual(out["reason"], "deadline_exceeded") + + def test_operator_abort_signal(self): + def interrupt(): + os.kill(os.getpid(), signal.SIGINT) + time.sleep(1) + return self.full["shares"][0] + session, receipt = self.run_session(answers=[self.h.age_identity, interrupt]) + out = self.finish(session, receipt) + self.assertEqual(out["reason"], "operator_abort") + self.assertTrue(out["teardown"]["container_absent_verified"]) + + def reaper_case(self, *, kill_parent: bool, deadline_in: float) -> dict: + session, receipt = self.run_session() + parent = subprocess.Popen(["sleep", "300"]) + R.sh(["docker", "run", "--detach", "--pull=never", "--name", session.name, "--network", "none", + "--entrypoint", "sleep", IMAGE, "300"]).check_returncode() + reaper = subprocess.Popen([sys.executable, str(R.ROOT / "scripts/openbao-isolated-restore/restore_adapter.py"), + "--reaper"], stdin=subprocess.PIPE, start_new_session=True) + header = {"parent_pid": parent.pid, "deadline_epoch": time.time() + deadline_in, "docker": "docker", + "receipt_path": str(receipt), "base_receipt": session.receipt} + reaper.stdin.write((json.dumps(header) + "\nC " + session.name + "\n").encode()) + reaper.stdin.flush() + if kill_parent: + parent.kill() + reaper.stdin.close() # parent death closes the pipe + self.assertEqual(reaper.wait(timeout=120), 0) + if not reaper.stdin.closed: + reaper.stdin.close() + parent.wait(timeout=20) + out = json.loads(receipt.read_text()) + self.assertEqual(out["teardown"]["performed_by"], "reaper") + self.assertEqual(R.Rehearsal.verify(receipt, self.root, "--expect-result", "aborted").returncode, 0) + return out + + def test_parent_loss_reaper_cleans_up(self): + self.assertEqual(self.reaper_case(kill_parent=True, deadline_in=600)["reason"], "parent_lost") + + def test_hard_deadline_reaper_kills_session(self): + self.assertEqual(self.reaper_case(kill_parent=False, deadline_in=3)["reason"], "deadline_exceeded") + + +if __name__ == "__main__": + unittest.main() diff --git a/workplans/RPF-WP-0064-isolated-original-openbao-restore.md b/workplans/RPF-WP-0064-isolated-original-openbao-restore.md index 463783e..bfd4bf7 100644 --- a/workplans/RPF-WP-0064-isolated-original-openbao-restore.md +++ b/workplans/RPF-WP-0064-isolated-original-openbao-restore.md @@ -4,7 +4,7 @@ type: workplan title: "Isolated original-OpenBao restore for secrets-engine consumer continuity" domain: financials repo: railiance-platform -status: proposed +status: active flavor: implementation owner: claude created: "2026-10-23" @@ -127,7 +127,7 @@ state_hub_task_id: "02b16f20-8ab3-547b-b8ac-1334139d3450" ```task id: RPF-WP-0064-T02 -status: todo +status: done priority: high depends_on: [RPF-WP-0064-T01] state_hub_task_id: "51adc483-5102-5723-ab64-e7a2698befe8" @@ -168,11 +168,42 @@ Design `scripts/openbao-isolated-restore/` (adapter plus doc). Fixed properties: Teardown also runs on success, unless phase B was admitted in the same window. The plaintext snapshot never touches persistent disk. +### Delivered (2026-10-23) + +`scripts/openbao-isolated-restore/` holds `run-isolated-restore.sh` (the +attended launcher), `preflight.py` (metadata-only, runs before any prompt) and +`restore_adapter.py` (the session plus a reaper). The design document is +`docs/openbao-isolated-restore.md`. Implemented as specified above, with these +recorded choices: + +- **Readback credential: generate-root from the two shares already entered.** + The OTP is decoded in memory, and the credential is revoked and verified dead + before teardown. The operator OIDC path cannot work without a network. The + admission names the choice (`readback_credential`), so the founder confirms it + by signing T05. +- OpenBao 2.5 disables unauthenticated generate-root by default. The target + enables it on its loopback-only listener, inside `--network none`. The image's + VOLUME paths (`/openbao/file`, `/openbao/logs`) are overlaid with tmpfs. That + is the only way to get no Docker volumes, and the first native run + failed closed on exactly this. +- Custody uses hidden TTY prompts. The age identity sits in a sealed memfd + (`/dev/fd/N` for `age`), and the decrypted snapshot sits in a sealed memfd + streamed to `snapshot restore -force /dev/stdin`. Shares are held in wiped + buffers. No custody value appears in argv, env, files or logs. +- The admission namespace is `railiance-openbao-restore-attempt`, with anchor + `/etc/railiance-openbao-restore/allowed_signers`. Original material refuses + if the anchor is writable by the runtime account, or if + `PINNED_IMAGE_DIGEST` (T06) is unset or differs from the admission. +- The 45-minute deadline is enforced in-session (every subprocess timeout) and + by a reaper on parent loss or at hard deadline + 60 s. The reaper removes + the container and writes the `aborted` receipt. +- Phase B is a design stub only (doc §Phase B). + ## T03 — Safe receipt schema and independent verifier ```task id: RPF-WP-0064-T03 -status: todo +status: done priority: high depends_on: [RPF-WP-0064-T02] state_hub_task_id: "69ef4a3b-7b0a-5b7b-b49a-c688f9d19554" @@ -194,11 +225,34 @@ Add a verifier that runs separately from the adapter. It rejects placeholders, unknown fields, missing teardown, `data` fields, and anything shaped like a token or share. Validate it with adversarial fixtures. +### Delivered (2026-10-23) + +- The v2 receipt shape is `docs/openbao-restore-receipt-v2.example.json` + (schema `railiance-openbao-restore-receipt/v2`, extends the drill template). + The admission shape is `docs/openbao-restore-admission.example.json`. All + digests are typed `sha256:`, and no key name sits next to a bare hex value. + gitleaks 8.30.1 finds nothing in any new file. +- The independent verifier is + `scripts/openbao-isolated-restore/verify-restore-receipt.py`, run with + `make openbao-verify-restore-receipt`. It does not import the adapter. It + takes a clean checkout at `adapter_commit` for the source-hash comparison, + and accepts `--not-after` and `--expect-result`. +- 17 offline tests in `tests/test_openbao_isolated_restore.py` + (`make openbao-isolated-restore-test`) cover: + - 28 adversarial receipt fixtures: unknown or missing fields, missing or + reaper-only teardown, `data` fields, KV `/data/` paths, + credential/share/hex/age/PEM shapes, placeholders, untyped digests, + missing claims, deadline overrun, unprotected anchor for original material, + result inconsistencies, phase B, source hash/commit mismatch, duplicate keys + and the `--not-after` window; + - admission refusals and signature namespace/byte binding; + - launcher and preflight refusals before any prompt. + ## T04 — Staged rehearsal with synthetic material ```task id: RPF-WP-0064-T04 -status: todo +status: done priority: high depends_on: [RPF-WP-0064-T03] state_hub_task_id: "9c10caca-d772-5608-954e-6e9f78d59a39" @@ -212,6 +266,59 @@ read it back, and tear it down. Rehearse each abort path (timeout, wrong ciphertext hash, network interface present, missing object). This step uses no real material. +### Rehearsal result (2026-10-23): native, generated material only + +`make openbao-isolated-restore-rehearsal` ran 12 tests, all passing, in about +156 s, against the real `quay.io/openbao/openbao@sha256:436eaf97…c115` image +(the local pull of tag 2.5.4; T06 still confirms the production digest). Every +container ran with `--network none` and tmpfs storage. + +Each run generates its own material: + +- an SSH admission key and anchor; +- an age identity; +- a 3-share, threshold-2 Shamir init; +- root credentials; +- a KV value. + +A source OpenBao is seeded from the real source JSON/HCL shapes. The JWT and +OIDC configs are written with `skip_jwks_validation`, because the source has +no network. That source is snapshotted, age-encrypted, restored through the +adapter into a fresh target, unsealed with shares 1 and 3, read back, and torn +down. Each receipt is checked by the independent verifier, and its text is +checked to hold none of the generated credentials, shares, identity or KV value. + +**Native coverage:** + +- `restored` end to end: sealed after restore, original shares unseal it, + cluster id and index match, the throwaway init is invalidated, the readback + credential is revoked; +- wrong age identity (aborts before any target exists); +- wrong ciphertext hash; +- wrong plaintext hash; +- wrong share (`unseal_refused`, target removed); +- missing JWT role and KV registration (`refused`/`object_missing`, not + repaired); +- source drift (`refused`/`object_mismatch`); +- in-session timeout (`deadline_exceeded`); +- operator Ctrl-C (`operator_abort`); +- parent loss (the reaper removes a real container and writes `parent_lost`); +- the reaper hard deadline (kills the session, `deadline_exceeded`). + +**Mocked (reduced coverage):** only the network-interface abort. A wrapper +around `docker` adds a `bridge` entry to the inspect output of a real +networkless container. Attaching a real network is out of bounds. The adapter +aborts `network_containment_failed`, and the verifier refuses any receipt that +claims such a network. + +**Not covered:** the founder TTY prompt itself, `run-isolated-restore.sh` past +its pre-prompt refusals, and `preflight.py` past admission refusal. All three +need a protected root-owned anchor and the T06 pin. The rehearsal drives the +same `Session` with injected prompts. + +After the run, no `rpf64-*` container, Docker volume or `/dev/shm` workdir +remained, and all generated material was deleted. + ## T05 — Founder-signed attempt admission ```task @@ -220,7 +327,7 @@ status: wait priority: high needs_human: true depends_on: [RPF-WP-0064-T04] -blocking_reason: "Founder must sign (SSH, secrets-engine founder-admission pattern) an attempt admission naming the snapshot ciphertext sha256, image digest, adapter commit, window start/end, abort operator and phase A only (phase B separately). Needs T04 rehearsal receipt first." +blocking_reason: "Founder signs (ssh-keygen -Y sign, namespace railiance-openbao-restore-attempt; allowed_signers at a root-owned /etc/railiance-openbao-restore/) the v1 admission in docs/openbao-restore-admission.example.json: attempt id, phase A only, window (<=24h) + 2700 s deadline, abort operator, T06 image digest, adapter commit + source_sha256 closure, snapshot ciphertext/plaintext sha256 + bytes + age recipient + source cluster id/applied index + source receipt sha256 (RPF-WP-0015-T03), unseal 2-of-3, readback via generate-root from entered shares. T04 rehearsal done; waits on T06 pin." state_hub_task_id: "a954eaf9-855b-5458-9f13-c6cfed289934" ``` @@ -231,7 +338,7 @@ id: RPF-WP-0064-T06 status: wait priority: low depends_on: [RPF-WP-0064-T02] -blocking_reason: "rapp-openbao confirms the chart 0.28.2 / image 2.5.4 pin (and digest) used by production at the 2026-10-14 snapshot time is the one the isolated target runs. Role/KV/auth definitions stay railiance-platform-owned; no rapp-openbao change requested." +blocking_reason: "rapp-openbao confirms the chart 0.28.2 / image 2.5.4 pin and digest used by production at the 2026-10-14 snapshot time (rehearsal pulled quay.io/openbao/openbao@sha256:436eaf9778cad75507ff70ea26ace30dcbe15606e619ac3823495663d7f7c115 for tag 2.5.4; not yet owner-confirmed). The confirmed digest is then written to PINNED_IMAGE_DIGEST in scripts/openbao-isolated-restore/restore_adapter.py; original-material runs refuse until it is set. Role/KV/auth definitions stay railiance-platform-owned; no rapp-openbao change requested." state_hub_task_id: "cb689628-9a5a-5f2b-a3ec-be37bd6cdb37" ```