From 624843e122d8196f508f7d97189fc3a1589c0658 Mon Sep 17 00:00:00 2001 From: codex Date: Sat, 24 Oct 2026 10:54:02 +0200 Subject: [PATCH] RPF-WP-0064-T07: record attempt -01 (unsealed, no leader in 60 s); leader wait 180 s with bounded diagnostics Co-Authored-By: Claude Opus 5.5 Assistant: claude-code Assistant-Model: opus Assistant-Process: 4384@bnt-lap001 Assistant-Session: 80d1f3da-7a91-4519-9737-9ba128b4da64 --- docs/openbao-restore-receipt-v2.example.json | 23 +++++- .../restore_adapter.py | 80 ++++++++++++++++--- .../verify-restore-receipt.py | 28 ++++++- tests/openbao_restore_rehearsal.py | 12 ++- tests/test_openbao_isolated_restore.py | 15 ++++ ...-0064-isolated-original-openbao-restore.md | 13 +++ 6 files changed, 155 insertions(+), 16 deletions(-) diff --git a/docs/openbao-restore-receipt-v2.example.json b/docs/openbao-restore-receipt-v2.example.json index e3ebc85..2f83f32 100644 --- a/docs/openbao-restore-receipt-v2.example.json +++ b/docs/openbao-restore-receipt-v2.example.json @@ -47,7 +47,28 @@ "unsealed_with_original_shares": true, "cluster_id_match": true, "applied_index_not_behind": true, - "throwaway_init_invalidated": true + "throwaway_init_invalidated": true, + "leader_wait": { + "waited_seconds": 3, + "polls": 4, + "elected": true, + "last_leader": { + "ha_enabled": true, + "is_self": true, + "leader_address": "http://127.0.0.1:8200", + "leader_cluster_address": "https://127.0.0.1:8201", + "raft_committed_index": 6040620, + "raft_applied_index": 6040620, + "active_time": "2026-10-24T09:00:00Z" + }, + "last_seal": { + "sealed": null, + "initialized": null, + "storage_type": null, + "version": null + }, + "last_error": null + } }, "network": { "network_mode": "none", diff --git a/scripts/openbao-isolated-restore/restore_adapter.py b/scripts/openbao-isolated-restore/restore_adapter.py index ad67ae7..cb1b970 100644 --- a/scripts/openbao-isolated-restore/restore_adapter.py +++ b/scripts/openbao-isolated-restore/restore_adapter.py @@ -54,6 +54,31 @@ IMAGE_REPOSITORY = "quay.io/openbao/openbao" # rehearsal pull. Original-material runs refuse any other image. PINNED_IMAGE_DIGEST: str | None = "sha256:436eaf9778cad75507ff70ea26ace30dcbe15606e619ac3823495663d7f7c115" MAX_DURATION_SECONDS = 2700 +LEADER_WAIT_SECONDS = 180 +LEADER_FIELDS = {"ha_enabled": bool, "is_self": bool, "leader_address": str, "leader_cluster_address": str, + "raft_committed_index": int, "raft_applied_index": int, "active_time": str} +SEAL_FIELDS = {"sealed": bool, "initialized": bool, "storage_type": str, "version": str} +_SECRETISH = re.compile(r"\b(?:hvs|hvb|hvr|s|b|r)\.[A-Za-z0-9_-]{8,}|[A-Za-z0-9+/=_-]{32,}") + + +def typed(value, kind): + """Keep a status value only if it has the expected type; bound strings and redact token-like text.""" + if kind is int: + return value if type(value) is int else None + if kind is bool: + return value if type(value) is bool else None + if isinstance(value, str): + return _SECRETISH.sub("[redacted]", value)[:200] + return None + + +def sanitize_status_line(raw: bytes) -> str | None: + """First non-empty stderr line, bounded, with token-like text redacted.""" + for line in raw.decode("utf-8", "replace").splitlines(): + line = line.strip() + if line: + return _SECRETISH.sub("[redacted]", line)[:200] + return None REAPER_GRACE_SECONDS = 60 DEFAULT_ANCHOR = Path("/etc/railiance-openbao-restore/allowed_signers") ANCHOR_ENV = "RAILIANCE_OPENBAO_RESTORE_ANCHOR" @@ -105,12 +130,17 @@ ALLOWED_WRITES = frozenset({"sys/unseal", "sys/generate-root/attempt", "sys/gene ABORT_REASONS = frozenset({ "admission_refused", "preflight_refused", "custody_identity_refused", "ciphertext_mismatch", "decrypt_failed", "plaintext_mismatch", "target_start_failed", "network_containment_failed", - "throwaway_init_failed", "restore_failed", "unseal_refused", "cluster_mismatch", + "throwaway_init_failed", "restore_failed", "unseal_refused", "leader_not_elected", "cluster_mismatch", "applied_index_behind", "readback_credential_failed", "readback_failed", "deadline_exceeded", "operator_abort", "parent_lost", "internal_error", }) +# Production runs the chart with setNodeId, so the snapshot's voter is the pod +# name. The target takes the same id; a native rehearsal showed a different id +# still elects a leader after a forced restore, so this is alignment, not the +# T07 attempt -01 cause. +RESTORE_NODE_ID = "openbao-0" SERVER_CONFIG = json.dumps({ - "storage": {"raft": {"path": "/openbao/data", "node_id": "isolated-restore"}}, + "storage": {"raft": {"path": "/openbao/data", "node_id": RESTORE_NODE_ID}}, # OpenBao 2.x disables unauthenticated generate-root by default; the # readback credential needs it, on a loopback-only listener without a network. "listener": {"tcp": {"address": "127.0.0.1:8200", "tls_disable": True, @@ -412,7 +442,8 @@ class Session: "ciphertext_verified": False, "plaintext_verified": False}, "restored": {"cluster_id": None, "applied_index": None, "sealed_after_restore": None, "unsealed_with_original_shares": False, "cluster_id_match": False, - "applied_index_not_behind": False, "throwaway_init_invalidated": False}, + "applied_index_not_behind": False, "throwaway_init_invalidated": False, + "leader_wait": None}, "network": {"network_mode": None, "docker_networks": [], "interfaces": [], "published_ports": [], "mounts": [], "egress_refused": False}, "objects": [], "result": None, "reason": None, @@ -625,7 +656,7 @@ class Session: result = self.write("sys/unseal", {"key": share.decode()}, reason="unseal_refused") require(result.get("sealed") is False, "unseal_refused") self.receipt["restored"]["unsealed_with_original_shares"] = True - leader = self.wait_active("unseal_refused") + leader = self.wait_active("leader_not_elected") status = self.read("sys/seal-status", reason="unseal_refused") or {} snap, restored = self.adm["snapshot"], self.receipt["restored"] restored["cluster_id"] = status.get("cluster_id") @@ -642,14 +673,39 @@ class Session: require(restored["applied_index_not_behind"], "applied_index_behind") def wait_active(self, reason: str) -> dict: - for _ in range(120): - leader = self.read("sys/leader", reason=reason) or {} - if leader.get("is_self") and isinstance(leader.get("raft_applied_index"), int) \ - and leader["raft_applied_index"] > 0: - return leader - time.sleep(0.5) - self.remaining() - raise Refused(reason) + # Bounded, non-secret diagnostics: T07 attempt -01 timed out here with no + # trace. Only typed status fields and a sanitized first error line are kept. + started, polls, leader, seal, error = time.monotonic(), 0, {}, {}, None + wait = self.receipt["restored"] + try: + while time.monotonic() - started < LEADER_WAIT_SECONDS: + polls += 1 + done = self.bao(["read", "-format=json", "sys/leader"], check=False, reason=reason) + if done.returncode == 0: + body = json.loads(done.stdout) + leader = body.get("data", body) if isinstance(body, dict) else {} + error = None + if leader.get("is_self") and isinstance(leader.get("raft_applied_index"), int) \ + and leader["raft_applied_index"] > 0: + return leader + else: + error = sanitize_status_line(done.stderr) + time.sleep(1) + self.remaining() + status = self.bao(["read", "-format=json", "sys/seal-status"], check=False, reason=reason) + if status.returncode == 0: + body = json.loads(status.stdout) + seal = body.get("data", body) if isinstance(body, dict) else {} + raise Refused(reason) + finally: + wait["leader_wait"] = { + "waited_seconds": int(time.monotonic() - started), "polls": polls, + "elected": bool(leader.get("is_self")) and isinstance(leader.get("raft_applied_index"), int) + and leader.get("raft_applied_index", 0) > 0, + "last_leader": {k: typed(leader.get(k), t) for k, t in LEADER_FIELDS.items()}, + "last_seal": {k: typed(seal.get(k), t) for k, t in SEAL_FIELDS.items()}, + "last_error": error, + } def generate_credential(self) -> None: self.say("Step 6/7: generate an in-memory readback credential from the entered shares.") diff --git a/scripts/openbao-isolated-restore/verify-restore-receipt.py b/scripts/openbao-isolated-restore/verify-restore-receipt.py index f5b7045..fd145b6 100644 --- a/scripts/openbao-isolated-restore/verify-restore-receipt.py +++ b/scripts/openbao-isolated-restore/verify-restore-receipt.py @@ -30,7 +30,19 @@ TOP = {"schema", "extends", "attempt_id", "material", "phase", "admission_sha256 SNAPSHOT = {"ciphertext_sha256", "ciphertext_bytes", "plaintext_sha256", "source_cluster_id", "source_applied_index", "source_receipt_sha256", "ciphertext_verified", "plaintext_verified"} RESTORED = {"cluster_id", "applied_index", "sealed_after_restore", "unsealed_with_original_shares", - "cluster_id_match", "applied_index_not_behind", "throwaway_init_invalidated"} + "cluster_id_match", "applied_index_not_behind", "throwaway_init_invalidated", "leader_wait"} +LEADER_WAIT = {"waited_seconds", "polls", "elected", "last_leader", "last_seal", "last_error"} +LEADER_FIELDS = {"ha_enabled": bool, "is_self": bool, "leader_address": str, "leader_cluster_address": str, + "raft_committed_index": int, "raft_applied_index": int, "active_time": str} +SEAL_FIELDS = {"sealed": bool, "initialized": bool, "storage_type": str, "version": str} + + +def typed_or_none(value: object, kind: type) -> bool: + if value is None: + return True + if kind is str: + return isinstance(value, str) and len(value) <= 200 + return type(value) is kind NETWORK = {"network_mode", "docker_networks", "interfaces", "published_ports", "mounts", "egress_refused"} TEARDOWN = {"container_removed", "container_absent_verified", "plaintext_memfd_closed", "custody_memfd_closed", "readback_credential_revoked", "docker_volumes_created", "performed_by"} @@ -44,7 +56,8 @@ CLAIMS = {"consumer_continuity", "live_jwt_login", "phase_b_network", "owner_acc "recurring_restore_assurance", "production_redirect"} ABORT = {"admission_refused", "preflight_refused", "custody_identity_refused", "ciphertext_mismatch", "decrypt_failed", "plaintext_mismatch", "target_start_failed", "network_containment_failed", - "throwaway_init_failed", "restore_failed", "unseal_refused", "cluster_mismatch", "applied_index_behind", + "throwaway_init_failed", "restore_failed", "unseal_refused", "leader_not_elected", "cluster_mismatch", + "applied_index_behind", "readback_credential_failed", "readback_failed", "deadline_exceeded", "operator_abort", "parent_lost", "internal_error"} DIGEST = re.compile(r"^sha256:[a-f0-9]{64}$") @@ -190,6 +203,17 @@ def verify(receipt: object, *, source_root: Path, not_after: datetime | None = N need(restored["cluster_id"] is None or (isinstance(restored["cluster_id"], str) and UUID.fullmatch(restored["cluster_id"])), "restored") need(restored["applied_index"] is None or type(restored["applied_index"]) is int, "restored") + wait = restored["leader_wait"] + if wait is not None: + need(isinstance(wait, dict) and set(wait) == LEADER_WAIT, "leader_wait") + need(type(wait["waited_seconds"]) is int and type(wait["polls"]) is int + and type(wait["elected"]) is bool, "leader_wait") + for name, spec in (("last_leader", LEADER_FIELDS), ("last_seal", SEAL_FIELDS)): + part = wait[name] + need(isinstance(part, dict) and set(part) == set(spec) + and all(typed_or_none(part[k], t) for k, t in spec.items()), "leader_wait") + need(wait["last_error"] is None or (isinstance(wait["last_error"], str) and len(wait["last_error"]) <= 200), + "leader_wait") network = receipt["network"] need(isinstance(network, dict) and set(network) == NETWORK, "network_fields") teardown = receipt["teardown"] diff --git a/tests/openbao_restore_rehearsal.py b/tests/openbao_restore_rehearsal.py index bdd8d82..74a0df7 100644 --- a/tests/openbao_restore_rehearsal.py +++ b/tests/openbao_restore_rehearsal.py @@ -67,6 +67,16 @@ def make_source_root(base: Path, mutate: dict[str, bytes] | None = None) -> tupl return root, commit +# The source mirrors production (rapp-openbao chart setNodeId: the pod name), +# built independently of the adapter's target config. A rehearsal with a target +# node id differing from this voter still restored and elected a leader, so the +# node id alone did not explain T07 attempt -01; it stays aligned with production. +PRODUCTION_NODE_ID = "openbao-0" +SOURCE_CONFIG = json.dumps({**json.loads(adapter.SERVER_CONFIG), + "storage": {"raft": {"path": "/openbao/data", "node_id": PRODUCTION_NODE_ID}}}, + separators=(",", ":")) + + class SyntheticBao: """A generated-material OpenBao in a networkless container (source side).""" @@ -81,7 +91,7 @@ class SyntheticBao: "--user", "100:1000", "--read-only", "--cap-drop", "ALL", "--security-opt", "no-new-privileges"] for target, options in adapter.TMPFS.items(): argv += ["--tmpfs", f"{target}:{options}"] - argv += ["-e", "BAO_LOCAL_CONFIG=" + adapter.SERVER_CONFIG, "-e", "BAO_ADDR=http://127.0.0.1:8200", + argv += ["-e", "BAO_LOCAL_CONFIG=" + SOURCE_CONFIG, "-e", "BAO_ADDR=http://127.0.0.1:8200", self.image, "server"] assert sh(argv).returncode == 0 try: diff --git a/tests/test_openbao_isolated_restore.py b/tests/test_openbao_isolated_restore.py index d286146..19c9a39 100644 --- a/tests/test_openbao_isolated_restore.py +++ b/tests/test_openbao_isolated_restore.py @@ -151,6 +151,21 @@ class AdmissionTests(Base): self.assertEqual(A.PINNED_IMAGE_DIGEST, "sha256:436eaf9778cad75507ff70ea26ace30dcbe15606e619ac3823495663d7f7c115") + def test_target_takes_production_raft_node_id(self): + # rapp-openbao runs the chart with setNodeId, so the snapshot's only voter is the pod name. + self.assertEqual(A.RESTORE_NODE_ID, "openbao-0") + self.assertEqual(json.loads(A.SERVER_CONFIG)["storage"]["raft"]["node_id"], "openbao-0") + self.assertIn("leader_not_elected", A.ABORT_REASONS) + + def test_leader_wait_diagnostics_are_typed_and_redacted(self): + line = A.sanitize_status_line(b"\nError: token hvs.CAESIabcdefghijklmnop local node not active\nmore") + self.assertNotIn("hvs.CAES", line) + self.assertIn("local node not active", line) + self.assertLessEqual(len(A.sanitize_status_line(b"x" * 500) or ""), 200) + self.assertIsNone(A.typed("5", int)) + self.assertIsNone(A.typed(1, bool)) + self.assertEqual(A.typed("http://openbao-0.openbao-internal:8200", str), "http://openbao-0.openbao-internal:8200") + def test_normalise_and_project(self): source = {"token_ttl": "5m", "token_policies": ["b", "a"], "_note": "x"} ok, projected = A.project(source, {"token_ttl": 300, "token_policies": ["a", "b"], "extra": 1}, ("_note",)) diff --git a/workplans/RPF-WP-0064-isolated-original-openbao-restore.md b/workplans/RPF-WP-0064-isolated-original-openbao-restore.md index dce0b76..e24c756 100644 --- a/workplans/RPF-WP-0064-isolated-original-openbao-restore.md +++ b/workplans/RPF-WP-0064-isolated-original-openbao-restore.md @@ -377,6 +377,19 @@ blocking_reason: "Requires the founder in attendance holding the offline railian state_hub_task_id: "96e88167-c484-527c-84aa-0464dd0f5597" ``` +2026-10-24 attempt `RPF-WP-0064-T07-20261024-01` (admission `796414c2…a3b9`, adapter `1701a3f`, window +08:00Z–14:00Z): `aborted`, reason `unseal_refused`, after 168 s; the receipt `1a7de4a4…abe6` (kept outside Git) +was independently verified from a clean clone at `1701a3f` as a genuine aborted result. It proved the off-host +ciphertext `8069449c…8c41` (375618 bytes) intact, decryption with the offline age identity, plaintext +`a6312444…590b`, and that the founder's two original Shamir shares unseal the restored barrier +(`unsealed_with_original_shares: true`); networkless containment and full teardown held, with no volumes. +The abort came after unseal: `sys/leader` never reported an active self with an applied index within 60 s, and the +adapter reused the `unseal_refused` reason. The first hypothesis (target raft node id `isolated-restore` versus +production voter `openbao-0`) was disproved by a native rehearsal, which restored and elected a leader anyway. +The adapter now aligns the node id with production, waits up to 180 s, aborts as `leader_not_elected`, and +records bounded typed `sys/leader`/`sys/seal-status` fields and a redacted first error line in +`restored.leader_wait`. Offline 19 and native 12 pass. A new `-02` admission is needed for the new adapter commit. + Run the admitted adapter once. The outcome is a T03 receipt verified independently. The receipt is kept outside Git and referenced by its hash. A missing object or failed check is a valid `refused` outcome; it is not repaired