RPF-WP-0064-T07: record attempt -01 (unsealed, no leader in 60 s); leader wait 180 s with bounded diagnostics

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

Assistant: claude-code
Assistant-Model: opus
Assistant-Process: 4384@bnt-lap001
Assistant-Session: 80d1f3da-7a91-4519-9737-9ba128b4da64

docs/openbao-restore-receipt-v2.example.json

@@ -47,7 +47,28 @@
     "unsealed_with_original_shares": true,
     "cluster_id_match": true,
     "applied_index_not_behind": true,
-    "throwaway_init_invalidated": true
+    "throwaway_init_invalidated": true,
+    "leader_wait": {
+      "waited_seconds": 3,
+      "polls": 4,
+      "elected": true,
+      "last_leader": {
+        "ha_enabled": true,
+        "is_self": true,
+        "leader_address": "http://127.0.0.1:8200",
+        "leader_cluster_address": "https://127.0.0.1:8201",
+        "raft_committed_index": 6040620,
+        "raft_applied_index": 6040620,
+        "active_time": "2026-10-24T09:00:00Z"
+      },
+      "last_seal": {
+        "sealed": null,
+        "initialized": null,
+        "storage_type": null,
+        "version": null
+      },
+      "last_error": null
+    }
   },
   "network": {
     "network_mode": "none",

scripts/openbao-isolated-restore/restore_adapter.py

@@ -54,6 +54,31 @@ IMAGE_REPOSITORY = "quay.io/openbao/openbao"
 # rehearsal pull. Original-material runs refuse any other image.
 PINNED_IMAGE_DIGEST: str | None = "sha256:436eaf9778cad75507ff70ea26ace30dcbe15606e619ac3823495663d7f7c115"
 MAX_DURATION_SECONDS = 2700
+LEADER_WAIT_SECONDS = 180
+LEADER_FIELDS = {"ha_enabled": bool, "is_self": bool, "leader_address": str, "leader_cluster_address": str,
+                 "raft_committed_index": int, "raft_applied_index": int, "active_time": str}
+SEAL_FIELDS = {"sealed": bool, "initialized": bool, "storage_type": str, "version": str}
+_SECRETISH = re.compile(r"\b(?:hvs|hvb|hvr|s|b|r)\.[A-Za-z0-9_-]{8,}|[A-Za-z0-9+/=_-]{32,}")
+
+
+def typed(value, kind):
+    """Keep a status value only if it has the expected type; bound strings and redact token-like text."""
+    if kind is int:
+        return value if type(value) is int else None
+    if kind is bool:
+        return value if type(value) is bool else None
+    if isinstance(value, str):
+        return _SECRETISH.sub("[redacted]", value)[:200]
+    return None
+
+
+def sanitize_status_line(raw: bytes) -> str | None:
+    """First non-empty stderr line, bounded, with token-like text redacted."""
+    for line in raw.decode("utf-8", "replace").splitlines():
+        line = line.strip()
+        if line:
+            return _SECRETISH.sub("[redacted]", line)[:200]
+    return None
 REAPER_GRACE_SECONDS = 60
 DEFAULT_ANCHOR = Path("/etc/railiance-openbao-restore/allowed_signers")
 ANCHOR_ENV = "RAILIANCE_OPENBAO_RESTORE_ANCHOR"
@@ -105,12 +130,17 @@ ALLOWED_WRITES = frozenset({"sys/unseal", "sys/generate-root/attempt", "sys/gene
 ABORT_REASONS = frozenset({
     "admission_refused", "preflight_refused", "custody_identity_refused", "ciphertext_mismatch",
     "decrypt_failed", "plaintext_mismatch", "target_start_failed", "network_containment_failed",
-    "throwaway_init_failed", "restore_failed", "unseal_refused", "cluster_mismatch",
+    "throwaway_init_failed", "restore_failed", "unseal_refused", "leader_not_elected", "cluster_mismatch",
     "applied_index_behind", "readback_credential_failed", "readback_failed", "deadline_exceeded",
     "operator_abort", "parent_lost", "internal_error",
 })
+# Production runs the chart with setNodeId, so the snapshot's voter is the pod
+# name. The target takes the same id; a native rehearsal showed a different id
+# still elects a leader after a forced restore, so this is alignment, not the
+# T07 attempt -01 cause.
+RESTORE_NODE_ID = "openbao-0"
 SERVER_CONFIG = json.dumps({
-    "storage": {"raft": {"path": "/openbao/data", "node_id": "isolated-restore"}},
+    "storage": {"raft": {"path": "/openbao/data", "node_id": RESTORE_NODE_ID}},
     # OpenBao 2.x disables unauthenticated generate-root by default; the
     # readback credential needs it, on a loopback-only listener without a network.
     "listener": {"tcp": {"address": "127.0.0.1:8200", "tls_disable": True,
@@ -412,7 +442,8 @@ class Session:
                          "ciphertext_verified": False, "plaintext_verified": False},
             "restored": {"cluster_id": None, "applied_index": None, "sealed_after_restore": None,
                          "unsealed_with_original_shares": False, "cluster_id_match": False,
-                         "applied_index_not_behind": False, "throwaway_init_invalidated": False},
+                         "applied_index_not_behind": False, "throwaway_init_invalidated": False,
+                         "leader_wait": None},
             "network": {"network_mode": None, "docker_networks": [], "interfaces": [], "published_ports": [],
                         "mounts": [], "egress_refused": False},
             "objects": [], "result": None, "reason": None,
@@ -625,7 +656,7 @@ class Session:
             result = self.write("sys/unseal", {"key": share.decode()}, reason="unseal_refused")
         require(result.get("sealed") is False, "unseal_refused")
         self.receipt["restored"]["unsealed_with_original_shares"] = True
-        leader = self.wait_active("unseal_refused")
+        leader = self.wait_active("leader_not_elected")
         status = self.read("sys/seal-status", reason="unseal_refused") or {}
         snap, restored = self.adm["snapshot"], self.receipt["restored"]
         restored["cluster_id"] = status.get("cluster_id")
@@ -642,14 +673,39 @@ class Session:
         require(restored["applied_index_not_behind"], "applied_index_behind")
 
     def wait_active(self, reason: str) -> dict:
-        for _ in range(120):
-            leader = self.read("sys/leader", reason=reason) or {}
-            if leader.get("is_self") and isinstance(leader.get("raft_applied_index"), int) \
-                    and leader["raft_applied_index"] > 0:
-                return leader
-            time.sleep(0.5)
-            self.remaining()
-        raise Refused(reason)
+        # Bounded, non-secret diagnostics: T07 attempt -01 timed out here with no
+        # trace. Only typed status fields and a sanitized first error line are kept.
+        started, polls, leader, seal, error = time.monotonic(), 0, {}, {}, None
+        wait = self.receipt["restored"]
+        try:
+            while time.monotonic() - started < LEADER_WAIT_SECONDS:
+                polls += 1
+                done = self.bao(["read", "-format=json", "sys/leader"], check=False, reason=reason)
+                if done.returncode == 0:
+                    body = json.loads(done.stdout)
+                    leader = body.get("data", body) if isinstance(body, dict) else {}
+                    error = None
+                    if leader.get("is_self") and isinstance(leader.get("raft_applied_index"), int) \
+                            and leader["raft_applied_index"] > 0:
+                        return leader
+                else:
+                    error = sanitize_status_line(done.stderr)
+                time.sleep(1)
+                self.remaining()
+            status = self.bao(["read", "-format=json", "sys/seal-status"], check=False, reason=reason)
+            if status.returncode == 0:
+                body = json.loads(status.stdout)
+                seal = body.get("data", body) if isinstance(body, dict) else {}
+            raise Refused(reason)
+        finally:
+            wait["leader_wait"] = {
+                "waited_seconds": int(time.monotonic() - started), "polls": polls,
+                "elected": bool(leader.get("is_self")) and isinstance(leader.get("raft_applied_index"), int)
+                and leader.get("raft_applied_index", 0) > 0,
+                "last_leader": {k: typed(leader.get(k), t) for k, t in LEADER_FIELDS.items()},
+                "last_seal": {k: typed(seal.get(k), t) for k, t in SEAL_FIELDS.items()},
+                "last_error": error,
+            }
 
     def generate_credential(self) -> None:
         self.say("Step 6/7: generate an in-memory readback credential from the entered shares.")

scripts/openbao-isolated-restore/verify-restore-receipt.py

@@ -30,7 +30,19 @@ TOP = {"schema", "extends", "attempt_id", "material", "phase", "admission_sha256
 SNAPSHOT = {"ciphertext_sha256", "ciphertext_bytes", "plaintext_sha256", "source_cluster_id",
             "source_applied_index", "source_receipt_sha256", "ciphertext_verified", "plaintext_verified"}
 RESTORED = {"cluster_id", "applied_index", "sealed_after_restore", "unsealed_with_original_shares",
-            "cluster_id_match", "applied_index_not_behind", "throwaway_init_invalidated"}
+            "cluster_id_match", "applied_index_not_behind", "throwaway_init_invalidated", "leader_wait"}
+LEADER_WAIT = {"waited_seconds", "polls", "elected", "last_leader", "last_seal", "last_error"}
+LEADER_FIELDS = {"ha_enabled": bool, "is_self": bool, "leader_address": str, "leader_cluster_address": str,
+                 "raft_committed_index": int, "raft_applied_index": int, "active_time": str}
+SEAL_FIELDS = {"sealed": bool, "initialized": bool, "storage_type": str, "version": str}
+
+
+def typed_or_none(value: object, kind: type) -> bool:
+    if value is None:
+        return True
+    if kind is str:
+        return isinstance(value, str) and len(value) <= 200
+    return type(value) is kind
 NETWORK = {"network_mode", "docker_networks", "interfaces", "published_ports", "mounts", "egress_refused"}
 TEARDOWN = {"container_removed", "container_absent_verified", "plaintext_memfd_closed", "custody_memfd_closed",
             "readback_credential_revoked", "docker_volumes_created", "performed_by"}
@@ -44,7 +56,8 @@ CLAIMS = {"consumer_continuity", "live_jwt_login", "phase_b_network", "owner_acc
           "recurring_restore_assurance", "production_redirect"}
 ABORT = {"admission_refused", "preflight_refused", "custody_identity_refused", "ciphertext_mismatch",
          "decrypt_failed", "plaintext_mismatch", "target_start_failed", "network_containment_failed",
-         "throwaway_init_failed", "restore_failed", "unseal_refused", "cluster_mismatch", "applied_index_behind",
+         "throwaway_init_failed", "restore_failed", "unseal_refused", "leader_not_elected", "cluster_mismatch",
+         "applied_index_behind",
          "readback_credential_failed", "readback_failed", "deadline_exceeded", "operator_abort", "parent_lost",
          "internal_error"}
 DIGEST = re.compile(r"^sha256:[a-f0-9]{64}$")
@@ -190,6 +203,17 @@ def verify(receipt: object, *, source_root: Path, not_after: datetime | None = N
     need(restored["cluster_id"] is None or (isinstance(restored["cluster_id"], str)
                                             and UUID.fullmatch(restored["cluster_id"])), "restored")
     need(restored["applied_index"] is None or type(restored["applied_index"]) is int, "restored")
+    wait = restored["leader_wait"]
+    if wait is not None:
+        need(isinstance(wait, dict) and set(wait) == LEADER_WAIT, "leader_wait")
+        need(type(wait["waited_seconds"]) is int and type(wait["polls"]) is int
+             and type(wait["elected"]) is bool, "leader_wait")
+        for name, spec in (("last_leader", LEADER_FIELDS), ("last_seal", SEAL_FIELDS)):
+            part = wait[name]
+            need(isinstance(part, dict) and set(part) == set(spec)
+                 and all(typed_or_none(part[k], t) for k, t in spec.items()), "leader_wait")
+        need(wait["last_error"] is None or (isinstance(wait["last_error"], str) and len(wait["last_error"]) <= 200),
+             "leader_wait")
     network = receipt["network"]
     need(isinstance(network, dict) and set(network) == NETWORK, "network_fields")
     teardown = receipt["teardown"]

tests/openbao_restore_rehearsal.py

@@ -67,6 +67,16 @@ def make_source_root(base: Path, mutate: dict[str, bytes] | None = None) -> tupl
     return root, commit
 
 
+# The source mirrors production (rapp-openbao chart setNodeId: the pod name),
+# built independently of the adapter's target config. A rehearsal with a target
+# node id differing from this voter still restored and elected a leader, so the
+# node id alone did not explain T07 attempt -01; it stays aligned with production.
+PRODUCTION_NODE_ID = "openbao-0"
+SOURCE_CONFIG = json.dumps({**json.loads(adapter.SERVER_CONFIG),
+                            "storage": {"raft": {"path": "/openbao/data", "node_id": PRODUCTION_NODE_ID}}},
+                           separators=(",", ":"))
+
+
 class SyntheticBao:
     """A generated-material OpenBao in a networkless container (source side)."""
 
@@ -81,7 +91,7 @@ class SyntheticBao:
                 "--user", "100:1000", "--read-only", "--cap-drop", "ALL", "--security-opt", "no-new-privileges"]
         for target, options in adapter.TMPFS.items():
             argv += ["--tmpfs", f"{target}:{options}"]
-        argv += ["-e", "BAO_LOCAL_CONFIG=" + adapter.SERVER_CONFIG, "-e", "BAO_ADDR=http://127.0.0.1:8200",
+        argv += ["-e", "BAO_LOCAL_CONFIG=" + SOURCE_CONFIG, "-e", "BAO_ADDR=http://127.0.0.1:8200",
                  self.image, "server"]
         assert sh(argv).returncode == 0
         try:

tests/test_openbao_isolated_restore.py

@@ -151,6 +151,21 @@ class AdmissionTests(Base):
         self.assertEqual(A.PINNED_IMAGE_DIGEST,
                          "sha256:436eaf9778cad75507ff70ea26ace30dcbe15606e619ac3823495663d7f7c115")
 
+    def test_target_takes_production_raft_node_id(self):
+        # rapp-openbao runs the chart with setNodeId, so the snapshot's only voter is the pod name.
+        self.assertEqual(A.RESTORE_NODE_ID, "openbao-0")
+        self.assertEqual(json.loads(A.SERVER_CONFIG)["storage"]["raft"]["node_id"], "openbao-0")
+        self.assertIn("leader_not_elected", A.ABORT_REASONS)
+
+    def test_leader_wait_diagnostics_are_typed_and_redacted(self):
+        line = A.sanitize_status_line(b"\nError: token hvs.CAESIabcdefghijklmnop local node not active\nmore")
+        self.assertNotIn("hvs.CAES", line)
+        self.assertIn("local node not active", line)
+        self.assertLessEqual(len(A.sanitize_status_line(b"x" * 500) or ""), 200)
+        self.assertIsNone(A.typed("5", int))
+        self.assertIsNone(A.typed(1, bool))
+        self.assertEqual(A.typed("http://openbao-0.openbao-internal:8200", str), "http://openbao-0.openbao-internal:8200")
+
     def test_normalise_and_project(self):
         source = {"token_ttl": "5m", "token_policies": ["b", "a"], "_note": "x"}
         ok, projected = A.project(source, {"token_ttl": 300, "token_policies": ["a", "b"], "extra": 1}, ("_note",))

workplans/RPF-WP-0064-isolated-original-openbao-restore.md

@@ -377,6 +377,19 @@ blocking_reason: "Requires the founder in attendance holding the offline railian
 state_hub_task_id: "96e88167-c484-527c-84aa-0464dd0f5597"
 ```
 
+2026-10-24 attempt `RPF-WP-0064-T07-20261024-01` (admission `796414c2…a3b9`, adapter `1701a3f`, window
+08:00Z–14:00Z): `aborted`, reason `unseal_refused`, after 168 s; the receipt `1a7de4a4…abe6` (kept outside Git)
+was independently verified from a clean clone at `1701a3f` as a genuine aborted result. It proved the off-host
+ciphertext `8069449c…8c41` (375618 bytes) intact, decryption with the offline age identity, plaintext
+`a6312444…590b`, and that the founder's two original Shamir shares unseal the restored barrier
+(`unsealed_with_original_shares: true`); networkless containment and full teardown held, with no volumes.
+The abort came after unseal: `sys/leader` never reported an active self with an applied index within 60 s, and the
+adapter reused the `unseal_refused` reason. The first hypothesis (target raft node id `isolated-restore` versus
+production voter `openbao-0`) was disproved by a native rehearsal, which restored and elected a leader anyway.
+The adapter now aligns the node id with production, waits up to 180 s, aborts as `leader_not_elected`, and
+records bounded typed `sys/leader`/`sys/seal-status` fields and a redacted first error line in
+`restored.leader_wait`. Offline 19 and native 12 pass. A new `-02` admission is needed for the new adapter commit.
+
 Run the admitted adapter once. The outcome is a T03 receipt verified
 independently. The receipt is kept outside Git and referenced by its hash. A
 missing object or failed check is a valid `refused` outcome; it is not repaired