Harden WP-0024 recovery execution gates

Assistant: codex
Assistant-Model: gpt-5.6-sol
Assistant-Session: 01a02669-87ee-7a31-b111-edc95a16e0fa

scripts/audit-core-recovery-preflight.py

@@ -111,6 +111,80 @@ def secret_key_names(description: str) -> set[str]:
     return keys
 
 
+def validate_snapshot_receipt(
+    path: Path,
+    *,
+    live_openbao: dict[str, Any],
+    now: datetime,
+    max_age_hours: float,
+) -> dict[str, Any]:
+    try:
+        receipt = json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError) as exc:
+        raise PreflightError("OpenBao snapshot receipt is unavailable or invalid JSON") from exc
+    if not isinstance(receipt, dict) or receipt.get("receipt_version") != 1:
+        raise PreflightError("OpenBao snapshot receipt has an unsupported shape/version")
+    required_strings = (
+        "receipt_id", "created_at", "operator", "source_cluster",
+        "source_namespace", "source_pod", "cluster_id", "snapshot_sha256",
+        "encrypted_snapshot_sha256", "encrypted_location_ref",
+    )
+    for key in required_strings:
+        if not isinstance(receipt.get(key), str) or not receipt[key].strip():
+            raise PreflightError(f"OpenBao snapshot receipt is missing {key}")
+    required_true = (
+        "snapshot_created", "source_initialized", "source_unsealed",
+        "snapshot_encrypted", "encrypted_copy_off_host", "encryption_verified",
+        "hash_verified", "no_secret_material_recorded",
+    )
+    for key in required_true:
+        if receipt.get(key) is not True:
+            raise PreflightError(f"OpenBao snapshot receipt requires {key}=true")
+    if (
+        receipt["source_cluster"] != "railiance01"
+        or receipt["source_namespace"] != "openbao"
+        or receipt["source_pod"] != "openbao-0"
+        or receipt["cluster_id"] != live_openbao.get("cluster_id")
+    ):
+        raise PreflightError("OpenBao snapshot receipt does not match the live source")
+    try:
+        created = rfc3339(receipt["created_at"])
+        snapshot_index = int(receipt["raft_applied_index"])
+        live_index = int(live_openbao["raft_applied_index"])
+    except (KeyError, TypeError, ValueError) as exc:
+        raise PreflightError("OpenBao snapshot receipt has invalid time/index metadata") from exc
+    age_hours = (now - created).total_seconds() / 3600
+    if age_hours < -(5 / 60) or age_hours > max_age_hours:
+        raise PreflightError("OpenBao snapshot receipt is outside the permitted age")
+    if snapshot_index <= 0 or snapshot_index > live_index:
+        raise PreflightError("OpenBao snapshot receipt has an impossible Raft index")
+    digest_pattern = re.compile(r"^sha256:[0-9a-f]{64}$")
+    digests = (receipt["snapshot_sha256"], receipt["encrypted_snapshot_sha256"])
+    if any(not digest_pattern.fullmatch(value) for value in digests):
+        raise PreflightError("OpenBao snapshot receipt has an invalid digest")
+    if any(len(set(value.removeprefix("sha256:"))) <= 1 for value in digests):
+        raise PreflightError("OpenBao snapshot receipt contains a placeholder digest")
+    if digests[0] == digests[1]:
+        raise PreflightError("plain and encrypted snapshot digests must differ")
+    encoded = json.dumps(receipt, sort_keys=True)
+    for marker in (
+        "BEGIN PRIVATE KEY", "BEGIN OPENSSH PRIVATE KEY", "AGE-SECRET-KEY-1",
+        "OPENBAO_ROOT_TOKEN", "VAULT_TOKEN", "hvs.", "<", "YYYY-MM-DD",
+    ):
+        if marker in encoded:
+            raise PreflightError("OpenBao snapshot receipt contains forbidden material/placeholder")
+    return {
+        "receipt_id": receipt["receipt_id"],
+        "created_at": created.isoformat().replace("+00:00", "Z"),
+        "age_hours": round(age_hours, 2),
+        "cluster_id_matches": True,
+        "raft_applied_index": snapshot_index,
+        "encrypted_copy_off_host": True,
+        "verified": True,
+        "secret_values_observed": False,
+    }
+
+
 def common_state(remote: Remote, now: datetime) -> dict[str, Any]:
     node_list = remote.kubectl_json(["get", "nodes"], label="read node state")
     nodes = node_list.get("items", [])
@@ -186,6 +260,8 @@ def common_state(remote: Remote, now: datetime) -> dict[str, Any]:
             "seal_type": bao_status.get("type"),
             "threshold": bao_status.get("t"),
             "shares": bao_status.get("n"),
+            "cluster_id": bao_status.get("cluster_id"),
+            "raft_applied_index": bao_status.get("raft_applied_index"),
         },
         "external_secret_stores": stores,
         "external_secrets": external_secrets,
@@ -220,6 +296,23 @@ def automated_common_pass(state: dict[str, Any]) -> bool:
     )
 
 
+def endpoint_status(remote: Remote, path: str) -> int:
+    code = remote.kubectl(
+        [
+            "-n", "audit-core", "exec", "deploy/audit-core", "--", "python", "-c",
+            "import urllib.request,urllib.error; "
+            f"u='http://127.0.0.1:8080{path}'; "
+            "\ntry:\n r=urllib.request.urlopen(u,timeout=3); print(r.status)"
+            "\nexcept urllib.error.HTTPError as e:\n print(e.code)",
+        ],
+        label=f"probe audit-core {path}",
+    )
+    try:
+        return int(code)
+    except ValueError as exc:
+        raise PreflightError(f"audit-core {path} returned an invalid status") from exc
+
+
 def database_lease_preflight(remote: Remote, args: argparse.Namespace) -> dict[str, Any]:
     now = datetime.now(UTC)
     state = common_state(remote, now)
@@ -232,14 +325,8 @@ def database_lease_preflight(remote: Remote, args: argparse.Namespace) -> dict[s
         label="read database Secret key names",
     )
     keys = secret_key_names(description)
-    health_code = remote.kubectl(
-        ["-n", "audit-core", "exec", "deploy/audit-core", "--", "python", "-c", 'import urllib.request; print(urllib.request.urlopen("http://127.0.0.1:8080/healthz", timeout=3).status)'],
-        label="probe audit-core health",
-    )
-    ready_code = remote.kubectl(
-        ["-n", "audit-core", "exec", "deploy/audit-core", "--", "python", "-c", 'import urllib.request; print(urllib.request.urlopen("http://127.0.0.1:8080/readyz", timeout=3).status)'],
-        label="probe audit-core readiness",
-    )
+    health_code = endpoint_status(remote, "/healthz")
+    ready_code = endpoint_status(remote, "/readyz")
     gates = {
         "approved_window_id_recorded": bool(args.approved_window_id),
         "audit_core_owner_acknowledged": args.audit_core_owner_ack,
@@ -247,7 +334,7 @@ def database_lease_preflight(remote: Remote, args: argparse.Namespace) -> dict[s
         "approved_synthetic_load_contract": bool(args.synthetic_load_id),
         "attended_abort_operator_named": bool(args.abort_operator),
     }
-    automated = automated_common_pass(state) and keys == EXPECTED_DB_KEYS and health_code == "200" and ready_code == "200"
+    automated = automated_common_pass(state) and keys == EXPECTED_DB_KEYS and health_code == 200 and ready_code == 200
     return {
         "procedure": "audit-core-database-lease-recovery",
         "preflight_only": True,
@@ -257,8 +344,8 @@ def database_lease_preflight(remote: Remote, args: argparse.Namespace) -> dict[s
         "baseline": {
             "database_secret_resource_version": secret_rv,
             "database_secret_keys": sorted(keys),
-            "health_status": int(health_code),
-            "readiness_status": int(ready_code),
+            "health_status": health_code,
+            "readiness_status": ready_code,
             "pod_uid": state["audit_core"]["pod_uid"],
             "restart_count": state["audit_core"]["restart_count"],
         },
@@ -276,9 +363,17 @@ def reboot_preflight(remote: Remote, args: argparse.Namespace) -> dict[str, Any]
     )
     latest = latest_completed_backup(backups, now)
     snapshot_evidence = Path(args.openbao_snapshot_evidence).resolve() if args.openbao_snapshot_evidence else None
+    snapshot_receipt = None
+    if snapshot_evidence:
+        snapshot_receipt = validate_snapshot_receipt(
+            snapshot_evidence,
+            live_openbao=state["openbao"],
+            now=now,
+            max_age_hours=args.max_snapshot_age_hours,
+        )
     gates = {
         "approved_window_id_recorded": bool(args.approved_window_id),
-        "openbao_snapshot_evidence_present": bool(snapshot_evidence and snapshot_evidence.is_file()),
+        "openbao_snapshot_evidence_present": bool(snapshot_receipt),
         "unseal_quorum_attested": args.unseal_quorum_attested,
         "provider_console_access_attested": args.provider_console_attested,
         "host_cluster_platform_database_audit_owners_acknowledged": args.all_owners_ack,
@@ -293,6 +388,8 @@ def reboot_preflight(remote: Remote, args: argparse.Namespace) -> dict[str, Any]
         "operator_gates": gates,
         "latest_platform_pg_backup": latest,
         "max_backup_age_hours": args.max_backup_age_hours,
+        "openbao_snapshot_receipt": snapshot_receipt,
+        "max_snapshot_age_hours": args.max_snapshot_age_hours,
         "state": state,
         "secret_values_observed": False,
     }
@@ -312,6 +409,7 @@ def main() -> int:
     parser.add_argument("--provider-console-attested", action="store_true")
     parser.add_argument("--all-owners-ack", action="store_true")
     parser.add_argument("--max-backup-age-hours", type=float, default=26.0)
+    parser.add_argument("--max-snapshot-age-hours", type=float, default=24.0)
     args = parser.parse_args()
     try:
         remote = Remote(args.remote)