From 3f9e4535d1b52d2dd1fc269ffc29820119afdccf Mon Sep 17 00:00:00 2001 From: codex Date: Sat, 22 Aug 2026 14:00:18 +0200 Subject: [PATCH] Harden WP-0024 recovery execution gates Assistant: codex Assistant-Model: gpt-5.6-sol Assistant-Session: 01a02669-87ee-7a31-b111-edc95a16e0fa --- Makefile | 19 +- ...-core-database-lease-approval.example.json | 29 ++ docs/audit-core-database-lease-recovery.md | 46 +- ...enbao-reboot-snapshot-receipt.example.json | 22 + docs/railiance01-coordinated-reboot.md | 7 +- scripts/audit-core-database-lease-recovery.py | 438 ++++++++++++++++++ scripts/audit-core-recovery-preflight.py | 122 ++++- ...test_audit_core_database_lease_recovery.py | 87 ++++ tests/test_audit_core_recovery_preflight.py | 76 ++- ...-core-custody-and-recovery-coordination.md | 24 + 10 files changed, 848 insertions(+), 22 deletions(-) create mode 100644 docs/audit-core-database-lease-approval.example.json create mode 100644 docs/openbao-reboot-snapshot-receipt.example.json create mode 100755 scripts/audit-core-database-lease-recovery.py create mode 100644 tests/test_audit_core_database_lease_recovery.py diff --git a/Makefile b/Makefile index 6534b45..d7de669 100644 --- a/Makefile +++ b/Makefile @@ -476,10 +476,23 @@ audit-whitehat-e2-cleanup: ## Remove the exact engagement credentials and reload python3 scripts/audit-core-whitehat-e2-credentials.py cleanup --confirm WH-ENG-20260822-AUDIT-E2-01:attended audit-database-lease-recovery-preflight: ## Read-only gate for the restart-free database lease exercise - python3 scripts/audit-core-recovery-preflight.py database-lease + python3 scripts/audit-core-recovery-preflight.py database-lease $(AUDIT_DATABASE_LEASE_PREFLIGHT_ARGS) + +audit-database-lease-approval-check: ## Validate an approved T02 receipt without mutation + @test -n "$(AUDIT_DATABASE_LEASE_APPROVAL)" || { echo 'set AUDIT_DATABASE_LEASE_APPROVAL=/path/to/approved-receipt.json' >&2; exit 2; } + python3 scripts/audit-core-database-lease-recovery.py validate-approval \ + --approval "$(AUDIT_DATABASE_LEASE_APPROVAL)" + +audit-database-lease-recovery-exercise: ## Run the separately approved attended T02 exercise + @test -n "$(AUDIT_DATABASE_LEASE_APPROVAL)" || { echo 'set AUDIT_DATABASE_LEASE_APPROVAL=/path/to/approved-receipt.json' >&2; exit 2; } + @test -n "$(AUDIT_DATABASE_LEASE_DRIVER)" || { echo 'set AUDIT_DATABASE_LEASE_DRIVER=/path/to/approved-load-driver' >&2; exit 2; } + python3 scripts/audit-core-database-lease-recovery.py exercise \ + --approval "$(AUDIT_DATABASE_LEASE_APPROVAL)" \ + --load-driver "$(AUDIT_DATABASE_LEASE_DRIVER)" \ + --confirm RAILIANCE-WP-0024-T02:attended railiance01-reboot-preflight: ## Read-only gate for the coordinated single-node reboot - python3 scripts/audit-core-recovery-preflight.py node-reboot + python3 scripts/audit-core-recovery-preflight.py node-reboot $(RAILIANCE01_REBOOT_PREFLIGHT_ARGS) -.PHONY: audit-whitehat-e2-preflight audit-whitehat-e2-project audit-whitehat-e2-cleanup audit-database-lease-recovery-preflight railiance01-reboot-preflight apps-pg-isolation-probe +.PHONY: audit-whitehat-e2-preflight audit-whitehat-e2-project audit-whitehat-e2-cleanup audit-database-lease-recovery-preflight audit-database-lease-approval-check audit-database-lease-recovery-exercise railiance01-reboot-preflight apps-pg-isolation-probe .PHONY: db-deploy db-status db-shell db-logs forgejo-db-deploy forgejo-db-status forgejo-db-shell apps-pg-deploy apps-pg-status apps-pg-shell apps-pg-logs net-kingdom-pg-inter-hub-networkpolicy-deploy pg-deploy pg-status pg-pgpool-check valkey-deploy valkey-status openbao-repo openbao-dry-run openbao-overlay-apply openbao-verify-login-overlay openbao-deploy openbao-status openbao-verify openbao-verify-post-unseal openbao-configure-initial openbao-configure-ssh openbao-verify-ssh openbao-verify-authenticated openbao-configure-external-secrets-issue-core openbao-configure-external-secrets-activity-core openbao-configure-external-secrets-forgejo openbao-validate-restore-evidence openbao-validate-emergency-evidence credential-grants-validate credential-change-applier-dry-run credential-change-applier-apply-plan credential-change-applier-apply credential-change-runbook credential-change-record-evidence credential-change-lifecycle-plan credential-change-lifecycle-event credential-change-import-inventory agent-high-risk-boundary-check openbao-credential-change-appliers-dry-run openbao-configure-credential-change-appliers openbao-token-grants-dry-run openbao-configure-token-grants openbao-verify-token-grants-dry-run openbao-verify-token-grants openbao-verify-token-grants-smoke credential-helper-dry-run credential-tests credential-exec-ops-warden-smoke argocd-bootstrap-dry-run argocd-bootstrap-deploy argocd-repo-apply argocd-status backup forgejo-backup forgejo-backup-dry-run forgejo-backup-status forgejo-package-prune forgejo-package-prune-dry-run reef-deployables apps-pg-evidence consumption-preflight help diff --git a/docs/audit-core-database-lease-approval.example.json b/docs/audit-core-database-lease-approval.example.json new file mode 100644 index 0000000..f6b72d8 --- /dev/null +++ b/docs/audit-core-database-lease-approval.example.json @@ -0,0 +1,29 @@ +{ + "procedure": "audit-core-database-lease-recovery", + "task_id": "RAILIANCE-WP-0024-T02", + "status": "pending", + "approval_id": null, + "window": { + "start": null, + "end": null + }, + "abort_operator": null, + "owners": { + "audit-core": { + "acknowledged": false, + "message_id": null + }, + "rapp-postgres": { + "acknowledged": false, + "message_id": null + }, + "railiance-platform": { + "acknowledged": false, + "message_id": null + } + }, + "synthetic_load": { + "contract_id": null, + "driver_revision": null + } +} diff --git a/docs/audit-core-database-lease-recovery.md b/docs/audit-core-database-lease-recovery.md index da25ff4..bfb56d3 100644 --- a/docs/audit-core-database-lease-recovery.md +++ b/docs/audit-core-database-lease-recovery.md @@ -29,6 +29,25 @@ The result must have `automated_checks_passed: true`. It will remain provided. The helper reads Secret metadata and key names only; it never reads Secret data. +The live harness is +`scripts/audit-core-database-lease-recovery.py`. Start from the deliberately +pending `docs/audit-core-database-lease-approval.example.json`; the actual +value-safe receipt must name a ≤15-minute window, approval id, abort operator, +State Hub acknowledgement ids from audit-core/rapp-postgres/platform, and an +approved synthetic-load contract plus driver revision. Validate without +mutation: + +```bash +make audit-database-lease-approval-check \ + AUDIT_DATABASE_LEASE_APPROVAL=/path/to/approved-receipt.json +``` + +The separately approved load driver is an executable that accepts `baseline`, +`expect-unavailable`, `expect-recovered`, and `cleanup`, plus +`--contract-id ID`. Each call returns only its exact value-safe JSON evidence +shape. Unexpected keys—including a bearer accidentally returned as `token`— +make the harness fail without echoing driver output. + ## Attended sequence Keep one terminal attached from baseline through recovery. A value-handling @@ -41,11 +60,14 @@ bodies and clear all transient JSON from memory or mode-`0600` temporary files. 2. Start bounded retrying synthetic traffic. Its evidence records only request identifier, attempt time, response class, and final accepted/duplicate result—never the bearer or payload. -3. Inside the attended process, read the current mounted username without - emitting it. List only lease handles below - `sys/leases/lookup/database/creds/audit-core-runtime`, look each up, and - retain the one handle whose lease username matches. Abort unless exactly one - live handle matches. Clear every lookup response and username immediately. +3. Inside the attended process, list only lease handles below + `sys/leases/lookup/database/creds/audit-core-runtime`. OpenBao's lease lookup + returns issue/expiry/TTL metadata but not the leased username, so exact + selection is deliberately stricter: there must be exactly one live handle + under that exact runtime prefix, and its issue/expiry interval must contain + the current database Secret refresh time. Abort for database-owner + disambiguation if zero or multiple handles exist. Retain only a one-way + handle fingerprint in evidence and clear the lookup response immediately. 4. Recheck that the Secret `resourceVersion`, pod UID, and selected handle have not changed. Revoke only that handle. Do not revoke the role prefix and do not revoke the ESO parent token. @@ -64,6 +86,20 @@ bodies and clear all transient JSON from memory or mode-`0600` temporary files. the predecessor lease is revoked. The replacement remains managed by ESO and expires through its normal database role TTL. +Only inside the approved window, with the receipt and reviewed driver: + +```bash +make audit-database-lease-recovery-exercise \ + AUDIT_DATABASE_LEASE_APPROVAL=/path/to/approved-receipt.json \ + AUDIT_DATABASE_LEASE_DRIVER=/path/to/approved-load-driver +``` + +The exact confirmation string is embedded in the target. The harness rechecks +the Secret/pod/lease immediately before revocation, requires at least five +minutes of remaining lease TTL to avoid the normal ESO refresh race, performs +best-effort forced reconciliation on an interrupted post-revocation run, and +never issues a rollout/restart command. + ## Time bounds and abort path - Planned outage observation: at most 120 seconds. diff --git a/docs/openbao-reboot-snapshot-receipt.example.json b/docs/openbao-reboot-snapshot-receipt.example.json new file mode 100644 index 0000000..e63ea17 --- /dev/null +++ b/docs/openbao-reboot-snapshot-receipt.example.json @@ -0,0 +1,22 @@ +{ + "receipt_version": 1, + "receipt_id": "replace-with-operator-receipt-id", + "created_at": "YYYY-MM-DDTHH:MM:SSZ", + "operator": "replace-with-operator-id", + "source_cluster": "railiance01", + "source_namespace": "openbao", + "source_pod": "openbao-0", + "cluster_id": "replace-with-live-openbao-cluster-id", + "raft_applied_index": 0, + "snapshot_created": false, + "source_initialized": false, + "source_unsealed": false, + "snapshot_sha256": "sha256:0000000000000000000000000000000000000000000000000000000000000000", + "snapshot_encrypted": false, + "encrypted_snapshot_sha256": "sha256:1111111111111111111111111111111111111111111111111111111111111111", + "encrypted_location_ref": "replace-with-approved-off-host-custody-handle", + "encrypted_copy_off_host": false, + "encryption_verified": false, + "hash_verified": false, + "no_secret_material_recorded": false +} diff --git a/docs/railiance01-coordinated-reboot.md b/docs/railiance01-coordinated-reboot.md index 535b2b9..6a5a873 100644 --- a/docs/railiance01-coordinated-reboot.md +++ b/docs/railiance01-coordinated-reboot.md @@ -38,7 +38,9 @@ the reviewed 1/1 Ready audit-core image. The following are human gates and make - approved window id and acknowledgements from all five owners; - a named attended driver/abort operator; - independent provider-console access, not SSH alone; -- a current non-secret OpenBao Raft snapshot evidence file; +- a current non-secret OpenBao Raft snapshot receipt that validates against + `docs/openbao-reboot-snapshot-receipt.example.json`, the live cluster id and + Raft index, and the default 24-hour freshness bound; - explicit availability of two of three Shamir shares through their approved out-of-band custody paths. @@ -58,6 +60,9 @@ State Hub. verify an on-demand backup before continuing. 5. Create and escrow a fresh OpenBao Raft snapshot through the approved encrypted backup lane. Record only its evidence receipt and verification. + Pass that receipt to the preflight with + `--openbao-snapshot-evidence /path/to/receipt.json`; a merely existing, + stale, placeholder, wrong-cluster, unencrypted, or on-host receipt is denied. 6. Have both Shamir share custodians present. Prove provider-console access and SSH access. Record the exact restart and abort operators. 7. Re-run the preflight with the approval and attestation flags. The result diff --git a/scripts/audit-core-database-lease-recovery.py b/scripts/audit-core-database-lease-recovery.py new file mode 100755 index 0000000..08b1ff8 --- /dev/null +++ b/scripts/audit-core-database-lease-recovery.py @@ -0,0 +1,438 @@ +#!/usr/bin/env python3 +"""Attended audit-core database lease failure/recovery exercise. + +The script is fail-closed and value-safe. It selects only a single live lease +under the exact audit-core runtime prefix, never emits its id, never reads a +database password, never restarts audit-core, and requires a separately +approved synthetic-load driver. +""" + +from __future__ import annotations + +import argparse +import hashlib +import importlib.util +import json +import os +import stat +import subprocess +import sys +import time +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import Any, Callable + + +ROOT = Path(__file__).resolve().parents[1] +TASK_ID = "RAILIANCE-WP-0024-T02" +PROCEDURE = "audit-core-database-lease-recovery" +LEASE_PREFIX = "database/creds/audit-core-runtime" +EXTERNAL_SECRET = "audit-core-database" +CONFIRM = f"{TASK_ID}:attended" +MAX_WINDOW_SECONDS = 15 * 60 +MIN_LEASE_TTL_SECONDS = 5 * 60 +EXPECTED_DRIVER_KEYS = { + "baseline": {"contract_id", "fixture_id", "status", "secret_values_observed"}, + "expect-unavailable": { + "contract_id", "fixture_id", "status", "http_status", "attempts", + "secret_values_observed", + }, + "expect-recovered": { + "contract_id", "fixture_id", "status", "http_status", "attempts", + "secret_values_observed", + }, + "cleanup": {"contract_id", "fixture_id", "status", "secret_values_observed"}, +} + + +class ProcedureError(RuntimeError): + pass + + +def load_preflight_module() -> Any: + spec = importlib.util.spec_from_file_location( + "audit_core_recovery_preflight", + ROOT / "scripts" / "audit-core-recovery-preflight.py", + ) + if not spec or not spec.loader: + raise ProcedureError("cannot load recovery preflight module") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +PREFLIGHT = load_preflight_module() + + +def parse_time(value: str) -> datetime: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + if parsed.tzinfo is None: + raise ProcedureError("approval timestamps must include a timezone") + return parsed.astimezone(UTC) + + +def load_approval(path: Path, now: datetime, *, require_open_window: bool) -> dict[str, Any]: + try: + document = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise ProcedureError("approval receipt is unavailable or invalid JSON") from exc + if not isinstance(document, dict): + raise ProcedureError("approval receipt must be an object") + if document.get("procedure") != PROCEDURE or document.get("task_id") != TASK_ID: + raise ProcedureError("approval receipt has the wrong procedure or task") + if document.get("status") != "approved" or not document.get("approval_id"): + raise ProcedureError("approval receipt is not approved") + window = document.get("window") or {} + if not window.get("start") or not window.get("end"): + raise ProcedureError("approval receipt has no complete window") + start, end = parse_time(window["start"]), parse_time(window["end"]) + if not start < end or (end - start).total_seconds() > MAX_WINDOW_SECONDS: + raise ProcedureError("approval window must be positive and at most 15 minutes") + if require_open_window and not start <= now <= end: + raise ProcedureError("current time is outside the approved window") + if not document.get("abort_operator"): + raise ProcedureError("approval receipt has no abort operator") + owners = document.get("owners") or {} + for owner in ("audit-core", "rapp-postgres", "railiance-platform"): + record = owners.get(owner) or {} + if record.get("acknowledged") is not True or not record.get("message_id"): + raise ProcedureError(f"approval receipt lacks {owner} acknowledgement") + load = document.get("synthetic_load") or {} + if not load.get("contract_id") or not load.get("driver_revision"): + raise ProcedureError("approval receipt lacks the synthetic-load contract") + return document + + +def safe_run( + command: list[str], *, label: str, env: dict[str, str] | None = None +) -> subprocess.CompletedProcess[str]: + completed = subprocess.run( + command, text=True, capture_output=True, env=env, check=False + ) + if completed.returncode != 0: + # Never attach stdout/stderr: Bao and load-driver processes may hold + # sensitive material even though their contract forbids emitting it. + raise ProcedureError(f"{label} failed (exit {completed.returncode})") + return completed + + +class Authority: + def __init__(self, token_file: Path) -> None: + if not token_file.is_file() or stat.S_IMODE(token_file.stat().st_mode) != 0o600: + raise ProcedureError("OpenBao token file must exist with mode 0600") + token = token_file.read_text(encoding="utf-8").splitlines()[0].strip() + if not token: + raise ProcedureError("OpenBao token file is empty") + self.env = dict(os.environ, BAO_ADDR="https://bao.coulomb.social", BAO_TOKEN=token) + + def bao(self, args: list[str], *, label: str) -> str: + return safe_run(["bao", *args], label=label, env=self.env).stdout.strip() + + def single_runtime_lease(self) -> dict[str, Any]: + raw = self.bao( + ["list", "-format=json", f"sys/leases/lookup/{LEASE_PREFIX}"], + label="list exact runtime lease handles", + ) + try: + handles = json.loads(raw) + except json.JSONDecodeError as exc: + raise ProcedureError("runtime lease list is invalid JSON") from exc + if not isinstance(handles, list) or len(handles) != 1: + count = len(handles) if isinstance(handles, list) else "unknown" + raise ProcedureError( + f"exact runtime prefix must contain one live handle (observed {count})" + ) + lease_id = f"{LEASE_PREFIX}/{handles[0]}" + lookup_raw = self.bao( + ["write", "-format=json", "sys/leases/lookup", f"lease_id={lease_id}"], + label="lookup exact runtime lease metadata", + ) + try: + lookup = json.loads(lookup_raw)["data"] + issue = parse_time(lookup["issue_time"]) + expires = parse_time(lookup["expire_time"]) + ttl = int(lookup["ttl"]) + except (json.JSONDecodeError, KeyError, TypeError, ValueError) as exc: + raise ProcedureError("runtime lease metadata is incomplete") from exc + return { + "id": lease_id, + "fingerprint": hashlib.sha256(lease_id.encode()).hexdigest()[:12], + "issue_time": issue, + "expire_time": expires, + "ttl": ttl, + } + + def revoke(self, lease_id: str) -> None: + self.bao(["lease", "revoke", lease_id], label="revoke exact runtime lease") + + +class LoadDriver: + def __init__(self, path: Path, contract_id: str, expected_revision: str) -> None: + if not path.is_file() or not os.access(path, os.X_OK): + raise ProcedureError("synthetic-load driver must be an executable file") + revision = "sha256:" + hashlib.sha256(path.read_bytes()).hexdigest() + if expected_revision != revision: + raise ProcedureError("synthetic-load driver does not match the approved revision") + self.path = path + self.contract_id = contract_id + self.revision = revision + + def run(self, phase: str) -> dict[str, Any]: + result = safe_run( + [str(self.path), phase, "--contract-id", self.contract_id], + label=f"synthetic-load {phase}", + ) + try: + payload = json.loads(result.stdout) + except json.JSONDecodeError as exc: + raise ProcedureError(f"synthetic-load {phase} returned invalid JSON") from exc + if not isinstance(payload, dict) or set(payload) != EXPECTED_DRIVER_KEYS[phase]: + raise ProcedureError(f"synthetic-load {phase} returned an unsafe evidence shape") + if payload.get("contract_id") != self.contract_id: + raise ProcedureError(f"synthetic-load {phase} returned the wrong contract") + if payload.get("secret_values_observed") is not False: + raise ProcedureError(f"synthetic-load {phase} did not attest value safety") + return payload + + +def secret_state(remote: Any) -> dict[str, Any]: + secret_rv = remote.kubectl( + ["-n", "audit-core", "get", "secret", EXTERNAL_SECRET, "-o", "jsonpath={.metadata.resourceVersion}"], + label="read database Secret resource version", + ) + external = remote.kubectl_json( + ["-n", "audit-core", "get", "externalsecret", EXTERNAL_SECRET], + label="read database ExternalSecret state", + ) + mount_generation = remote.kubectl( + ["-n", "audit-core", "exec", "deploy/audit-core", "--", "readlink", "/etc/audit-core/db/..data"], + label="read mounted credential generation", + ) + refresh = external.get("status", {}).get("refreshTime") + ready = PREFLIGHT.resource_condition(external)["ready"] + if not refresh: + raise ProcedureError("database ExternalSecret has no refresh time") + return { + "resource_version": secret_rv, + "mount_generation": mount_generation, + "refresh_time": parse_time(refresh), + "ready": ready, + } + + +def pod_state(remote: Any) -> dict[str, Any]: + pods = remote.kubectl_json( + ["-n", "audit-core", "get", "pods", "-l", "app.kubernetes.io/name=audit-core"], + label="read audit-core pod metadata", + ).get("items", []) + if len(pods) != 1: + raise ProcedureError(f"expected one audit-core pod, observed {len(pods)}") + statuses = pods[0].get("status", {}).get("containerStatuses", []) + return { + "uid": pods[0]["metadata"]["uid"], + "restart_count": sum(int(item.get("restartCount", 0)) for item in statuses), + } + + +def assert_lease_matches_refresh(lease: dict[str, Any], secret: dict[str, Any]) -> None: + delta = abs((lease["issue_time"] - secret["refresh_time"]).total_seconds()) + if ( + delta > 5 + or secret["refresh_time"] < lease["issue_time"] - timedelta(seconds=5) + or secret["refresh_time"] > lease["expire_time"] + ): + raise ProcedureError("single runtime lease is not coherent with the mounted Secret refresh") + if lease["ttl"] < MIN_LEASE_TTL_SECONDS: + raise ProcedureError("runtime lease is too close to automatic refresh; wait for the next sync") + + +def wait_for( + predicate: Callable[[], bool], *, label: str, timeout: int = 120, interval: float = 2 +) -> None: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if predicate(): + return + time.sleep(interval) + raise ProcedureError(f"timed out waiting for {label}") + + +def force_refresh(remote: Any, baseline: dict[str, Any]) -> dict[str, Any]: + remote.kubectl( + [ + "-n", "audit-core", "annotate", "externalsecret", EXTERNAL_SECRET, + f"railiance.io/force-sync={int(time.time())}", "--overwrite", + ], + label="force database ExternalSecret reconciliation", + ) + latest: dict[str, Any] = {} + + def changed() -> bool: + nonlocal latest + latest = secret_state(remote) + return bool( + latest["ready"] + and latest["resource_version"] != baseline["resource_version"] + and latest["mount_generation"] != baseline["mount_generation"] + and latest["refresh_time"] > baseline["refresh_time"] + ) + + wait_for(changed, label="database Secret and mounted generation refresh") + return latest + + +def exercise(args: argparse.Namespace) -> dict[str, Any]: + approval = load_approval(args.approval, datetime.now(UTC), require_open_window=True) + if args.confirm != CONFIRM: + raise ProcedureError(f"live exercise requires --confirm {CONFIRM}") + remote = PREFLIGHT.Remote(args.remote) + authority = Authority(args.token_file) + driver = LoadDriver( + args.load_driver, + approval["synthetic_load"]["contract_id"], + approval["synthetic_load"]["driver_revision"], + ) + + preflight_args = argparse.Namespace( + approved_window_id=approval["approval_id"], + audit_core_owner_ack=True, + rapp_postgres_owner_ack=True, + synthetic_load_id=approval["synthetic_load"]["contract_id"], + abort_operator=approval["abort_operator"], + ) + preflight = PREFLIGHT.database_lease_preflight(remote, preflight_args) + if not preflight["ready_for_live_execution"]: + raise ProcedureError("database lease recovery preflight is not ready") + + before_pod = pod_state(remote) + before_secret = secret_state(remote) + lease = authority.single_runtime_lease() + assert_lease_matches_refresh(lease, before_secret) + baseline_load = driver.run("baseline") + if baseline_load.get("status") != "ready": + raise ProcedureError("synthetic-load baseline is not ready") + revoked = False + recovered_secret: dict[str, Any] | None = None + unavailable: dict[str, Any] | None = None + recovered: dict[str, Any] | None = None + cleanup_result: dict[str, Any] | None = None + completed = False + try: + # Close the race with ESO's ordinary refresh before the destructive step. + if secret_state(remote) != before_secret or pod_state(remote) != before_pod: + raise ProcedureError("baseline changed before revocation") + current = authority.single_runtime_lease() + if current["fingerprint"] != lease["fingerprint"]: + raise ProcedureError("runtime lease changed before revocation") + authority.revoke(lease["id"]) + revoked = True + wait_for( + lambda: PREFLIGHT.endpoint_status(remote, "/healthz") == 200 + and PREFLIGHT.endpoint_status(remote, "/readyz") == 503, + label="health 200 and readiness 503 after revocation", + timeout=60, + ) + unavailable = driver.run("expect-unavailable") + if ( + unavailable.get("http_status") != 503 + or unavailable.get("status") != "retryable_unavailable" + or not isinstance(unavailable.get("attempts"), int) + or unavailable["attempts"] < 1 + or unavailable.get("fixture_id") != baseline_load.get("fixture_id") + ): + raise ProcedureError("synthetic load did not prove retryable 503") + recovered_secret = force_refresh(remote, before_secret) + wait_for( + lambda: PREFLIGHT.endpoint_status(remote, "/readyz") == 200, + label="audit-core readiness recovery", + ) + recovered = driver.run("expect-recovered") + if ( + recovered.get("status") not in {"accepted", "duplicate"} + or recovered.get("http_status") not in {200, 202} + or recovered.get("fixture_id") != baseline_load.get("fixture_id") + ): + raise ProcedureError("synthetic load did not prove accepted/duplicate recovery") + after_pod = pod_state(remote) + if after_pod != before_pod: + raise ProcedureError("audit-core pod identity or restart count changed") + completed = True + finally: + if revoked and recovered_secret is None: + try: + force_refresh(remote, before_secret) + except Exception: + pass + try: + cleanup_result = driver.run("cleanup") + if ( + cleanup_result.get("status") != "clean" + or cleanup_result.get("fixture_id") != baseline_load.get("fixture_id") + ): + raise ProcedureError("synthetic-load cleanup did not confirm exact fixture removal") + except Exception: + if completed: + raise + return { + "procedure": PROCEDURE, + "task_id": TASK_ID, + "approval_id": approval["approval_id"], + "synthetic_load_contract": approval["synthetic_load"]["contract_id"], + "synthetic_load_driver_revision": driver.revision, + "lease_handle_fingerprint": lease["fingerprint"], + "lease_revoked": revoked, + "health_during_failure": 200, + "readiness_during_failure": 503, + "synthetic_unavailable_attempts": unavailable["attempts"] if unavailable else None, + "secret_resource_version_changed": bool( + recovered_secret + and recovered_secret["resource_version"] != before_secret["resource_version"] + ), + "mount_generation_changed": bool( + recovered_secret + and recovered_secret["mount_generation"] != before_secret["mount_generation"] + ), + "readiness_recovered": PREFLIGHT.endpoint_status(remote, "/readyz") == 200, + "synthetic_recovery_status": recovered["status"] if recovered else None, + "same_pod_uid": pod_state(remote)["uid"] == before_pod["uid"], + "restart_count_unchanged": pod_state(remote)["restart_count"] == before_pod["restart_count"], + "load_cleanup_status": cleanup_result["status"] if cleanup_result else None, + "completed_at": datetime.now(UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z"), + "secret_values_observed": False, + } + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("command", choices=["validate-approval", "exercise"]) + parser.add_argument("--approval", type=Path, required=True) + parser.add_argument("--remote", default="railiance01") + parser.add_argument( + "--token-file", type=Path, + default=Path.home() / ".local/openbao/platform-admin.token", + ) + parser.add_argument("--load-driver", type=Path) + parser.add_argument("--confirm") + args = parser.parse_args() + try: + if args.command == "validate-approval": + approval = load_approval(args.approval, datetime.now(UTC), require_open_window=False) + result = { + "procedure": PROCEDURE, + "approval_id": approval["approval_id"], + "approval_receipt_valid": True, + "secret_values_observed": False, + } + else: + if args.load_driver is None: + raise ProcedureError("live exercise requires --load-driver") + result = exercise(args) + except (OSError, IndexError, ProcedureError, PREFLIGHT.PreflightError) as exc: + print(f"database lease recovery failed: {exc}", file=sys.stderr) + return 1 + print(json.dumps(result, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/audit-core-recovery-preflight.py b/scripts/audit-core-recovery-preflight.py index 0f2c5ff..b2cbb5b 100755 --- a/scripts/audit-core-recovery-preflight.py +++ b/scripts/audit-core-recovery-preflight.py @@ -111,6 +111,80 @@ def secret_key_names(description: str) -> set[str]: return keys +def validate_snapshot_receipt( + path: Path, + *, + live_openbao: dict[str, Any], + now: datetime, + max_age_hours: float, +) -> dict[str, Any]: + try: + receipt = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise PreflightError("OpenBao snapshot receipt is unavailable or invalid JSON") from exc + if not isinstance(receipt, dict) or receipt.get("receipt_version") != 1: + raise PreflightError("OpenBao snapshot receipt has an unsupported shape/version") + required_strings = ( + "receipt_id", "created_at", "operator", "source_cluster", + "source_namespace", "source_pod", "cluster_id", "snapshot_sha256", + "encrypted_snapshot_sha256", "encrypted_location_ref", + ) + for key in required_strings: + if not isinstance(receipt.get(key), str) or not receipt[key].strip(): + raise PreflightError(f"OpenBao snapshot receipt is missing {key}") + required_true = ( + "snapshot_created", "source_initialized", "source_unsealed", + "snapshot_encrypted", "encrypted_copy_off_host", "encryption_verified", + "hash_verified", "no_secret_material_recorded", + ) + for key in required_true: + if receipt.get(key) is not True: + raise PreflightError(f"OpenBao snapshot receipt requires {key}=true") + if ( + receipt["source_cluster"] != "railiance01" + or receipt["source_namespace"] != "openbao" + or receipt["source_pod"] != "openbao-0" + or receipt["cluster_id"] != live_openbao.get("cluster_id") + ): + raise PreflightError("OpenBao snapshot receipt does not match the live source") + try: + created = rfc3339(receipt["created_at"]) + snapshot_index = int(receipt["raft_applied_index"]) + live_index = int(live_openbao["raft_applied_index"]) + except (KeyError, TypeError, ValueError) as exc: + raise PreflightError("OpenBao snapshot receipt has invalid time/index metadata") from exc + age_hours = (now - created).total_seconds() / 3600 + if age_hours < -(5 / 60) or age_hours > max_age_hours: + raise PreflightError("OpenBao snapshot receipt is outside the permitted age") + if snapshot_index <= 0 or snapshot_index > live_index: + raise PreflightError("OpenBao snapshot receipt has an impossible Raft index") + digest_pattern = re.compile(r"^sha256:[0-9a-f]{64}$") + digests = (receipt["snapshot_sha256"], receipt["encrypted_snapshot_sha256"]) + if any(not digest_pattern.fullmatch(value) for value in digests): + raise PreflightError("OpenBao snapshot receipt has an invalid digest") + if any(len(set(value.removeprefix("sha256:"))) <= 1 for value in digests): + raise PreflightError("OpenBao snapshot receipt contains a placeholder digest") + if digests[0] == digests[1]: + raise PreflightError("plain and encrypted snapshot digests must differ") + encoded = json.dumps(receipt, sort_keys=True) + for marker in ( + "BEGIN PRIVATE KEY", "BEGIN OPENSSH PRIVATE KEY", "AGE-SECRET-KEY-1", + "OPENBAO_ROOT_TOKEN", "VAULT_TOKEN", "hvs.", "<", "YYYY-MM-DD", + ): + if marker in encoded: + raise PreflightError("OpenBao snapshot receipt contains forbidden material/placeholder") + return { + "receipt_id": receipt["receipt_id"], + "created_at": created.isoformat().replace("+00:00", "Z"), + "age_hours": round(age_hours, 2), + "cluster_id_matches": True, + "raft_applied_index": snapshot_index, + "encrypted_copy_off_host": True, + "verified": True, + "secret_values_observed": False, + } + + def common_state(remote: Remote, now: datetime) -> dict[str, Any]: node_list = remote.kubectl_json(["get", "nodes"], label="read node state") nodes = node_list.get("items", []) @@ -186,6 +260,8 @@ def common_state(remote: Remote, now: datetime) -> dict[str, Any]: "seal_type": bao_status.get("type"), "threshold": bao_status.get("t"), "shares": bao_status.get("n"), + "cluster_id": bao_status.get("cluster_id"), + "raft_applied_index": bao_status.get("raft_applied_index"), }, "external_secret_stores": stores, "external_secrets": external_secrets, @@ -220,6 +296,23 @@ def automated_common_pass(state: dict[str, Any]) -> bool: ) +def endpoint_status(remote: Remote, path: str) -> int: + code = remote.kubectl( + [ + "-n", "audit-core", "exec", "deploy/audit-core", "--", "python", "-c", + "import urllib.request,urllib.error; " + f"u='http://127.0.0.1:8080{path}'; " + "\ntry:\n r=urllib.request.urlopen(u,timeout=3); print(r.status)" + "\nexcept urllib.error.HTTPError as e:\n print(e.code)", + ], + label=f"probe audit-core {path}", + ) + try: + return int(code) + except ValueError as exc: + raise PreflightError(f"audit-core {path} returned an invalid status") from exc + + def database_lease_preflight(remote: Remote, args: argparse.Namespace) -> dict[str, Any]: now = datetime.now(UTC) state = common_state(remote, now) @@ -232,14 +325,8 @@ def database_lease_preflight(remote: Remote, args: argparse.Namespace) -> dict[s label="read database Secret key names", ) keys = secret_key_names(description) - health_code = remote.kubectl( - ["-n", "audit-core", "exec", "deploy/audit-core", "--", "python", "-c", 'import urllib.request; print(urllib.request.urlopen("http://127.0.0.1:8080/healthz", timeout=3).status)'], - label="probe audit-core health", - ) - ready_code = remote.kubectl( - ["-n", "audit-core", "exec", "deploy/audit-core", "--", "python", "-c", 'import urllib.request; print(urllib.request.urlopen("http://127.0.0.1:8080/readyz", timeout=3).status)'], - label="probe audit-core readiness", - ) + health_code = endpoint_status(remote, "/healthz") + ready_code = endpoint_status(remote, "/readyz") gates = { "approved_window_id_recorded": bool(args.approved_window_id), "audit_core_owner_acknowledged": args.audit_core_owner_ack, @@ -247,7 +334,7 @@ def database_lease_preflight(remote: Remote, args: argparse.Namespace) -> dict[s "approved_synthetic_load_contract": bool(args.synthetic_load_id), "attended_abort_operator_named": bool(args.abort_operator), } - automated = automated_common_pass(state) and keys == EXPECTED_DB_KEYS and health_code == "200" and ready_code == "200" + automated = automated_common_pass(state) and keys == EXPECTED_DB_KEYS and health_code == 200 and ready_code == 200 return { "procedure": "audit-core-database-lease-recovery", "preflight_only": True, @@ -257,8 +344,8 @@ def database_lease_preflight(remote: Remote, args: argparse.Namespace) -> dict[s "baseline": { "database_secret_resource_version": secret_rv, "database_secret_keys": sorted(keys), - "health_status": int(health_code), - "readiness_status": int(ready_code), + "health_status": health_code, + "readiness_status": ready_code, "pod_uid": state["audit_core"]["pod_uid"], "restart_count": state["audit_core"]["restart_count"], }, @@ -276,9 +363,17 @@ def reboot_preflight(remote: Remote, args: argparse.Namespace) -> dict[str, Any] ) latest = latest_completed_backup(backups, now) snapshot_evidence = Path(args.openbao_snapshot_evidence).resolve() if args.openbao_snapshot_evidence else None + snapshot_receipt = None + if snapshot_evidence: + snapshot_receipt = validate_snapshot_receipt( + snapshot_evidence, + live_openbao=state["openbao"], + now=now, + max_age_hours=args.max_snapshot_age_hours, + ) gates = { "approved_window_id_recorded": bool(args.approved_window_id), - "openbao_snapshot_evidence_present": bool(snapshot_evidence and snapshot_evidence.is_file()), + "openbao_snapshot_evidence_present": bool(snapshot_receipt), "unseal_quorum_attested": args.unseal_quorum_attested, "provider_console_access_attested": args.provider_console_attested, "host_cluster_platform_database_audit_owners_acknowledged": args.all_owners_ack, @@ -293,6 +388,8 @@ def reboot_preflight(remote: Remote, args: argparse.Namespace) -> dict[str, Any] "operator_gates": gates, "latest_platform_pg_backup": latest, "max_backup_age_hours": args.max_backup_age_hours, + "openbao_snapshot_receipt": snapshot_receipt, + "max_snapshot_age_hours": args.max_snapshot_age_hours, "state": state, "secret_values_observed": False, } @@ -312,6 +409,7 @@ def main() -> int: parser.add_argument("--provider-console-attested", action="store_true") parser.add_argument("--all-owners-ack", action="store_true") parser.add_argument("--max-backup-age-hours", type=float, default=26.0) + parser.add_argument("--max-snapshot-age-hours", type=float, default=24.0) args = parser.parse_args() try: remote = Remote(args.remote) diff --git a/tests/test_audit_core_database_lease_recovery.py b/tests/test_audit_core_database_lease_recovery.py new file mode 100644 index 0000000..198b013 --- /dev/null +++ b/tests/test_audit_core_database_lease_recovery.py @@ -0,0 +1,87 @@ +from __future__ import annotations + +import importlib.util +import json +import os +import tempfile +import unittest +from datetime import UTC, datetime, timedelta +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +SPEC = importlib.util.spec_from_file_location( + "audit_core_database_lease_recovery", + ROOT / "scripts" / "audit-core-database-lease-recovery.py", +) +assert SPEC and SPEC.loader +module = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(module) + + +def approved_receipt(now: datetime) -> dict: + return { + "procedure": module.PROCEDURE, + "task_id": module.TASK_ID, + "status": "approved", + "approval_id": "operator-window-1", + "window": { + "start": (now - timedelta(minutes=1)).isoformat(), + "end": (now + timedelta(minutes=9)).isoformat(), + }, + "abort_operator": "operator-a", + "owners": { + owner: {"acknowledged": True, "message_id": f"ack-{owner}"} + for owner in ("audit-core", "rapp-postgres", "railiance-platform") + }, + "synthetic_load": {"contract_id": "load-1", "driver_revision": "abc123"}, + } + + +class DatabaseLeaseRecoveryTests(unittest.TestCase): + def test_approval_requires_exact_owner_and_window_contract(self) -> None: + now = datetime(2026, 8, 22, 12, 0, tzinfo=UTC) + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "approval.json" + path.write_text(json.dumps(approved_receipt(now)), encoding="utf-8") + result = module.load_approval(path, now, require_open_window=True) + self.assertEqual("operator-window-1", result["approval_id"]) + + def test_pending_or_overlong_approval_is_rejected(self) -> None: + now = datetime(2026, 8, 22, 12, 0, tzinfo=UTC) + receipt = approved_receipt(now) + receipt["status"] = "pending" + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "approval.json" + path.write_text(json.dumps(receipt), encoding="utf-8") + with self.assertRaises(module.ProcedureError): + module.load_approval(path, now, require_open_window=True) + + def test_lease_must_align_to_refresh_and_have_headroom(self) -> None: + issue = datetime(2026, 8, 22, 12, 0, tzinfo=UTC) + lease = { + "issue_time": issue, + "expire_time": issue + timedelta(minutes=15), + "ttl": 800, + } + secret = {"refresh_time": issue + timedelta(seconds=1)} + module.assert_lease_matches_refresh(lease, secret) + lease["ttl"] = 20 + with self.assertRaises(module.ProcedureError): + module.assert_lease_matches_refresh(lease, secret) + + def test_load_driver_rejects_extra_evidence_fields(self) -> None: + with tempfile.TemporaryDirectory() as directory: + driver = Path(directory) / "driver" + driver.write_text( + "#!/bin/sh\nprintf '%s\\n' '{\"contract_id\":\"load-1\",\"fixture_id\":\"f-1\",\"status\":\"ready\",\"secret_values_observed\":false,\"token\":\"forbidden\"}'\n", + encoding="utf-8", + ) + os.chmod(driver, 0o700) + revision = "sha256:" + module.hashlib.sha256(driver.read_bytes()).hexdigest() + with self.assertRaises(module.ProcedureError): + module.LoadDriver(driver, "load-1", revision).run("baseline") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_audit_core_recovery_preflight.py b/tests/test_audit_core_recovery_preflight.py index 597175d..4632629 100644 --- a/tests/test_audit_core_recovery_preflight.py +++ b/tests/test_audit_core_recovery_preflight.py @@ -1,8 +1,10 @@ from __future__ import annotations import importlib.util +import json +import tempfile import unittest -from datetime import UTC, datetime +from datetime import UTC, datetime, timedelta from pathlib import Path @@ -96,6 +98,78 @@ host: 50 bytes {"username", "password", "host"}, module.secret_key_names(description) ) + def test_snapshot_receipt_matches_live_cluster_and_freshness(self) -> None: + now = datetime(2026, 8, 22, 12, 0, tzinfo=UTC) + receipt = { + "receipt_version": 1, + "receipt_id": "snapshot-20260822", + "created_at": (now - timedelta(hours=1)).isoformat(), + "operator": "operator-a", + "source_cluster": "railiance01", + "source_namespace": "openbao", + "source_pod": "openbao-0", + "cluster_id": "cluster-1", + "raft_applied_index": 900, + "snapshot_created": True, + "source_initialized": True, + "source_unsealed": True, + "snapshot_sha256": "sha256:" + "ab" * 32, + "snapshot_encrypted": True, + "encrypted_snapshot_sha256": "sha256:" + "cd" * 32, + "encrypted_location_ref": "custody:openbao:snapshot-20260822", + "encrypted_copy_off_host": True, + "encryption_verified": True, + "hash_verified": True, + "no_secret_material_recorded": True, + } + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "receipt.json" + path.write_text(json.dumps(receipt), encoding="utf-8") + result = module.validate_snapshot_receipt( + path, + live_openbao={"cluster_id": "cluster-1", "raft_applied_index": 1000}, + now=now, + max_age_hours=24, + ) + self.assertTrue(result["verified"]) + self.assertEqual(1.0, result["age_hours"]) + self.assertFalse(result["secret_values_observed"]) + + def test_snapshot_receipt_rejects_stale_or_secret_material(self) -> None: + now = datetime(2026, 8, 22, 12, 0, tzinfo=UTC) + receipt = { + "receipt_version": 1, + "receipt_id": "snapshot-old", + "created_at": (now - timedelta(hours=25)).isoformat(), + "operator": "operator-a", + "source_cluster": "railiance01", + "source_namespace": "openbao", + "source_pod": "openbao-0", + "cluster_id": "cluster-1", + "raft_applied_index": 900, + "snapshot_created": True, + "source_initialized": True, + "source_unsealed": True, + "snapshot_sha256": "sha256:" + "ab" * 32, + "snapshot_encrypted": True, + "encrypted_snapshot_sha256": "sha256:" + "cd" * 32, + "encrypted_location_ref": "custody:old", + "encrypted_copy_off_host": True, + "encryption_verified": True, + "hash_verified": True, + "no_secret_material_recorded": True, + } + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "receipt.json" + path.write_text(json.dumps(receipt), encoding="utf-8") + with self.assertRaises(module.PreflightError): + module.validate_snapshot_receipt( + path, + live_openbao={"cluster_id": "cluster-1", "raft_applied_index": 1000}, + now=now, + max_age_hours=24, + ) + if __name__ == "__main__": unittest.main() diff --git a/workplans/RAILIANCE-WP-0024-audit-core-custody-and-recovery-coordination.md b/workplans/RAILIANCE-WP-0024-audit-core-custody-and-recovery-coordination.md index 9dabbd8..90a7e94 100644 --- a/workplans/RAILIANCE-WP-0024-audit-core-custody-and-recovery-coordination.md +++ b/workplans/RAILIANCE-WP-0024-audit-core-custody-and-recovery-coordination.md @@ -113,6 +113,20 @@ checks without observing Secret data. The live revocation still waits for an approved window, audit-core/rapp-postgres acknowledgement, a separately approved synthetic-load contract, and a named abort operator. +**Harness hardening (2026-08-22):** +`scripts/audit-core-database-lease-recovery.py` now implements the attended +orchestration but cannot run from repository access alone. It requires a +value-safe approved receipt based on +`docs/audit-core-database-lease-approval.example.json`, a revision-pinned +synthetic-load driver, exact confirmation, an open ≤15-minute window, all +three owner acknowledgements, and at least five minutes before the ordinary +lease refresh. OpenBao metadata inspection corrected the selection contract: +lease lookup exposes issue/expiry/TTL, not a database username. The harness +therefore aborts unless the exact runtime prefix has one live handle whose +issue time coheres with the ExternalSecret refresh. A live read-only check +observed exactly one handle, 0.073 seconds of issue/refresh skew, and no secret +values. It was not revoked. + ## T03 — Define the coordinated railiance01 reboot exercise ```task @@ -144,6 +158,16 @@ the intentionally non-automated gates: an approved window and owner set, provider-console access, a current OpenBao Raft snapshot receipt, 2-of-3 Shamir quorum availability, and a named abort operator. +**Snapshot gate hardening (2026-08-22):** reboot preflight no longer accepts a +file-existence assertion as snapshot evidence. A receipt based on +`docs/openbao-reboot-snapshot-receipt.example.json` must be no more than 24 +hours old, match the live OpenBao cluster id and a possible Raft applied index, +carry distinct non-placeholder plain/encrypted hashes, and attest encryption, +off-host persistence, verification, and absence of secret material. The +pending example intentionally fails. Creating the secret-bearing snapshot and +its off-host copy remains an attended platform-custody action near the future +reboot window; no snapshot was created in this implementation step. + ## T04 — Route the contracts and approval gates ```task