net-kingdom/runbooks/openbao-restore-drill/pack.yaml

spec: runbook-pack/v0.2
id: nk.openbao-restore-drill
title: OpenBao restore drill
owner: net-kingdom
outcome: A Raft snapshot was taken, encrypted, restored into an isolated instance and verified, and the non-secret evidence validates.
engine: native
confirm_phrase: I understand this cannot be undone
framing:
  - Never restore into namespace openbao; use a disposable cluster, VM or namespace.
  - The plaintext snapshot is removed once encrypted; only hashes and flags go into the evidence.
  - Execute after initial configuration exists and before live secrets move in.
  - Ported from the security bootstrap console runbook "Restore drill".
actors:
  - id: operator
    role: OpenBao ceremony operator
    description: Takes the snapshot and runs the isolated restore.
    privileges: [openbao-root-or-sudo]
datapoints:
  - id: drill_dir
    label: Drill workspace
    type: path
    default: /tmp/netkingdom-openbao-restore-drill
    dimensions: {actor: operator, kind: location, subsystem: openbao}
    ask_in: snapshot
  - id: age_recipient
    label: Custodian age recipient (public key)
    type: string
    default: ""
    help: The age public key; never a private key.
    dimensions: {actor: operator, kind: reference, subsystem: sops-age}
    ask_in: snapshot
stages:
  - id: snapshot
    title: Snapshot and encrypt
    goal: An encrypted snapshot with recorded hashes exists and no plaintext remains.
    steps:
      - id: workspace
        title: Prepare the workspace
        actor: operator
        owner: workstation
        uses: [drill_dir]
        command: mkdir -p {{drill_dir}} && chmod 700 {{drill_dir}}
        idempotent: idempotent
        verify:
          done_when: The directory exists
          command: test -d {{drill_dir}}
          expect: exit-0
      - id: take
        title: Take and copy out the snapshot
        actor: operator
        owner: openbao
        requires: [workspace]
        uses: [drill_dir]
        command: |
          printf "OpenBao token: " >&2; read -rs OPENBAO_TOKEN; printf "\n" >&2
          printf "%s\n" "$OPENBAO_TOKEN" | kubectl exec -i -n openbao openbao-0 -- sh -c 'read -r BAO_TOKEN; export BAO_TOKEN; bao operator raft snapshot save /tmp/openbao-raft.snap'
          unset OPENBAO_TOKEN
          kubectl cp openbao/openbao-0:/tmp/openbao-raft.snap {{drill_dir}}/openbao-raft.snap
          kubectl exec -n openbao openbao-0 -- rm -f /tmp/openbao-raft.snap
          sha256sum {{drill_dir}}/openbao-raft.snap | tee {{drill_dir}}/openbao-raft.snap.sha256
        idempotent: idempotent
        idempotent_note: Each run overwrites the local copy with a fresh snapshot.
        auto_confirm: manual-only
        rollback: Delete the local snapshot and its hash file.
        verify:
          done_when: The snapshot exists locally
          command: test -s {{drill_dir}}/openbao-raft.snap
          expect: exit-0
      - id: encrypt
        title: Encrypt and remove the plaintext
        actor: operator
        owner: workstation
        requires: [take]
        uses: [drill_dir, age_recipient]
        command: |
          age -r {{age_recipient}} -o {{drill_dir}}/openbao-raft.snap.age {{drill_dir}}/openbao-raft.snap
          sha256sum {{drill_dir}}/openbao-raft.snap.age | tee {{drill_dir}}/openbao-raft.snap.age.sha256
          shred -u {{drill_dir}}/openbao-raft.snap
        reversible: false
        idempotent: non-idempotent
        verify:
          done_when: The encrypted file exists and the plaintext is gone
          command: test -s {{drill_dir}}/openbao-raft.snap.age && test ! -e {{drill_dir}}/openbao-raft.snap
          expect: exit-0
  - id: restore
    title: Restore in isolation
    goal: The snapshot restores into a fresh isolated instance and verifies.
    requires: [snapshot]
    steps:
      - id: isolated-restore
        title: Run the isolated restore proof
        actor: operator
        owner: openbao
        description: "1. Use a disposable cluster, VM or namespace, never namespace=openbao. 2. Deploy a fresh OpenBao with empty storage. 3. Decrypt the snapshot only inside the isolated workspace. 4. bao operator raft snapshot restore -force /tmp/openbao-raft.snap. 5. Unseal with the current drill shares. 6. Verify status, mounts, auth methods, policies and a non-production test secret read. 7. Destroy the isolated environment."
        idempotent: non-idempotent
        auto_confirm: manual-only
        rollback: Destroy the isolated environment.
        verify:
          done_when: Status, mounts, auth, policies and a test secret read verified, and the environment destroyed
          expect: manual
    gates:
      - id: evidence-valid
        title: Restore evidence validates
        goal: recoverable-custody
        blocking: true
        downstream_risk: A backup that was never restored is not a backup.
        checks:
          - done_when: The non-secret evidence JSON validates
            command: make -C ../railiance-platform openbao-validate-restore-evidence OPENBAO_RESTORE_EVIDENCE={{drill_dir}}/evidence.json
            expect: exit-0
  - id: after
    title: Post-restore readiness
    goal: Post-unseal checks pass.
    requires: ["gate:evidence-valid"]
    steps:
      - id: post-unseal
        title: Run the post-restore readiness check
        actor: operator
        owner: railiance-platform
        command: make -C ../railiance-platform openbao-verify-post-unseal
        idempotent: idempotent
        verify:
          done_when: The readiness check passes
          command: make -C ../railiance-platform openbao-verify-post-unseal
          expect: exit-0