Add run records, step state machine, verification and exercise-status diff (RBT-WP-0003)

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

Assistant: claude-code
Assistant-Model: sonnet
Assistant-Process: 295952@bnt-lap001
Assistant-Session: e93f64ad-516c-46eb-9666-aad8d300c477

.gitignore

@@ -180,3 +180,4 @@ cython_debug/
 !.claude/rules/
 !.claude/rules/*.md
 
+.local/

docs/run-spec.md

@@ -0,0 +1,26 @@
+# Run record `runbook-run/v0.1`
+
+One run of one native pack, stored as JSON in
+`<owning repo>/.local/runbook-runs/<run_id>.json` (git-ignored). Non-secret only.
+
+| Field | Meaning |
+| --- | --- |
+| `run_id` | Timestamp plus random suffix |
+| `pack_id`, `pack_path` | The pack run |
+| `operator` | One word: the person who ran it |
+| `params` | Resolved parameter values (validated, never secret) |
+| `steps.<id>.state` | `todo`, `running`, `done`, `failed`, `skipped` |
+| `steps.<id>.verification` | `kind`, `passed`, and for commands `exit_code`, `output_bytes`, `seconds`. Output text is never stored |
+| `steps.<id>.evidence` | The facts the step declared, supplied by the operator |
+
+## Rules
+
+- Steps run in order; a step starts only after its predecessors are done or skipped.
+- A step completes only with a passing verification and its declared evidence.
+- A verify command runs only when the operator confirms (`--yes`), with a 60 s timeout.
+- Only read-only steps may be skipped, and a run with skips never exercises a pack.
+- Values matching the secret markers are refused everywhere they could be recorded.
+- The engine never edits `pack.yaml`. `rtut run finish` shows the exercise-status
+  diff; `--apply` writes it, and only for a run in which every step is done.
+
+CLI: `rtut run new|list|status|cmd|start|verify|done|fail|retry|skip|finish <pack.yaml> ...`

rtut/__main__.py

@@ -12,12 +12,66 @@ def _packs(root: str) -> list[P.Pack]:
     return [P.load(p) for p in P.discover(Path(root))]
 
 
+def _run(pk, a) -> int:
+    from . import run as R
+    try:
+        if a.action == "new":
+            r = R.create(pk, a.operator, dict(kv.split("=", 1) for kv in a.param))
+            print(r.data["run_id"])
+            return 0
+        if a.action == "list":
+            print("\n".join(R.list_runs(pk)))
+            return 0
+        r = R.load(pk, a.run_id)
+        if a.action == "status":
+            for sid, st in r.data["steps"].items():
+                v = st.get("verification", {})
+                print(f"{st['state']:8} {sid}  {'verified' if v.get('passed') else ''}")
+            return 0
+        if a.action == "cmd":
+            print(R.command_for(r, a.step)); return 0
+        if a.action == "start":
+            R.start(r, a.step); print(R.command_for(r, a.step) or ""); return 0
+        if a.action == "verify":
+            res = R.verify(r, a.step, confirm=a.yes, manual_ok=a.passed or None)
+            print("passed" if res["passed"] else "FAILED"); return 0 if res["passed"] else 1
+        if a.action == "done":
+            R.complete(r, a.step, dict(kv.split("=", 1) for kv in a.evidence)); return 0
+        if a.action == "fail":
+            R.fail(r, a.step, a.note); return 0
+        if a.action == "retry":
+            R.retry(r, a.step); return 0
+        if a.action == "skip":
+            R.skip(r, a.step, a.note); return 0
+        old, new = R.exercise_update(r)
+        import difflib
+        print("".join(difflib.unified_diff(old.splitlines(True), new.splitlines(True), "pack.yaml", "pack.yaml")))
+        if a.apply:
+            pk.path.write_text(new); print("written")
+        return 0
+    except (R.RunError, KeyError, ValueError) as e:
+        print(f"error: {e}", file=sys.stderr)
+        return 1
+
+
 def main(argv: list[str] | None = None) -> int:
     ap = argparse.ArgumentParser(prog="rtut")
     sub = ap.add_subparsers(dest="cmd", required=True)
     for name in ("list", "validate"):
         sp = sub.add_parser(name)
         sp.add_argument("root", nargs="?", default=".")
+    sp = sub.add_parser("run", help="record a run of a native pack")
+    sp.add_argument("action", choices=["new", "list", "status", "cmd", "start", "verify", "done", "fail", "retry", "skip", "finish"])
+    sp.add_argument("manifest")
+    sp.add_argument("run_id", nargs="?")
+    sp.add_argument("step", nargs="?")
+    sp.add_argument("--operator", "-o", default="")
+    sp.add_argument("--param", "-p", action="append", default=[], help="name=value")
+    sp.add_argument("--evidence", "-e", action="append", default=[], help="name=value (non-secret)")
+    sp.add_argument("--note", default="")
+    sp.add_argument("--yes", action="store_true", help="confirm running a verify command")
+    sp.add_argument("--pass", dest="passed", action="store_true", help="manual check: operator confirms it passed")
+    sp.add_argument("--apply", action="store_true", help="finish: write the exercise-status change")
     sp = sub.add_parser("launch"); sp.add_argument("manifest")
     sp.add_argument("--timeout", type=float, default=30.0)
     sp = sub.add_parser("show"); sp.add_argument("manifest")
@@ -45,6 +99,8 @@ def main(argv: list[str] | None = None) -> int:
     if not pk.ok:
         print("\n".join(pk.errors), file=sys.stderr)
         return 1
+    if a.cmd == "run":
+        return _run(pk, a)
     if a.cmd == "launch":
         from . import legacy
         try:

rtut/run.py

@@ -0,0 +1,230 @@
+"""Run records (runbook-run/v0.1): non-secret progress and evidence for one pack run."""
+from __future__ import annotations
+
+import json
+import re
+import subprocess
+import time
+import uuid
+from dataclasses import dataclass
+from datetime import date, datetime, timezone
+from pathlib import Path
+
+from .legacy import repo_root
+from .pack import PLACEHOLDER, SECRET_RE, Pack, render
+
+STATES = ("todo", "running", "done", "failed", "skipped")
+VERIFY_TIMEOUT = 60
+
+
+class RunError(Exception):
+    pass
+
+
+def _now() -> str:
+    return datetime.now(timezone.utc).isoformat(timespec="seconds")
+
+
+def _clean(value: str, what: str) -> str:
+    if SECRET_RE.search(value):
+        raise RunError(f"{what} contains a secret-looking marker; not recorded")
+    return value
+
+
+def runs_dir(pack: Pack) -> Path:
+    return repo_root(pack) / ".local" / "runbook-runs"
+
+
+@dataclass
+class Run:
+    pack: Pack
+    data: dict
+
+    @property
+    def path(self) -> Path:
+        return runs_dir(self.pack) / f"{self.data['run_id']}.json"
+
+    def save(self) -> None:
+        self.data["updated"] = _now()
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        self.path.write_text(json.dumps(self.data, indent=2, sort_keys=True) + "\n")
+
+    def step_def(self, step_id: str) -> dict:
+        for s in self.pack.data.get("steps", []):
+            if s["id"] == step_id:
+                return s
+        raise RunError(f"no such step: {step_id}")
+
+    def state(self, step_id: str) -> dict:
+        return self.data["steps"][step_id]
+
+    @property
+    def complete(self) -> bool:
+        return all(v["state"] in ("done", "skipped") for v in self.data["steps"].values())
+
+
+def create(pack: Pack, operator: str, params: dict[str, str] | None = None) -> Run:
+    if not pack.ok or pack.data.get("engine") != "native":
+        raise RunError("only valid native packs can be run")
+    operator = _clean(operator.strip(), "operator")
+    if not re.fullmatch(r"\S+", operator):
+        raise RunError("operator must be one word (name or handle)")
+    values = dict(params or {})
+    known = {p["id"] for p in pack.data.get("parameters", [])}
+    for k in values:
+        if k not in known:
+            raise RunError(f"unknown parameter {k}")
+    resolved: dict[str, str] = {}
+    for p in pack.data.get("parameters", []):
+        # render() validates values/pattern/unsafe characters and applies defaults
+        try:
+            resolved[p["id"]] = render("{{%s}}" % p["id"], values, pack)
+        except (KeyError, ValueError) as e:
+            raise RunError(str(e)) from e
+        _clean(resolved[p["id"]], f"parameter {p['id']}")
+    data = {
+        "spec": "runbook-run/v0.1",
+        "run_id": time.strftime("%Y%m%d-%H%M%S-") + uuid.uuid4().hex[:6],
+        "pack_id": pack.id,
+        "pack_path": str(pack.path),
+        "operator": operator,
+        "created": _now(),
+        "params": resolved,
+        "steps": {s["id"]: {"state": "todo"} for s in pack.data["steps"]},
+    }
+    run = Run(pack, data)
+    run.save()
+    return run
+
+
+def load(pack: Pack, run_id: str) -> Run:
+    f = runs_dir(pack) / f"{run_id}.json"
+    if not f.exists():
+        raise RunError(f"no such run: {run_id}")
+    return Run(pack, json.loads(f.read_text()))
+
+
+def list_runs(pack: Pack) -> list[str]:
+    d = runs_dir(pack)
+    return sorted(p.stem for p in d.glob("*.json")) if d.exists() else []
+
+
+def _order(run: Run, step_id: str) -> None:
+    for s in run.pack.data["steps"]:
+        if s["id"] == step_id:
+            return
+        if run.state(s["id"])["state"] not in ("done", "skipped"):
+            raise RunError(f"step {s['id']} must be done or skipped before {step_id}")
+
+
+def command_for(run: Run, step_id: str) -> str | None:
+    cmd = run.step_def(step_id).get("command")
+    return render(cmd, run.data["params"], run.pack) if cmd else None
+
+
+def start(run: Run, step_id: str) -> None:
+    run.step_def(step_id)
+    _order(run, step_id)
+    st = run.state(step_id)
+    if st["state"] in ("done", "skipped"):
+        raise RunError(f"step {step_id} already {st['state']}")
+    st.update(state="running", started=_now())
+    run.save()
+
+
+def verify(run: Run, step_id: str, confirm: bool = False, manual_ok: bool | None = None) -> dict:
+    """Run the step's check. Commands run only when the operator confirms.
+
+    Stored: exit code, duration, output size, and whether 'contains' matched.
+    Output text is never stored.
+    """
+    v = run.step_def(step_id)["verify"]
+    st = run.state(step_id)
+    if st["state"] != "running":
+        raise RunError(f"step {step_id} is not running")
+    expect = v.get("expect", "manual")
+    if expect == "manual":
+        if manual_ok is None:
+            raise RunError("manual verification needs the operator's yes/no")
+        result = {"kind": "manual", "passed": bool(manual_ok)}
+    else:
+        if not confirm:
+            raise RunError("running a verify command needs operator confirmation")
+        cmd = render(v["command"], run.data["params"], run.pack)
+        t0 = time.monotonic()
+        try:
+            cp = subprocess.run(cmd, shell=True, capture_output=True, text=True,
+                                timeout=VERIFY_TIMEOUT)
+            code, out = cp.returncode, cp.stdout + cp.stderr
+        except subprocess.TimeoutExpired:
+            code, out = -1, ""
+        result = {"kind": expect, "exit_code": code, "output_bytes": len(out.encode()),
+                  "seconds": round(time.monotonic() - t0, 2)}
+        if expect == "exit-0":
+            result["passed"] = code == 0
+        else:
+            needle = render(v["contains"], run.data["params"], run.pack)
+            result["passed"] = code == 0 and needle in out
+    result["at"] = _now()
+    st["verification"] = result
+    run.save()
+    return result
+
+
+def complete(run: Run, step_id: str, evidence: dict[str, str] | None = None) -> None:
+    sd = run.step_def(step_id)
+    st = run.state(step_id)
+    if st["state"] != "running":
+        raise RunError(f"step {step_id} is not running")
+    if not st.get("verification", {}).get("passed"):
+        raise RunError(f"step {step_id} has no passing verification")
+    evidence = evidence or {}
+    missing = [n for n in sd.get("evidence", []) if n not in evidence]
+    if missing:
+        raise RunError(f"evidence required: {', '.join(missing)}")
+    st["evidence"] = {k: _clean(str(v), f"evidence {k}") for k, v in evidence.items()}
+    st.update(state="done", finished=_now())
+    run.save()
+
+
+def fail(run: Run, step_id: str, note: str) -> None:
+    st = run.state(step_id)
+    if st["state"] != "running":
+        raise RunError(f"step {step_id} is not running")
+    st.update(state="failed", note=_clean(note, "note"), finished=_now())
+    run.save()
+
+
+def retry(run: Run, step_id: str) -> None:
+    st = run.state(step_id)
+    if st["state"] != "failed":
+        raise RunError(f"step {step_id} has not failed")
+    st.pop("verification", None)
+    st.update(state="running", started=_now())
+    run.save()
+
+
+def skip(run: Run, step_id: str, reason: str) -> None:
+    sd = run.step_def(step_id)
+    if sd.get("risk", "read-only") != "read-only":
+        raise RunError("only read-only steps may be skipped")
+    _order(run, step_id)
+    run.state(step_id).update(state="skipped", note=_clean(reason, "reason"), finished=_now())
+    run.save()
+
+
+STATUS_LINE = re.compile(r"^exercise_status:.*$", re.M)
+
+
+def exercise_update(run: Run, today: date | None = None) -> tuple[str, str]:
+    """Return (old, new) pack.yaml text with exercised status. Does not write.
+
+    A run counts only if every step is done (skips do not exercise the pack).
+    """
+    if any(v["state"] != "done" for v in run.data["steps"].values()):
+        raise RunError("every step must be done (not skipped) to exercise the pack")
+    today = today or date.today()
+    old = run.pack.path.read_text()
+    new = STATUS_LINE.sub(f"exercise_status: exercised {today.isoformat()} by {run.data['operator']}",
+                          old, count=1)
+    return old, new

tests/test_run.py

@@ -0,0 +1,82 @@
+import shutil
+from datetime import date
+from pathlib import Path
+
+import pytest
+
+from rtut import pack as P, run as R
+
+ROOT = Path(__file__).resolve().parent.parent
+
+
+@pytest.fixture
+def pk(tmp_path):
+    d = tmp_path / "runbooks" / "hello"
+    d.mkdir(parents=True)
+    shutil.copy(ROOT / "runbooks/example-hello/pack.yaml", d / "pack.yaml")
+    return P.load(d / "pack.yaml")
+
+
+def test_create_and_resume(pk):
+    r = R.create(pk, "bernd", {"host": "example.org"})
+    assert r.data["params"] == {"host": "example.org", "count": "1"}
+    assert R.list_runs(pk) == [r.data["run_id"]]
+    assert R.load(pk, r.data["run_id"]).data == r.data
+    assert (pk.path.parents[2] / ".local/runbook-runs").is_dir()
+
+
+def test_bad_parameters_and_operator(pk):
+    with pytest.raises(R.RunError):
+        R.create(pk, "bernd", {"host": "a;b"})
+    with pytest.raises(R.RunError):
+        R.create(pk, "bernd", {"nope": "x"})
+    with pytest.raises(R.RunError):
+        R.create(pk, "two words")
+    with pytest.raises(R.RunError):
+        R.create(pk, "hvs.ABCDEFGHIJKLMN")
+
+
+def test_full_run_and_exercise(pk):
+    r = R.create(pk, "bernd", {"host": "localhost"})
+    assert R.command_for(r, "ping") == "ping -c 1 localhost"
+    R.start(r, "ping")
+    with pytest.raises(R.RunError):          # command needs confirmation
+        R.verify(r, "ping")
+    with pytest.raises(R.RunError):          # cannot complete without passing check
+        R.complete(r, "ping")
+    res = R.verify(r, "ping", confirm=True)
+    assert res["kind"] == "exit-0" and "output_bytes" in res
+    assert "stdout" not in res and "output" not in res
+    if not res["passed"]:
+        pytest.skip("ping unavailable in this environment")
+    R.complete(r, "ping")
+    assert r.complete
+    old, new = R.exercise_update(r, date(2026, 9, 29))
+    assert "exercise_status: exercised 2026-09-29 by bernd" in new and old != new
+    assert pk.path.read_text() == old        # never written by the engine
+
+
+def test_order_fail_retry_skip(pk):
+    r = R.create(pk, "bernd")
+    with pytest.raises(R.RunError):
+        R.fail(r, "ping", "x")               # not running
+    R.start(r, "ping")
+    R.fail(r, "ping", "no route")
+    assert r.state("ping")["state"] == "failed"
+    R.retry(r, "ping")
+    assert r.state("ping")["state"] == "running" and "verification" not in r.state("ping")
+    with pytest.raises(R.RunError):
+        R.fail(r, "ping", "-----BEGIN RSA PRIVATE KEY-----")
+    r2 = R.create(pk, "bernd")
+    R.skip(r2, "ping", "not needed")
+    with pytest.raises(R.RunError):
+        R.exercise_update(r2)                # skips never exercise a pack
+
+
+def test_manual_verification_needs_answer(pk):
+    r = R.create(pk, "bernd")
+    pk.data["steps"][0]["verify"] = {"done_when": "checked by eye", "expect": "manual"}
+    R.start(r, "ping")
+    with pytest.raises(R.RunError):
+        R.verify(r, "ping")
+    assert R.verify(r, "ping", manual_ok=True)["passed"]

workplans/RBT-WP-0003-engine-core-run-state-and-evidence.md

@@ -4,7 +4,7 @@ type: workplan
 title: "Engine core: run state and evidence"
 domain: infotech
 repo: runbook-tutorials
-status: ready
+status: finished
 flavor: implementation
 owner: claude
 topic_slug: stack
@@ -24,12 +24,12 @@ exercise status.
 
 ```task
 id: RBT-WP-0003-T01
-status: todo
+status: done
 priority: high
 state_hub_task_id: "e61fbba9-d69f-59d4-ad86-c1fab69e8171"
 ```
 
-Specify `runbook-run/v0.1`: pack id and revision, operator label, parameter
+Specified in `docs/run-spec.md`: `runbook-run/v0.1`: pack id and revision, operator label, parameter
 values, per-step state (`todo|running|done|failed|skipped`), timestamps,
 evidence facts, verification results. Non-secret only, with the same secret-marker
 rejection as packs. Stored under `.local/runbook-runs/` in the repo being operated.
@@ -38,7 +38,7 @@ rejection as packs. Stored under `.local/runbook-runs/` in the repo being operat
 
 ```task
 id: RBT-WP-0003-T02
-status: todo
+status: done
 priority: high
 state_hub_task_id: "fb10ebd1-d122-5b5f-9fd5-83d08539f8c1"
 ```
@@ -51,7 +51,7 @@ resume.
 
 ```task
 id: RBT-WP-0003-T03
-status: todo
+status: done
 priority: medium
 state_hub_task_id: "870980df-c1e8-51e5-9a00-643f63404783"
 ```
@@ -64,7 +64,7 @@ it, with a timeout and captured output size only (no output stored).
 
 ```task
 id: RBT-WP-0003-T04
-status: todo
+status: done
 priority: medium
 state_hub_task_id: "bad99b81-2b64-520f-b92f-1b8c6457659b"
 ```