From 1c89c8586293059676425ce633e0074dcbbac5d3 Mon Sep 17 00:00:00 2001 From: tegwick Date: Tue, 29 Sep 2026 00:43:38 +0200 Subject: [PATCH] Add run records, step state machine, verification and exercise-status diff (RBT-WP-0003) Co-Authored-By: Claude Sonnet 5.5 Assistant: claude-code Assistant-Model: sonnet Assistant-Process: 295952@bnt-lap001 Assistant-Session: e93f64ad-516c-46eb-9666-aad8d300c477 --- .gitignore | 1 + docs/run-spec.md | 26 ++ rtut/__main__.py | 56 +++++ rtut/run.py | 230 ++++++++++++++++++ tests/test_run.py | 82 +++++++ ...0003-engine-core-run-state-and-evidence.md | 12 +- 6 files changed, 401 insertions(+), 6 deletions(-) create mode 100644 docs/run-spec.md create mode 100644 rtut/run.py create mode 100644 tests/test_run.py diff --git a/.gitignore b/.gitignore index 88737ca..2dd1faf 100644 --- a/.gitignore +++ b/.gitignore @@ -180,3 +180,4 @@ cython_debug/ !.claude/rules/ !.claude/rules/*.md +.local/ diff --git a/docs/run-spec.md b/docs/run-spec.md new file mode 100644 index 0000000..2a98618 --- /dev/null +++ b/docs/run-spec.md @@ -0,0 +1,26 @@ +# Run record `runbook-run/v0.1` + +One run of one native pack, stored as JSON in +`/.local/runbook-runs/.json` (git-ignored). Non-secret only. + +| Field | Meaning | +| --- | --- | +| `run_id` | Timestamp plus random suffix | +| `pack_id`, `pack_path` | The pack run | +| `operator` | One word: the person who ran it | +| `params` | Resolved parameter values (validated, never secret) | +| `steps..state` | `todo`, `running`, `done`, `failed`, `skipped` | +| `steps..verification` | `kind`, `passed`, and for commands `exit_code`, `output_bytes`, `seconds`. Output text is never stored | +| `steps..evidence` | The facts the step declared, supplied by the operator | + +## Rules + +- Steps run in order; a step starts only after its predecessors are done or skipped. +- A step completes only with a passing verification and its declared evidence. +- A verify command runs only when the operator confirms (`--yes`), with a 60 s timeout. +- Only read-only steps may be skipped, and a run with skips never exercises a pack. +- Values matching the secret markers are refused everywhere they could be recorded. +- The engine never edits `pack.yaml`. `rtut run finish` shows the exercise-status + diff; `--apply` writes it, and only for a run in which every step is done. + +CLI: `rtut run new|list|status|cmd|start|verify|done|fail|retry|skip|finish ...` diff --git a/rtut/__main__.py b/rtut/__main__.py index 40c8adb..36ad4e1 100644 --- a/rtut/__main__.py +++ b/rtut/__main__.py @@ -12,12 +12,66 @@ def _packs(root: str) -> list[P.Pack]: return [P.load(p) for p in P.discover(Path(root))] +def _run(pk, a) -> int: + from . import run as R + try: + if a.action == "new": + r = R.create(pk, a.operator, dict(kv.split("=", 1) for kv in a.param)) + print(r.data["run_id"]) + return 0 + if a.action == "list": + print("\n".join(R.list_runs(pk))) + return 0 + r = R.load(pk, a.run_id) + if a.action == "status": + for sid, st in r.data["steps"].items(): + v = st.get("verification", {}) + print(f"{st['state']:8} {sid} {'verified' if v.get('passed') else ''}") + return 0 + if a.action == "cmd": + print(R.command_for(r, a.step)); return 0 + if a.action == "start": + R.start(r, a.step); print(R.command_for(r, a.step) or ""); return 0 + if a.action == "verify": + res = R.verify(r, a.step, confirm=a.yes, manual_ok=a.passed or None) + print("passed" if res["passed"] else "FAILED"); return 0 if res["passed"] else 1 + if a.action == "done": + R.complete(r, a.step, dict(kv.split("=", 1) for kv in a.evidence)); return 0 + if a.action == "fail": + R.fail(r, a.step, a.note); return 0 + if a.action == "retry": + R.retry(r, a.step); return 0 + if a.action == "skip": + R.skip(r, a.step, a.note); return 0 + old, new = R.exercise_update(r) + import difflib + print("".join(difflib.unified_diff(old.splitlines(True), new.splitlines(True), "pack.yaml", "pack.yaml"))) + if a.apply: + pk.path.write_text(new); print("written") + return 0 + except (R.RunError, KeyError, ValueError) as e: + print(f"error: {e}", file=sys.stderr) + return 1 + + def main(argv: list[str] | None = None) -> int: ap = argparse.ArgumentParser(prog="rtut") sub = ap.add_subparsers(dest="cmd", required=True) for name in ("list", "validate"): sp = sub.add_parser(name) sp.add_argument("root", nargs="?", default=".") + sp = sub.add_parser("run", help="record a run of a native pack") + sp.add_argument("action", choices=["new", "list", "status", "cmd", "start", "verify", "done", "fail", "retry", "skip", "finish"]) + sp.add_argument("manifest") + sp.add_argument("run_id", nargs="?") + sp.add_argument("step", nargs="?") + sp.add_argument("--operator", "-o", default="") + sp.add_argument("--param", "-p", action="append", default=[], help="name=value") + sp.add_argument("--evidence", "-e", action="append", default=[], help="name=value (non-secret)") + sp.add_argument("--note", default="") + sp.add_argument("--yes", action="store_true", help="confirm running a verify command") + sp.add_argument("--pass", dest="passed", action="store_true", help="manual check: operator confirms it passed") + sp.add_argument("--apply", action="store_true", help="finish: write the exercise-status change") sp = sub.add_parser("launch"); sp.add_argument("manifest") sp.add_argument("--timeout", type=float, default=30.0) sp = sub.add_parser("show"); sp.add_argument("manifest") @@ -45,6 +99,8 @@ def main(argv: list[str] | None = None) -> int: if not pk.ok: print("\n".join(pk.errors), file=sys.stderr) return 1 + if a.cmd == "run": + return _run(pk, a) if a.cmd == "launch": from . import legacy try: diff --git a/rtut/run.py b/rtut/run.py new file mode 100644 index 0000000..3c6a65b --- /dev/null +++ b/rtut/run.py @@ -0,0 +1,230 @@ +"""Run records (runbook-run/v0.1): non-secret progress and evidence for one pack run.""" +from __future__ import annotations + +import json +import re +import subprocess +import time +import uuid +from dataclasses import dataclass +from datetime import date, datetime, timezone +from pathlib import Path + +from .legacy import repo_root +from .pack import PLACEHOLDER, SECRET_RE, Pack, render + +STATES = ("todo", "running", "done", "failed", "skipped") +VERIFY_TIMEOUT = 60 + + +class RunError(Exception): + pass + + +def _now() -> str: + return datetime.now(timezone.utc).isoformat(timespec="seconds") + + +def _clean(value: str, what: str) -> str: + if SECRET_RE.search(value): + raise RunError(f"{what} contains a secret-looking marker; not recorded") + return value + + +def runs_dir(pack: Pack) -> Path: + return repo_root(pack) / ".local" / "runbook-runs" + + +@dataclass +class Run: + pack: Pack + data: dict + + @property + def path(self) -> Path: + return runs_dir(self.pack) / f"{self.data['run_id']}.json" + + def save(self) -> None: + self.data["updated"] = _now() + self.path.parent.mkdir(parents=True, exist_ok=True) + self.path.write_text(json.dumps(self.data, indent=2, sort_keys=True) + "\n") + + def step_def(self, step_id: str) -> dict: + for s in self.pack.data.get("steps", []): + if s["id"] == step_id: + return s + raise RunError(f"no such step: {step_id}") + + def state(self, step_id: str) -> dict: + return self.data["steps"][step_id] + + @property + def complete(self) -> bool: + return all(v["state"] in ("done", "skipped") for v in self.data["steps"].values()) + + +def create(pack: Pack, operator: str, params: dict[str, str] | None = None) -> Run: + if not pack.ok or pack.data.get("engine") != "native": + raise RunError("only valid native packs can be run") + operator = _clean(operator.strip(), "operator") + if not re.fullmatch(r"\S+", operator): + raise RunError("operator must be one word (name or handle)") + values = dict(params or {}) + known = {p["id"] for p in pack.data.get("parameters", [])} + for k in values: + if k not in known: + raise RunError(f"unknown parameter {k}") + resolved: dict[str, str] = {} + for p in pack.data.get("parameters", []): + # render() validates values/pattern/unsafe characters and applies defaults + try: + resolved[p["id"]] = render("{{%s}}" % p["id"], values, pack) + except (KeyError, ValueError) as e: + raise RunError(str(e)) from e + _clean(resolved[p["id"]], f"parameter {p['id']}") + data = { + "spec": "runbook-run/v0.1", + "run_id": time.strftime("%Y%m%d-%H%M%S-") + uuid.uuid4().hex[:6], + "pack_id": pack.id, + "pack_path": str(pack.path), + "operator": operator, + "created": _now(), + "params": resolved, + "steps": {s["id"]: {"state": "todo"} for s in pack.data["steps"]}, + } + run = Run(pack, data) + run.save() + return run + + +def load(pack: Pack, run_id: str) -> Run: + f = runs_dir(pack) / f"{run_id}.json" + if not f.exists(): + raise RunError(f"no such run: {run_id}") + return Run(pack, json.loads(f.read_text())) + + +def list_runs(pack: Pack) -> list[str]: + d = runs_dir(pack) + return sorted(p.stem for p in d.glob("*.json")) if d.exists() else [] + + +def _order(run: Run, step_id: str) -> None: + for s in run.pack.data["steps"]: + if s["id"] == step_id: + return + if run.state(s["id"])["state"] not in ("done", "skipped"): + raise RunError(f"step {s['id']} must be done or skipped before {step_id}") + + +def command_for(run: Run, step_id: str) -> str | None: + cmd = run.step_def(step_id).get("command") + return render(cmd, run.data["params"], run.pack) if cmd else None + + +def start(run: Run, step_id: str) -> None: + run.step_def(step_id) + _order(run, step_id) + st = run.state(step_id) + if st["state"] in ("done", "skipped"): + raise RunError(f"step {step_id} already {st['state']}") + st.update(state="running", started=_now()) + run.save() + + +def verify(run: Run, step_id: str, confirm: bool = False, manual_ok: bool | None = None) -> dict: + """Run the step's check. Commands run only when the operator confirms. + + Stored: exit code, duration, output size, and whether 'contains' matched. + Output text is never stored. + """ + v = run.step_def(step_id)["verify"] + st = run.state(step_id) + if st["state"] != "running": + raise RunError(f"step {step_id} is not running") + expect = v.get("expect", "manual") + if expect == "manual": + if manual_ok is None: + raise RunError("manual verification needs the operator's yes/no") + result = {"kind": "manual", "passed": bool(manual_ok)} + else: + if not confirm: + raise RunError("running a verify command needs operator confirmation") + cmd = render(v["command"], run.data["params"], run.pack) + t0 = time.monotonic() + try: + cp = subprocess.run(cmd, shell=True, capture_output=True, text=True, + timeout=VERIFY_TIMEOUT) + code, out = cp.returncode, cp.stdout + cp.stderr + except subprocess.TimeoutExpired: + code, out = -1, "" + result = {"kind": expect, "exit_code": code, "output_bytes": len(out.encode()), + "seconds": round(time.monotonic() - t0, 2)} + if expect == "exit-0": + result["passed"] = code == 0 + else: + needle = render(v["contains"], run.data["params"], run.pack) + result["passed"] = code == 0 and needle in out + result["at"] = _now() + st["verification"] = result + run.save() + return result + + +def complete(run: Run, step_id: str, evidence: dict[str, str] | None = None) -> None: + sd = run.step_def(step_id) + st = run.state(step_id) + if st["state"] != "running": + raise RunError(f"step {step_id} is not running") + if not st.get("verification", {}).get("passed"): + raise RunError(f"step {step_id} has no passing verification") + evidence = evidence or {} + missing = [n for n in sd.get("evidence", []) if n not in evidence] + if missing: + raise RunError(f"evidence required: {', '.join(missing)}") + st["evidence"] = {k: _clean(str(v), f"evidence {k}") for k, v in evidence.items()} + st.update(state="done", finished=_now()) + run.save() + + +def fail(run: Run, step_id: str, note: str) -> None: + st = run.state(step_id) + if st["state"] != "running": + raise RunError(f"step {step_id} is not running") + st.update(state="failed", note=_clean(note, "note"), finished=_now()) + run.save() + + +def retry(run: Run, step_id: str) -> None: + st = run.state(step_id) + if st["state"] != "failed": + raise RunError(f"step {step_id} has not failed") + st.pop("verification", None) + st.update(state="running", started=_now()) + run.save() + + +def skip(run: Run, step_id: str, reason: str) -> None: + sd = run.step_def(step_id) + if sd.get("risk", "read-only") != "read-only": + raise RunError("only read-only steps may be skipped") + _order(run, step_id) + run.state(step_id).update(state="skipped", note=_clean(reason, "reason"), finished=_now()) + run.save() + + +STATUS_LINE = re.compile(r"^exercise_status:.*$", re.M) + + +def exercise_update(run: Run, today: date | None = None) -> tuple[str, str]: + """Return (old, new) pack.yaml text with exercised status. Does not write. + + A run counts only if every step is done (skips do not exercise the pack). + """ + if any(v["state"] != "done" for v in run.data["steps"].values()): + raise RunError("every step must be done (not skipped) to exercise the pack") + today = today or date.today() + old = run.pack.path.read_text() + new = STATUS_LINE.sub(f"exercise_status: exercised {today.isoformat()} by {run.data['operator']}", + old, count=1) + return old, new diff --git a/tests/test_run.py b/tests/test_run.py new file mode 100644 index 0000000..c6bb628 --- /dev/null +++ b/tests/test_run.py @@ -0,0 +1,82 @@ +import shutil +from datetime import date +from pathlib import Path + +import pytest + +from rtut import pack as P, run as R + +ROOT = Path(__file__).resolve().parent.parent + + +@pytest.fixture +def pk(tmp_path): + d = tmp_path / "runbooks" / "hello" + d.mkdir(parents=True) + shutil.copy(ROOT / "runbooks/example-hello/pack.yaml", d / "pack.yaml") + return P.load(d / "pack.yaml") + + +def test_create_and_resume(pk): + r = R.create(pk, "bernd", {"host": "example.org"}) + assert r.data["params"] == {"host": "example.org", "count": "1"} + assert R.list_runs(pk) == [r.data["run_id"]] + assert R.load(pk, r.data["run_id"]).data == r.data + assert (pk.path.parents[2] / ".local/runbook-runs").is_dir() + + +def test_bad_parameters_and_operator(pk): + with pytest.raises(R.RunError): + R.create(pk, "bernd", {"host": "a;b"}) + with pytest.raises(R.RunError): + R.create(pk, "bernd", {"nope": "x"}) + with pytest.raises(R.RunError): + R.create(pk, "two words") + with pytest.raises(R.RunError): + R.create(pk, "hvs.ABCDEFGHIJKLMN") + + +def test_full_run_and_exercise(pk): + r = R.create(pk, "bernd", {"host": "localhost"}) + assert R.command_for(r, "ping") == "ping -c 1 localhost" + R.start(r, "ping") + with pytest.raises(R.RunError): # command needs confirmation + R.verify(r, "ping") + with pytest.raises(R.RunError): # cannot complete without passing check + R.complete(r, "ping") + res = R.verify(r, "ping", confirm=True) + assert res["kind"] == "exit-0" and "output_bytes" in res + assert "stdout" not in res and "output" not in res + if not res["passed"]: + pytest.skip("ping unavailable in this environment") + R.complete(r, "ping") + assert r.complete + old, new = R.exercise_update(r, date(2026, 9, 29)) + assert "exercise_status: exercised 2026-09-29 by bernd" in new and old != new + assert pk.path.read_text() == old # never written by the engine + + +def test_order_fail_retry_skip(pk): + r = R.create(pk, "bernd") + with pytest.raises(R.RunError): + R.fail(r, "ping", "x") # not running + R.start(r, "ping") + R.fail(r, "ping", "no route") + assert r.state("ping")["state"] == "failed" + R.retry(r, "ping") + assert r.state("ping")["state"] == "running" and "verification" not in r.state("ping") + with pytest.raises(R.RunError): + R.fail(r, "ping", "-----BEGIN RSA PRIVATE KEY-----") + r2 = R.create(pk, "bernd") + R.skip(r2, "ping", "not needed") + with pytest.raises(R.RunError): + R.exercise_update(r2) # skips never exercise a pack + + +def test_manual_verification_needs_answer(pk): + r = R.create(pk, "bernd") + pk.data["steps"][0]["verify"] = {"done_when": "checked by eye", "expect": "manual"} + R.start(r, "ping") + with pytest.raises(R.RunError): + R.verify(r, "ping") + assert R.verify(r, "ping", manual_ok=True)["passed"] diff --git a/workplans/RBT-WP-0003-engine-core-run-state-and-evidence.md b/workplans/RBT-WP-0003-engine-core-run-state-and-evidence.md index be1581a..80f7196 100644 --- a/workplans/RBT-WP-0003-engine-core-run-state-and-evidence.md +++ b/workplans/RBT-WP-0003-engine-core-run-state-and-evidence.md @@ -4,7 +4,7 @@ type: workplan title: "Engine core: run state and evidence" domain: infotech repo: runbook-tutorials -status: ready +status: finished flavor: implementation owner: claude topic_slug: stack @@ -24,12 +24,12 @@ exercise status. ```task id: RBT-WP-0003-T01 -status: todo +status: done priority: high state_hub_task_id: "e61fbba9-d69f-59d4-ad86-c1fab69e8171" ``` -Specify `runbook-run/v0.1`: pack id and revision, operator label, parameter +Specified in `docs/run-spec.md`: `runbook-run/v0.1`: pack id and revision, operator label, parameter values, per-step state (`todo|running|done|failed|skipped`), timestamps, evidence facts, verification results. Non-secret only, with the same secret-marker rejection as packs. Stored under `.local/runbook-runs/` in the repo being operated. @@ -38,7 +38,7 @@ rejection as packs. Stored under `.local/runbook-runs/` in the repo being operat ```task id: RBT-WP-0003-T02 -status: todo +status: done priority: high state_hub_task_id: "fb10ebd1-d122-5b5f-9fd5-83d08539f8c1" ``` @@ -51,7 +51,7 @@ resume. ```task id: RBT-WP-0003-T03 -status: todo +status: done priority: medium state_hub_task_id: "870980df-c1e8-51e5-9a00-643f63404783" ``` @@ -64,7 +64,7 @@ it, with a timeout and captured output size only (no output stored). ```task id: RBT-WP-0003-T04 -status: todo +status: done priority: medium state_hub_task_id: "bad99b81-2b64-520f-b92f-1b8c6457659b" ```