coulomb/runbook-tutorials
5
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Retire exercise_status: maturity is the only notion; add CLI tests (RBT-WP-0009-T01)

Browse source 
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

Assistant: claude-code
Assistant-Model: sonnet
Assistant-Process: 295952@bnt-lap001
Assistant-Session: e93f64ad-516c-46eb-9666-aad8d300c477
This commit is contained in:
tegwick 2026-09-29 21:28:56 +02:00
parent b89e8eaa74
commit 6d3414d8b7
29 changed files with 444 additions and 218 deletions
Download patch file Download diff file Expand all files Collapse all files

99
tests/test_cli.py Normal file
View file

@ -0,0 +1,99 @@
"""The command line, end to end. Each command runs through main() against a throwaway repository."""
import shutil
import sys
from pathlib import Path
import pytest
import yaml
from rtut.__main__ import main
ROOT = Path(__file__).resolve().parent.parent
@pytest.fixture
def repo(tmp_path):
for name, src in (("v1", "showcase-lighthouse"), ("v2", "showcase-lighthouse-v2")):
d = tmp_path / "runbooks" / name; d.mkdir(parents=True)
shutil.copy(ROOT / f"runbooks/{src}/pack.yaml", d / "pack.yaml")
(tmp_path / "attribute-sets").mkdir()
shutil.copy(ROOT / "attribute-sets/harbour-defaults.yaml", tmp_path / "attribute-sets/harbour-defaults.yaml")
(tmp_path / "project-templates").mkdir()
shutil.copy(ROOT / "project-templates/lighthouse-watch.yaml", tmp_path / "project-templates/lighthouse-watch.yaml")
return tmp_path
def run(capsys, *argv):
code = main([str(a) for a in argv])
out = capsys.readouterr()
return code, out.out, out.err
def test_validate_and_list_show_maturity_not_exercise_status(repo, capsys):
code, out, _ = run(capsys, "validate", repo)
assert code == 0 and out.count("ok ") == 2
code, out, _ = run(capsys, "list", repo)
assert code == 0 and "rtut.showcase-lighthouse\tnative\tdraft\t" in out and "exercise" not in out
code, out, _ = run(capsys, "show", repo / "runbooks/v1/pack.yaml")
assert code == 0 and "[draft]" in out and "engine=native" in out
def test_render_and_score_and_maturity(repo, capsys):
code, out, _ = run(capsys, "render", repo / "runbooks/v1/pack.yaml", "meet-the-keeper", "-p", "keeper=Bea")
assert code == 0 and out.strip().startswith('echo "Bea climbs the stairs')
code, out, _ = run(capsys, "score", repo)
assert code == 0 and "rtut.showcase-lighthouse\ttried 0" in out and out.rstrip().endswith("[draft]")
code, out, _ = run(capsys, "maturity", repo / "runbooks/v1/pack.yaml")
assert code == 0 and out.startswith("rtut.showcase-lighthouse: draft") and "reviewed: False" in out
def test_review_then_maturity(repo, capsys):
pack = repo / "runbooks/v1/pack.yaml"
assert run(capsys, "review", pack, "--by", "alice", "--note", "read it")[0] == 0
code, out, _ = run(capsys, "maturity", pack)
assert "rtut.showcase-lighthouse: reviewed" in out and "reviewed: True" in out
code, _, err = run(capsys, "review", pack, "--by", "two words")
assert code == 1 and "one word" in err
def test_run_commands_walk_a_v01_pack(repo, capsys):
pack = repo / "runbooks/v1/pack.yaml"
code, rid, _ = run(capsys, "run", "new", pack, "-o", "bernd", "-p", "logdir=/tmp/rtut-cli-x")
rid = rid.strip()
assert code == 0 and rid
assert run(capsys, "run", "start", pack, rid, "read-the-sign")[0] == 0
assert run(capsys, "run", "verify", pack, rid, "read-the-sign", "--pass")[0] == 0
assert run(capsys, "run", "done", pack, rid, "read-the-sign")[0] == 0
code, out, _ = run(capsys, "run", "status", pack, rid)
assert "done read-the-sign" in out and "todo meet-the-keeper" in out
code, out, _ = run(capsys, "run", "finalize", pack, rid, "--outcome", "abandoned", "--note", "cli test")
assert code == 0 and out.strip() == "abandoned"
code, _, err = run(capsys, "run", "start", pack, rid, "meet-the-keeper")
assert code == 1 and "finished" in err
assert run(capsys, "run", "feedback", pack, rid, "--rating", "poor", "--note", "x")[0] == 1 # closed runs are locked
def test_the_retired_finish_action_is_gone(repo):
with pytest.raises(SystemExit):
main(["run", "finish", str(repo / "runbooks/v1/pack.yaml"), "x"])
def test_project_commands(repo, capsys):
code, out, _ = run(capsys, "project", "list", repo)
assert code == 0 and "lighthouse-watch\ttemplate" in out
code, out, _ = run(capsys, "project", "new", repo, "lighthouse-watch", "demo", "-e", "rehearsal", "--title", "Demo watch",
"--description", "A demo", "-s", "keeper=Ada", "-s", "logdir=/tmp/rtut-cli-p")
assert code == 0 and out.strip().endswith("projects/demo/project.yaml")
made = yaml.safe_load((repo / "projects/demo/project.yaml").read_text())
assert made["title"] == "Demo watch" and made["description"] == "A demo" and made["template"] == "lighthouse-watch"
# the template's example members need packs this fixture does not have, so validation reports it rather than crashing
code, out, _ = run(capsys, "project", "validate", repo, "demo")
assert code == 1 and "FAIL demo" in out and "rtut.example-hello not found" in out
code, _, err = run(capsys, "project", "status", repo, "nope")
assert code == 1 and "no project nope" in err
def test_ui_and_launch_arguments_parse(repo):
from rtut.__main__ import main as m
with pytest.raises(SystemExit):
m(["ui", "--port", "not-a-port"])

2
tests/test_legacy.py
View file

@ -17,7 +17,7 @@ def make_repo(tmp_path, command, url, cwd="."):
d.mkdir(parents=True)
(d / "pack.yaml").write_text(yaml.safe_dump({
"spec": "runbook-pack/v0.1", "id": "t.old", "title": "Old console", "owner": "t",
"exercise_status": "unexercised", "engine": "legacy-command",
"engine": "legacy-command",
"legacy": {"command": command, "cwd": cwd, "url": url}}))
return P.load(d / "pack.yaml")

6
tests/test_maturity.py
View file

@ -33,8 +33,10 @@ def test_digest_is_stable_ignores_exercise_status_and_reviews_but_changes_with_c
d1 = P.digest(pk)
assert d1 == P.digest(pk)
text = pk.path.read_text()
pk.path.write_text(text.replace("exercise_status: unexercised", "exercise_status: exercised 2026-09-29 by x"))
assert P.digest(pk) == d1 # recording an exercise is not a new revision
assert "engine: native\n" in text
pk.path.write_text(text.replace("engine: native\n", "exercise_status: exercised 2026-09-29 by x\nengine: native\n", 1)) # as an old pack carries it
assert P.digest(pk) == d1 # the deprecated field is ignored: dropping it is not a new revision
pk.path.write_text(text)
M.add_review(pk, "alice", "read it")
assert P.digest(pk) == d1 # neither is a review
pk.path.write_text(text + "\n# changed\n")

14
tests/test_pack.py
View file

@ -29,8 +29,6 @@ def mut(fn):
@pytest.mark.parametrize("fn,frag", [
(lambda d: d.pop("exercise_status"), "exercise_status"),
(lambda d: d.update(exercise_status="done"), "exercise_status"),
(lambda d: d["parameters"][0].update(secret=True), "secret"),
(lambda d: d["steps"][0].pop("verify"), "verify"),
(lambda d: d["steps"][0].update(command="x {{nope}}"), "undefined placeholder"),
@ -49,7 +47,7 @@ def test_invalid(tmp_path, fn, frag):
def test_legacy_requires_command(tmp_path):
d = {"spec": "runbook-pack/v0.1", "id": "x.y", "title": "Legacy x", "owner": "o",
"exercise_status": "unexercised", "engine": "legacy-command"}
"engine": "legacy-command"}
assert not check(tmp_path, d).ok
d["legacy"] = {"command": ["true"]}
assert check(tmp_path, d).ok
@ -74,3 +72,13 @@ def test_text_parameter_allows_spaces_but_not_metacharacters():
with pytest.raises(ValueError):
P.render(d["steps"][0]["command"], {"host": "two words"}, pk)
def test_the_deprecated_exercise_status_is_accepted_and_ignored_so_old_packs_still_load(tmp_path):
d = copy.deepcopy(GOOD)
d["exercise_status"] = "exercised 2026-01-01 by someone"
assert check(tmp_path, d).ok
d["exercise_status"] = "whatever text" # deprecated: not validated, not used
assert check(tmp_path, d).ok
d.pop("exercise_status")
assert check(tmp_path, d).ok # and it is not required any more

2
tests/test_projectrun.py
View file

@ -19,7 +19,7 @@ def v2_pack(pid, extra_step=None):
{"id": "second", "title": "Second step", "actor": "op", "owner": "t", "verify": {"done_when": "second done", "expect": "manual"}}]
if extra_step:
steps.append(extra_step)
return {"spec": "runbook-pack/v0.2", "id": pid, "title": f"Pack {pid}", "owner": "t", "exercise_status": "unexercised", "engine": "native",
return {"spec": "runbook-pack/v0.2", "id": pid, "title": f"Pack {pid}", "owner": "t", "engine": "native",
"framing": ["keep in mind"], "actors": [{"id": "op", "role": "Operator", "description": "does it"}],
"datapoints": [{"id": "site", "label": "Site", "type": "string", "pattern": "[a-z]+", "default": "north", "ask_in": "main"}],
"stages": [{"id": "main", "title": "Main", "goal": "do the things", "steps": steps}]}

8
tests/test_run.py
View file

@ -51,9 +51,7 @@ def test_full_run_and_exercise(pk):
pytest.skip("ping unavailable in this environment")
R.complete(r, "ping")
assert r.complete
old, new = R.exercise_update(r, date(2026, 9, 29))
assert "exercise_status: exercised 2026-09-29 by bernd" in new and old != new
assert pk.path.read_text() == old # never written by the engine
assert pk.path.read_text() == P.load(pk.path).path.read_text() # the engine never writes a pack
def test_order_fail_retry_skip(pk):
@ -69,8 +67,8 @@ def test_order_fail_retry_skip(pk):
R.fail(r, "ping", "-----BEGIN RSA PRIVATE KEY-----")
r2 = R.create(pk, "bernd")
R.skip(r2, "ping", "not needed")
with pytest.raises(R.RunError):
R.exercise_update(r2) # skips never exercise a pack
with pytest.raises(R.RunError, match="succeeds only when every step is done"):
R.finalize(r2, "succeeded") # a skipped step never counts as done
def test_manual_verification_needs_answer(pk):

29
tests/test_run2.py
View file

@ -60,6 +60,14 @@ def pass_gate(r, gid):
R.gate_check(r, gid, i, confirm=True, manual_ok=True)
def counts_towards_maturity(r) -> bool:
"""Would this run be clean maturity evidence? (succeeded, no deviation, with an environment)"""
from rtut import maturity as M
r.data["outcome"] = "succeeded"
r.data["context"] = {"project": "p", "environment": "rehearsal"}
return M._evidence(r.pack, r.data.get("pack_digest"), [r])["clean_runs"] == 1
# ---- datapoints -----------------------------------------------------------------------------------
def test_create_sets_datapoint_states(tmp_path):
@ -165,15 +173,11 @@ def test_waiver_only_where_declared_with_reason_and_recorded(tmp_path):
R.waive(r, "night-on-record", "try")
def test_deviations_block_exercise_but_not_finalize(tmp_path):
def test_deviations_keep_a_run_out_of_maturity(tmp_path):
r = new(tmp_path)
R.start(r, "warn", alt="sound-horn", override_reason="probe", acknowledge_risk=True)
r.data["steps"]["warn"]["state"] = "failed"
assert r.data["deviations"]
with pytest.raises(R1.RunError, match="deviations"):
for k in r.data["steps"]:
r.data["steps"][k]["state"] = "done"
R1.exercise_update(r)
assert r.data["deviations"] and not counts_towards_maturity(r)
assert counts_towards_maturity(new(tmp_path)) # the same without a deviation would count
# ---- confirmation styles ------------------------------------------------------------------------------
@ -264,10 +268,7 @@ def test_auto_confirm_push_through_records_deviation_and_moves_on(tmp_path):
assert dev["kind"] == "push-through" and dev["mode"] == "auto-confirm" and dev["risk_acknowledged"] is False
nxt = R.auto_confirm_tick(r, late) # continues to the manual-only step
assert nxt["item"] == "close-log" and nxt["action"] == "awaiting-operator"
with pytest.raises(R1.RunError, match="deviations"):
for k in r.data["steps"]:
r.data["steps"][k]["state"] = "done"
R1.exercise_update(r)
assert not counts_towards_maturity(r) # a pushed-through run is not evidence
def test_auto_confirm_gate_push_through_and_stop(tmp_path):
@ -460,8 +461,4 @@ def test_a_run_that_changed_a_used_value_never_counts_towards_maturity(tmp_path)
r = new(tmp_path)
do_step(r, "read-the-sky")
R.set_datapoint(r, "weather", "clear", override_reason="changed my mind", acknowledge_risk=True)
assert r.data["deviations"]
with pytest.raises(R1.RunError, match="deviations"):
for k in r.data["steps"]:
r.data["steps"][k]["state"] = "done"
R1.exercise_update(r)
assert r.data["deviations"] and not counts_towards_maturity(r)

18
tests/test_score.py
View file

@ -64,17 +64,6 @@ def test_scorecard_counts_every_outcome_and_stale_open_runs(pk):
assert c["feedback"]["good"] == 1
def test_finalizing_success_can_record_exercise_status(pk):
r = R.create(pk, "bernd"); play_all(r)
with pytest.raises(R.RunError):
R.apply_exercise(r) # not finalized yet
R.finalize(r, "succeeded")
status = R.apply_exercise(r)
assert status.startswith("exercised ") and status.endswith(" by bernd")
reloaded = P.load(pk.path)
assert reloaded.ok and reloaded.data["exercise_status"] == status
def test_capture_documentation_and_limits(pk):
r = R.create(pk, "bernd")
with pytest.raises(R.RunError, match="start the step"):
@ -166,7 +155,7 @@ def test_closing_records_feedback_with_the_outcome_and_validates_before_changing
assert R.load(pk, r.data["run_id"]).data["feedback"][0]["rating"] == "poor"
def test_a_closed_record_cannot_be_saved_with_changes_but_the_exercise_marker_is_allowed(pk):
def test_a_closed_record_cannot_be_saved_with_changes(pk):
r = R.create(pk, "bernd"); R.finalize(r, "abandoned")
r2 = R.load(pk, r.data["run_id"])
r2.data["steps"]["read-the-sign"]["state"] = "done"
@ -177,10 +166,7 @@ def test_a_closed_record_cannot_be_saved_with_changes_but_the_exercise_marker_is
with pytest.raises(R.RunError, match="operator"):
r3.save()
assert R.load(pk, r.data["run_id"]).data["operator"] == "bernd" # nothing reached the disk
r4 = R.load(pk, r.data["run_id"])
r4.data["exercise_recorded"] = "x"
r4.save(allow=("exercise_recorded",)) # the one permitted later update
assert R.load(pk, r.data["run_id"]).data["exercise_recorded"] == "x" and R.seal_state(R.load(pk, r.data["run_id"])) == "sealed"
assert R.seal_state(R.load(pk, r.data["run_id"])) == "sealed" # and the seal still holds
def test_closing_seals_the_file_read_only_and_tampering_is_detected(pk):

4
tests/test_showcase.py
View file

@ -35,8 +35,8 @@ def test_scripted_playthrough(pk):
R.complete(r, sid, {n: "1" for n in s.get("evidence", [])})
assert r.complete
assert not Path(r.data["params"]["logdir"]).exists()
old, new = R.exercise_update(r)
assert "exercised" in new
R.finalize(r, "succeeded") # every step done: the run may succeed
assert r.data["outcome"] == "succeeded"
def test_bad_logdir_rejected(pk):

11
tests/test_ui.py
View file

@ -245,17 +245,11 @@ def test_score_runs_list_and_finalize_flow(srv):
assert req(srv, "POST", base + "/finalize", {"_t": t, "outcome": "succeeded", "note": "smooth"})[0] == 303
_, _, body = req(srv, "GET", base)
assert "Run finished: succeeded" in body and "smooth" in body
assert "Record pack as exercised" in body and "would set it to" in body # explicit, with the diff in words
assert "Record pack as exercised" not in body and "would set it to" not in body # retired: maturity is derived from runs
assert "Start this step" not in body # read-only now
_, _, body = req(srv, "GET", "/")
assert "succeeded 1" in body
# exercised status is only written on the explicit click
pack_file = srv.app.packs[i].path
assert "exercise_status: unexercised" in pack_file.read_text()
assert req(srv, "POST", base + "/exercise", {"_t": t})[0] == 303
assert "exercise_status: exercised" in pack_file.read_text()
_, _, body = req(srv, "GET", base)
assert "Pack recorded as" in body and "Record pack as exercised" not in body
assert req(srv, "POST", base + "/exercise", {"_t": t})[0] == 400 # the route is gone
def test_abandon_and_actions_refused_after_finish(srv):
@ -264,7 +258,6 @@ def test_abandon_and_actions_refused_after_finish(srv):
assert req(srv, "POST", base + "/finalize", {"_t": t, "outcome": "abandoned"})[0] == 303
st, _, body = req(srv, "POST", base + "/read-the-sign/start", {"_t": t})
assert st == 400 and "finished" in body
assert req(srv, "POST", base + "/exercise", {"_t": t})[0] == 400 # only a succeeded run may exercise the pack
def test_capture_and_pasted_output_check(srv):

2
tests/test_ui_projectrun.py
View file

@ -19,7 +19,7 @@ def v2_pack(pid, manual_only_last=False):
if manual_only_last:
steps.append({"id": "careful", "title": "Careful step", "actor": "op", "owner": "t", "auto_confirm": "manual-only",
"verify": {"done_when": "you did it", "expect": "manual"}})
return {"spec": "runbook-pack/v0.2", "id": pid, "title": f"Pack {pid}", "owner": "t", "exercise_status": "unexercised", "engine": "native",
return {"spec": "runbook-pack/v0.2", "id": pid, "title": f"Pack {pid}", "owner": "t", "engine": "native",
"framing": ["keep in mind"], "actors": [{"id": "op", "role": "Operator", "description": "does it"}],
"datapoints": [{"id": "site", "label": "Site", "type": "string", "pattern": "[a-z]+", "default": "north", "ask_in": "main"}],
"stages": [{"id": "main", "title": "Main", "goal": "do the things", "steps": steps}]}

9
tests/test_v2.py
View file

@ -106,7 +106,7 @@ def test_three_levels_allowed():
def test_legacy_v02_needs_only_schema():
d = {"spec": "runbook-pack/v0.2", "id": "x.y", "title": "Legacy x", "owner": "o",
"exercise_status": "unexercised", "engine": "legacy-command", "legacy": {"command": ["true"]}}
"engine": "legacy-command", "legacy": {"command": ["true"]}}
assert v2.validate(d) == []
@ -133,3 +133,10 @@ def test_idempotency_is_declared_validated_and_summarized():
assert flow.pack_idempotency(d) == "unknown" # one undeclared step makes the whole unknown
assert errs(lambda x: step(x, "warn").update(idempotent="sometimes"))
assert errs(lambda x: step(x, "warn").update(idempotent=True)) # a YAML "yes" is refused, not silently accepted
def test_v02_accepts_the_deprecated_exercise_status_and_does_not_require_it():
d = copy.deepcopy(GOOD)
assert "exercise_status" not in d and v2.validate(d) == []
d["exercise_status"] = "exercised 2026-01-01 by someone"
assert v2.validate(d) == []