Run outcomes, score, explicit exercise recording, and step captures (RBT-WP-0004-T13)

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

Assistant: claude-code
Assistant-Model: sonnet
Assistant-Process: 295952@bnt-lap001
Assistant-Session: e93f64ad-516c-46eb-9666-aad8d300c477

tests/test_score.py

@@ -0,0 +1,104 @@
+import shutil
+import subprocess
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+
+import pytest
+
+from rtut import pack as P, run as R
+
+ROOT = Path(__file__).resolve().parent.parent
+
+
+@pytest.fixture
+def pk(tmp_path):
+    d = tmp_path / "runbooks" / "show"; d.mkdir(parents=True)
+    shutil.copy(ROOT / "runbooks/showcase-lighthouse/pack.yaml", d / "pack.yaml")
+    return P.load(d / "pack.yaml")
+
+
+def play_all(r):
+    for s in r.pack.data["steps"]:
+        sid = s["id"]; R.start(r, sid)
+        cmd = R.command_for(r, sid)
+        if cmd:
+            subprocess.run(cmd, shell=True, check=True, capture_output=True)
+        R.verify(r, sid, confirm=True, manual_ok=True if s["verify"]["expect"] == "manual" else None)
+        R.complete(r, sid, {n: "1" for n in s.get("evidence", [])})
+
+
+def test_finalize_rules(pk):
+    r = R.create(pk, "bernd")
+    with pytest.raises(R.RunError):
+        R.finalize(r, "succeeded")                 # nothing done yet
+    with pytest.raises(R.RunError):
+        R.finalize(r, "failed")                    # nothing failed: abandon instead
+    with pytest.raises(R.RunError):
+        R.finalize(r, "great")
+    R.start(r, "read-the-sign"); R.fail(r, "read-the-sign", "could not read it")
+    R.finalize(r, "failed", "gave up on step 1")
+    assert r.data["outcome"] == "failed" and r.data["outcome_note"]
+    with pytest.raises(R.RunError):
+        R.finalize(r, "abandoned")                 # already finished
+
+
+def test_finished_run_is_read_only_but_takes_feedback(pk):
+    r = R.create(pk, "bernd"); R.finalize(r, "abandoned")
+    for call in (lambda: R.start(r, "read-the-sign"), lambda: R.add_capture(r, "read-the-sign", "x")):
+        with pytest.raises(R.RunError, match="finished"):
+            call()
+    R.add_feedback(r, "poor", "gave up: unclear")   # feedback stays possible
+
+
+def test_scorecard_counts_every_outcome_and_stale_open_runs(pk):
+    ok = R.create(pk, "a"); play_all(ok); R.finalize(ok, "succeeded"); R.add_feedback(ok, "good")
+    bad = R.create(pk, "b"); R.start(bad, "read-the-sign"); R.fail(bad, "read-the-sign", "x"); R.finalize(bad, "failed")
+    gone = R.create(pk, "c"); R.finalize(gone, "abandoned")
+    open_ = R.create(pk, "d")
+    stale = R.create(pk, "e")
+    stale.data["updated"] = (datetime.now(timezone.utc) - timedelta(days=9)).isoformat()
+    stale.path.write_text(__import__("json").dumps(stale.data))
+    c = R.scorecard(pk)
+    assert (c["tried"], c["succeeded"], c["failed"], c["abandoned"], c["open"]) == (5, 1, 1, 2, 1)
+    assert c["feedback"]["good"] == 1
+
+
+def test_finalizing_success_can_record_exercise_status(pk):
+    r = R.create(pk, "bernd"); play_all(r)
+    with pytest.raises(R.RunError):
+        R.apply_exercise(r)                        # not finalized yet
+    R.finalize(r, "succeeded")
+    status = R.apply_exercise(r)
+    assert status.startswith("exercised ") and status.endswith(" by bernd")
+    reloaded = P.load(pk.path)
+    assert reloaded.ok and reloaded.data["exercise_status"] == status
+
+
+def test_capture_documentation_and_limits(pk):
+    r = R.create(pk, "bernd")
+    with pytest.raises(R.RunError, match="start the step"):
+        R.add_capture(r, "read-the-sign", "out")
+    R.start(r, "read-the-sign")
+    R.add_capture(r, "read-the-sign", "Tower of stone", command="printf poem")
+    assert R.load(pk, r.data["run_id"]).state("read-the-sign")["captures"][0]["output"] == "Tower of stone"
+    with pytest.raises(R.RunError, match="too large"):
+        R.add_capture(r, "read-the-sign", "x" * 20001)
+    with pytest.raises(R.RunError, match="secret-looking"):
+        R.add_capture(r, "read-the-sign", "hvs.ABCDEFGHIJKLMN")
+    with pytest.raises(R.RunError, match="nothing"):
+        R.add_capture(r, "read-the-sign", "  ")
+
+
+def test_verify_against_pasted_output(pk):
+    r = R.create(pk, "bernd", {"keeper": "Ada"})
+    R.start(r, "read-the-sign")
+    with pytest.raises(R.RunError, match="output-contains"):
+        R.verify_pasted(r, "read-the-sign", "x")                  # manual step: not applicable
+    R.complete  # (step 2 is output-contains)
+    R.verify(r, "read-the-sign", manual_ok=True); R.complete(r, "read-the-sign")
+    R.start(r, "meet-the-keeper")
+    assert not R.verify_pasted(r, "meet-the-keeper", "nothing useful", "echo ...")["passed"]
+    ok = R.verify_pasted(r, "meet-the-keeper", "Ada climbs the stairs. Motto on the wall - x")
+    assert ok["passed"] and ok["source"] == "pasted"
+    caps = R.load(pk, r.data["run_id"]).state("meet-the-keeper")["captures"]
+    assert len(caps) == 2 and caps[0]["source"] == "pasted-for-verification"

tests/test_ui.py

@@ -177,7 +177,9 @@ def test_done_steps_fold_to_the_title_line_and_others_stay_open(srv):
     _, _, body = req(srv, "GET", base)
     assert "<details class='step done'>" in body                        # done: no 'open' attribute, so folded
     assert "1. Read the poem" in body.split("<details class='step done'>")[1].split("</summary>")[0]   # title line stays in the summary
-    assert "Comment: fine" in body.split("<details class='step done'>")[1].split("</details>")[0]      # the rest is inside the fold
+    rest = body.split("<details class='step done'>")[1]
+    step_end = min(x for x in (rest.find("<details id=current"), rest.find("<details class='step"), rest.find("<div id=close")) if x >= 0)
+    assert "Comment: fine" in rest[:step_end]                                                           # the rest is inside the fold
 
 
 def _finish_showcase(srv, t, base):
@@ -223,3 +225,65 @@ def test_feedback_validation(srv):
     st, _, body = req(srv, "POST", base + "/feedback", {"_t": t, "rating": "good", "comment": "hvs.ABCDEFGHIJKLMN"})
     assert st == 400 and "secret-looking" in body
     assert req(srv, "POST", base + "/feedback", {"rating": "good"})[0] == 403      # CSRF token required
+
+
+def test_score_runs_list_and_finalize_flow(srv):
+    t, loc, _ = _showcase_run(srv)
+    base = loc.split("#")[0]
+    i = int(base.split("/")[2])
+    _, _, body = req(srv, "GET", f"/pack/{i}")
+    assert "tried 1 · succeeded 0" in body and "open 1" in body and "Runs" in body
+    _, _, body = req(srv, "GET", base)
+    assert "Abandon run" in body and "Finish: succeeded" not in body       # cannot succeed before every step is done
+    _finish_showcase(srv, t, base)
+    _, _, body = req(srv, "GET", base)
+    assert "Finish: succeeded" in body
+    assert req(srv, "POST", base + "/finalize", {"_t": t, "outcome": "succeeded", "note": "smooth"})[0] == 303
+    _, _, body = req(srv, "GET", base)
+    assert "Run finished: succeeded" in body and "smooth" in body
+    assert "Record pack as exercised" in body and "would set it to" in body   # explicit, with the diff in words
+    assert "Start this step" not in body                                       # read-only now
+    _, _, body = req(srv, "GET", "/")
+    assert "succeeded 1" in body
+    # exercised status is only written on the explicit click
+    pack_file = srv.app.packs[i].path
+    assert "exercise_status: unexercised" in pack_file.read_text()
+    assert req(srv, "POST", base + "/exercise", {"_t": t})[0] == 303
+    assert "exercise_status: exercised" in pack_file.read_text()
+    _, _, body = req(srv, "GET", base)
+    assert "Pack recorded as" in body and "Record pack as exercised" not in body
+
+
+def test_abandon_and_actions_refused_after_finish(srv):
+    t, loc, _ = _showcase_run(srv)
+    base = loc.split("#")[0]
+    assert req(srv, "POST", base + "/finalize", {"_t": t, "outcome": "abandoned"})[0] == 303
+    st, _, body = req(srv, "POST", base + "/read-the-sign/start", {"_t": t})
+    assert st == 400 and "finished" in body
+    assert req(srv, "POST", base + "/exercise", {"_t": t})[0] == 400          # only a succeeded run may exercise the pack
+
+
+def test_capture_and_pasted_output_check(srv):
+    t, loc, _ = _showcase_run(srv)
+    base = loc.split("#")[0]
+    st, _, body = req(srv, "POST", base + "/meet-the-keeper/capture", {"_t": t, "output": "x"})
+    assert st == 400 and "start the step" in body
+    for sid in ("read-the-sign",):
+        req(srv, "POST", f"{base}/{sid}/start", {"_t": t})
+        req(srv, "POST", f"{base}/{sid}/verify", {"_t": t, "ok": "1"})
+        req(srv, "POST", f"{base}/{sid}/done", {"_t": t})
+    req(srv, "POST", base + "/meet-the-keeper/start", {"_t": t})
+    _, _, body = req(srv, "GET", base)
+    assert "Capture command and output" in body and "Check against this output" in body
+    assert req(srv, "POST", base + "/meet-the-keeper/capture", {"_t": t, "command": "echo hi", "output": "hello there"})[0] == 303
+    _, _, body = req(srv, "GET", base)
+    assert "hello there" in body and "$ echo hi" in body and "1 saved" in body
+    st, _, body = req(srv, "POST", base + "/meet-the-keeper/verify-pasted", {"_t": t, "output": "wrong text"})
+    assert st == 303
+    _, _, body = req(srv, "GET", base)
+    assert "Check not passed" in body
+    req(srv, "POST", base + "/meet-the-keeper/verify-pasted", {"_t": t, "output": "Ada climbs the stairs. Motto on the wall - x"})
+    _, _, body = req(srv, "GET", base)
+    assert "Check passed" in body
+    st, _, body = req(srv, "POST", base + "/meet-the-keeper/capture", {"_t": t, "output": "hvs.ABCDEFGHIJKLMN"})
+    assert st == 400 and "secret-looking" in body