tegwick
f5619e1a52
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> Assistant: claude-code Assistant-Model: sonnet Assistant-Process: 295952@bnt-lap001 Assistant-Session: e93f64ad-516c-46eb-9666-aad8d300c477
104 lines
4.5 KiB
Python
104 lines
4.5 KiB
Python
import shutil
|
|
import subprocess
|
|
from datetime import datetime, timedelta, timezone
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from rtut import pack as P, run as R
|
|
|
|
ROOT = Path(__file__).resolve().parent.parent
|
|
|
|
|
|
@pytest.fixture
|
|
def pk(tmp_path):
|
|
d = tmp_path / "runbooks" / "show"; d.mkdir(parents=True)
|
|
shutil.copy(ROOT / "runbooks/showcase-lighthouse/pack.yaml", d / "pack.yaml")
|
|
return P.load(d / "pack.yaml")
|
|
|
|
|
|
def play_all(r):
|
|
for s in r.pack.data["steps"]:
|
|
sid = s["id"]; R.start(r, sid)
|
|
cmd = R.command_for(r, sid)
|
|
if cmd:
|
|
subprocess.run(cmd, shell=True, check=True, capture_output=True)
|
|
R.verify(r, sid, confirm=True, manual_ok=True if s["verify"]["expect"] == "manual" else None)
|
|
R.complete(r, sid, {n: "1" for n in s.get("evidence", [])})
|
|
|
|
|
|
def test_finalize_rules(pk):
|
|
r = R.create(pk, "bernd")
|
|
with pytest.raises(R.RunError):
|
|
R.finalize(r, "succeeded") # nothing done yet
|
|
with pytest.raises(R.RunError):
|
|
R.finalize(r, "failed") # nothing failed: abandon instead
|
|
with pytest.raises(R.RunError):
|
|
R.finalize(r, "great")
|
|
R.start(r, "read-the-sign"); R.fail(r, "read-the-sign", "could not read it")
|
|
R.finalize(r, "failed", "gave up on step 1")
|
|
assert r.data["outcome"] == "failed" and r.data["outcome_note"]
|
|
with pytest.raises(R.RunError):
|
|
R.finalize(r, "abandoned") # already finished
|
|
|
|
|
|
def test_finished_run_is_read_only_but_takes_feedback(pk):
|
|
r = R.create(pk, "bernd"); R.finalize(r, "abandoned")
|
|
for call in (lambda: R.start(r, "read-the-sign"), lambda: R.add_capture(r, "read-the-sign", "x")):
|
|
with pytest.raises(R.RunError, match="finished"):
|
|
call()
|
|
R.add_feedback(r, "poor", "gave up: unclear") # feedback stays possible
|
|
|
|
|
|
def test_scorecard_counts_every_outcome_and_stale_open_runs(pk):
|
|
ok = R.create(pk, "a"); play_all(ok); R.finalize(ok, "succeeded"); R.add_feedback(ok, "good")
|
|
bad = R.create(pk, "b"); R.start(bad, "read-the-sign"); R.fail(bad, "read-the-sign", "x"); R.finalize(bad, "failed")
|
|
gone = R.create(pk, "c"); R.finalize(gone, "abandoned")
|
|
stalled = R.create(pk, "d")
|
|
stale = R.create(pk, "e")
|
|
stale.data["updated"] = (datetime.now(timezone.utc) - timedelta(days=9)).isoformat()
|
|
stale.path.write_text(__import__("json").dumps(stale.data))
|
|
c = R.scorecard(pk)
|
|
assert (c["tried"], c["succeeded"], c["failed"], c["abandoned"], c["stalled"]) == (5, 1, 1, 2, 1)
|
|
assert c["feedback"]["good"] == 1
|
|
|
|
|
|
def test_finalizing_success_can_record_exercise_status(pk):
|
|
r = R.create(pk, "bernd"); play_all(r)
|
|
with pytest.raises(R.RunError):
|
|
R.apply_exercise(r) # not finalized yet
|
|
R.finalize(r, "succeeded")
|
|
status = R.apply_exercise(r)
|
|
assert status.startswith("exercised ") and status.endswith(" by bernd")
|
|
reloaded = P.load(pk.path)
|
|
assert reloaded.ok and reloaded.data["exercise_status"] == status
|
|
|
|
|
|
def test_capture_documentation_and_limits(pk):
|
|
r = R.create(pk, "bernd")
|
|
with pytest.raises(R.RunError, match="start the step"):
|
|
R.add_capture(r, "read-the-sign", "out")
|
|
R.start(r, "read-the-sign")
|
|
R.add_capture(r, "read-the-sign", "Tower of stone", command="printf poem")
|
|
assert R.load(pk, r.data["run_id"]).state("read-the-sign")["captures"][0]["output"] == "Tower of stone"
|
|
with pytest.raises(R.RunError, match="too large"):
|
|
R.add_capture(r, "read-the-sign", "x" * 20001)
|
|
with pytest.raises(R.RunError, match="secret-looking"):
|
|
R.add_capture(r, "read-the-sign", "hvs.ABCDEFGHIJKLMN")
|
|
with pytest.raises(R.RunError, match="nothing"):
|
|
R.add_capture(r, "read-the-sign", " ")
|
|
|
|
|
|
def test_verify_against_pasted_output(pk):
|
|
r = R.create(pk, "bernd", {"keeper": "Ada"})
|
|
R.start(r, "read-the-sign")
|
|
with pytest.raises(R.RunError, match="output-contains"):
|
|
R.verify_pasted(r, "read-the-sign", "x") # manual step: not applicable
|
|
R.complete # (step 2 is output-contains)
|
|
R.verify(r, "read-the-sign", manual_ok=True); R.complete(r, "read-the-sign")
|
|
R.start(r, "meet-the-keeper")
|
|
assert not R.verify_pasted(r, "meet-the-keeper", "nothing useful", "echo ...")["passed"]
|
|
ok = R.verify_pasted(r, "meet-the-keeper", "Ada climbs the stairs. Motto on the wall - x")
|
|
assert ok["passed"] and ok["source"] == "pasted"
|
|
caps = R.load(pk, r.data["run_id"]).state("meet-the-keeper")["captures"]
|
|
assert len(caps) == 2 and caps[0]["source"] == "pasted-for-verification"
|