tegwick
6d3414d8b7
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> Assistant: claude-code Assistant-Model: sonnet Assistant-Process: 295952@bnt-lap001 Assistant-Session: e93f64ad-516c-46eb-9666-aad8d300c477
101 lines
4.7 KiB
Python
101 lines
4.7 KiB
Python
import shutil
|
|
import subprocess
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from rtut import maturity as M, pack as P, run as R
|
|
|
|
ROOT = Path(__file__).resolve().parent.parent
|
|
|
|
|
|
@pytest.fixture
|
|
def pk(tmp_path):
|
|
d = tmp_path / "runbooks" / "show"; d.mkdir(parents=True)
|
|
shutil.copy(ROOT / "runbooks/showcase-lighthouse/pack.yaml", d / "pack.yaml")
|
|
return P.load(d / "pack.yaml")
|
|
|
|
|
|
def succeed(pk, operator, context):
|
|
r = R.create(pk, operator, {"logdir": f"/tmp/rtut-mat-{operator}-{context.get('project', 'x')}".lower()}, context)
|
|
for s in pk.data["steps"]:
|
|
sid = s["id"]; R.start(r, sid)
|
|
cmd = R.command_for(r, sid)
|
|
if cmd:
|
|
subprocess.run(cmd, shell=True, check=True, capture_output=True)
|
|
R.verify(r, sid, confirm=True, manual_ok=True if s["verify"]["expect"] == "manual" else None)
|
|
R.complete(r, sid, {n: "1" for n in s.get("evidence", [])})
|
|
R.finalize(r, "succeeded")
|
|
return r
|
|
|
|
|
|
def test_digest_is_stable_ignores_exercise_status_and_reviews_but_changes_with_content(pk):
|
|
d1 = P.digest(pk)
|
|
assert d1 == P.digest(pk)
|
|
text = pk.path.read_text()
|
|
assert "engine: native\n" in text
|
|
pk.path.write_text(text.replace("engine: native\n", "exercise_status: exercised 2026-09-29 by x\nengine: native\n", 1)) # as an old pack carries it
|
|
assert P.digest(pk) == d1 # the deprecated field is ignored: dropping it is not a new revision
|
|
pk.path.write_text(text)
|
|
M.add_review(pk, "alice", "read it")
|
|
assert P.digest(pk) == d1 # neither is a review
|
|
pk.path.write_text(text + "\n# changed\n")
|
|
assert P.digest(pk) != d1
|
|
|
|
|
|
def test_runs_carry_digest_and_context_and_validate_environment(pk):
|
|
r = R.create(pk, "bob", None, {"project": "p1", "environment": "rehearsal", "target": "t1"})
|
|
assert r.data["pack_digest"] == P.digest(pk) and r.data["context"] == {"project": "p1", "environment": "rehearsal", "target": "t1"}
|
|
with pytest.raises(R.RunError, match="environment must be one of"):
|
|
R.create(pk, "bob", None, {"environment": "prod"})
|
|
assert R.create(pk, "bob").data["context"] == {}
|
|
|
|
|
|
def test_maturity_ladder_draft_reviewed_rehearsed_proven(pk):
|
|
assert M.maturity(pk)["level"] == "draft"
|
|
M.add_review(pk, "alice")
|
|
assert M.maturity(pk)["level"] == "reviewed"
|
|
succeed(pk, "bob", {"project": "lab", "environment": "rehearsal"})
|
|
m = M.maturity(pk)
|
|
assert m["level"] == "rehearsed" and m["evidence"]["rehearsal_runs"] == 1
|
|
succeed(pk, "bob", {"project": "stage", "environment": "staging"}) # second environment, same operator
|
|
assert M.maturity(pk)["level"] == "rehearsed"
|
|
succeed(pk, "carol", {"project": "prod", "environment": "production"}) # second operator
|
|
m = M.maturity(pk)
|
|
assert m["level"] == "proven" and m["evidence"]["operators"] == ["bob", "carol"] and len(m["evidence"]["environments"]) == 3
|
|
|
|
|
|
def test_proven_needs_a_review_and_a_rehearsal(pk):
|
|
succeed(pk, "bob", {"project": "a", "environment": "staging"})
|
|
succeed(pk, "carol", {"project": "b", "environment": "production"})
|
|
assert M.maturity(pk)["level"] == "draft" # clean runs in two environments, but no rehearsal and no review
|
|
M.add_review(pk, "alice")
|
|
m = M.maturity(pk)
|
|
assert m["level"] == "reviewed" # reviewed, yet never rehearsed: not proven
|
|
assert m["evidence"]["clean_runs"] == 2 and m["evidence"]["rehearsal_runs"] == 0
|
|
succeed(pk, "dave", {"project": "lab", "environment": "rehearsal"})
|
|
assert M.maturity(pk)["level"] == "proven" # review + rehearsal + 3 environments + 3 operators
|
|
|
|
|
|
def test_runs_without_context_or_with_deviations_never_count(pk):
|
|
succeed(pk, "bob", {}) # no environment: not attributable
|
|
assert M.maturity(pk)["evidence"]["clean_runs"] == 0
|
|
r = R.create(pk, "bob", {"logdir": "/tmp/rtut-mat-dev"}, {"project": "lab", "environment": "rehearsal"})
|
|
r.data["outcome"] = "succeeded"; r.data["deviations"] = [{"kind": "override"}]; r.save()
|
|
assert M.maturity(pk)["evidence"]["clean_runs"] == 0
|
|
|
|
|
|
def test_maturity_goes_stale_when_the_pack_changes(pk):
|
|
M.add_review(pk, "alice")
|
|
succeed(pk, "bob", {"project": "lab", "environment": "rehearsal"})
|
|
assert M.maturity(pk)["level"] == "rehearsed"
|
|
pk.path.write_text(pk.path.read_text() + "\n# a change after the proof\n")
|
|
m = M.maturity(pk)
|
|
assert m["level"] == "stale" and m["stale_from"] == "rehearsed" and m["evidence"]["level"] == "draft"
|
|
|
|
|
|
def test_review_input_is_validated(pk):
|
|
with pytest.raises(M.ReviewError):
|
|
M.add_review(pk, "two words")
|
|
with pytest.raises(M.ReviewError):
|
|
M.add_review(pk, "alice", "hvs.ABCDEFGHIJKLMN")
|