runbook-tutorials/tests/test_maturity.py

import shutil
import subprocess
from pathlib import Path

import pytest

from rtut import maturity as M, pack as P, run as R

ROOT = Path(__file__).resolve().parent.parent


@pytest.fixture
def pk(tmp_path):
    d = tmp_path / "runbooks" / "show"; d.mkdir(parents=True)
    shutil.copy(ROOT / "runbooks/showcase-lighthouse/pack.yaml", d / "pack.yaml")
    return P.load(d / "pack.yaml")


def succeed(pk, operator, context):
    r = R.create(pk, operator, {"logdir": f"/tmp/rtut-mat-{operator}-{context.get('project', 'x')}".lower()}, context)
    for s in pk.data["steps"]:
        sid = s["id"]; R.start(r, sid)
        cmd = R.command_for(r, sid)
        if cmd:
            subprocess.run(cmd, shell=True, check=True, capture_output=True)
        R.verify(r, sid, confirm=True, manual_ok=True if s["verify"]["expect"] == "manual" else None)
        R.complete(r, sid, {n: "1" for n in s.get("evidence", [])})
    R.finalize(r, "succeeded")
    return r


def test_digest_is_stable_ignores_exercise_status_and_reviews_but_changes_with_content(pk):
    d1 = P.digest(pk)
    assert d1 == P.digest(pk)
    text = pk.path.read_text()
    assert "engine: native\n" in text
    pk.path.write_text(text.replace("engine: native\n", "exercise_status: exercised 2026-09-29 by x\nengine: native\n", 1))   # as an old pack carries it
    assert P.digest(pk) == d1                                     # the deprecated field is ignored: dropping it is not a new revision
    pk.path.write_text(text)
    M.add_review(pk, "alice", "read it")
    assert P.digest(pk) == d1                                     # neither is a review
    pk.path.write_text(text + "\n# changed\n")
    assert P.digest(pk) != d1


def test_runs_carry_digest_and_context_and_validate_environment(pk):
    r = R.create(pk, "bob", None, {"project": "p1", "environment": "rehearsal", "target": "t1"})
    assert r.data["pack_digest"] == P.digest(pk) and r.data["context"] == {"project": "p1", "environment": "rehearsal", "target": "t1"}
    with pytest.raises(R.RunError, match="environment must be one of"):
        R.create(pk, "bob", None, {"environment": "prod"})
    assert R.create(pk, "bob").data["context"] == {}


def test_maturity_ladder_draft_reviewed_rehearsed_proven(pk):
    assert M.maturity(pk)["level"] == "draft"
    M.add_review(pk, "alice")
    assert M.maturity(pk)["level"] == "reviewed"
    succeed(pk, "bob", {"project": "lab", "environment": "rehearsal"})
    m = M.maturity(pk)
    assert m["level"] == "rehearsed" and m["evidence"]["rehearsal_runs"] == 1
    succeed(pk, "bob", {"project": "stage", "environment": "staging"})                 # second environment, same operator
    assert M.maturity(pk)["level"] == "rehearsed"
    succeed(pk, "carol", {"project": "prod", "environment": "production"})            # second operator
    m = M.maturity(pk)
    assert m["level"] == "proven" and m["evidence"]["operators"] == ["bob", "carol"] and len(m["evidence"]["environments"]) == 3


def test_proven_needs_a_review_and_a_rehearsal(pk):
    succeed(pk, "bob", {"project": "a", "environment": "staging"})
    succeed(pk, "carol", {"project": "b", "environment": "production"})
    assert M.maturity(pk)["level"] == "draft"                     # clean runs in two environments, but no rehearsal and no review
    M.add_review(pk, "alice")
    m = M.maturity(pk)
    assert m["level"] == "reviewed"                               # reviewed, yet never rehearsed: not proven
    assert m["evidence"]["clean_runs"] == 2 and m["evidence"]["rehearsal_runs"] == 0
    succeed(pk, "dave", {"project": "lab", "environment": "rehearsal"})
    assert M.maturity(pk)["level"] == "proven"                    # review + rehearsal + 3 environments + 3 operators


def test_runs_without_context_or_with_deviations_never_count(pk):
    succeed(pk, "bob", {})                                        # no environment: not attributable
    assert M.maturity(pk)["evidence"]["clean_runs"] == 0
    r = R.create(pk, "bob", {"logdir": "/tmp/rtut-mat-dev"}, {"project": "lab", "environment": "rehearsal"})
    r.data["outcome"] = "succeeded"; r.data["deviations"] = [{"kind": "override"}]; r.save()
    assert M.maturity(pk)["evidence"]["clean_runs"] == 0


def test_maturity_goes_stale_when_the_pack_changes(pk):
    M.add_review(pk, "alice")
    succeed(pk, "bob", {"project": "lab", "environment": "rehearsal"})
    assert M.maturity(pk)["level"] == "rehearsed"
    pk.path.write_text(pk.path.read_text() + "\n# a change after the proof\n")
    m = M.maturity(pk)
    assert m["level"] == "stale" and m["stale_from"] == "rehearsed" and m["evidence"]["level"] == "draft"


def test_review_input_is_validated(pk):
    with pytest.raises(M.ReviewError):
        M.add_review(pk, "two words")
    with pytest.raises(M.ReviewError):
        M.add_review(pk, "alice", "hvs.ABCDEFGHIJKLMN")