clay-borg/tools/design.py

#!/usr/bin/env python3
"""design — report the finding register (CB-WP-0022 T05).

Implements the metrics in `specs/GameDesign.md` §4 over the register in
`specs/FindingRegister.md`. ADR-0012 D8 retired `design-baseline.py`, which
was a hand-maintained dict counting itself; the difference that matters is
that **every number here is computed over rows that name real files, and
the reproduction check stats the file.**

`design-baseline.py`'s `has_reproduction` was `p.startswith("scenarios/")`
and nothing else, so its own positive control was green against a path
that had been deleted. The self-test below asserts the opposite property
directly: a row citing a nonexistent file must NOT count as reproduced.
"""

import os, re, sys, subprocess, datetime

ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
REGISTER = os.path.join(ROOT, "specs", "FindingRegister.md")

BEGIN = "<!-- design-register:begin -->"
END = "<!-- design-register:end -->"

KINDS = {"underdetermined", "inconsistent", "inert", "degenerate", "unplayed"}
OPEN_STATES = {"raised", "reported", "ruled"}   # the queue: not yet applied
CLOSED_STATES = {"applied", "withdrawn"}    # the log
NOTE = "note"
NOTE_EXPIRY_DAYS = 30  # GameDesign §3.1, same figure as rule-coverage.py


def parse(text):
    """Rows between the register markers. Raises if the block is absent —
    a register that silently reports zero findings is worse than one that
    fails."""
    try:
        block = text.split(BEGIN)[1].split(END)[0]
    except IndexError:
        raise SystemExit(f"no design-register block in {REGISTER}")
    rows = []
    for line in block.splitlines():
        line = line.strip()
        if not line.startswith("|") or line.startswith("|---"):
            continue
        cells = [c.strip() for c in line.strip("|").split("|")]
        if len(cells) != 7 or cells[0] in ("id",):
            continue
        rows.append(dict(zip(
            ("id", "kind", "state", "repro", "role", "raised", "owner"), cells)))
    return rows


def reproduced(row, root=ROOT):
    """GameDesign §1.1: the artifact must resolve. A named test is admitted
    by its `crate::module::name` shape; anything else must be a real path
    on disk, and that is checked by stat, not by prefix."""
    p = row["repro"]
    if p in ("", "—", "-"):
        return False
    if "::" in p:               # a named test
        return True
    return os.path.exists(os.path.join(root, p))


_SIM = None


def sim_passes(root=ROOT):
    """`make sim` is the authority on whether scenarios pass. Run once and
    cache — asking per row costs a full suite per finding."""
    global _SIM
    if _SIM is None:
        r = subprocess.run(["make", "sim"], cwd=root, capture_output=True, text=True)
        _SIM = r.returncode == 0
    return _SIM


def alarming(row, root=ROOT):
    """§1.3, and the distinction the backfill discovered: **only a
    counterexample alarms when green.**

    A `default` reproduction encodes a provisional choice and is SUPPOSED
    to pass — U2's scenario is green because the default it documents is
    implemented, which says nothing about whether ground-game agrees. A
    `counterexample` is the model-checker shape: it demonstrates the
    defect, so it must be red while the finding is open. GR-E01's went
    green when the edition landed and nothing noticed for four days.
    """
    if row["role"] != "counterexample":
        return False
    if not row["repro"].startswith("scenarios/"):
        return False
    return sim_passes(root)


def report(root=ROOT, today=None):
    today = today or datetime.date.today()
    rows = parse(open(os.path.join(root, "specs", "FindingRegister.md")).read())

    findings = [r for r in rows if r["state"] != NOTE]
    notes = [r for r in rows if r["state"] == NOTE]
    queue = [r for r in findings if r["state"] in OPEN_STATES]
    closed = [r for r in findings if r["state"] in CLOSED_STATES]

    print("design — the finding register\n")
    print("  QUEUE (open findings)")
    for r in sorted(queue, key=lambda r: r["raised"]):
        age = (today - datetime.date.fromisoformat(r["raised"])).days
        mark = "repro" if reproduced(r, root) else "  -  "
        print(f"    {mark}  {r['id']:<4} {r['kind']:<16} {r['state']:<9} {age:>4}d  {r['owner']}")

    if notes:
        print("\n  NOTES (not reportable — GameDesign §3.1)")
        for r in sorted(notes, key=lambda r: r["raised"]):
            age = (today - datetime.date.fromisoformat(r["raised"])).days
            flag = "  EXPIRED" if age > NOTE_EXPIRY_DAYS else ""
            print(f"           {r['id']:<4} {r['kind']:<16} {age:>4}d{flag}")

    # ---- §4 metrics. The log is reported on request, not by default,
    # because a default view mixing open and closed loses the queue
    # property (ADR-0012 D5).
    n = len(findings)
    repro = sum(1 for r in findings if reproduced(r, root))
    debt = [r for r in queue if not reproduced(r, root)]
    expired = [r for r in notes
               if (today - datetime.date.fromisoformat(r["raised"])).days > NOTE_EXPIRY_DAYS]
    unresolved_green = [r for r in queue if reproduced(r, root) and alarming(r, root)]

    print(f"\n  findings                       {n}   (+{len(notes)} note(s))")
    print(f"  with a resolving reproduction  {repro}/{n}"
          f"{'' if not n else f' = {100*repro//n}%'}   target 100%")
    print(f"  open, lacking a reproduction   {len(debt)}   target 0"
          + (f"   [{', '.join(r['id'] for r in debt)}]" if debt else ""))
    print(f"  reproductions green while open {len(unresolved_green)}   target 0"
          + ("   <-- ALARM, GameDesign §1.3" if unresolved_green else ""))
    print(f"  notes past {NOTE_EXPIRY_DAYS} days             {len(expired)}   target 0")
    print(f"  closed (log)                   {len(closed)}"
          f"   [{', '.join(r['id'] for r in closed)}]" if closed else "")

    bad = [r for r in rows if r["kind"] not in KINDS]
    if bad:
        print(f"\n  UNKNOWN KIND: {', '.join(r['id'] for r in bad)}"
              " — a sixth kind means the taxonomy was invented (ADR-0012 D4)")
    return 0


def self_test():
    ok = True

    def check(name, cond, detail=""):
        nonlocal ok
        ok = ok and bool(cond)
        print(f"  {'ok  ' if cond else 'FAIL'}  {name}{'  ' + detail if detail else ''}")

    rows = parse(open(REGISTER).read())
    check("the register parses", len(rows) >= 14, f"{len(rows)} row(s)")
    check("every kind is one of the five",
          all(r["kind"] in KINDS for r in rows),
          "a sixth kind means the taxonomy was invented")
    check("every state is known",
          all(r["state"] in OPEN_STATES | CLOSED_STATES | {NOTE} for r in rows))

    # THE control design-baseline.py did not have. Its YES-control passed a
    # path that had been deleted and still returned True.
    check("a nonexistent reproduction does NOT count",
          not reproduced({"repro": "scenarios/ground/gr-e01-threshold-unreachable-2p.yaml"}),
          "this exact path was deleted by 2da19a4 and the old tool said yes")
    check("a real reproduction DOES count",
          reproduced({"repro": "scenarios/ground/gr-p05-solve-legality.yaml"}))
    check("a named test counts", reproduced({"repro": "games_ground::view::tests::a_spectator_sees_no_hands"}))
    check("an em-dash does not count", not reproduced({"repro": "—"}))

    # The distinction the backfill discovered: a green DEFAULT is expected,
    # a green COUNTEREXAMPLE is the alarm. Without this the report cried
    # wolf over U2, whose scenario is green precisely because the
    # provisional default it documents is implemented.
    check("a green default does not alarm",
          not alarming({"role": "default", "repro": "scenarios/ground/gr-d01-darvo-trigger.yaml"}))
    check("role is recorded for every reproduced row",
          all(r["role"] in ("counterexample", "default")
              for r in rows if r["repro"] not in ("—", "-", "")))

    # CB-WP-0026 T03. A scenario may claim a U-item only if it names it,
    # because two plausible mappings were written and both were wrong on
    # reading what the scenario actually exercises (gr-a04 asserts consent
    # is required, not U1's timing; gr-d05 is the unrejected REVERSE, not
    # U5's rejected one). A claim nobody can check is how three wrong
    # premises reached ground-game.
    import glob
    for path in glob.glob(os.path.join(ROOT, "scenarios", "ground", "*.yaml")):
        body = open(path).read()
        m = re.search(r"^encodes_u_item:\s*(\S+)", body, re.M)
        if m:
            check(f"{os.path.basename(path)} names the U-item it claims",
                  re.search(rf"\b{m.group(1)}\b", body) is not None, m.group(1))

    # The measured claim: exactly one U-item has a scenario encoding it.
    #
    # **Asserted on the DECLARATION, not on a mention.** The first version
    # grepped for `\bU<n>\b` and went red the moment two scenarios recorded
    # *why they do not* encode U1 and U5 — a mention is not a claim, and a
    # measurement that cannot tell them apart is the loose proxy that let
    # "six of the ten have provisional scenarios" stand unchallenged.
    claimed = sorted({re.search(r"^encodes_u_item:\s*(\S+)", open(p).read(), re.M).group(1)
                      for p in glob.glob(os.path.join(ROOT, "scenarios", "ground", "*.yaml"))
                      if re.search(r"^encodes_u_item:\s*\S+", open(p).read(), re.M)})
    check("exactly one U-item is encoded by a scenario", claimed == ["U2"], f"{claimed}")

    print("design self-test (positive control)")
    return 0 if ok else 1


if __name__ == "__main__":
    sys.exit(self_test() if "--self-test" in sys.argv else report())