clay-borg/tools/size-metrics.py

#!/usr/bin/env python3
"""AM-2 and AM-3: the size metrics, which measured nothing until now.

CB-WP-0006 T02. Both rows were `unmutatable` in the first M-D1-MUT run for
the same reason: `make loc` prints line counts and nothing divides,
compares, or gates.

**AM-2 is the anti-gaming pair for AM-1.** 100% rule coverage means
nothing if the rules are trivially small, which is the specific risk
MetricsAndScenarios §1 names when it introduces M-D1-SPL. An
uninstrumented AM-2 therefore leaves AM-1 gameable — and AM-1 is the
headline this project has reported as `met` since CB-WP-0001.

**AM-3 is a different situation and this tool refuses to blur them.** It
measures "LOC to express the CB-RES-0001 synthetic game on our kernel"
against a boardgame.io baseline of ~36 LOC for a declarative 3p
commit/reveal game object. That artifact **has never been built**:
`games/` contains only `ground`, and `benches/synthetic.rs` *drives*
GROUND rather than *defining* a synthetic game. Measuring GROUND's 1,575
impl lines against boardgame.io's 36-line synthetic game object would
compare two different games and call it a D1 result.

So the measurement mechanism ships ready — a marker-delimited region — and
the row reports **blocked**, naming the missing artifact. A number would
have been worse than a blank.

Usage:
  python3 tools/size-metrics.py
  python3 tools/size-metrics.py --self-test
"""
import os
import re
import sys

from repo import enter_root

# The rules code AM-2 divides. Everything before the first `#[cfg(test)]`
# is implementation; tests are excluded because AM-2 asks how much code a
# rule costs, not how much it is exercised.
RULES_SRC = "games/ground/src/lib.rs"
RULES_SPEC = "specs/GroundRules.md"
AM2_MAX_LOC_PER_RULE = 40.0

# AM-3's subject, once it exists, is delimited by these markers so the
# measurement is unambiguous and cannot drift as the file grows.
AM3_BEGIN = "// AM-3:BEGIN"
AM3_END = "// AM-3:END"
AM3_MAX_LOC = 50
AM3_SEARCH = ("games", "benches", "crates")


def code_lines(lines):
    """Non-blank, non-comment lines. Block comments handled."""
    n, in_block = 0, False
    for line in lines:
        s = line.strip()
        if not s:
            continue
        if in_block:
            if "*/" in s:
                in_block = False
            continue
        if s.startswith("/*"):
            if "*/" not in s:
                in_block = True
            continue
        if s.startswith("//"):
            continue
        n += 1
    return n


def impl_region(path=RULES_SRC):
    """Lines of `path` before the first `#[cfg(test)]`."""
    src = open(path).read().split("\n")
    for i, line in enumerate(src):
        if line.strip().startswith("#[cfg(test)]"):
            return src[:i]
    return src


def rule_count(path=RULES_SPEC):
    return len(set(re.findall(r"\*\*(GR-[A-Z]+\d+)", open(path).read())))


def am3_region():
    """(path, lines) for the marked AM-3 subject, or None if unbuilt."""
    for root in AM3_SEARCH:
        if not os.path.isdir(root):
            continue
        for dirpath, dirnames, files in os.walk(root):
            dirnames[:] = [d for d in dirnames if d != "target"]
            for f in sorted(files):
                if not f.endswith(".rs"):
                    continue
                p = os.path.join(dirpath, f)
                text = open(p).read()
                if AM3_BEGIN in text and AM3_END in text:
                    body = text.split(AM3_BEGIN, 1)[1].split(AM3_END, 1)[0]
                    return p, body.split("\n")
    return None


def report():
    loc = code_lines(impl_region())
    rules = rule_count()

    # Positive control: refuse to divide by a denominator we failed to
    # parse, and refuse to report a ratio over zero implementation.
    if not rules:
        print(f"ERROR — no GR-rules parsed from {RULES_SPEC}; refusing to "
              f"report AM-2", file=sys.stderr)
        return 1
    if loc <= 0:
        print(f"ERROR — no implementation lines found in {RULES_SRC}; "
              f"refusing to report AM-2", file=sys.stderr)
        return 1

    ratio = loc / rules
    ok = ratio <= AM2_MAX_LOC_PER_RULE
    print("AM-2 / M-D1-SPL — implementation cost per rule")
    print(f"  {RULES_SRC}: {loc:,} code lines before the first #[cfg(test)]")
    print(f"  {RULES_SPEC}: {rules} numbered rules")
    print(f"  AM-2: {ratio:.1f} LOC/rule  [{'ok  ' if ok else 'FAIL'} "
          f"target <= {AM2_MAX_LOC_PER_RULE:.0f}]  "
          f"({AM2_MAX_LOC_PER_RULE / ratio:.2f}x headroom)")
    print("  NOTE: this is AM-1's anti-gaming pair — 100% rule coverage means")
    print("        nothing if the rules are trivially small.")

    print("\nAM-3 — LOC to express the CB-RES-0001 synthetic game on our kernel")
    region = am3_region()
    if region is None:
        print("  BLOCKED — the artifact this row measures does not exist.")
        print(f"  No {AM3_BEGIN} / {AM3_END} region found under "
              f"{'/, '.join(AM3_SEARCH)}/.")
        print("  `games/` contains only `ground`, and benches/synthetic.rs")
        print("  *drives* GROUND rather than *defining* a synthetic game.")
        print("  The baseline is a declarative 3p commit/reveal game object")
        print("  (~36 LOC, boardgame.io). Measuring GROUND's implementation")
        print("  against it would compare two different games and call the")
        print("  difference a D1 result.")
        print("  Instrument is ready: mark the region when the artifact exists.")
        # Deliberately not an error. A blocked row is a true report; the
        # decision to build or withdraw belongs to a workplan, not here.
        return 0 if ok else 2

    path, lines = region
    n = code_lines(lines)
    good = n <= AM3_MAX_LOC
    print(f"  {path}: {n} code lines between the markers")
    print(f"  AM-3: {n} LOC  [{'ok  ' if good else 'FAIL'} "
          f"target <= {AM3_MAX_LOC}]")
    if not good:
        ok = False
    return 0 if ok else 2


def self_test():
    """Each assertion pins a failure this tool must detect."""
    results = []

    def check(name, cond, detail=""):
        results.append((name, cond, detail))

    check("blank and comment lines are not counted",
          code_lines(["", "  ", "// x", "let a = 1;"]) == 1)
    check("block comments are not counted",
          code_lines(["/* a", "b", "*/", "let a = 1;"]) == 1)
    check("a one-line block comment does not swallow the file",
          code_lines(["/* a */", "let a = 1;", "let b = 2;"]) == 2)
    check("code after a block comment is counted",
          code_lines(["/*", "*/", "x", "y"]) == 2)

    # The defect this tool exists to end: reporting a ratio over nothing.
    check("zero rules is detectable (main aborts on it)",
          len(set(re.findall(r"\*\*(GR-[A-Z]+\d+)", "no rules"))) == 0)
    check("zero implementation is detectable", code_lines(["", "// c"]) == 0)

    # The real measurement must be non-trivial, or the guard above fires
    # on everything and nothing is ever reported.
    loc, rules = code_lines(impl_region()), rule_count()
    check("real implementation measures non-zero", loc > 100, f"{loc:,} LOC")
    check("real rule count measures non-zero", rules > 10, f"{rules} rules")

    # The test region must actually be excluded — AM-2 asks what a rule
    # costs, not how much it is exercised, and lib.rs is ~30% tests.
    full = code_lines(open(RULES_SRC).read().split("\n"))
    check("the #[cfg(test)] region is excluded", loc < full,
          f"{loc:,} impl vs {full:,} whole file")

    # AM-3's marker mechanism must work, so that a future artifact is
    # measured automatically rather than needing this tool changed.
    import tempfile
    with tempfile.TemporaryDirectory() as d:
        os.makedirs(os.path.join(d, "games"))
        p = os.path.join(d, "games", "x.rs")
        with open(p, "w") as fh:
            fh.write(f"pre\n{AM3_BEGIN}\nlet a = 1;\n// c\n\nlet b = 2;\n"
                     f"{AM3_END}\npost\n")
        here = os.getcwd()
        try:
            os.chdir(d)
            got = am3_region()
            check("AM-3 markers delimit a region and count only code",
                  got is not None and code_lines(got[1]) == 2,
                  f"{code_lines(got[1]) if got else 'none'} lines")
        finally:
            os.chdir(here)
    check("AM-3 reports blocked when no region is marked",
          am3_region() is None,
          "the synthetic game has not been built; a number here would be wrong")

    print("size-metrics self-test (positive control)")
    ok = True
    for name, passed, det in results:
        print(f"  [{'ok ' if passed else 'FAIL'}] {name}"
              + (f"  — {det}" if det else ""))
        ok &= passed
    return 0 if ok else 1


def main():
    enter_root()
    if "--self-test" in sys.argv:
        return self_test()
    return report()


if __name__ == "__main__":
    sys.exit(main())