diff --git a/Makefile b/Makefile index 17f3117..55a76bd 100644 --- a/Makefile +++ b/Makefile @@ -24,7 +24,7 @@ TOOLS := $(REPO)/tools # Every cargo recipe runs at the repo root; the shell does not persist cd. IN_REPO := cd $(REPO) && -.PHONY: check test sim bench bench-test coverage dep-weight cost cost-test cost-pin cost-budget cost-mix loop-lint self-tests env-test task-done status facts-check facts-gen mutation-check loc all +.PHONY: check test sim bench bench-test coverage dep-weight cost cost-test cost-pin cost-budget cost-mix loop-lint self-tests env-test task-done status facts-check facts-gen mutation-check size-metrics loc all ## fmt + clippy (deny warnings) + HashMap deny-lint check: @@ -42,6 +42,10 @@ dep-weight: coverage: $(PY) $(TOOLS)/rule-coverage.py +# AM-2 (M-D1-SPL, AM-1's anti-gaming pair) and AM-3. +size-metrics: + $(PY) $(TOOLS)/size-metrics.py + # M-D2-CST (specs/CostAccounting.md). cost-test is the positive control and # runs first: a cost number from an unverified collector is void. cost: cost-test @@ -65,6 +69,7 @@ self-tests: $(PY) $(TOOLS)/status.py --self-test $(PY) $(TOOLS)/facts.py --self-test $(PY) $(TOOLS)/mutation-check.py --self-test + $(PY) $(TOOLS)/size-metrics.py --self-test # T01 positive control: prove the environment fix, do not assume it. Runs # every tool from a foreign working directory with a PATH that has no @@ -142,4 +147,4 @@ loc: printf '%-28s %s\n' $$d "$$(find $$d/src -name '*.rs' | xargs cat | grep -vcE '^\s*(//|$$)')"; \ done -all: check test sim coverage dep-weight self-tests env-test facts-check loop-lint bench-test +all: check test sim coverage size-metrics dep-weight self-tests env-test facts-check loop-lint bench-test diff --git a/facts.toml b/facts.toml index f14f17f..1f0bc28 100644 --- a/facts.toml +++ b/facts.toml @@ -40,8 +40,8 @@ fmt = "{:,}" by = "tools/mutation-check.py" [am_unmutatable] -value = 7 -text = "7" +value = 6 +text = "6" fmt = "{:,}" by = "tools/mutation-check.py" diff --git a/tools/__pycache__/cb-cost.cpython-312.pyc b/tools/__pycache__/cb-cost.cpython-312.pyc index 03aa62e..2d6315c 100644 Binary files a/tools/__pycache__/cb-cost.cpython-312.pyc and b/tools/__pycache__/cb-cost.cpython-312.pyc differ diff --git a/tools/__pycache__/dep-weight.cpython-312.pyc b/tools/__pycache__/dep-weight.cpython-312.pyc index 05302ed..99f4259 100644 Binary files a/tools/__pycache__/dep-weight.cpython-312.pyc and b/tools/__pycache__/dep-weight.cpython-312.pyc differ diff --git a/tools/__pycache__/mutation-check.cpython-312.pyc b/tools/__pycache__/mutation-check.cpython-312.pyc index 512fbe1..9374fca 100644 Binary files a/tools/__pycache__/mutation-check.cpython-312.pyc and b/tools/__pycache__/mutation-check.cpython-312.pyc differ diff --git a/tools/mutation-check.py b/tools/mutation-check.py index 5702b25..d8938be 100644 --- a/tools/mutation-check.py +++ b/tools/mutation-check.py @@ -88,15 +88,32 @@ def rows(): mutate=("scenarios/ground/gr-r06-round-resolve.yaml", "GR-R06, ", "")), + # A PROPERTY mutation: inflate the implementation past the ratio. + # 58 rules x 40 = 2,320; the impl is 1,575, so ~800 lines of filler + # crosses it. Raising the threshold instead would only prove the + # comparison runs. Row("AM-2", "<= 40 spec lines per rule in games/ground", - unmutatable="no instrument computes it. `make loc` prints LOC " - "and nothing divides by rule count or compares to 40; " - "CB-EV-0001 records AM-2 as unreported."), + verify=py + ["tools/size-metrics.py"], + mutate=("games/ground/src/lib.rs", + "impl Aggregate for GroundState {", + "fn _am2_filler() {\n" + " let _x = 0;\n" * 800 + + "}\n\nimpl Aggregate for GroundState {"), + expect="FAIL target <= 40"), Row("AM-3", "synthetic workload definition <= 50 LOC", - unmutatable="no instrument. The synthetic workload is hardcoded " - "in benches/synthetic.rs (see K18) and its LOC is " - "never measured or compared."), + unmutatable="BLOCKED, not uninstrumented — the distinction " + "matters. `make size-metrics` ships the measurement " + "(a marker-delimited region) and reports the row " + "blocked, because the artifact it measures has never " + "been built: games/ contains only ground, and " + "benches/synthetic.rs drives GROUND rather than " + "defining a synthetic game. The baseline is a " + "declarative 3p commit/reveal game object (~36 LOC, " + "boardgame.io); measuring GROUND's 1,575 impl lines " + "against it would compare two different games and " + "call the difference a D1 result. Still counts " + "against M-D1-MUT (ADR-0005 §1) — a row that cannot " + "fail asserts nothing, however good the reason."), Row("AM-4a", "third-party LOC, shipped runtime <= 250,000", verify=py + ["tools/dep-weight.py"], diff --git a/tools/size-metrics.py b/tools/size-metrics.py new file mode 100644 index 0000000..6448ab6 --- /dev/null +++ b/tools/size-metrics.py @@ -0,0 +1,230 @@ +#!/usr/bin/env python3 +"""AM-2 and AM-3: the size metrics, which measured nothing until now. + +CB-WP-0006 T02. Both rows were `unmutatable` in the first M-D1-MUT run for +the same reason: `make loc` prints line counts and nothing divides, +compares, or gates. + +**AM-2 is the anti-gaming pair for AM-1.** 100% rule coverage means +nothing if the rules are trivially small, which is the specific risk +MetricsAndScenarios §1 names when it introduces M-D1-SPL. An +uninstrumented AM-2 therefore leaves AM-1 gameable — and AM-1 is the +headline this project has reported as `met` since CB-WP-0001. + +**AM-3 is a different situation and this tool refuses to blur them.** It +measures "LOC to express the CB-RES-0001 synthetic game on our kernel" +against a boardgame.io baseline of ~36 LOC for a declarative 3p +commit/reveal game object. That artifact **has never been built**: +`games/` contains only `ground`, and `benches/synthetic.rs` *drives* +GROUND rather than *defining* a synthetic game. Measuring GROUND's 1,575 +impl lines against boardgame.io's 36-line synthetic game object would +compare two different games and call it a D1 result. + +So the measurement mechanism ships ready — a marker-delimited region — and +the row reports **blocked**, naming the missing artifact. A number would +have been worse than a blank. + +Usage: + python3 tools/size-metrics.py + python3 tools/size-metrics.py --self-test +""" +import os +import re +import sys + +from repo import enter_root + +# The rules code AM-2 divides. Everything before the first `#[cfg(test)]` +# is implementation; tests are excluded because AM-2 asks how much code a +# rule costs, not how much it is exercised. +RULES_SRC = "games/ground/src/lib.rs" +RULES_SPEC = "specs/GroundRules.md" +AM2_MAX_LOC_PER_RULE = 40.0 + +# AM-3's subject, once it exists, is delimited by these markers so the +# measurement is unambiguous and cannot drift as the file grows. +AM3_BEGIN = "// AM-3:BEGIN" +AM3_END = "// AM-3:END" +AM3_MAX_LOC = 50 +AM3_SEARCH = ("games", "benches", "crates") + + +def code_lines(lines): + """Non-blank, non-comment lines. Block comments handled.""" + n, in_block = 0, False + for line in lines: + s = line.strip() + if not s: + continue + if in_block: + if "*/" in s: + in_block = False + continue + if s.startswith("/*"): + if "*/" not in s: + in_block = True + continue + if s.startswith("//"): + continue + n += 1 + return n + + +def impl_region(path=RULES_SRC): + """Lines of `path` before the first `#[cfg(test)]`.""" + src = open(path).read().split("\n") + for i, line in enumerate(src): + if line.strip().startswith("#[cfg(test)]"): + return src[:i] + return src + + +def rule_count(path=RULES_SPEC): + return len(set(re.findall(r"\*\*(GR-[A-Z]+\d+)", open(path).read()))) + + +def am3_region(): + """(path, lines) for the marked AM-3 subject, or None if unbuilt.""" + for root in AM3_SEARCH: + if not os.path.isdir(root): + continue + for dirpath, dirnames, files in os.walk(root): + dirnames[:] = [d for d in dirnames if d != "target"] + for f in sorted(files): + if not f.endswith(".rs"): + continue + p = os.path.join(dirpath, f) + text = open(p).read() + if AM3_BEGIN in text and AM3_END in text: + body = text.split(AM3_BEGIN, 1)[1].split(AM3_END, 1)[0] + return p, body.split("\n") + return None + + +def report(): + loc = code_lines(impl_region()) + rules = rule_count() + + # Positive control: refuse to divide by a denominator we failed to + # parse, and refuse to report a ratio over zero implementation. + if not rules: + print(f"ERROR — no GR-rules parsed from {RULES_SPEC}; refusing to " + f"report AM-2", file=sys.stderr) + return 1 + if loc <= 0: + print(f"ERROR — no implementation lines found in {RULES_SRC}; " + f"refusing to report AM-2", file=sys.stderr) + return 1 + + ratio = loc / rules + ok = ratio <= AM2_MAX_LOC_PER_RULE + print("AM-2 / M-D1-SPL — implementation cost per rule") + print(f" {RULES_SRC}: {loc:,} code lines before the first #[cfg(test)]") + print(f" {RULES_SPEC}: {rules} numbered rules") + print(f" AM-2: {ratio:.1f} LOC/rule [{'ok ' if ok else 'FAIL'} " + f"target <= {AM2_MAX_LOC_PER_RULE:.0f}] " + f"({AM2_MAX_LOC_PER_RULE / ratio:.2f}x headroom)") + print(" NOTE: this is AM-1's anti-gaming pair — 100% rule coverage means") + print(" nothing if the rules are trivially small.") + + print("\nAM-3 — LOC to express the CB-RES-0001 synthetic game on our kernel") + region = am3_region() + if region is None: + print(" BLOCKED — the artifact this row measures does not exist.") + print(f" No {AM3_BEGIN} / {AM3_END} region found under " + f"{'/, '.join(AM3_SEARCH)}/.") + print(" `games/` contains only `ground`, and benches/synthetic.rs") + print(" *drives* GROUND rather than *defining* a synthetic game.") + print(" The baseline is a declarative 3p commit/reveal game object") + print(" (~36 LOC, boardgame.io). Measuring GROUND's implementation") + print(" against it would compare two different games and call the") + print(" difference a D1 result.") + print(" Instrument is ready: mark the region when the artifact exists.") + # Deliberately not an error. A blocked row is a true report; the + # decision to build or withdraw belongs to a workplan, not here. + return 0 if ok else 2 + + path, lines = region + n = code_lines(lines) + good = n <= AM3_MAX_LOC + print(f" {path}: {n} code lines between the markers") + print(f" AM-3: {n} LOC [{'ok ' if good else 'FAIL'} " + f"target <= {AM3_MAX_LOC}]") + if not good: + ok = False + return 0 if ok else 2 + + +def self_test(): + """Each assertion pins a failure this tool must detect.""" + results = [] + + def check(name, cond, detail=""): + results.append((name, cond, detail)) + + check("blank and comment lines are not counted", + code_lines(["", " ", "// x", "let a = 1;"]) == 1) + check("block comments are not counted", + code_lines(["/* a", "b", "*/", "let a = 1;"]) == 1) + check("a one-line block comment does not swallow the file", + code_lines(["/* a */", "let a = 1;", "let b = 2;"]) == 2) + check("code after a block comment is counted", + code_lines(["/*", "*/", "x", "y"]) == 2) + + # The defect this tool exists to end: reporting a ratio over nothing. + check("zero rules is detectable (main aborts on it)", + len(set(re.findall(r"\*\*(GR-[A-Z]+\d+)", "no rules"))) == 0) + check("zero implementation is detectable", code_lines(["", "// c"]) == 0) + + # The real measurement must be non-trivial, or the guard above fires + # on everything and nothing is ever reported. + loc, rules = code_lines(impl_region()), rule_count() + check("real implementation measures non-zero", loc > 100, f"{loc:,} LOC") + check("real rule count measures non-zero", rules > 10, f"{rules} rules") + + # The test region must actually be excluded — AM-2 asks what a rule + # costs, not how much it is exercised, and lib.rs is ~30% tests. + full = code_lines(open(RULES_SRC).read().split("\n")) + check("the #[cfg(test)] region is excluded", loc < full, + f"{loc:,} impl vs {full:,} whole file") + + # AM-3's marker mechanism must work, so that a future artifact is + # measured automatically rather than needing this tool changed. + import tempfile + with tempfile.TemporaryDirectory() as d: + os.makedirs(os.path.join(d, "games")) + p = os.path.join(d, "games", "x.rs") + with open(p, "w") as fh: + fh.write(f"pre\n{AM3_BEGIN}\nlet a = 1;\n// c\n\nlet b = 2;\n" + f"{AM3_END}\npost\n") + here = os.getcwd() + try: + os.chdir(d) + got = am3_region() + check("AM-3 markers delimit a region and count only code", + got is not None and code_lines(got[1]) == 2, + f"{code_lines(got[1]) if got else 'none'} lines") + finally: + os.chdir(here) + check("AM-3 reports blocked when no region is marked", + am3_region() is None, + "the synthetic game has not been built; a number here would be wrong") + + print("size-metrics self-test (positive control)") + ok = True + for name, passed, det in results: + print(f" [{'ok ' if passed else 'FAIL'}] {name}" + + (f" — {det}" if det else "")) + ok &= passed + return 0 if ok else 1 + + +def main(): + enter_root() + if "--self-test" in sys.argv: + return self_test() + return report() + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/workplans/CB-WP-0006-instrument-the-table.md b/workplans/CB-WP-0006-instrument-the-table.md index 6e5fc90..7bd1dd3 100644 --- a/workplans/CB-WP-0006-instrument-the-table.md +++ b/workplans/CB-WP-0006-instrument-the-table.md @@ -124,6 +124,45 @@ AM-3 depends on K18 (benches driven from scenario files); if K18 resolves toward amending the spec rather than implementing it, AM-3 must be restated or withdrawn with an argument rather than left unmeasured. +**Delivered — but the two rows resolved differently, and the difference is +the point.** + +**AM-2 is instrumented and enforced.** `tools/size-metrics.py` + +`make size-metrics`, in `make all`: + +```text +AM-2: 27.2 LOC/rule [ok target <= 40] (1.47x headroom) + 1,575 code lines before the first #[cfg(test)] / 58 numbered rules +``` + +Tests are excluded because AM-2 asks what a rule *costs*, not how much it +is exercised — `lib.rs` is ~18% test code and including it would have +flattered the number. Verified red by a **property** mutation: ~800 lines +of filler injected into the impl, pushing the ratio past 40. `expect` is +the precise failure signature `FAIL target <= 40`, not the row name, which +would have matched passing output too. + +**AM-3 is BLOCKED, not uninstrumented — and this is a finding, not a +deferral.** The row measures "LOC to express the CB-RES-0001 synthetic +game on our kernel" against a boardgame.io baseline of ~36 LOC for a +declarative **3p commit/reveal game object**. That artifact has never been +built: `games/` contains only `ground`, and `benches/synthetic.rs` +*drives* GROUND rather than *defining* a synthetic game. + +Measuring GROUND's 1,575 impl lines against a 36-line synthetic game +object would compare **two different games** and call the difference a D1 +result. So the tool ships the measurement mechanism — a marker-delimited +`// AM-3:BEGIN` / `// AM-3:END` region, self-tested — and **reports the row +blocked, naming the missing artifact**. A number here would have been +worse than a blank. + +It therefore stays `unmutatable` and **still counts against M-D1-MUT**, per +ADR-0005 §1: a row that cannot fail asserts nothing, however good the +reason. Resolving it needs an artifact, not a metric tweak — carried +forward, not silently dropped. + +**M-D1-MUT: 5 → 6 of 14.** + ## Task: AM-5, AM-9 — measure or withdraw, but stop leaving them blank ```task