#!/usr/bin/env python3 """AM-1 / M-D1-COV: every numbered GR-rule needs >=1 scenario. Compares the rule IDs declared in specs/GroundRules.md against the `covers:` lists in scenarios/ground/*.yaml. Exits non-zero when a scenario claims a rule the spec does not define, so coverage can never be inflated by a typo'd or invented rule ID. Stated limit (InnerLoop implementation rule 4): this gate counts tags. It proves no rule is unclaimed and no claimed rule is invented. It does NOT prove a scenario exercises the rule it names. Positive control (InnerLoop v1.1 §Step 5): the run asserts it actually found rules and scenarios. Before this was added, a broken spec regex yielded rules=[] and missing=[] and the tool exited 0 reporting "0/0" — the harness-does-nothing class, in the tool that reports our headline coverage number. Usage: python3 tools/rule-coverage.py python3 tools/rule-coverage.py --self-test """ import glob import re import sys RULE_RE = r"\*\*(GR-[A-Z]+\d+)" COVERS_RE = r"covers: \[(.*?)\]" AGGREGATE = "games/ground/src/lib.rs" # CB-WP-0003 T08: a provisional default with no expiry can shape the kernel # indefinitely while looking handled. CI warns; it does not break the build, # because the ruling is a ground-game decision we cannot make for them. PROVISIONAL_WARN_DAYS = 30 ID_RE = r"GR-[A-Z]+\d+" def parse_rules(spec_text): return sorted(set(re.findall(RULE_RE, spec_text))) def parse_code_ids(text): """Rule IDs named anywhere in the aggregate source (T09).""" return set(re.findall(ID_RE, text)) def provisional_items(paths): """(path, owner, raised) for every scenario encoding a U-item default.""" out = [] for path in paths: text = open(path).read() if not re.search(r"^provisional:\s*true", text, re.M): continue owner = re.search(r"^provisional_owner:\s*(\S+)", text, re.M) raised = re.search(r"^provisional_raised:\s*(\S+)", text, re.M) out.append((path, owner.group(1) if owner else None, raised.group(1) if raised else None)) return out def parse_covers(text): match = re.search(COVERS_RE, text, re.S) if not match: return set() return {c.strip() for c in match.group(1).split(",") if c.strip()} def self_test(): """Each assertion pins a failure this tool must detect.""" results = [] def check(name, ok, detail=""): results.append((name, ok, detail)) # The defect that motivated this control: a spec that parses to zero # rules must not be reportable as coverage. check("zero rules detected as a failure", parse_rules("no rules here") == [], "empty spec yields no rules; main() now aborts on this") # The matcher must actually match the real format. check("rule matcher works on real spec format", parse_rules("**GR-R06** something\n**GR-A12** other") == ["GR-A12", "GR-R06"]) # covers: parsing, including the empty case. check("covers matcher works", parse_covers("covers: [GR-R06, GR-A12]") == {"GR-R06", "GR-A12"}) check("missing covers yields empty set", parse_covers("no covers key") == set()) # T09: the spec -> code link must be detectable. check("code-id matcher finds ids in source", parse_code_ids("// GR-R06: lead first\nfn f(){} // GR-A12") == {"GR-R06", "GR-A12"}) check("code-id matcher finds none in unmarked source", parse_code_ids("fn f() { let x = 1; }") == set()) # T08: every provisional scenario must carry an owner and a date. import glob as _g prov = provisional_items(sorted(_g.glob("scenarios/ground/*.yaml"))) check("every provisional item has an owner and a raised date", bool(prov) and all(o and r for _, o, r in prov), f"{len(prov)} provisional item(s)") print("rule-coverage self-test (positive control)") ok = True for name, passed, detail in results: print(f" [{'ok ' if passed else 'FAIL'}] {name}" + (f" — {detail}" if detail else "")) ok &= passed return 0 if ok else 1 def main(): if "--self-test" in sys.argv: return self_test() rules = parse_rules(open("specs/GroundRules.md").read()) paths = sorted(glob.glob("scenarios/ground/*.yaml")) # Positive control: refuse to report a percentage over nothing. if not rules: print("ERROR — no GR-rules parsed from specs/GroundRules.md; " "refusing to report coverage", file=sys.stderr) return 1 if not paths: print("ERROR — no scenarios found in scenarios/ground/; " "refusing to report coverage", file=sys.stderr) return 1 covered = set() for path in paths: covered |= parse_covers(open(path).read()) known = set(rules) hit = sorted(known & covered) missing = [r for r in rules if r not in covered] invented = sorted(covered - known) # T09: the spec -> code -> scenario chain, made mechanical. A rule a # scenario claims should also be named in the aggregate, or the claim # rests on nothing but a tag. code_ids = parse_code_ids(open(AGGREGATE).read()) unlinked = sorted((known & covered) - code_ids) phantom = sorted(code_ids - known) pct = 100 * len(hit) // len(rules) linked = len((known & covered) & code_ids) print(f"AM-1 rule coverage: {len(hit)}/{len(rules)} ({pct}%) " f"over {len(paths)} scenarios") print(f"AM-1b spec->code link: {linked}/{len(hit)} claimed rules also " f"named in {AGGREGATE}") print(" NOTE: counts tags; does not prove a scenario exercises what it names") if unlinked: print(" unlinked (claimed by a scenario, absent from the aggregate):") print(" ", " ".join(unlinked)) if phantom: print(" ERROR — rule id in code that the spec does not define:", " ".join(phantom), file=sys.stderr) return 1 # T08: provisional items are reported with an owner and an age. prov = provisional_items(paths) if prov: import datetime today = datetime.date.today() print(f"\nprovisional U-item defaults: {len(prov)}") unowned, stale = [], [] for path, owner, raised in prov: age = "?" if raised: try: age = (today - datetime.date.fromisoformat(raised)).days except ValueError: age = "?" name = path.split("/")[-1] print(f" {name:<34} owner={owner or 'NONE':<12} age={age}d") if not owner: unowned.append(name) if isinstance(age, int) and age > PROVISIONAL_WARN_DAYS: stale.append(f"{name} ({age}d)") if unowned: print(" WARN — provisional with no owner:", " ".join(unowned)) if stale: print(f" WARN — provisional for over {PROVISIONAL_WARN_DAYS} days:", " ".join(stale)) print(" NOTE: evidence files must list these; a ruling flips the " "scenario, not the kernel") if missing: print(" uncovered:", " ".join(missing)) if invented: print(" ERROR — claimed but not defined in the spec:", " ".join(invented), file=sys.stderr) return 1 return 0 if not missing else 2 if __name__ == "__main__": sys.exit(main())