#!/usr/bin/env python3 """AM-1 / M-D1-COV: every numbered GR-rule needs >=1 scenario. Compares the rule IDs declared in specs/GroundRules.md against the `covers:` lists in scenarios/ground/*.yaml. Exits non-zero when a scenario claims a rule the spec does not define, so coverage can never be inflated by a typo'd or invented rule ID. Stated limit (InnerLoop implementation rule 4): this gate counts tags. It proves no rule is unclaimed and no claimed rule is invented. It does NOT prove a scenario exercises the rule it names. Positive control (InnerLoop v1.1 §Step 5): the run asserts it actually found rules and scenarios. Before this was added, a broken spec regex yielded rules=[] and missing=[] and the tool exited 0 reporting "0/0" — the harness-does-nothing class, in the tool that reports our headline coverage number. Usage: python3 tools/rule-coverage.py python3 tools/rule-coverage.py --self-test """ import glob import re import sys RULE_RE = r"\*\*(GR-[A-Z]+\d+)" COVERS_RE = r"covers: \[(.*?)\]" def parse_rules(spec_text): return sorted(set(re.findall(RULE_RE, spec_text))) def parse_covers(text): match = re.search(COVERS_RE, text, re.S) if not match: return set() return {c.strip() for c in match.group(1).split(",") if c.strip()} def self_test(): """Each assertion pins a failure this tool must detect.""" results = [] def check(name, ok, detail=""): results.append((name, ok, detail)) # The defect that motivated this control: a spec that parses to zero # rules must not be reportable as coverage. check("zero rules detected as a failure", parse_rules("no rules here") == [], "empty spec yields no rules; main() now aborts on this") # The matcher must actually match the real format. check("rule matcher works on real spec format", parse_rules("**GR-R06** something\n**GR-A12** other") == ["GR-A12", "GR-R06"]) # covers: parsing, including the empty case. check("covers matcher works", parse_covers("covers: [GR-R06, GR-A12]") == {"GR-R06", "GR-A12"}) check("missing covers yields empty set", parse_covers("no covers key") == set()) print("rule-coverage self-test (positive control)") ok = True for name, passed, detail in results: print(f" [{'ok ' if passed else 'FAIL'}] {name}" + (f" — {detail}" if detail else "")) ok &= passed return 0 if ok else 1 def main(): if "--self-test" in sys.argv: return self_test() rules = parse_rules(open("specs/GroundRules.md").read()) paths = sorted(glob.glob("scenarios/ground/*.yaml")) # Positive control: refuse to report a percentage over nothing. if not rules: print("ERROR — no GR-rules parsed from specs/GroundRules.md; " "refusing to report coverage", file=sys.stderr) return 1 if not paths: print("ERROR — no scenarios found in scenarios/ground/; " "refusing to report coverage", file=sys.stderr) return 1 covered = set() for path in paths: covered |= parse_covers(open(path).read()) known = set(rules) hit = sorted(known & covered) missing = [r for r in rules if r not in covered] invented = sorted(covered - known) pct = 100 * len(hit) // len(rules) print(f"AM-1 rule coverage: {len(hit)}/{len(rules)} ({pct}%) " f"over {len(paths)} scenarios") print(" NOTE: counts tags; does not prove a scenario exercises what it names") if missing: print(" uncovered:", " ".join(missing)) if invented: print(" ERROR — claimed but not defined in the spec:", " ".join(invented), file=sys.stderr) return 1 return 0 if not missing else 2 if __name__ == "__main__": sys.exit(main())