Add the AM-1 rule-coverage gate
make coverage compares the GR-rule IDs defined in specs/GroundRules.md against the covers: lists in scenarios/ground/*.yaml. It exits 1 if a scenario claims a rule the spec does not define, so coverage cannot be inflated by an invented ID, and exits 2 while rules remain uncovered. Current reading: 34/58 (58%). AM-1 requires 100%. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
parent
27b7fe4329
commit
27016af216
2 changed files with 37 additions and 0 deletions
3
Makefile
3
Makefile
|
|
@ -15,6 +15,9 @@ test:
|
|||
$(CARGO) test --workspace
|
||||
|
||||
## run all GROUND scenarios through cb-sim
|
||||
coverage:
|
||||
python3 tools/rule-coverage.py
|
||||
|
||||
sim:
|
||||
$(CARGO) run -q -p cb-sim -- scenarios/ground/*.yaml
|
||||
|
||||
|
|
|
|||
34
tools/rule-coverage.py
Executable file
34
tools/rule-coverage.py
Executable file
|
|
@ -0,0 +1,34 @@
|
|||
#!/usr/bin/env python3
|
||||
"""AM-1 / M-D1-COV: every numbered GR-rule needs >=1 scenario.
|
||||
|
||||
Compares the rule IDs declared in specs/GroundRules.md against the
|
||||
`covers:` lists in scenarios/ground/*.yaml. Exits non-zero when a
|
||||
scenario claims a rule the spec does not define, so coverage can never
|
||||
be inflated by a typo'd or invented rule ID.
|
||||
"""
|
||||
import glob
|
||||
import re
|
||||
import sys
|
||||
|
||||
spec = open("specs/GroundRules.md").read()
|
||||
rules = sorted(set(re.findall(r"\*\*(GR-[A-Z]+\d+)", spec)))
|
||||
|
||||
covered = set()
|
||||
for path in sorted(glob.glob("scenarios/ground/*.yaml")):
|
||||
match = re.search(r"covers: \[(.*?)\]", open(path).read(), re.S)
|
||||
if match:
|
||||
covered |= {c.strip() for c in match.group(1).split(",") if c.strip()}
|
||||
|
||||
known = set(rules)
|
||||
hit = sorted(known & covered)
|
||||
missing = [r for r in rules if r not in covered]
|
||||
invented = sorted(covered - known)
|
||||
|
||||
pct = 100 * len(hit) // len(rules) if rules else 0
|
||||
print(f"AM-1 rule coverage: {len(hit)}/{len(rules)} ({pct}%)")
|
||||
if missing:
|
||||
print(" uncovered:", " ".join(missing))
|
||||
if invented:
|
||||
print(" ERROR — claimed but not defined in the spec:", " ".join(invented))
|
||||
sys.exit(1)
|
||||
sys.exit(0 if not missing else 2)
|
||||
Loading…
Add table
Add a link
Reference in a new issue