clay-borg/tools/rule-coverage.py

#!/usr/bin/env python3
"""AM-1 / M-D1-COV: every numbered GR-rule needs >=1 scenario.

Compares the rule IDs declared in specs/GroundRules.md against the
`covers:` lists in scenarios/ground/*.yaml. Exits non-zero when a
scenario claims a rule the spec does not define, so coverage can never
be inflated by a typo'd or invented rule ID.

Stated limit (InnerLoop implementation rule 4): this gate counts tags. It
proves no rule is unclaimed and no claimed rule is invented. It does NOT
prove a scenario exercises the rule it names.

Positive control (InnerLoop v1.1 §Step 5): the run asserts it actually
found rules and scenarios. Before this was added, a broken spec regex
yielded rules=[] and missing=[] and the tool exited 0 reporting "0/0" —
the harness-does-nothing class, in the tool that reports our headline
coverage number.

Usage:
  python3 tools/rule-coverage.py
  python3 tools/rule-coverage.py --self-test
"""
import glob
import re
import sys

RULE_RE = r"\*\*(GR-[A-Z]+\d+)"
COVERS_RE = r"covers: \[(.*?)\]"


def parse_rules(spec_text):
    return sorted(set(re.findall(RULE_RE, spec_text)))


def parse_covers(text):
    match = re.search(COVERS_RE, text, re.S)
    if not match:
        return set()
    return {c.strip() for c in match.group(1).split(",") if c.strip()}


def self_test():
    """Each assertion pins a failure this tool must detect."""
    results = []

    def check(name, ok, detail=""):
        results.append((name, ok, detail))

    # The defect that motivated this control: a spec that parses to zero
    # rules must not be reportable as coverage.
    check("zero rules detected as a failure", parse_rules("no rules here") == [],
          "empty spec yields no rules; main() now aborts on this")
    # The matcher must actually match the real format.
    check("rule matcher works on real spec format",
          parse_rules("**GR-R06** something\n**GR-A12** other")
          == ["GR-A12", "GR-R06"])
    # covers: parsing, including the empty case.
    check("covers matcher works", parse_covers("covers: [GR-R06, GR-A12]")
          == {"GR-R06", "GR-A12"})
    check("missing covers yields empty set", parse_covers("no covers key") == set())

    print("rule-coverage self-test (positive control)")
    ok = True
    for name, passed, detail in results:
        print(f"  [{'ok ' if passed else 'FAIL'}] {name}"
              + (f"  — {detail}" if detail else ""))
        ok &= passed
    return 0 if ok else 1


def main():
    if "--self-test" in sys.argv:
        return self_test()

    rules = parse_rules(open("specs/GroundRules.md").read())
    paths = sorted(glob.glob("scenarios/ground/*.yaml"))

    # Positive control: refuse to report a percentage over nothing.
    if not rules:
        print("ERROR — no GR-rules parsed from specs/GroundRules.md; "
              "refusing to report coverage", file=sys.stderr)
        return 1
    if not paths:
        print("ERROR — no scenarios found in scenarios/ground/; "
              "refusing to report coverage", file=sys.stderr)
        return 1

    covered = set()
    for path in paths:
        covered |= parse_covers(open(path).read())

    known = set(rules)
    hit = sorted(known & covered)
    missing = [r for r in rules if r not in covered]
    invented = sorted(covered - known)

    pct = 100 * len(hit) // len(rules)
    print(f"AM-1 rule coverage: {len(hit)}/{len(rules)} ({pct}%) "
          f"over {len(paths)} scenarios")
    print("  NOTE: counts tags; does not prove a scenario exercises what it names")
    if missing:
        print("  uncovered:", " ".join(missing))
    if invented:
        print("  ERROR — claimed but not defined in the spec:", " ".join(invented),
              file=sys.stderr)
        return 1
    return 0 if not missing else 2


if __name__ == "__main__":
    sys.exit(main())