clay-borg/tools/loop-lint.py

#!/usr/bin/env python3
"""Executable checks for specs/InnerLoop.md rules (CB-WP-0003 T01).

The loop's own rules were prose. A rule nobody can run is a suggestion,
and the audit in history/260731-inner-loop-rule-audit.md found several
that were already being violated with no signal. This makes the
mechanically-checkable ones fail a command.

Each check names the InnerLoop rule it enforces. Checks that cannot be
made mechanical are recorded in the audit as `checkable` or `decorative`
and are deliberately absent here — see the audit for why.

Positive control (InnerLoop v1.1 §Step 5): --self-test asserts each check
actually detects its failure, using fixtures with known answers. A linter
that passes everything because its matcher is broken is the same defect
class as a benchmark timing rejected work.

Usage:
  python3 tools/loop-lint.py              # lint the repo
  python3 tools/loop-lint.py --self-test  # positive control
"""

import os
import re
import sys

REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
LOADABILITY_LIMIT = 400

# Artifact classes the loop produces. history/ is an append-only trail
# (verbatim challenge text is not something to split), so it is exempt.
LOOP_DIRS = ("specs", "research", "decisions", "evidence", "workplans")


class Finding:
    def __init__(self, rule, path, detail):
        self.rule, self.path, self.detail = rule, path, detail

    def __str__(self):
        return f"  [{self.rule}] {self.path}\n      {self.detail}"


def _md_files(root=REPO):
    """Loop artifacts only.

    Vendored third-party trees (a baseline harness ships its own
    node_modules) are not artifacts this loop produces, and linting them
    buries the two real findings under eighteen irrelevant ones.
    """
    for d in LOOP_DIRS:
        base = os.path.join(root, d)
        for dirpath, dirnames, files in os.walk(base):
            dirnames[:] = [x for x in dirnames if x != "node_modules"]
            for f in sorted(files):
                if f.endswith(".md"):
                    yield os.path.relpath(os.path.join(dirpath, f), root)


def check_loadability(root=REPO):
    """§Agentic-efficiency 1 — every loop artifact stays under ~400 lines."""
    out = []
    for rel in _md_files(root):
        with open(os.path.join(root, rel)) as fh:
            n = sum(1 for _ in fh)
        if n > LOADABILITY_LIMIT:
            out.append(
                Finding(
                    "loadability",
                    rel,
                    f"{n} lines exceeds the ~{LOADABILITY_LIMIT}-line limit; "
                    f"split and link with relative paths",
                )
            )
    return out


def check_evidence_no_unmeasured(root=REPO):
    """§Rubric — `unmeasured` is legal in a survey, illegal in an evidence file."""
    out = []
    base = os.path.join(root, "evidence")
    if not os.path.isdir(base):
        return out
    for f in sorted(os.listdir(base)):
        if not f.endswith(".md"):
            continue
        rel = os.path.join("evidence", f)
        for i, line in enumerate(open(os.path.join(root, rel)), 1):
            # A row asserting the verdict, not prose discussing the word.
            if re.search(r"\|\s*unmeasured\s*\|", line):
                out.append(
                    Finding("evidence-unmeasured", f"{rel}:{i}",
                            "verdict `unmeasured` in an evidence table")
                )
    return out


def check_survey_tier_and_chaos(root=REPO):
    """§Loop tiers — tier declared, and the chaos roll recorded every time."""
    out = []
    base = os.path.join(root, "research")
    if not os.path.isdir(base):
        return out
    for f in sorted(os.listdir(base)):
        if not f.endswith(".md"):
            continue
        rel = os.path.join("research", f)
        text = open(os.path.join(root, rel)).read()
        if not re.search(r"^tier:\s*[SML]\b", text, re.M):
            out.append(Finding("tier-declared", rel, "no `tier:` declaration"))
        elif "chaos" not in text.lower():
            out.append(
                Finding("chaos-recorded", rel,
                        "tier declared without the chaos roll; the rule requires "
                        "recording it even when it changes nothing")
            )
    return out


def check_review_trail(root=REPO):
    """§Step 2 — a tier-L survey carries research/challenge/response history."""
    out = []
    base = os.path.join(root, "research")
    hist = os.path.join(root, "history")
    if not (os.path.isdir(base) and os.path.isdir(hist)):
        return out
    files = os.listdir(hist)
    for f in sorted(os.listdir(base)):
        if not f.endswith(".md"):
            continue
        rel = os.path.join("research", f)
        text = open(os.path.join(root, rel)).read()
        if not re.search(r"^tier:\s*L\b", text, re.M):
            continue
        if not re.search(r"^status:\s*approved", text, re.M):
            continue
        for kind in ("challenge", "response"):
            if not any(x.endswith(f"-{kind}.md") and kind in x for x in files):
                out.append(
                    Finding("review-trail", rel,
                            f"tier-L approved survey with no history/*-{kind}.md")
                )
    return out


def check_reporting_tools_self_test(root=REPO):
    """§Step 5 v1.1 — every tool that reports a number exposes --self-test."""
    out = []
    base = os.path.join(root, "tools")
    if not os.path.isdir(base):
        return out
    for f in sorted(os.listdir(base)):
        if not f.endswith(".py"):
            continue
        rel = os.path.join("tools", f)
        text = open(os.path.join(root, rel)).read()
        if "--self-test" not in text:
            out.append(
                Finding("self-test", rel,
                        "reporting tool with no --self-test entry point; "
                        "nothing verifies its positive control still works")
            )
    return out


CHECKS = (
    check_loadability,
    check_evidence_no_unmeasured,
    check_survey_tier_and_chaos,
    check_review_trail,
    check_reporting_tools_self_test,
)


def self_test():
    """Each check must DETECT its failure, not merely run."""
    import shutil
    import tempfile

    results = []

    def check(name, ok, detail=""):
        results.append((name, ok, detail))

    tmp = tempfile.mkdtemp()
    try:
        for d in LOOP_DIRS + ("tools", "history"):
            os.makedirs(os.path.join(tmp, d), exist_ok=True)

        # loadability: 401 lines must trip, 400 must not.
        with open(os.path.join(tmp, "specs", "Big.md"), "w") as fh:
            fh.write("x\n" * (LOADABILITY_LIMIT + 1))
        with open(os.path.join(tmp, "specs", "Ok.md"), "w") as fh:
            fh.write("x\n" * LOADABILITY_LIMIT)
        f = check_loadability(tmp)
        check("loadability detects overlong artifact",
              len(f) == 1 and "Big.md" in f[0].path,
              f"{len(f)} finding(s)")

        # evidence: a table verdict trips; the word in prose does not.
        with open(os.path.join(tmp, "evidence", "E.md"), "w") as fh:
            fh.write("| AC-1 | x | unmeasured |\n"
                     "the word unmeasured appearing in prose is fine\n")
        f = check_evidence_no_unmeasured(tmp)
        check("evidence-unmeasured detects a table verdict, not prose",
              len(f) == 1, f"{len(f)} finding(s), expected exactly 1")

        # tier/chaos: missing tier trips; tier without chaos trips.
        with open(os.path.join(tmp, "research", "A.md"), "w") as fh:
            fh.write("# survey\nno tier here\n")
        with open(os.path.join(tmp, "research", "B.md"), "w") as fh:
            fh.write("tier: L (structural L)\n")
        f = check_survey_tier_and_chaos(tmp)
        rules = sorted(x.rule for x in f)
        check("tier/chaos detects both omissions",
              rules == ["chaos-recorded", "tier-declared"], f"{rules}")

        # self-test: a tool without the flag trips.
        with open(os.path.join(tmp, "tools", "silent.py"), "w") as fh:
            fh.write("print(42)\n")
        f = check_reporting_tools_self_test(tmp)
        check("self-test detects a tool lacking --self-test",
              len(f) == 1 and "silent.py" in f[0].path, f"{len(f)} finding(s)")

        # review trail: approved tier-L survey with no history trips.
        with open(os.path.join(tmp, "research", "C.md"), "w") as fh:
            fh.write("tier: L (structural L, chaos 3)\nstatus: approved\n")
        f = check_review_trail(tmp)
        check("review-trail detects a missing challenge/response",
              len(f) == 2, f"{len(f)} finding(s), expected 2")
    finally:
        shutil.rmtree(tmp, ignore_errors=True)

    print("loop-lint self-test (positive control)")
    ok = True
    for name, passed, detail in results:
        print(f"  [{'ok ' if passed else 'FAIL'}] {name}"
              + (f"  — {detail}" if detail else ""))
        ok &= passed
    return 0 if ok else 1


def main():
    if "--self-test" in sys.argv:
        return self_test()

    findings = []
    for c in CHECKS:
        findings.extend(c())

    print("loop-lint — executable InnerLoop rules")
    if not findings:
        print("  no findings")
        return 0
    by_rule = {}
    for f in findings:
        by_rule.setdefault(f.rule, []).append(f)
    for rule, fs in sorted(by_rule.items()):
        print(f"\n{rule} ({len(fs)}):")
        for f in fs:
            print(str(f))
    print(f"\n{len(findings)} finding(s)")
    return 1


if __name__ == "__main__":
    sys.exit(main())