tegwick
3f1dbac164
CB-RES-0003 measured 84 turns and $15.33 — the largest mechanical category — spent prefixing commands with `cd` and `export PATH="$HOME/.cargo/bin:$PATH"`. Both causes are now fixed once instead of per-leaf. tools/repo.py resolves the repo root from __file__ and cargo from PATH then the standard rustup locations. Every tool imports ROOT from it, so the repo path is stated once rather than redefined in four files — single source of fact, the rule DFD earned in InnerLoop v1.2. rule-coverage and dep-weight now call enter_root(), which is why their relative paths did not need rewriting one by one. The Makefile derives REPO from MAKEFILE_LIST and resolves CARGO the same way, so `make -C <repo> <target>` works from any directory with no prefix. make env-test is the positive control, and is in `make all`: every tool runs from / with PATH=/usr/bin:/bin. Without it this fix could regress silently and invalidate T05's measurement — the whole point of the control loop. dep-weight's "cargo not on PATH" error is kept rather than deleted. It should now be unreachable, and --self-test asserts cargo_bin() resolves unaided; a control that never fires is cheaper than a regression. loop-lint failed on repo.py on its first run — a reporting tool with a positive control but no --self-test entry point. Second time the gate has caught work from its own pass within the hour. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
266 lines
9.4 KiB
Python
266 lines
9.4 KiB
Python
#!/usr/bin/env python3
|
|
"""Executable checks for specs/InnerLoop.md rules (CB-WP-0003 T01).
|
|
|
|
The loop's own rules were prose. A rule nobody can run is a suggestion,
|
|
and the audit in history/260731-inner-loop-rule-audit.md found several
|
|
that were already being violated with no signal. This makes the
|
|
mechanically-checkable ones fail a command.
|
|
|
|
Each check names the InnerLoop rule it enforces. Checks that cannot be
|
|
made mechanical are recorded in the audit as `checkable` or `decorative`
|
|
and are deliberately absent here — see the audit for why.
|
|
|
|
Positive control (InnerLoop v1.1 §Step 5): --self-test asserts each check
|
|
actually detects its failure, using fixtures with known answers. A linter
|
|
that passes everything because its matcher is broken is the same defect
|
|
class as a benchmark timing rejected work.
|
|
|
|
Usage:
|
|
python3 tools/loop-lint.py # lint the repo
|
|
python3 tools/loop-lint.py --self-test # positive control
|
|
"""
|
|
|
|
import os
|
|
import re
|
|
import sys
|
|
|
|
from repo import ROOT as REPO # noqa: E402 (single source of fact, T01)
|
|
LOADABILITY_LIMIT = 400
|
|
|
|
# Artifact classes the loop produces. history/ is an append-only trail
|
|
# (verbatim challenge text is not something to split), so it is exempt.
|
|
LOOP_DIRS = ("specs", "research", "decisions", "evidence", "workplans")
|
|
|
|
|
|
class Finding:
|
|
def __init__(self, rule, path, detail):
|
|
self.rule, self.path, self.detail = rule, path, detail
|
|
|
|
def __str__(self):
|
|
return f" [{self.rule}] {self.path}\n {self.detail}"
|
|
|
|
|
|
def _md_files(root=REPO):
|
|
"""Loop artifacts only.
|
|
|
|
Vendored third-party trees (a baseline harness ships its own
|
|
node_modules) are not artifacts this loop produces, and linting them
|
|
buries the two real findings under eighteen irrelevant ones.
|
|
"""
|
|
for d in LOOP_DIRS:
|
|
base = os.path.join(root, d)
|
|
for dirpath, dirnames, files in os.walk(base):
|
|
dirnames[:] = [x for x in dirnames if x != "node_modules"]
|
|
for f in sorted(files):
|
|
if f.endswith(".md"):
|
|
yield os.path.relpath(os.path.join(dirpath, f), root)
|
|
|
|
|
|
def check_loadability(root=REPO):
|
|
"""§Agentic-efficiency 1 — every loop artifact stays under ~400 lines."""
|
|
out = []
|
|
for rel in _md_files(root):
|
|
with open(os.path.join(root, rel)) as fh:
|
|
n = sum(1 for _ in fh)
|
|
if n > LOADABILITY_LIMIT:
|
|
out.append(
|
|
Finding(
|
|
"loadability",
|
|
rel,
|
|
f"{n} lines exceeds the ~{LOADABILITY_LIMIT}-line limit; "
|
|
f"split and link with relative paths",
|
|
)
|
|
)
|
|
return out
|
|
|
|
|
|
def check_evidence_no_unmeasured(root=REPO):
|
|
"""§Rubric — `unmeasured` is legal in a survey, illegal in an evidence file."""
|
|
out = []
|
|
base = os.path.join(root, "evidence")
|
|
if not os.path.isdir(base):
|
|
return out
|
|
for f in sorted(os.listdir(base)):
|
|
if not f.endswith(".md"):
|
|
continue
|
|
rel = os.path.join("evidence", f)
|
|
for i, line in enumerate(open(os.path.join(root, rel)), 1):
|
|
# A row asserting the verdict, not prose discussing the word.
|
|
if re.search(r"\|\s*unmeasured\s*\|", line):
|
|
out.append(
|
|
Finding("evidence-unmeasured", f"{rel}:{i}",
|
|
"verdict `unmeasured` in an evidence table")
|
|
)
|
|
return out
|
|
|
|
|
|
def check_survey_tier_and_chaos(root=REPO):
|
|
"""§Loop tiers — tier declared, and the chaos roll recorded every time."""
|
|
out = []
|
|
base = os.path.join(root, "research")
|
|
if not os.path.isdir(base):
|
|
return out
|
|
for f in sorted(os.listdir(base)):
|
|
if not f.endswith(".md"):
|
|
continue
|
|
rel = os.path.join("research", f)
|
|
text = open(os.path.join(root, rel)).read()
|
|
if not re.search(r"^tier:\s*[SML]\b", text, re.M):
|
|
out.append(Finding("tier-declared", rel, "no `tier:` declaration"))
|
|
elif "chaos" not in text.lower():
|
|
out.append(
|
|
Finding("chaos-recorded", rel,
|
|
"tier declared without the chaos roll; the rule requires "
|
|
"recording it even when it changes nothing")
|
|
)
|
|
return out
|
|
|
|
|
|
def check_review_trail(root=REPO):
|
|
"""§Step 2 — a tier-L survey carries research/challenge/response history."""
|
|
out = []
|
|
base = os.path.join(root, "research")
|
|
hist = os.path.join(root, "history")
|
|
if not (os.path.isdir(base) and os.path.isdir(hist)):
|
|
return out
|
|
files = os.listdir(hist)
|
|
for f in sorted(os.listdir(base)):
|
|
if not f.endswith(".md"):
|
|
continue
|
|
rel = os.path.join("research", f)
|
|
text = open(os.path.join(root, rel)).read()
|
|
if not re.search(r"^tier:\s*L\b", text, re.M):
|
|
continue
|
|
if not re.search(r"^status:\s*approved", text, re.M):
|
|
continue
|
|
for kind in ("challenge", "response"):
|
|
if not any(x.endswith(f"-{kind}.md") and kind in x for x in files):
|
|
out.append(
|
|
Finding("review-trail", rel,
|
|
f"tier-L approved survey with no history/*-{kind}.md")
|
|
)
|
|
return out
|
|
|
|
|
|
def check_reporting_tools_self_test(root=REPO):
|
|
"""§Step 5 v1.1 — every tool that reports a number exposes --self-test."""
|
|
out = []
|
|
base = os.path.join(root, "tools")
|
|
if not os.path.isdir(base):
|
|
return out
|
|
for f in sorted(os.listdir(base)):
|
|
if not f.endswith(".py"):
|
|
continue
|
|
rel = os.path.join("tools", f)
|
|
text = open(os.path.join(root, rel)).read()
|
|
if "--self-test" not in text:
|
|
out.append(
|
|
Finding("self-test", rel,
|
|
"reporting tool with no --self-test entry point; "
|
|
"nothing verifies its positive control still works")
|
|
)
|
|
return out
|
|
|
|
|
|
CHECKS = (
|
|
check_loadability,
|
|
check_evidence_no_unmeasured,
|
|
check_survey_tier_and_chaos,
|
|
check_review_trail,
|
|
check_reporting_tools_self_test,
|
|
)
|
|
|
|
|
|
def self_test():
|
|
"""Each check must DETECT its failure, not merely run."""
|
|
import shutil
|
|
import tempfile
|
|
|
|
results = []
|
|
|
|
def check(name, ok, detail=""):
|
|
results.append((name, ok, detail))
|
|
|
|
tmp = tempfile.mkdtemp()
|
|
try:
|
|
for d in LOOP_DIRS + ("tools", "history"):
|
|
os.makedirs(os.path.join(tmp, d), exist_ok=True)
|
|
|
|
# loadability: 401 lines must trip, 400 must not.
|
|
with open(os.path.join(tmp, "specs", "Big.md"), "w") as fh:
|
|
fh.write("x\n" * (LOADABILITY_LIMIT + 1))
|
|
with open(os.path.join(tmp, "specs", "Ok.md"), "w") as fh:
|
|
fh.write("x\n" * LOADABILITY_LIMIT)
|
|
f = check_loadability(tmp)
|
|
check("loadability detects overlong artifact",
|
|
len(f) == 1 and "Big.md" in f[0].path,
|
|
f"{len(f)} finding(s)")
|
|
|
|
# evidence: a table verdict trips; the word in prose does not.
|
|
with open(os.path.join(tmp, "evidence", "E.md"), "w") as fh:
|
|
fh.write("| AC-1 | x | unmeasured |\n"
|
|
"the word unmeasured appearing in prose is fine\n")
|
|
f = check_evidence_no_unmeasured(tmp)
|
|
check("evidence-unmeasured detects a table verdict, not prose",
|
|
len(f) == 1, f"{len(f)} finding(s), expected exactly 1")
|
|
|
|
# tier/chaos: missing tier trips; tier without chaos trips.
|
|
with open(os.path.join(tmp, "research", "A.md"), "w") as fh:
|
|
fh.write("# survey\nno tier here\n")
|
|
with open(os.path.join(tmp, "research", "B.md"), "w") as fh:
|
|
fh.write("tier: L (structural L)\n")
|
|
f = check_survey_tier_and_chaos(tmp)
|
|
rules = sorted(x.rule for x in f)
|
|
check("tier/chaos detects both omissions",
|
|
rules == ["chaos-recorded", "tier-declared"], f"{rules}")
|
|
|
|
# self-test: a tool without the flag trips.
|
|
with open(os.path.join(tmp, "tools", "silent.py"), "w") as fh:
|
|
fh.write("print(42)\n")
|
|
f = check_reporting_tools_self_test(tmp)
|
|
check("self-test detects a tool lacking --self-test",
|
|
len(f) == 1 and "silent.py" in f[0].path, f"{len(f)} finding(s)")
|
|
|
|
# review trail: approved tier-L survey with no history trips.
|
|
with open(os.path.join(tmp, "research", "C.md"), "w") as fh:
|
|
fh.write("tier: L (structural L, chaos 3)\nstatus: approved\n")
|
|
f = check_review_trail(tmp)
|
|
check("review-trail detects a missing challenge/response",
|
|
len(f) == 2, f"{len(f)} finding(s), expected 2")
|
|
finally:
|
|
shutil.rmtree(tmp, ignore_errors=True)
|
|
|
|
print("loop-lint self-test (positive control)")
|
|
ok = True
|
|
for name, passed, detail in results:
|
|
print(f" [{'ok ' if passed else 'FAIL'}] {name}"
|
|
+ (f" — {detail}" if detail else ""))
|
|
ok &= passed
|
|
return 0 if ok else 1
|
|
|
|
|
|
def main():
|
|
if "--self-test" in sys.argv:
|
|
return self_test()
|
|
|
|
findings = []
|
|
for c in CHECKS:
|
|
findings.extend(c())
|
|
|
|
print("loop-lint — executable InnerLoop rules")
|
|
if not findings:
|
|
print(" no findings")
|
|
return 0
|
|
by_rule = {}
|
|
for f in findings:
|
|
by_rule.setdefault(f.rule, []).append(f)
|
|
for rule, fs in sorted(by_rule.items()):
|
|
print(f"\n{rule} ({len(fs)}):")
|
|
for f in fs:
|
|
print(str(f))
|
|
print(f"\n{len(findings)} finding(s)")
|
|
return 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|