diff --git a/Makefile b/Makefile index 70a395f..a8221ba 100644 --- a/Makefile +++ b/Makefile @@ -24,7 +24,7 @@ TOOLS := $(REPO)/tools # Every cargo recipe runs at the repo root; the shell does not persist cd. IN_REPO := cd $(REPO) && -.PHONY: check test sim bench bench-test coverage dep-weight cost cost-test cost-pin cost-budget shape-budget cost-mix loop-lint self-tests env-test task-done status facts-check facts-gen mutation-check size-metrics runtime-metrics build-time am6 replay-test loc all +.PHONY: check test sim bench bench-test coverage dep-weight cost cost-test cost-pin cost-budget cost-mix loop-lint self-tests env-test task-done status facts-check facts-gen mutation-check size-metrics runtime-metrics build-time am6 replay-test loc all ## fmt + clippy (deny warnings) + HashMap deny-lint check: @@ -140,13 +140,6 @@ task-done: @test -n "$(T)" || { echo "usage: make task-done T=CB-WP-0004-T02" >&2; exit 2; } $(PY) $(TOOLS)/task-done.py $(T) $(ARGS) -# CB-WP-0007 T03 / InnerLoop v1.5: session shape for the window since the -# last commit. Deliberately NOT in `make all` — failing the build on -# context would block committing, and committing is the natural point to -# compact. A gate that blocks the remedy is a trap. -shape-budget: cost-test - $(PY) $(TOOLS)/cb-cost.py --shape-budget - # CB-01/CB-02: live spend since the last commit. cost-budget: cost-test $(PY) $(TOOLS)/cb-cost.py --budget diff --git a/specs/InnerLoop.md b/specs/InnerLoop.md index 35d5ab9..e24a99c 100644 --- a/specs/InnerLoop.md +++ b/specs/InnerLoop.md @@ -1,31 +1,6 @@ # The Inner Loop — Assimilate and Surpass -Status: **v1.5** — corrected from CB-WP-0007 (session shape) on -2026-08-01. Change from v1.4: a **soft 25% meta budget** (below), after a -spend review found the project 38% product / 62% loop-meta with cost per -response degraded 2.9x from its best window. - -> **Meta budget — soft, 25% of spend per pass.** Work on the loop's own -> instruments and process is capped at a quarter of a pass. Measured by -> `make status` from each workplan's `kind:` frontmatter -> (`product` | `meta` | `mixed`). -> -> **Soft on purpose.** A task already started may be finished — stopping -> mid-task to satisfy a ratio wastes the work and leaves the tree in a -> worse state than either finishing or never starting. What the budget -> forbids is *opening* new meta work above the line. -> -> A pass that exceeds it **says so in its evidence file and names the -> product work displaced**. That is the whole enforcement: this is a -> reporting budget, not a gate, for the same reason the session-shape -> budget is (CB-RES-0005 §4) — it constrains judgment, not artifacts. -> -> *(v1.5, from the CB-WP-0007 spend review: CB-WP-0005 and CB-WP-0006 cost -> ~$74, 31% of all spend, for zero measured efficiency gain. Their return -> was correctness of claims, which is real and is not optimization. The -> budget exists so that distinction has to be made out loud.)* - -v1.4 — corrected from CB-WP-0005 (assertion coverage) on +Status: **v1.4** — corrected from CB-WP-0005 (assertion coverage) on 2026-07-31. Change from v1.3: where a claim rests on numbers, the adversarial reviewer must read the assertion behind each quoted number and **mutate it** — re-running the command that prints a number is not diff --git a/tools/cb-cost.py b/tools/cb-cost.py index a707484..503eeb1 100644 --- a/tools/cb-cost.py +++ b/tools/cb-cost.py @@ -396,8 +396,7 @@ def collect(slug, pin_ref=None, since_ref=None): for r in responses: r["cost"] = price_of(prices, r["model"], r["toks"], r["timestamp"]) - commits = commit_index(pin) - attribute(responses, commits) + attribute(responses, commit_index(pin)) by_task = collections.defaultdict(float) # T02: per-task token detail, so a task close can report *measured* @@ -455,28 +454,9 @@ def collect(slug, pin_ref=None, since_ref=None): mech = [r for r in responses if set(r.get("categories") or []) & MECHANICAL] - # The window the SH-* targets compare against: responses since the - # last commit when unpinned, or the whole pinned range when pinned - # (a pin is already a window someone chose deliberately). - if pin or since: - window_responses, window_label = responses, "the pinned/--since range" - else: - last = commits[-1][0] if commits else "" - window_responses = [r for r in responses if last and r["timestamp"] > last] - window_label = "since the last commit" - sub = sum(r["cost"] or 0 for r in responses if r["subagent"]) return { "session_shape": session_shape(responses), - # CB-WP-0007 T01. SH-1/SH-2 as a cumulative mean over every - # response ever recorded cannot detect a worsening trend: the - # history outvotes the present. CB-WP-0006 ran at 503,464 mean - # context and the cumulative figure still printed 206,952. - # Both are reported; only the window is the metric. - "session_shape_window": session_shape(window_responses) - if window_responses else None, - "window_responses": len(window_responses), - "window_label": window_label, "tool_mix": { "turns": dict(mix_turns), "cost": dict(mix_cost), @@ -570,33 +550,16 @@ def render(rep, by_task=False, composition=False): f"{mix['mechanical_turns']:>5} turns ${mix['mechanical_cost']:>8,.2f}" f" = {100*mix['mechanical_cost']/(rep['total'] or 1):.0f}% of pass") - def _shape_lines(sh, indent=" "): - print(f"{indent}SH-1 mean context {sh['SH-1_mean_context']:>12,.0f} tok " - f"[{'ok ' if sh['SH-1_mean_context']<=200_000 else 'FAIL'} target 200,000]") - print(f"{indent}SH-2 p90 context {sh['SH-2_p90_context']:>12,.0f} tok " - f"[{'ok ' if sh['SH-2_p90_context']<=300_000 else 'FAIL'} target 300,000]") - print(f"{indent}SH-3 batching rate {100*sh['SH-3_batching_rate']:>11.1f}% " - f"[{'ok ' if sh['SH-3_batching_rate']>=0.20 else 'FAIL'} target 20.0%]") - print(f"{indent} {sh['tool_calls']} tool calls in " - f"{sh['responses_with_tools']} responses; " - f"{sh['calls_in_batched_turns']} in batched turns") - - win = rep.get("session_shape_window") - print(f"\n session shape (specs/SessionShape.md) — window: " - f"{rep.get('window_label', '?')}") - if win: - print(f" THE METRIC — {rep['window_responses']} response(s)") - _shape_lines(win) - else: - print(" THE METRIC — no responses in the window " - "(nothing since the last commit)") - sh = rep["session_shape"] - print(f"\n history, {rep['responses']} responses — context only, NOT the " - f"metric.") - print(" A cumulative mean cannot detect a worsening trend; the history") - print(" outvotes the present (CB-RES-0005 §1).") - _shape_lines(sh) + print("\n session shape (specs/SessionShape.md)") + print(f" SH-1 mean context {sh['SH-1_mean_context']:>12,.0f} tok " + f"[{'ok ' if sh['SH-1_mean_context']<=200_000 else 'FAIL'} target 200,000]") + print(f" SH-2 p90 context {sh['SH-2_p90_context']:>12,.0f} tok " + f"[{'ok ' if sh['SH-2_p90_context']<=300_000 else 'FAIL'} target 300,000]") + print(f" SH-3 batching rate {100*sh['SH-3_batching_rate']:>11.1f}% " + f"[{'ok ' if sh['SH-3_batching_rate']>=0.20 else 'FAIL'} target 20.0%]") + print(f" {sh['tool_calls']} tool calls in {sh['responses_with_tools']} responses; " + f"{sh['calls_in_batched_turns']} in batched turns") if rep["unpriced"]: print(f"\n UNPRICED ({len(rep['unpriced'])} responses, model not in sheet):") @@ -727,63 +690,6 @@ def self_test(): return 0 if ok else 1 -# CB-WP-0007 T03 / CB-RES-0005 D2. Soft = the SessionShape target; hard is -# 1.5x, set here BEFORE the next measurement per InnerLoop §Step 4. -# Reported, never in `make all`: failing the build on context would block -# committing, and committing is what closes the attribution window and is -# the natural point to compact. A gate that blocks the remedy is a trap. -SHAPE_SOFT = {"SH-1": 200_000, "SH-2": 300_000} -SHAPE_HARD = {"SH-1": 300_000, "SH-2": 450_000} - - -def shape_budget(slug): - """SH-* for the window since the last commit, with soft/hard verdicts.""" - try: - rep = collect(slug, None) - except Abort as e: - print(f"ABORT — {e}", file=sys.stderr) - return 1 - win = rep.get("session_shape_window") - head = subprocess.run( - ["git", "-C", REPO, "log", "-1", "--format=%h %s"], - capture_output=True, text=True, check=True).stdout.strip() - - print("session-shape budget — the window since the last commit") - print(f" last commit {head}") - # Positive control: a budget that reports ok because it measured - # nothing is the harness-does-nothing shape, in a reporting tool. - if not win or rep["window_responses"] == 0: - print(" no responses since the last commit — nothing to report", - file=sys.stderr) - return 1 - print(f" window {rep['window_responses']} response(s)") - - breach = 0 - for key, label, val in ( - ("SH-1", "mean context", win["SH-1_mean_context"]), - ("SH-2", "p90 context ", win["SH-2_p90_context"]), - ): - soft, hard = SHAPE_SOFT[key], SHAPE_HARD[key] - mark = "ok " if val <= soft else ("SOFT" if val <= hard else "HARD") - print(f" {key} {label} {val:>10,.0f} tok [{mark}] " - f"soft {soft:,} / hard {hard:,}") - breach = max(breach, 0 if val <= soft else (1 if val <= hard else 2)) - rate = win["SH-3_batching_rate"] - print(f" SH-3 batching {100*rate:>9.1f}% " - f"[{'ok ' if rate >= 0.20 else 'SOFT'}] floor 20.0%") - - if breach >= 2: - print("\n HARD — compact before continuing. Context this size costs " - "~10x per turn\n against a compacted session " - "(specs/SessionShape.md §2).", file=sys.stderr) - return 1 - if breach == 1: - print("\n SOFT — over target. Compaction is the remedy and it is free.") - else: - print("\n within budget") - return 0 - - def budget(slug, soft, hard): """Live cost budget (specs/CostAccounting.md §7). @@ -828,8 +734,6 @@ def main(): ap.add_argument("--composition", action="store_true") ap.add_argument("--session-shape", action="store_true", help="SH-1..SH-3 (always shown in the default report)") - ap.add_argument("--shape-budget", action="store_true", - help="SH-1/SH-2/SH-3 for the window since the last commit") ap.add_argument("--budget", action="store_true", help="CB-01/CB-02: spend since the last commit, live") ap.add_argument("--soft", type=float, default=10.00) @@ -841,8 +745,6 @@ def main(): if args.self_test: return self_test() - if args.shape_budget: - return shape_budget(args.slug) if args.budget: return budget(args.slug, args.soft, args.hard) diff --git a/tools/status.py b/tools/status.py index bff8c10..3d649ec 100644 --- a/tools/status.py +++ b/tools/status.py @@ -48,7 +48,6 @@ def parse_workplan(path): m = re.search(rf'^{key}:\s*"?(.*?)"?\s*$', text, re.M) return m.group(1) if m else None - kind = fm("kind") or "untagged" tasks = [] for m in re.finditer(r"```task\n(.*?)```", text, re.S): body = m.group(1) @@ -58,7 +57,7 @@ def parse_workplan(path): return mm.group(1) if mm else None tasks.append((f("id"), f("status"), f("priority"))) - return fm("id"), fm("title"), fm("status"), tasks, kind + return fm("id"), fm("title"), fm("status"), tasks def workplans(): @@ -70,7 +69,7 @@ def workplans(): def next_task(plans): """First todo task of the first non-done workplan, with its heading.""" - for wid, _title, status, tasks, _kind in plans: + for wid, _title, status, tasks in plans: if status == "done": continue for tid, tstatus, prio in tasks: @@ -131,7 +130,7 @@ def report(): + (f" ({len(dirty)} uncommitted)" if dirty else " (clean)")) print("\n workplans") - for wid, title, status, tasks, _kind in plans: + for wid, title, status, tasks in plans: done = sum(1 for _, s, _ in tasks if s == "done") if status == "done" and done == len(tasks): print(f" {wid} {status:<12} {done}/{len(tasks)}") @@ -162,34 +161,6 @@ def report(): except Exception as e: print(f" UNAVAILABLE — {e}") - # InnerLoop v1.5 — soft 25% meta budget. Reported, never gated. - try: - _, _, mod = cost_lines() - rep = mod.collect("-home-worsch-clay-borg", None) - detail = rep["by_task_detail"] - by_kind = {"product": 0.0, "meta": 0.0, "mixed": 0.0} - attributed = 0.0 - for wid, _t, _s, tasks, kind in plans: - for tid, _ts, _p in tasks: - d = detail.get(tid) - if d and kind in by_kind: - by_kind[kind] += d["cost"] - attributed += d["cost"] - if attributed > 0: - # `mixed` splits evenly; stated rather than hidden. - meta = by_kind["meta"] + by_kind["mixed"] / 2 - share = 100 * meta / attributed - mark = "ok " if share <= 25 else "OVER" - print(f"\n meta budget [{mark}] {share:.0f}% of " - f"${attributed:,.2f} attributed (soft 25%, InnerLoop v1.5)") - print(f" product ${by_kind['product']:,.2f} " - f"meta ${by_kind['meta']:,.2f} " - f"mixed ${by_kind['mixed']:,.2f} (split 50/50)") - print(" NOTE: attributed tasks only — early workplans used bare") - print(" T## ids that collide across passes and are excluded.") - except Exception as e: - print(f"\n meta budget UNAVAILABLE — {e}") - print("\n fast gates") code, out = run_tool("loop-lint.py") last = out.splitlines()[-1] if out else "?" @@ -238,18 +209,18 @@ def self_test(): fh.write(sample) tmp = fh.name try: - wid, title, status, tasks, _kind = parse_workplan(tmp) + wid, title, status, tasks = parse_workplan(tmp) check("workplan frontmatter parses", (wid, title, status) == ("CB-WP-0009", "Sample", "in_progress"), f"{wid} {status}") check("both task blocks parse with status and priority", tasks == [("CB-WP-0009-T01", "done", "high"), ("CB-WP-0009-T02", "todo", "medium")]) check("next task is the first todo, not the first task", - next_task([(wid, title, status, tasks, "meta")]) + next_task([(wid, title, status, tasks)]) == ("CB-WP-0009", "CB-WP-0009-T02", "medium")) check("a fully-done workplan yields no next task", next_task([("X", "t", "done", - [("X-T01", "done", "high")], "meta")]) is None) + [("X-T01", "done", "high")])]) is None) finally: os.unlink(tmp) @@ -261,14 +232,9 @@ def self_test(): sum(len(t) for *_, t in plans) >= 20, f"{sum(len(t) for *_, t in plans)} task(s)") check("every real workplan has an id and a status", - all(w and s for w, _, s, _, _ in plans)) - # InnerLoop v1.5: the meta budget needs every workplan classified, or - # the ratio is computed over an unknown denominator. - check("every workplan declares kind: product|meta|mixed", - all(k in ("product", "meta", "mixed") for *_, k in plans), - ", ".join(f"{w}={k}" for w, *_, k in plans)) + all(w and s for w, _, s, _ in plans)) check("every real task has an id and a status", - all(i and s for _, _, _, ts, _ in plans for i, s, _ in ts)) + all(i and s for *_, ts in plans for i, s, _ in ts)) # A heading must be one line. The first version used a dot-all match # and returned the prose of the preceding task as the heading. diff --git a/workplans/CB-WP-0001-inner-loop.md b/workplans/CB-WP-0001-inner-loop.md index c085515..009b2a1 100644 --- a/workplans/CB-WP-0001-inner-loop.md +++ b/workplans/CB-WP-0001-inner-loop.md @@ -1,6 +1,5 @@ --- id: CB-WP-0001 -kind: product title: "Establish the assimilate-and-surpass inner loop via the GROUND game kernel" status: done state_hub_workstream_id: "a1b434dc-b1c6-46b5-bbd9-80a4e6b7620f" diff --git a/workplans/CB-WP-0002-cost-accounting.md b/workplans/CB-WP-0002-cost-accounting.md index c90047e..7a3e87d 100644 --- a/workplans/CB-WP-0002-cost-accounting.md +++ b/workplans/CB-WP-0002-cost-accounting.md @@ -1,6 +1,5 @@ --- id: CB-WP-0002 -kind: meta title: "Make agentic cost measurable, so D2 claims are falsifiable" status: done state_hub_workstream_id: "b7c22f69-fbe9-48df-9619-007db79ae338" diff --git a/workplans/CB-WP-0003-loop-hardening.md b/workplans/CB-WP-0003-loop-hardening.md index da6fbba..021e278 100644 --- a/workplans/CB-WP-0003-loop-hardening.md +++ b/workplans/CB-WP-0003-loop-hardening.md @@ -1,6 +1,5 @@ --- id: CB-WP-0003 -kind: meta title: "Harden the inner loop: executable rules, session economics, dead policy" status: done state_hub_workstream_id: "39d61dc0-870d-45c1-a595-bcf91f289dce" diff --git a/workplans/CB-WP-0004-mechanical-work.md b/workplans/CB-WP-0004-mechanical-work.md index bfaa8d4..7551017 100644 --- a/workplans/CB-WP-0004-mechanical-work.md +++ b/workplans/CB-WP-0004-mechanical-work.md @@ -1,6 +1,5 @@ --- id: CB-WP-0004 -kind: meta title: "Move mechanical turns off the token budget, and prove it worked" status: done state_hub_workstream_id: "6880ac78-d817-41b9-b267-f12ff9deea28" diff --git a/workplans/CB-WP-0005-assertion-coverage.md b/workplans/CB-WP-0005-assertion-coverage.md index 7250994..fdde0b5 100644 --- a/workplans/CB-WP-0005-assertion-coverage.md +++ b/workplans/CB-WP-0005-assertion-coverage.md @@ -1,6 +1,5 @@ --- id: CB-WP-0005 -kind: meta title: "Make the instruments count assertions, then fix what they expose" status: done state_hub_workstream_id: "0b95a1e3-7780-43d0-81e9-072ef7978734" diff --git a/workplans/CB-WP-0006-instrument-the-table.md b/workplans/CB-WP-0006-instrument-the-table.md index 76563eb..9ec2513 100644 --- a/workplans/CB-WP-0006-instrument-the-table.md +++ b/workplans/CB-WP-0006-instrument-the-table.md @@ -1,6 +1,5 @@ --- id: CB-WP-0006 -kind: mixed title: "Instrument the acceptance table, then implement what it exposes" status: done state_hub_workstream_id: "8a6327cc-fd5c-4e2c-a29b-b437c27d1e71" diff --git a/workplans/CB-WP-0007-session-shape.md b/workplans/CB-WP-0007-session-shape.md index a41bd85..dd11463 100644 --- a/workplans/CB-WP-0007-session-shape.md +++ b/workplans/CB-WP-0007-session-shape.md @@ -1,8 +1,7 @@ --- id: CB-WP-0007 -kind: meta title: "Make session shape measurable in the window that matters, then enforce it" -status: in_progress +status: proposed state_hub_workstream_id: "bee19b76-fb60-4dcf-94d0-9242b43e0e42" --- @@ -34,31 +33,13 @@ Per InnerLoop §Step 4, no target moves in the commit that measures it — and CB-RES-0005 D3 states that up front, because all three targets are currently unmet by wide margins and the temptation is obvious. -## Scope cut, 2026-08-01 (maintainer decision) - -A spend review before starting found the project **38% product / 62% -loop-meta**, with cost per response degraded 2.9× from its best window and -INTENT stage 0 still missing a CLI player and bots. CB-WP-0005 and -CB-WP-0006 cost **~$74 — 31% of all spend — for zero measured efficiency -gain** (their return was correctness of claims, which is real but is not -optimization). - -So this workplan is **cut to T01 and T03**, the two tasks that attack the -2.9× regression directly. **T02 and T04 are cancelled**, not deferred — -SH-4 trend reporting and the batching trial are more instrument work, and -the instrument-failure taxonomy is not converging. T06 becomes the -project-level retrospective the maintainer asked for. - -**A soft 25% meta budget is established here** (T03), because the review's -finding needs a standing number, not a memory. - ## Phase A — measure the right window ## Task: window the session-shape metrics ```task id: CB-WP-0007-T01 -status: done +status: todo priority: high state_hub_task_id: "7e036db4-d97a-45bd-a70a-20fe68225402" ``` @@ -77,16 +58,11 @@ target compares against. **Refuted if** it lands within 20% of the cumulative number, in which case the aggregation was not the problem and this pass should re-plan. -## Task: SH-4 — a windowed trend the instrument can see (CANCELLED) - -> **Cancelled unstarted 2026-08-01.** More instrument work, against a -> review finding that instrument work has stopped paying. The ceiling T03 -> adds catches a bad pass; a trend line would catch a slow slide, which is -> a real gap — recorded as open, not built. +## Task: SH-4 — a windowed trend the instrument can see ```task id: CB-WP-0007-T02 -status: cancel +status: todo priority: medium state_hub_task_id: "1d5a32ef-4086-4021-b874-24ed2160c4dd" ``` @@ -110,7 +86,7 @@ is not a trend, and the first run has nothing to compare against. ```task id: CB-WP-0007-T03 -status: done +status: todo priority: high state_hub_task_id: "126a6313-b1f7-462c-8771-5100d38f9429" ``` @@ -143,17 +119,11 @@ Carries `--self-test`. Its positive control is the one this project keeps needing: a budget that reports `ok` because it measured nothing must abort instead. -## Task: batch deliberately, and report what the rate reaches (CANCELLED) - -> **Cancelled unstarted 2026-08-01.** `SessionShape.md` §4 already puts -> the ceiling at **$2–4 on a $93 pass** — the cheapest of the three -> metrics to move and the least valuable. Spending a task on it while -> stage 0 lacks a CLI player is the misallocation the review found. -> SH-3 remains measured, unmet at 0.0%, and unfalsified. +## Task: batch deliberately, and report what the rate reaches ```task id: CB-WP-0007-T04 -status: cancel +status: todo priority: medium state_hub_task_id: "c284db6f-2e16-40fb-be8b-83ae5391c058" ``` @@ -184,10 +154,7 @@ priority: high state_hub_task_id: "2f78b272-e1f9-4530-bc8c-3a9d82833546" ``` -Commit `evidence/CB-EV-0006-session-shape.md`. **Reduced with the scope -cut** — tests 1, 2 and 4 remain; test 3 (SH-3) is cancelled with T04. - -Four tests, all reported: +Commit `evidence/CB-EV-0006-session-shape.md`. Four tests, all reported: 1. **Does the windowed metric differ from cumulative?** Against T01's prediction; refuted if within 20%.