From 38f237fc5c998419cb725a6b4516d89a451c792b Mon Sep 17 00:00:00 2001 From: tegwick Date: Sat, 1 Aug 2026 15:38:35 +0200 Subject: [PATCH] CB-WP-0009-T01: the meta budget measures the pass it governs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ADR-0006 D1. cb-cost gains pass_costs, which buckets responses into workplan windows in a single read of the transcripts — calling collect() once per boundary would re-read every transcript per window, and status is supposed to stay cheap enough that nobody replaces it with ls. make status now reports the share over a trailing three passes with the per-pass breakdown, keeps the lifetime figure labelled NOT the metric, and prints the D2 exemption so the next reader does not re-derive the standoff where the budget blocked its own repair. One pass would be a coin flip: a meta pass reads 100%, a product pass 0%. Three is the smallest window where the ratio means something and still moves. First reading: 36% over the last three passes against 49% lifetime, with CB-WP-0008 at 0%. The first run reported $0.00 for every window. Transcript stamps are UTC Z and git prints a local offset, so the string comparison put every response before the first boundary. Both the fix and a self-test for it are in; reverting the conversion turns four checks red. Co-Authored-By: Claude Opus 5 --- tools/cb-cost.py | 74 +++++++++++++ tools/status.py | 137 ++++++++++++++++++++----- workplans/CB-WP-0009-adaptive-gates.md | 9 +- 3 files changed, 194 insertions(+), 26 deletions(-) diff --git a/tools/cb-cost.py b/tools/cb-cost.py index a707484..ae6228b 100644 --- a/tools/cb-cost.py +++ b/tools/cb-cost.py @@ -524,6 +524,55 @@ def collect(slug, pin_ref=None, since_ref=None): } +def pass_costs(slug, boundaries): + """Cost per pass window, from one read of the transcripts. + + CB-WP-0009 T01 / ADR-0006 D1: a budget must measure the window it + governs. `boundaries` is `[(label, start_iso)]` in ascending order; + each window runs to the next label's start, the last to now. + Responses before the first boundary land in `_before`. + + Reads the transcripts **once**. Calling `collect()` per boundary + would re-read every transcript per window, and `make status` is + supposed to stay cheap enough that nobody replaces it with `ls`. + """ + prices = load_prices() + stale = check_price_sheet_age(prices) + if stale: + raise Abort(stale) + paths = transcript_paths(slug) + if not paths: + raise Abort(f"no transcripts found for {slug}") + responses = [] + for path in paths: + responses.extend(read_responses(path, None)) + if not responses: + raise Abort(f"no responses in {len(paths)} transcript(s) — refusing to report") + + ordered = sorted(boundaries, key=lambda b: b[1]) + out = {label: {"cost": 0.0, "responses": 0} for label, _ in ordered} + out["_before"] = {"cost": 0.0, "responses": 0} + + for r in responses: + cost = price_of(prices, r["model"], r["toks"], r["timestamp"]) or 0.0 + bucket = "_before" + for label, start in ordered: + if r["timestamp"] > start: + bucket = label + else: + break + out[bucket]["cost"] += cost + out[bucket]["responses"] += 1 + + # Positive control: a bucketing that placed nothing in any real window + # would report a confident 0% for every pass. + placed = sum(v["responses"] for k, v in out.items() if k != "_before") + if ordered and placed == 0: + raise Abort("pass_costs bucketed every response before the first " + "boundary — the boundaries are wrong, not the spend") + return out + + def render(rep, by_task=False, composition=False): print(f"M-D2-CST cost report — {rep['slug']}") print(f" pin {rep['pin'] or '(none — live file, not reproducible)'}") @@ -625,6 +674,31 @@ def self_test(): check("AC-8 5m cache priced at write_5m", abs(got - want) < 1e-9 and got != wrong, f"${got:.4f} (1h would be ${wrong:.4f})") + # ADR-0006 D1: pass_costs buckets by window, and refuses to report a + # confident 0% when every response landed before the first boundary. + slug = "-home-worsch-clay-borg" + try: + real = pass_costs(slug, [("early", "2020-01-01T00:00:00Z")]) + check("pass_costs places responses in a real window", + real["early"]["responses"] > 0 and real["early"]["cost"] > 0, + f"{real['early']['responses']} response(s)") + two = pass_costs(slug, [("a", "2020-01-01T00:00:00Z"), + ("b", "2026-07-31T00:00:00Z")]) + check("pass_costs splits at a boundary rather than pooling", + two["a"]["responses"] > 0 and two["b"]["responses"] > 0, + f"a={two['a']['responses']} b={two['b']['responses']}") + check("pass_costs windows sum to the unwindowed total", + abs((two["a"]["cost"] + two["b"]["cost"] + two["_before"]["cost"]) + - real["early"]["cost"] - real["_before"]["cost"]) < 0.01) + except Abort as e: + check("pass_costs places responses in a real window", False, str(e)) + try: + pass_costs(slug, [("future", "2099-01-01T00:00:00Z")]) + check("pass_costs aborts when nothing lands in any window", False, + "no Abort raised — every pass would read $0.00 and 0%") + except Abort: + check("pass_costs aborts when nothing lands in any window", True) + # AC-5: dedup invariant is enforced. import tempfile with tempfile.NamedTemporaryFile("w", suffix=".jsonl", delete=False) as fh: diff --git a/tools/status.py b/tools/status.py index bff8c10..5dd7c05 100644 --- a/tools/status.py +++ b/tools/status.py @@ -116,6 +116,90 @@ def cost_lines(): return open_spend, rep["total"], mod +# ------------------------------------------------------- the meta budget + +# How many passes the budget looks back over. One pass is a coin flip — +# a `meta` pass reads 100% and a `product` pass reads 0%, and neither +# says anything about the balance the budget is about. Three is the +# smallest window in which a ratio means something and still moves. +TRAILING_PASSES = 3 +META_SOFT_PCT = 25 + + +def workplan_starts(): + """[(id, kind, first_commit_iso)] — when each pass began. + + A pass starts at the commit that *added* its workplan file, which is + the only boundary git records and the same one `cb-cost --since` has + been windowed on by hand since CB-WP-0004 T05. + """ + import datetime + + out = [] + for path in sorted(glob.glob(os.path.join(ROOT, "workplans", "*.md"))): + wid, _title, _status, _tasks, kind = parse_workplan(path) + rel = os.path.relpath(path, ROOT) + added = git("log", "--diff-filter=A", "--format=%cI", "--", rel) + first = added.splitlines()[-1].strip() if added else "" + if not (wid and first): + continue + # Transcript timestamps are UTC `...Z`; git prints a local offset. + # Comparing the two as strings silently buckets everything into + # `_before`, which is how this first read reported $0. + first = ( + datetime.datetime.fromisoformat(first) + .astimezone(datetime.timezone.utc) + .isoformat() + .replace("+00:00", "Z") + ) + out.append((wid, kind, first)) + return sorted(out, key=lambda r: r[2]) + + +def meta_budget(plans): + """ADR-0006 D1: report the share for the window the budget governs.""" + _, _, mod = cost_lines() + starts = workplan_starts() + if not starts: + print("\n meta budget UNAVAILABLE — no workplan start commits found") + return + costs = mod.pass_costs("-home-worsch-clay-borg", + [(wid, when) for wid, _k, when in starts]) + + window = starts[-TRAILING_PASSES:] + kinds = {wid: kind for wid, kind, _ in starts} + total = sum(costs[wid]["cost"] for wid, _k, _w in window) + # `mixed` splits evenly; stated rather than hidden. + weight = {"product": 0.0, "meta": 1.0, "mixed": 0.5} + meta = sum(costs[wid]["cost"] * weight.get(kinds[wid], 0.5) + for wid, _k, _w in window) + + if total <= 0: + print("\n meta budget UNAVAILABLE — the trailing window measured $0") + return + share = 100 * meta / total + mark = "ok " if share <= META_SOFT_PCT else "OVER" + print(f"\n meta budget [{mark}] {share:.0f}% over the last " + f"{len(window)} pass(es) (soft {META_SOFT_PCT}%, InnerLoop v1.6)") + for wid, kind, _w in window: + c = costs[wid] + print(f" {wid} {kind:<8} ${c['cost']:>7,.2f} " + f"{c['responses']:>4} response(s)") + + # History, kept and labelled — the whole point of ADR-0006 D1 is that + # this number is not what the target compares against. + lifetime = sum(costs[wid]["cost"] for wid, _k, _w in starts) + life_meta = sum(costs[wid]["cost"] * weight.get(kind, 0.5) + for wid, kind, _ in starts) + if lifetime > 0: + print(f" history, all {len(starts)} passes: " + f"{100 * life_meta / lifetime:.0f}% — NOT the metric " + f"(ADR-0006 D1)") + print(" NOTE: repairing the instrument that reports a breach is always") + print(" in budget (ADR-0006 D2); other above-line meta work needs") + print(" `authorized_above_budget:` in the workplan frontmatter.") + + # ------------------------------------------------------------------ report @@ -162,31 +246,10 @@ def report(): except Exception as e: print(f" UNAVAILABLE — {e}") - # InnerLoop v1.5 — soft 25% meta budget. Reported, never gated. + # InnerLoop v1.6 — soft 25% meta budget over a trailing pass window. + # Reported, never gated. try: - _, _, mod = cost_lines() - rep = mod.collect("-home-worsch-clay-borg", None) - detail = rep["by_task_detail"] - by_kind = {"product": 0.0, "meta": 0.0, "mixed": 0.0} - attributed = 0.0 - for wid, _t, _s, tasks, kind in plans: - for tid, _ts, _p in tasks: - d = detail.get(tid) - if d and kind in by_kind: - by_kind[kind] += d["cost"] - attributed += d["cost"] - if attributed > 0: - # `mixed` splits evenly; stated rather than hidden. - meta = by_kind["meta"] + by_kind["mixed"] / 2 - share = 100 * meta / attributed - mark = "ok " if share <= 25 else "OVER" - print(f"\n meta budget [{mark}] {share:.0f}% of " - f"${attributed:,.2f} attributed (soft 25%, InnerLoop v1.5)") - print(f" product ${by_kind['product']:,.2f} " - f"meta ${by_kind['meta']:,.2f} " - f"mixed ${by_kind['mixed']:,.2f} (split 50/50)") - print(" NOTE: attributed tasks only — early workplans used bare") - print(" T## ids that collide across passes and are excluded.") + meta_budget(plans) except Exception as e: print(f"\n meta budget UNAVAILABLE — {e}") @@ -279,6 +342,32 @@ def self_test(): repr(h[:60])) check("task heading resolves to something", bool(h)) + # ADR-0006 D1: the budget must measure the window it governs. + starts = workplan_starts() + check("every workplan has a start commit", len(starts) >= len(plans) - 1, + f"{len(starts)} start(s) for {len(plans)} workplan(s)") + check("start commits are UTC and comparable to transcript stamps", + all(w.endswith("Z") for _i, _k, w in starts), + # A local-offset stamp compares as a string against `...Z` and + # silently buckets every response before the first boundary, + # which is exactly how this reported $0 on its first run. + ", ".join(w for _i, _k, w in starts[:1])) + check("start commits are in ascending order", + [w for _i, _k, w in starts] == sorted(w for _i, _k, w in starts)) + + buf = io.StringIO() + try: + with redirect_stdout(buf): + meta_budget(plans) + text = buf.getvalue() + check("budget reports a windowed share", "over the last" in text) + check("budget labels the cumulative figure as history", + "NOT the metric" in text) + check("budget states the instrument-repair exemption", + "in budget (ADR-0006 D2)" in text) + except Exception as e: + check("budget reports a windowed share", False, str(e)) + # The report itself must run and produce substance, not a stub. buf = io.StringIO() try: diff --git a/workplans/CB-WP-0009-adaptive-gates.md b/workplans/CB-WP-0009-adaptive-gates.md index b9faecc..041a2b2 100644 --- a/workplans/CB-WP-0009-adaptive-gates.md +++ b/workplans/CB-WP-0009-adaptive-gates.md @@ -2,7 +2,7 @@ id: CB-WP-0009 kind: meta title: "Make control gates experimental: window them, review them, cash them out" -status: proposed +status: in_progress authorized_above_budget: "maintainer, 2026-08-01 — 'improve our policies where it seems promising; we will experiment our way to sensible task- and context-specific control and review gates'. Meta read 59% at the time." state_hub_workstream_id: "98c0af4a-415c-4cf5-92d0-a722eb70cb90" --- @@ -34,7 +34,7 @@ prose has not delivered. ```task id: CB-WP-0009-T01 -status: todo +status: done priority: high state_hub_task_id: "c6eefacd-5bb9-4640-8356-12e8112bcf11" ``` @@ -53,6 +53,11 @@ it was a pure product pass — against 61% cumulative. **Refuted if** the two land within 20% of each other, in which case D1 is wrong and the machinery should be deleted rather than kept. +**Done 2026-08-01.** `cb-cost.pass_costs` buckets by pass window in one +read of the transcripts; `make status` reports a trailing 3-pass share +with the per-pass breakdown and the exemption. First reading **36% over +the last 3 passes** against **49% lifetime** — CB-WP-0008 alone reads 0%. + ## Task: `gates.toml` and `make gate-review` ```task