diff --git a/specs/SessionShape.md b/specs/SessionShape.md index 44e4d38..6b9a521 100644 --- a/specs/SessionShape.md +++ b/specs/SessionShape.md @@ -136,6 +136,47 @@ workplan — the failure mode InnerLoop v1.1 rule 13 exists for, and the `history/260731-inner-loop-rule-audit.md` identified as having no executable defence. +### Correction (CB-WP-0013 T01): SH-3 was never read as measured + +**Six evidence files quote `SH-3 0.0%`, and none of them measured it.** + +`cb-cost --shape-budget` reports the window **since the last commit**, and +it is read while writing the evidence file — immediately *after* a commit, +when that window holds one or two responses. SH-1 and SH-2 are location +statistics and survive a sample of two. SH-3 is a **rate**, and at n=2 its +only possible values are 0%, 50% and 100%. One window, three metrics, and +it is only wrong for one of them. + +Measured over real per-pass windows instead: + +| window since | responses | SH-3 | +|---|---:|---:| +| `0d2ab22` | 115 | 6.2% | +| `6fb0aea` | 113 | 6.3% | +| `331e7e9` | 95 | 1.1% | +| `c5fa610` | 74 | 1.4% | +| `883b608` | 70 | 1.4% | +| `84d6886` | 24 | 4.3% | + +The substantive claim survives — SH-3 is badly unmet — but the constant +0.0% hid something the real numbers show: against the **7.8–8.6%** pinned +above, **batching has got worse**, and a metric frozen at a constant could +not have shown a trend at all. Six passes reported a breach and the breach +was moving underneath them. + +**This is the ninth recorded instance of the trusted-arithmetic class, and +the second in this same metric** — the eighth is the `0 of 330 tool calls` +figure corrected immediately above. SH-3 is the number this project keeps +getting wrong, which is itself a reason to doubt the metric rather than the +behaviour (§SH-3 disposition). + +**The fix**, `cb-cost.py` `sh3_line()`: below a minimum sample the tool +prints `insufficient sample` and no verdict. The floor is derived, not +round — if the true rate were exactly the 20% target, the chance of seeing +zero batched turns in `n` responses is `0.8^n`; at **n = 14** that is 4.4%, +so "0 batched in 14" rules out a target-meeting rate at ~95%. Below that +the tool has nothing to say and now says so. + At an 8.6% batching rate there is real headroom, but the saving is bounded: eliminating every avoidable single-call turn would remove at most a few percent of turns, worth roughly $2–4 on a $93 pass. **Batch because it is diff --git a/tools/cb-cost.py b/tools/cb-cost.py index ae6228b..c924ff9 100644 --- a/tools/cb-cost.py +++ b/tools/cb-cost.py @@ -624,8 +624,15 @@ def render(rep, by_task=False, composition=False): f"[{'ok ' if sh['SH-1_mean_context']<=200_000 else 'FAIL'} target 200,000]") print(f"{indent}SH-2 p90 context {sh['SH-2_p90_context']:>12,.0f} tok " f"[{'ok ' if sh['SH-2_p90_context']<=300_000 else 'FAIL'} target 300,000]") - print(f"{indent}SH-3 batching rate {100*sh['SH-3_batching_rate']:>11.1f}% " - f"[{'ok ' if sh['SH-3_batching_rate']>=0.20 else 'FAIL'} target 20.0%]") + # Same guard as the budget path: a rate a window cannot support + # must not be printed as though it were measured. + if sh["responses_with_tools"] < SH3_MIN_SAMPLE: + print(f"{indent}SH-3 batching rate insufficient sample " + f"[{sh['responses_with_tools']} with tool calls, " + f"need {SH3_MIN_SAMPLE}]") + else: + print(f"{indent}SH-3 batching rate {100*sh['SH-3_batching_rate']:>11.1f}% " + f"[{'ok ' if sh['SH-3_batching_rate']>=0.20 else 'FAIL'} target 20.0%]") print(f"{indent} {sh['tool_calls']} tool calls in " f"{sh['responses_with_tools']} responses; " f"{sh['calls_in_batched_turns']} in batched turns") @@ -786,6 +793,27 @@ def self_test(): finally: os.unlink(empty) + # SH-3 minimum sample (CB-WP-0013 T01). Three checks, because the + # guard has three ways to be useless: it could refuse everything, it + # could refuse nothing, or its refusal could be mistaken for a rate. + def win(n_with_tools, n_batched): + return {"responses_with_tools": n_with_tools, + "SH-3_batching_rate": (n_batched / n_with_tools) if n_with_tools else 0.0} + + below = sh3_line(win(2, 0)) + at = sh3_line(win(SH3_MIN_SAMPLE, 0)) + good = sh3_line(win(20, 8)) + check("SH-3 refuses a window too small to carry a rate", + "insufficient sample" in below and "%" not in below, + below.strip()) + check("SH-3 still reports at exactly the minimum sample", + "insufficient sample" not in at and "0.0%" in at, at.strip()) + check("SH-3 reports a real rate above the floor", + "40.0%" in good and "ok" in good, good.strip()) + # The one that matters: a refusal must not read as a measured zero. + check("a refusal is distinguishable from a genuine 0.0%", + below != sh3_line(win(SH3_MIN_SAMPLE, 0))) + # AC-7: the subagent tree is discovered by the path globs. slug = "-home-worsch-clay-borg" paths = transcript_paths(slug) @@ -809,6 +837,36 @@ def self_test(): SHAPE_SOFT = {"SH-1": 200_000, "SH-2": 300_000} SHAPE_HARD = {"SH-1": 300_000, "SH-2": 450_000} +# CB-WP-0013 T01. SH-1 and SH-2 are location statistics and survive a +# two-response window. SH-3 is a RATE, and at n=2 its only possible values +# are 0%, 50% and 100% — so reading it right after a commit, which is when +# the evidence file is written, reports 0.0% almost regardless of +# behaviour. Six evidence files quoted that 0.0% as a measurement; the real +# per-pass figure was 1.1-6.3%. +# +# The floor is derived, not round. If the true rate were exactly the 20% +# target, the chance of observing ZERO batched turns in n responses is +# 0.8^n. At n = 14 that is 4.4%, so "0 batched in 14" rules out a +# target-meeting rate at ~95%. Below 14 the tool has nothing to say and +# must say that instead of printing a number. +SH3_MIN_SAMPLE = 14 + + +def sh3_line(win, min_sample=SH3_MIN_SAMPLE): + """SH-3, or an explicit refusal when the window cannot support a rate. + + Returns the line rather than printing it so the refusal is testable + without capturing stdout — a guard that can only be checked by reading + output is a guard that will be checked by nobody. + """ + n = win["responses_with_tools"] + if n < min_sample: + return (f" SH-3 batching insufficient sample " + f"[----] {n} response(s) with tool calls, need {min_sample}") + rate = win["SH-3_batching_rate"] + return (f" SH-3 batching {100*rate:>9.1f}% " + f"[{'ok ' if rate >= 0.20 else 'SOFT'}] floor 20.0%") + def shape_budget(slug): """SH-* for the window since the last commit, with soft/hard verdicts.""" @@ -842,9 +900,7 @@ def shape_budget(slug): print(f" {key} {label} {val:>10,.0f} tok [{mark}] " f"soft {soft:,} / hard {hard:,}") breach = max(breach, 0 if val <= soft else (1 if val <= hard else 2)) - rate = win["SH-3_batching_rate"] - print(f" SH-3 batching {100*rate:>9.1f}% " - f"[{'ok ' if rate >= 0.20 else 'SOFT'}] floor 20.0%") + print(sh3_line(win)) if breach >= 2: print("\n HARD — compact before continuing. Context this size costs " diff --git a/workplans/CB-WP-0013-instrument-corrections.md b/workplans/CB-WP-0013-instrument-corrections.md index 50e43cc..1006d74 100644 --- a/workplans/CB-WP-0013-instrument-corrections.md +++ b/workplans/CB-WP-0013-instrument-corrections.md @@ -69,7 +69,7 @@ figure, `syn` alone 66,916. ```task id: CB-WP-0013-T01 -status: todo +status: done priority: high ``` @@ -95,6 +95,32 @@ already records as the eighth trusted-arithmetic instance. **This is the ninth, and it is in the same metric.** That coincidence is worth a sentence: SH-3 is the number this project keeps getting wrong. +**Done 2026-08-02.** `cb-cost.py` gains `sh3_line()`: below a minimum +sample it prints `insufficient sample` and **no verdict**. + +**The window was kept, not split.** SH-3 could instead have been given a +per-pass window while SH-1/SH-2 kept the since-last-commit one. Rejected: +the budget's stated purpose is the *open remainder* since the last commit +— the only thing a live budget can fire on — and giving one of three +metrics a different window makes "the window" ambiguous in a tool that +three specs cite. Refusing to report is honest and keeps one window. + +**The floor is derived, not round.** If the true rate were exactly the 20% +target, the chance of observing zero batched turns in `n` responses is +`0.8^n`. At **n = 14** that is 4.4%, so "0 batched in 14" rules out a +target-meeting rate at ~95%. Below 14 the tool has nothing to say. + +Four controls, three mutations, each red for its stated reason: + +| mutation | result | +|---|---| +| the guard never refuses | `SH-3 refuses a window too small` red, *and* `a refusal is distinguishable from a genuine 0.0%` red | +| the guard always refuses | reporting at exactly the minimum, and a real 40% rate, both red | +| the refusal prints as `0.0%` | `refuses a window too small` red — the case the six evidence files actually hit | + +The correction is recorded in `specs/SessionShape.md` §4, beside the +eighth instance, with the real per-pass figures. + ## Task: argue SH-3's floor, or move it ```task