T04: specs/SessionShape.md — compaction is the lever, not session length
The task's original premise was wrong and is recorded rather than
deleted. It was written to prescribe one task per session; measurement
says the variable is context, not turn count.
Measured:
- compaction cut context 27x (542,991 -> 19,974) and cost/turn 3.1x;
the 202 turns after C1 cost less than half the 136 before it
- a turn costs $0.010 at 20k context and $0.270 at 540k
- break-even for a compaction is 2-11 turns, so: compact whenever
context exceeds ~300k and work remains
- a fresh session is NOT free -- cold start floors at ~51k and must
then re-read the artifacts a compact summary already holds (~66k).
Prefer compaction to continue work; prefer a fresh session when the
task changes, because then prior context is pure overhead.
cb-cost now emits SH-1..SH-3 so the targets come from the instrument
rather than from analysis, per InnerLoop v1.1. All three are UNMET
(mean context 232,982 vs 200,000; p90 492,042 vs 300,000; batching 7.8%
vs 20%) and are reported unmet rather than retargeted -- retargeting in
the commit that first measures is precisely what T07 exists to prevent.
Eighth error instance found while writing this: CB-WP-0001's claim that
"0 of 330 tool calls were batched" is wrong. 330 was the count of
single-call responses, not the total; 31 responses batched, covering 76
calls. It was carried into this workplan unverified. Trusted-arithmetic
class -- the one the T01 audit flagged as having no executable defence,
confirming that finding within hours of making it.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
parent
39583357f4
commit
d382fd4555
3 changed files with 229 additions and 1 deletions
|
|
@ -147,6 +147,14 @@ def read_responses(path, pin=None):
|
|||
)
|
||||
toks["output"] = max(t["output"] for t in per_row)
|
||||
head = rows[0]
|
||||
# Tool calls are spread across the group's lines, so they are counted
|
||||
# over the whole group — one response may carry several (SS-05).
|
||||
tool_calls = sum(
|
||||
1
|
||||
for r in rows
|
||||
for c in (r["message"].get("content") or [])
|
||||
if c.get("type") == "tool_use"
|
||||
)
|
||||
out.append(
|
||||
{
|
||||
"request_id": rid,
|
||||
|
|
@ -154,12 +162,33 @@ def read_responses(path, pin=None):
|
|||
"timestamp": head.get("timestamp") or "",
|
||||
"session": head.get("sessionId") or os.path.basename(path),
|
||||
"toks": toks,
|
||||
"tool_calls": tool_calls,
|
||||
"subagent": "/subagents/" in path,
|
||||
}
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def session_shape(responses):
|
||||
"""SH-1..SH-3 from specs/SessionShape.md."""
|
||||
import statistics
|
||||
|
||||
ctx = sorted(r["toks"]["cache_read"] for r in responses)
|
||||
with_tools = [r for r in responses if r["tool_calls"] > 0]
|
||||
batched = [r for r in with_tools if r["tool_calls"] > 1]
|
||||
calls = sum(r["tool_calls"] for r in with_tools)
|
||||
pct = statistics.quantiles(ctx, n=100) if len(ctx) > 1 else [ctx[0]] * 99
|
||||
return {
|
||||
"SH-1_mean_context": statistics.mean(ctx) if ctx else 0,
|
||||
"SH-2_p90_context": pct[89],
|
||||
"SH-3_batching_rate": (len(batched) / len(with_tools)) if with_tools else 0.0,
|
||||
"p50_context": pct[49],
|
||||
"responses_with_tools": len(with_tools),
|
||||
"tool_calls": calls,
|
||||
"calls_in_batched_turns": calls - (len(with_tools) - len(batched)),
|
||||
}
|
||||
|
||||
|
||||
# ------------------------------------------------------------- attribution
|
||||
|
||||
|
||||
|
|
@ -293,6 +322,7 @@ def collect(slug, pin_ref=None):
|
|||
|
||||
sub = sum(r["cost"] or 0 for r in responses if r["subagent"])
|
||||
return {
|
||||
"session_shape": session_shape(responses),
|
||||
"slug": slug,
|
||||
"pin": pin,
|
||||
"responses": len(responses),
|
||||
|
|
@ -344,6 +374,17 @@ def render(rep, by_task=False, composition=False):
|
|||
f" A per-task table is a view over {100*(1-un/tot):.0f}% of spend."
|
||||
)
|
||||
|
||||
sh = rep["session_shape"]
|
||||
print("\n session shape (specs/SessionShape.md)")
|
||||
print(f" SH-1 mean context {sh['SH-1_mean_context']:>12,.0f} tok "
|
||||
f"[{'ok ' if sh['SH-1_mean_context']<=200_000 else 'FAIL'} target 200,000]")
|
||||
print(f" SH-2 p90 context {sh['SH-2_p90_context']:>12,.0f} tok "
|
||||
f"[{'ok ' if sh['SH-2_p90_context']<=300_000 else 'FAIL'} target 300,000]")
|
||||
print(f" SH-3 batching rate {100*sh['SH-3_batching_rate']:>11.1f}% "
|
||||
f"[{'ok ' if sh['SH-3_batching_rate']>=0.20 else 'FAIL'} target 20.0%]")
|
||||
print(f" {sh['tool_calls']} tool calls in {sh['responses_with_tools']} responses; "
|
||||
f"{sh['calls_in_batched_turns']} in batched turns")
|
||||
|
||||
if rep["unpriced"]:
|
||||
print(f"\n UNPRICED ({len(rep['unpriced'])} responses, model not in sheet):")
|
||||
for u in rep["unpriced"]:
|
||||
|
|
@ -446,6 +487,8 @@ def main():
|
|||
ap.add_argument("--pin", help="commit-ish or ISO Z instant (CA-07)")
|
||||
ap.add_argument("--by-task", action="store_true")
|
||||
ap.add_argument("--composition", action="store_true")
|
||||
ap.add_argument("--session-shape", action="store_true",
|
||||
help="SH-1..SH-3 (always shown in the default report)")
|
||||
ap.add_argument("--self-test", action="store_true")
|
||||
ap.add_argument("--json", action="store_true")
|
||||
args = ap.parse_args()
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue