CB-WP-0009-T01: the meta budget measures the pass it governs
ADR-0006 D1. cb-cost gains pass_costs, which buckets responses into workplan windows in a single read of the transcripts — calling collect() once per boundary would re-read every transcript per window, and status is supposed to stay cheap enough that nobody replaces it with ls. make status now reports the share over a trailing three passes with the per-pass breakdown, keeps the lifetime figure labelled NOT the metric, and prints the D2 exemption so the next reader does not re-derive the standoff where the budget blocked its own repair. One pass would be a coin flip: a meta pass reads 100%, a product pass 0%. Three is the smallest window where the ratio means something and still moves. First reading: 36% over the last three passes against 49% lifetime, with CB-WP-0008 at 0%. The first run reported $0.00 for every window. Transcript stamps are UTC Z and git prints a local offset, so the string comparison put every response before the first boundary. Both the fix and a self-test for it are in; reverting the conversion turns four checks red. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
parent
f4dccb6f99
commit
38f237fc5c
3 changed files with 194 additions and 26 deletions
|
|
@ -524,6 +524,55 @@ def collect(slug, pin_ref=None, since_ref=None):
|
|||
}
|
||||
|
||||
|
||||
def pass_costs(slug, boundaries):
|
||||
"""Cost per pass window, from one read of the transcripts.
|
||||
|
||||
CB-WP-0009 T01 / ADR-0006 D1: a budget must measure the window it
|
||||
governs. `boundaries` is `[(label, start_iso)]` in ascending order;
|
||||
each window runs to the next label's start, the last to now.
|
||||
Responses before the first boundary land in `_before`.
|
||||
|
||||
Reads the transcripts **once**. Calling `collect()` per boundary
|
||||
would re-read every transcript per window, and `make status` is
|
||||
supposed to stay cheap enough that nobody replaces it with `ls`.
|
||||
"""
|
||||
prices = load_prices()
|
||||
stale = check_price_sheet_age(prices)
|
||||
if stale:
|
||||
raise Abort(stale)
|
||||
paths = transcript_paths(slug)
|
||||
if not paths:
|
||||
raise Abort(f"no transcripts found for {slug}")
|
||||
responses = []
|
||||
for path in paths:
|
||||
responses.extend(read_responses(path, None))
|
||||
if not responses:
|
||||
raise Abort(f"no responses in {len(paths)} transcript(s) — refusing to report")
|
||||
|
||||
ordered = sorted(boundaries, key=lambda b: b[1])
|
||||
out = {label: {"cost": 0.0, "responses": 0} for label, _ in ordered}
|
||||
out["_before"] = {"cost": 0.0, "responses": 0}
|
||||
|
||||
for r in responses:
|
||||
cost = price_of(prices, r["model"], r["toks"], r["timestamp"]) or 0.0
|
||||
bucket = "_before"
|
||||
for label, start in ordered:
|
||||
if r["timestamp"] > start:
|
||||
bucket = label
|
||||
else:
|
||||
break
|
||||
out[bucket]["cost"] += cost
|
||||
out[bucket]["responses"] += 1
|
||||
|
||||
# Positive control: a bucketing that placed nothing in any real window
|
||||
# would report a confident 0% for every pass.
|
||||
placed = sum(v["responses"] for k, v in out.items() if k != "_before")
|
||||
if ordered and placed == 0:
|
||||
raise Abort("pass_costs bucketed every response before the first "
|
||||
"boundary — the boundaries are wrong, not the spend")
|
||||
return out
|
||||
|
||||
|
||||
def render(rep, by_task=False, composition=False):
|
||||
print(f"M-D2-CST cost report — {rep['slug']}")
|
||||
print(f" pin {rep['pin'] or '(none — live file, not reproducible)'}")
|
||||
|
|
@ -625,6 +674,31 @@ def self_test():
|
|||
check("AC-8 5m cache priced at write_5m", abs(got - want) < 1e-9 and got != wrong,
|
||||
f"${got:.4f} (1h would be ${wrong:.4f})")
|
||||
|
||||
# ADR-0006 D1: pass_costs buckets by window, and refuses to report a
|
||||
# confident 0% when every response landed before the first boundary.
|
||||
slug = "-home-worsch-clay-borg"
|
||||
try:
|
||||
real = pass_costs(slug, [("early", "2020-01-01T00:00:00Z")])
|
||||
check("pass_costs places responses in a real window",
|
||||
real["early"]["responses"] > 0 and real["early"]["cost"] > 0,
|
||||
f"{real['early']['responses']} response(s)")
|
||||
two = pass_costs(slug, [("a", "2020-01-01T00:00:00Z"),
|
||||
("b", "2026-07-31T00:00:00Z")])
|
||||
check("pass_costs splits at a boundary rather than pooling",
|
||||
two["a"]["responses"] > 0 and two["b"]["responses"] > 0,
|
||||
f"a={two['a']['responses']} b={two['b']['responses']}")
|
||||
check("pass_costs windows sum to the unwindowed total",
|
||||
abs((two["a"]["cost"] + two["b"]["cost"] + two["_before"]["cost"])
|
||||
- real["early"]["cost"] - real["_before"]["cost"]) < 0.01)
|
||||
except Abort as e:
|
||||
check("pass_costs places responses in a real window", False, str(e))
|
||||
try:
|
||||
pass_costs(slug, [("future", "2099-01-01T00:00:00Z")])
|
||||
check("pass_costs aborts when nothing lands in any window", False,
|
||||
"no Abort raised — every pass would read $0.00 and 0%")
|
||||
except Abort:
|
||||
check("pass_costs aborts when nothing lands in any window", True)
|
||||
|
||||
# AC-5: dedup invariant is enforced.
|
||||
import tempfile
|
||||
with tempfile.NamedTemporaryFile("w", suffix=".jsonl", delete=False) as fh:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue