T11: dated rates and staleness become data, ahead of the 2026-08-31 flip
The price sheet had two defects of one shape -- a schema that could not
hold the fact it needed, the same criticism the cost survey levelled at
the State Hub.
CA-16 time-boxed rates are DATA. Sonnet's intro price lived in a
`# intro ...` comment and was invisible to the collector that
reads the file. Now promo_input/promo_output/promo_until,
applied per response at its own timestamp.
CA-17 the 90-day staleness rule was prose in MetricsAndScenarios 1a
that every M-D2-CST verdict silently inherited. Now `recorded`
+ `max_age_days` in the sheet, and a stale sheet ABORTS.
Both are exercised by make cost-test: the promo rate must apply before
2026-08-31 and lapse after, and a 102-day-old sheet must trip.
Applying CA-16 moved AC-1 from $93.32 to $93.15 -- the $0.17 CB-EV-0002
predicted, now collected rather than noted. That is a legitimate
retarget under T07's distinction: the instrument disproved the target,
and its output is in this commit. The number has now been stated five
times ($248.46, $92.21, $92.87, $93.32, $93.15), each correction from a
different mechanism.
Evidence tables regenerated from the tool rather than hand-patched,
per CA-15 -- which is the rule that exists because hand-typed tables
were the only thing the adversarial review found wrong.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
parent
06628e83e1
commit
db731aa0bc
6 changed files with 138 additions and 32 deletions
Binary file not shown.
|
|
@ -72,16 +72,53 @@ def components(usage):
|
|||
}
|
||||
|
||||
|
||||
def price_of(prices, model, toks):
|
||||
def rates_at(prices, model, when=None):
|
||||
"""Input/output rate in force for `model` at ISO instant `when` (CA-16).
|
||||
|
||||
Promotional rates are data, not comments. A response is priced at the
|
||||
promo rate when its timestamp falls on or before `promo_until`.
|
||||
"""
|
||||
pr = prices.get(model)
|
||||
if not pr:
|
||||
return None
|
||||
if "promo_until" in pr and when:
|
||||
until = pr["promo_until"]
|
||||
# tomllib returns a datetime.date for a bare TOML date.
|
||||
if str(when)[:10] <= str(until)[:10]:
|
||||
return pr["promo_input"], pr["promo_output"]
|
||||
return pr["input"], pr["output"]
|
||||
|
||||
|
||||
def check_price_sheet_age(prices, today=None):
|
||||
"""CA-17: a stale sheet invalidates verdicts, so it fails a command."""
|
||||
import datetime as _dt
|
||||
|
||||
recorded = prices.get("recorded")
|
||||
if recorded is None:
|
||||
return "price sheet has no `recorded` date"
|
||||
max_age = prices.get("max_age_days", 90)
|
||||
today = today or _dt.date.today()
|
||||
if isinstance(recorded, _dt.datetime):
|
||||
recorded = recorded.date()
|
||||
age = (today - recorded).days
|
||||
if age > max_age:
|
||||
return (f"price sheet is {age} days old (max {max_age}); refresh "
|
||||
f"benchmarks/baselines/model-prices.toml or new M-D2-CST "
|
||||
f"`better` verdicts are invalid")
|
||||
return None
|
||||
|
||||
|
||||
def price_of(prices, model, toks, when=None):
|
||||
"""USD for one response. Returns None when the model is unpriced (CA-05)."""
|
||||
pr = prices.get(model)
|
||||
if not pr:
|
||||
return None
|
||||
rin, rout = rates_at(prices, model, when)
|
||||
cache = prices["cache"]
|
||||
unit = pr["input"] / 1e6
|
||||
unit = rin / 1e6
|
||||
return (
|
||||
toks["input"] * unit
|
||||
+ toks["output"] * pr["output"] / 1e6
|
||||
+ toks["output"] * rout / 1e6
|
||||
+ toks["cache_read"] * unit * cache["read"]
|
||||
+ toks["write_5m"] * unit * cache["write_5m"]
|
||||
+ toks["write_1h"] * unit * cache["write_1h"]
|
||||
|
|
@ -281,8 +318,12 @@ def collect(slug, pin_ref=None):
|
|||
# as though it were an answer.
|
||||
raise Abort(f"no responses in {len(paths)} transcript(s) — refusing to report")
|
||||
|
||||
stale = check_price_sheet_age(prices)
|
||||
if stale:
|
||||
raise Abort(stale)
|
||||
|
||||
for r in responses:
|
||||
r["cost"] = price_of(prices, r["model"], r["toks"])
|
||||
r["cost"] = price_of(prices, r["model"], r["toks"], r["timestamp"])
|
||||
|
||||
attribute(responses, commit_index(pin))
|
||||
|
||||
|
|
@ -298,11 +339,11 @@ def collect(slug, pin_ref=None):
|
|||
continue
|
||||
by_task[r["task"]] += r["cost"]
|
||||
by_model[r["model"]] += r["cost"]
|
||||
pr = prices[r["model"]]
|
||||
unit = pr["input"] / 1e6
|
||||
rin, rout = rates_at(prices, r["model"], r["timestamp"])
|
||||
unit = rin / 1e6
|
||||
rates = {
|
||||
"input": unit,
|
||||
"output": pr["output"] / 1e6,
|
||||
"output": rout / 1e6,
|
||||
"cache_read": unit * cache["read"],
|
||||
"write_5m": unit * cache["write_5m"],
|
||||
"write_1h": unit * cache["write_1h"],
|
||||
|
|
@ -456,6 +497,21 @@ def self_test():
|
|||
finally:
|
||||
os.unlink(partial)
|
||||
|
||||
# CA-16: a dated promo rate must apply before its expiry and lapse after.
|
||||
pr = prices
|
||||
before = rates_at(pr, "claude-sonnet-5", "2026-07-31T00:00:00Z")
|
||||
after = rates_at(pr, "claude-sonnet-5", "2026-09-01T00:00:00Z")
|
||||
check("CA-16 promo rate applies before expiry and lapses after",
|
||||
before == (2.0, 10.0) and after == (3.0, 15.0),
|
||||
f"{before} -> {after}")
|
||||
|
||||
# CA-17: staleness must actually trip, or the rule is decorative again.
|
||||
import datetime as _dt
|
||||
fresh = check_price_sheet_age(pr, _dt.date(2026, 8, 1))
|
||||
stale = check_price_sheet_age(pr, _dt.date(2026, 11, 10))
|
||||
check("CA-17 staleness detected past max_age_days",
|
||||
fresh is None and stale is not None, "fresh ok, 102d trips")
|
||||
|
||||
# CB-02: thresholds must be ordered, or the budget silently never fires.
|
||||
ap_defaults = {"soft": 10.00, "hard": 22.00}
|
||||
check("CB-02 budget thresholds ordered and positive",
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue