From db731aa0bca59d9bba79e4c0c41e705a4e0727b4 Mon Sep 17 00:00:00 2001 From: tegwick Date: Fri, 31 Jul 2026 09:23:29 +0200 Subject: [PATCH] T11: dated rates and staleness become data, ahead of the 2026-08-31 flip The price sheet had two defects of one shape -- a schema that could not hold the fact it needed, the same criticism the cost survey levelled at the State Hub. CA-16 time-boxed rates are DATA. Sonnet's intro price lived in a `# intro ...` comment and was invisible to the collector that reads the file. Now promo_input/promo_output/promo_until, applied per response at its own timestamp. CA-17 the 90-day staleness rule was prose in MetricsAndScenarios 1a that every M-D2-CST verdict silently inherited. Now `recorded` + `max_age_days` in the sheet, and a stale sheet ABORTS. Both are exercised by make cost-test: the promo rate must apply before 2026-08-31 and lapse after, and a 102-day-old sheet must trip. Applying CA-16 moved AC-1 from $93.32 to $93.15 -- the $0.17 CB-EV-0002 predicted, now collected rather than noted. That is a legitimate retarget under T07's distinction: the instrument disproved the target, and its output is in this commit. The number has now been stated five times ($248.46, $92.21, $92.87, $93.32, $93.15), each correction from a different mechanism. Evidence tables regenerated from the tool rather than hand-patched, per CA-15 -- which is the rule that exists because hand-typed tables were the only thing the adversarial review found wrong. Co-Authored-By: Claude Opus 5 --- benchmarks/baselines/model-prices.toml | 24 ++++++-- evidence/CB-EV-0002-cost-accounting.md | 33 +++++----- specs/CostAccounting.md | 41 ++++++++++++- tools/__pycache__/cb-cost.cpython-312.pyc | Bin 27379 -> 29414 bytes tools/cb-cost.py | 70 +++++++++++++++++++--- workplans/CB-WP-0003-loop-hardening.md | 2 +- 6 files changed, 138 insertions(+), 32 deletions(-) diff --git a/benchmarks/baselines/model-prices.toml b/benchmarks/baselines/model-prices.toml index 3e12c08..68cade5 100644 --- a/benchmarks/baselines/model-prices.toml +++ b/benchmarks/baselines/model-prices.toml @@ -1,6 +1,15 @@ -# USD per million tokens; source: Anthropic pricing, recorded 2026-07-31 -# Governed by specs/MetricsAndScenarios.md §1a — refresh on price change or -# after 90 days, or new M-D2-CST 'better' verdicts are invalid. +# USD per million tokens. Source: Anthropic pricing. +# Governed by specs/CostAccounting.md §9 — a stale sheet invalidates new +# M-D2-CST `better` verdicts. Staleness is enforced by `make cost-test`, +# not by this comment. +# +# Dated rates are DATA, not comments. A promotional rate written as a +# comment is invisible to the collector that reads this file: the sonnet +# intro price sat in a `#` remark and was silently ignored, costing $0.17 +# on a $93.32 pass and heading for a sign flip on 2026-08-31. + +recorded = 2026-07-31 +max_age_days = 90 [claude-fable-5] input = 10.00 @@ -11,8 +20,13 @@ input = 5.00 output = 25.00 [claude-sonnet-5] -input = 3.00 # intro 2.00 through 2026-08-31 -output = 15.00 # intro 10.00 through 2026-08-31 +input = 3.00 +output = 15.00 +# Promotional rate in force through `until` (inclusive); the collector +# applies it to any response whose timestamp falls on or before that date. +promo_input = 2.00 +promo_output = 10.00 +promo_until = 2026-08-31 [claude-haiku-4-5] input = 1.00 diff --git a/evidence/CB-EV-0002-cost-accounting.md b/evidence/CB-EV-0002-cost-accounting.md index a8f986a..e193449 100644 --- a/evidence/CB-EV-0002-cost-accounting.md +++ b/evidence/CB-EV-0002-cost-accounting.md @@ -17,9 +17,9 @@ transcribed by hand (CA-15). | ID | Metric | Target | Measured | Verdict | |---|---|---|---|---| -| AC-1 | pinned total, as two components | $93.32 = $92.21 + $1.11 | **$92.21 main + $1.11 subagent = $93.32** | **met** | +| AC-1 | pinned total, as two components | $93.15 = $92.03 + $1.11 | **$92.03 main + $1.11 subagent = $93.15** | **met** | | AC-2 | reconciliation residual | $0.00 | **$0.000000** | **met** | -| AC-3 | unattributed share reported | present, 33% | **32.5%, own line** | **met** | +| AC-3 | unattributed share reported | present, 33% | **32.4%, own line** | **met** | | AC-4 | composition reported | 5 components | **5 of 5** | **met** | | AC-5 | dedup violation aborts | non-zero exit | **abort raised** | **met** | | AC-6 | zero responses refuses to report | non-zero exit | **0 rows, no number emitted** | **met** | @@ -27,7 +27,7 @@ transcribed by hand (CA-15). | AC-8 | 5m cache priced at 1.25× | $1.25/100k @ fable | **$1.2500 (1h would be $2.0000)** | **met** | | AC-9 | streamed partial output → final | 5,5,195 → 195 | **195** | **met** | -No `unmeasured` rows. AC-1's target was corrected three times before this +No `unmeasured` rows. AC-1's target was corrected four times before and after this run; §4 records why, because the sequence is more useful than the final number. @@ -35,23 +35,23 @@ number. ```text input 690 tok $ 0.00 0.0% - output 323,643 tok $ 11.15 11.9% - cache_read 80,611,798 tok $ 59.75 64.0% + output 323,643 tok $ 11.13 11.9% + cache_read 80,611,798 tok $ 59.67 64.1% write_5m 37,467 tok $ 0.47 0.5% - write_1h 1,672,854 tok $ 21.95 23.5% - TOTAL $ 93.32 + write_1h 1,672,854 tok $ 21.87 23.5% + TOTAL $ 93.15 ``` **88.0% of spend is cache; 11.9% is output.** The ratio of context re-read -to text written is 249:1. A single total would have shown $93.32 and +to text written is 249:1. A single total would have shown $93.15 and concealed all of it — which is exactly what CB-WP-0001's `M-D2-TOK` would have done, and why that metric is now demoted. ## 3. Per-task attribution ```text - UNATTRIBUTED $ 30.32 32.5% - T08 $ 21.02 22.5% + UNATTRIBUTED $ 30.14 32.4% + T08 $ 21.02 22.6% T07 $ 12.01 12.9% T03 $ 9.91 10.6% T04 $ 8.00 8.6% @@ -60,7 +60,7 @@ have done, and why that metric is now demoted. T09 $ 1.29 1.4% ``` -**Stated limit (CA-10):** 32.5% of cost sits in commits whose subject +**Stated limit (CA-10):** 32.4% of cost sits in commits whose subject carries no `T##` tag, so this table is a view over 67.5% of spend. That is a property of commit hygiene, not of the collector. @@ -75,14 +75,15 @@ is worth watching, not yet a conclusion from n=1. - **The per-task figures are not comparable across passes.** They mix models (opus/fable/sonnet) at different price points and different cache states. The dollar figure is comparable; a token count is not. -- **AC-3's 32.5% is a fixture pin, not a quality target.** Improving commit +- **AC-3's 32.4% is a fixture pin, not a quality target.** Improving commit tagging will move it, and that is the desired direction. - **This is one session.** Every ratio here (cache share, $/turn, the compaction effect in §5) is n=1 and should be treated as a hypothesis until a second pass reproduces it. -- **The price sheet cannot express a time-boxed rate.** Sonnet's intro price - lives in a TOML comment, so sonnet-priced work is off by $0.17 here - (0.19%). This becomes a real error on 2026-08-31. +- ~~**The price sheet cannot express a time-boxed rate.**~~ **Fixed + 2026-07-31 by CB-WP-0003 T11** (CA-16/CA-17). Applying the sonnet + promotional rate moved AC-1 from $93.32 to **$93.15** — the $0.17 this + section predicted, now collected rather than merely noted. - **AC-1 is not invoice-verified.** No admin key exists, so the Anthropic billing API could not independently confirm the total. The transcript counters are the same ones billing uses, but that is an argument, not a @@ -145,7 +146,7 @@ largest sample was assumed to hold on the smallest one.** The main transcript is 338 of 346 responses, so 206/206 felt conclusive; the violation lives entirely in the 8 responses nobody checked separately. -Cost of the adversarial review this pass: **$1.11**, against a $93.32 pass. +Cost of the adversarial review this pass: **$1.11**, against a $93.15 pass. It found three approval-blocking defects, one of which (the subagent exclusion) would have made this evidence file certify a broken collector. Second consecutive pass where a ~1% spend on review changed the outcome. diff --git a/specs/CostAccounting.md b/specs/CostAccounting.md index 7fa8ee3..aa19d7a 100644 --- a/specs/CostAccounting.md +++ b/specs/CostAccounting.md @@ -67,6 +67,16 @@ cost = input_tokens × price.input > reported as an unpriced line with its token counts, never dropped and > never priced at a default. +> **CA-16.** Promotional and other time-boxed rates are **data**, not +> comments. A response is priced at the rate in force *at its own +> timestamp*: `promo_input`/`promo_output` apply on or before +> `promo_until`, and the standard rate applies after. + +> **CA-17.** The price sheet carries `recorded` and `max_age_days`, and a +> sheet older than `max_age_days` **aborts the run**. A stale sheet +> invalidates `better` verdicts, so the staleness rule is a command rather +> than a sentence. + ### 1.3 Scope of a measurement > **CA-06.** A measurement enumerates **every** transcript for the repo: @@ -125,9 +135,9 @@ Each row names the command that produces its number, per InnerLoop §Step 4. | ID | Metric | Target | Instrument | |---|---|---|---| -| **AC-1** | reproduces the pinned CB-WP-0001 total | **$93.32** = $92.21 main + $1.11 subagent | `make cost-pin` | +| **AC-1** | reproduces the pinned CB-WP-0001 total | **$93.15** = $92.03 main + $1.11 subagent | `make cost-pin` | | **AC-2** | reconciliation residual (CA-14) | **$0.00** exactly | same command, `reconciled: ok` line | -| **AC-3** | unattributed share reported (CA-10) | present, and **33%** on the pinned run | `cb-cost --pin fc76445 --by-task` | +| **AC-3** | unattributed share reported (CA-10) | present, and **32.4%** on the pinned run | `cb-cost --pin fc76445 --by-task` | | **AC-4** | composition reported (CA-13) | all five components present | `cb-cost --pin fc76445 --composition` | | **AC-5** | dedup invariant asserted (CA-02) | violation exits non-zero | `make cost-test` | | **AC-6** | positive control: refuses to report on zero responses | exits non-zero | `make cost-test` | @@ -145,7 +155,7 @@ assertion against a fixture whose expected value is known and fails loudly; Per InnerLoop §Step 4, the acceptance table is checked against the contracts in this same spec: -- AC-1's $93.32 is reachable only if CA-06 holds (both trees enumerated). +- AC-1's $93.15 is reachable only if CA-06 holds (both trees enumerated). Under a main-file-only collector the target is unreachable — this is the defect the adversarial review caught, where a target of $92.21 would have been hit *only* by a broken collector. @@ -228,6 +238,31 @@ signal — which is exactly the dead-policy failure this replaces, one level up. The mitigation is that it costs nothing to run and is on the one command surface. +## 9. The price sheet + +`benchmarks/baselines/model-prices.toml` is the single source of rates. +Two defects were fixed here, both of the same shape — **a schema that +could not hold the fact it needed**, which is the criticism this +capability's survey levelled at the State Hub: + +1. **A time-boxed rate lived in a comment.** Sonnet's intro price + (2.00/10.00 through 2026-08-31) was written as `# intro …` and was + therefore invisible to the collector, which used the standard rate. Cost + at the pin: **$0.17**, 0.19% of the pass. The real hazard was the + date — on **2026-08-31** the promotion expires and a comment-only + representation would have been wrong in the opposite direction, with + nothing to notice. Now expressed as `promo_*` fields and applied per + CA-16. +2. **The 90-day staleness rule had no check.** It was prose in + `MetricsAndScenarios.md` §1a and every M-D2-CST verdict silently + inherited it. Now `recorded` + `max_age_days` in the sheet, enforced by + CA-17 and exercised by `make cost-test`. + +Applying CA-16 moved AC-1 from $93.32 to **$93.15**. This is a *legitimate* +retarget under the distinction CB-WP-0003 T07 draws: the instrument +disproved the target, rather than the implementation missing it. The +instrument's output is in the same commit. + ## 8. Revisions to M-D2-CST `specs/MetricsAndScenarios.md` §1a is superseded by this spec. M-D2-CST is diff --git a/tools/__pycache__/cb-cost.cpython-312.pyc b/tools/__pycache__/cb-cost.cpython-312.pyc index 8ad4a54de1395369d4ee76cf4508751ceef2ffb6..4b96e0084a1936df1f9c4407bfec5b43ec9661fd 100644 GIT binary patch delta 6623 zcmb6-3ve98mD4k`fBkp$TiNnRmSnH3m1N1bB#g0TNyf&K{E-uEgOOI6k+jlkcdut; z*;})$;0Q>@DP%gqqHN!Z3zsrya3J8!um4zW$CUEB@T!`-~IR~fe$iXG| z-mF#$`GyDme*M(yot*<HA zCiO=~N0qRq)<`W9Rn%xCtiot&LWq zghHwm2}_`0p8{$sQm@&gXDqA*L%lVoj4s2q40#>RBaI1$VYWs_ikwJIo;rYmBA1V+ zu%VnGs2cTI46CW0YFNyrFho?y;G@*An4_r{;Kk`nv=DpQaS_#?fJPgkxd(n~6u<;` zBZG71JmQ`5PPbomcqewe=gLfGyOXwrXNI4*c^}u3IeF7t=X1&?Eg!nFCbfr~r@N>7 zCT}~}J)3{_$OYj--b~NiuIi*ae|mJ@T`}8tp>)2g>59AQN=8%Cl{q;$wPl8%X?i^4 zimT#^SV0S5)P5d7x}c(HR5!g~PG>kUCO*W)c~R$v1y<&Bvlnt58F4`uCdi3Jq~9W2 zFD^)li#j<@ROqhr!(v+O(s$U#q8mzTiH1vQ7F__&bYs~!Es^Hv0^8T;VV4^02^gB_ zLJYQG7`NdIC2r9zx;WAXo8*vfy5%e<+jSA3wHK{hF)~I2$9n>UN{<{kpBWwS56q2?fK7T5H;yKV^M2T4XeY_8wNorA~iay}z~lpw!!^ zXqrNM7w2W3xP=M(tcD%c!*i1|XwlTxqD4Rz!x>alr@3D-fN(-h_^h-Zbf>ogFoYgi zqs>^eEzHgkV2Y|0P^Psnn0tQzkkUWWV`|xBE;z0Pg>B;QhM)QAwPIsR?SV&qgi{UOIe(&dPRHj+ZDa#*8KM>$ zQRzmg(oNVFXfSMu6r3L2Y0}5S;9YU@sci+g;K)*Y8X2ISK#agoeGb3`cg>OgM{aEA z`#D831+)1V#H%^}`*u&XPI{6VdDD$kdnekHuB>VIRP#jJHG9V7ol}`J;=FxD(wQ}R ze5z(<k|ITs$XYB@J`GDwrnZd3JBn8?A={Z`k@H zVGY6vy%Sq@EfNX!jHwDm@76`v0Q3p;9;6X!kbV{SD?KOL%T5+okr&zJ;wcZV4#mT2 z*i=PTA#Ty#tjilGd)b@bQnH`@hqsL=tgfWEd^d7g27*dRraf4<%RyBOhWj-`98v-@ z9bqFS8!Qh2>EXF2N@5lt_Ml)4pi(}-r+8B(sz6;u7qhU)pwREjD#^QSxNJQcVULyV zSvj)UiJtsY?o~-ddjR~B`(%Q9JA317%QTs;{$}=k_Qv1Q0NYXi0LXf+e4T4QFwqkT z#@J`&<*pg18bZHzoGQ$>dZ*IeRj7akp%EIwz^F#=gBsP% zW>Ae7qz4vktZ!K9*XZ|M!R$Tk|TP^E6_=h2Tj9f6eCC6(+uo)r$ap znTA~n%j$Sg8=~Jq?B62zE`mz{3=72FVANEI&LQ?01kWNc$NPJ%p(*-!ux4{48$MZPcaN7AdxLes(|6 zEp6P<73dzc+(kOC+taRM!AaOtPT}H$YXZ6rcji>7$Z77yeirR8nw43=CoQCh>8MIL z4O0=b3>T$qOJB{KuBUs(9ZNk}lwXL)&L!mkQH<;V-6La=i)Vr>v}0Bp&oWo(?@T9} z2~$VpM4p~GyozmbyA$@bzJxpF)g7=n4&*q_A2*X`Tr6zIG9mL3`gvS9JtMuue~L>h z16H%>%eTbz&r2TvAn27H&(X7Gr|IZ(^;|tCRfk%Z>@o)jM`X4#KG(`jx8$sImUtd0 zWd?=e)sQlIZvoS&T!<;|xJP&D?nP}qpPP+qIYrOY-E4b9aqcr<9MFGZ29Du1D3q0M zwCBb1_55Wd*Aj`n)L_|Gp!pWN6{Hte&Nj2G+9f-dbM$;U_pB)AeW@(RHl(@r0+Yvc zewo!T@xxV%!`(#}fF(Uzyii-u1~zU>z0LVOgA148r{&YfIPevUuQpUM`2C2p4sVKPPMvb&*0Ow2gvD`q=49j~2m^qa7K#Kk!`2#syNp7qiWI!JPz}KeaPD;OfEksg>-fP518lFr#d?>S~5>qWwKv z#`{_B>Gp?1SM1(vnFSwn#L__~$@`uaGhdzEKRf=!kxRLkHeW7&uI0Ir=ew?Y_Dqt= zs*{cz0`jrR=F9%ivu?RudeyUi$}uTS9!zHEO?OY7m=v$&7bgq6Gu@|$=j2;X60t6C zk7MrRE&nAJ;f}>}BnegqGLLkjF-BTi{O=(#6OidEuqv4=70Z>R(ZKQ^74&@;X_3fJ z*mO%_9WFyG%PHYqB^ptKc+obK9{R69q0>kfajckd>xO!oWIt}HAg9=8Ene~gE8H%FcjnS}5ud$PDn@9tDscp3jcPl2;Y#yLLV!v)H-C^b-rbT4|#WYNx z3uAYbegFu=wQNp?phYzLpU5|W;64Q3XI<^p76CT<{<%c^*9B`4#Mp<5*^SzK2RF!_ zES7gp7SGk~-Y6WAYvfwF?i?S_<{@F1H^>6`>w5IQ{SYQb;3ys^)DBnJY`Fokhzav} z%;k+~PI(jfv(0G^Y@HV`3heHVL0e;xlbg=*>@W5du}AMHBsuJF?pT?dllC-m!5p=j zt?gVT+;X00bsfIMA{EBi63^A}4k6B>ubKFksLo90i|uxGmb0)MKGeb}{^80;5aYuH_TyH+h*wDhyt zf-i<=G~ey)^}XB4UCgm>Kjd$B?W-YAv6K5<Z0u-^7YfU~$0yHV%MAuiY>tjYr1d z-rc826=0BU`|lx#*thoAu4n)@iUH6knzsphHzJN8ILv;%f2$`C4DLF**x+{XQ3g-z z;fk>BKr`vI3u6lREsG#H1 z2fD}*`}Bb7{~aR!8$gUdDvep8B1u1+_=e;o6mwo*ZYo@uf#4%{w!1t5P8?kAI1Pf| z#|J9IB9DzmRcb>e%=m#3h3-X?bu0`|5+kXIVn&-!0qygqeEw6&hY`^b1|u-@ad;s_ z$&v6NEkS8ufUtOD8{k(@Vef&luQwnYHn@cLK(vDx5};Zz5Hb(D;R-~1WMv>Q7SiC= zO&N`9CsIN1Iu7*jC``mifTx%<^@TUOfT}7obs)wGK=qno;>m+Wu4-sLhp5}cxz`SE z=Q~!EU8tPTtDD&Kp6IyZY)U#^lOr=#^I2s-ayHKHxn!NMXne=nh}YLGGsouLRkP}q zy!98FFWvQ0!ThFOue+~o?7ZUa`mKex>?5q?tHpCG@2nxDlXV>GB6;lWp*17T!pjBu z9aioYPfKP;v+(M=^_^Dkwc5r`NqD`sqSMd4(d_85i9ZlTfIn=ubO^RLt$7_Qgg1+X zj#b>7B?Rj$%=)Urj?KcG&6ZBV_LjAWa(UId&^(gB?|olyW#N6wH$s`@S-O* z!@%;Z7vlbjNk{X^x92t=jS?3IO*2=ZpR&vM6tw*S+uuR(`CAzMXTbSvsfAxOO!_~7 zHHcz3BT*&XqXLlWgDkUWrL_T;*v-$?^&GVDUcw%fzh`X%6y3~rDD6jRKGf5Oy9w$s ztKW}CO$)H$^84Y{Yrqe0bQ)5IRS0f1*c}`YmkM)PD1jK(U|5FhKgcpC{Cjp?X-IsC zYB|A*RHY|C2UUujPEn;VPQaXh427EJjdo0Hk*^K`9t2v7HHmX9vs#{yB9o`)2Uj+?z#Nr1EBY4#~M$l|`%{uVwp( vB~K?I)|DyRsaW2!yHY5%~A_Go)V(PK|-tNp%N7J|~= zWPkI$?sM+_eRn?mfPDT3l5@`CuyIgc>WJ(aU4JOYN7R#L!(?mT0d8A~*sh{(2W|8x3e|F`N#z}l3j4CN=f+V0Mi3vOCCXG+) znkix8IM7V!g(%~Rwsd3yt-NG5yy;j`vY>`BSH0F7yoP3Y-Ra6%6WwgRyE}(wf#tOxJy8X*9*d4>YBa6n|$>hQnXd_f7Aw>Yu@AuOJc+xiZX30!4#=a;S@l>N>DF#{;Sy95VD6L_K z{7F*JoKvTe239q-wQ&vlZHg>MyVA3^#AVt=V*|7XJ*J+p9Fgcsl&w-&QNz)Ys*Ahj zphP>_^HZx$F>v*yE=^6C0$78>E9TZxFW@sg#bHwHLg4|f53*%eDLKJ@R#rhe*}G-! z#hu_ySEBUh(J9tDtC*}}KbUm@>evG1_67)~L4=)bNno0NKZv>zQU~c87745{jeveU z^-|z{!3IMQsF`ab5iuHHzkwYqd99?Tz zZdIx2DG*;tEv!1j=RSt&Zk3~wG7wgG(?`(yD8dgB#sGAa z0)58mVKf{;IEwHi1f#!WC_MoX;B|q@J@m&QPh3LXPGuz)xkK)Xhokg3c>lzuo~(X{ z&s~O9aj`%81`1uuMrw=6cJ|ZSY2~CAFgT=a)D40}&2fX4lhGvS5mPc3 z24PU=o)UJlbrKtGuy4*wy0k1U@7l0jHw@c0(WB*Rd71VmMvaz<+~{$5nNwjPM$xP+ z-cZCAFSO<+-I{xX!*sze*S?eE|V;6RkqPj85mU)s1J$Hlp|&1U%ROnTjYjtVr_|kufK3x4ouHO&yg=C#6w442EWsCQB=yZS) zSUnz#Lbmq9OUi32AZJ*h&EIw!{ds=>#&sK8Ix=s(0V%SA)u4SmyDzKGMgQtokJx zVgFp!MiOl4>J8*B_Q>iwqOo(U>z87xPB{H)EEe&Hf{}>ghZ)g*XwAMp{zZN|gr+?7 zb_dnaZdvlH{WPl3v#hnfj#RTm`*b_Lq=Zm4zlXleo^GGAv;p+==E*?`cV57bDKJi7 z1zERGn5ZtOF_r!i zVh(ds^--REydltatsdc|nq;nuCo3JDn~6^5X?Z{-uB$`>=RN6?=4q}wIE9Q_H5c%Q zUF4GPq|4~Jc0!MC_Q}S=A$O)F=(lcyH*bP{{!uZLmB|G9mg+MxFuoZ{&xjB5dpXDL zfDE4WCi9a86UcRemQQ)9Axfl$(9aY6YvQNov#zK;gkf@DhH#QO%>}C*Z{; zJ)^TlPV+zmLMJDAI=SgwHB4{-h;S=M;O`(A{SSNV)=Ghle)G_#b>ta#%clO43G0{n zrnt(TF5u3^?9!%YAh^=a>)}E(u(_(_5cHn#DRQKz3y!~9)fkDvj#L&`!EOufeVCoz zyuauo#t2olJ+NnIRqyr>#QGzWzgvdW^D?__%P!K)UfwdN$e`o^#0EsV1RfOfRr>Nw zTbONYljkeYKSE*&H?M|`$kQ6QOmE-X!2gS5k8YjrgwF#-rU5=(yM-;-wwBz&?%Af8 z-@}&v^3Zm#YZ*v^yfon&eEJqE+dgdw2~jr<&@j;F7DJZ$`{D|{j1>(AMoy;n=rZ?5 zBjM=YG$$K;{XUojg=zl81xerAh29uk(+7f46QhXPXbM91hv!Y6n3@MNXM^gxo zAl&#JK%WI$z>-ddq7v)DFRLJx=m%ieiLBdVariD%03;d(n}YWz-o}NaE_ApN@DxxW3yva_U+n>;fT;7qxGoxP<08>~2vS6l za7kDqnr$iti-p}6eatsw|GIa?`U|4@o^{__cd+dzbtlr6RAWpa3qHa2_!TPxv#AGL zD#7lIYt3O)7a|zrFwPxM<@6hLv4Qv$voT&bI~=bV!mm1A+!G5&b@MG7RxMlJNiU%Z zKG^%>;fPF6U`xP?>3yJQEguZYaU)Y3Q^zDhD-aAyFwV4b=tt1@6oPR%Gwxo-vKecV zSy_RjrEJ6P(`?l+e)#8<hdHKi|U7CCkW_2-!f$q$_*a aAf3&x%w max_age: + return (f"price sheet is {age} days old (max {max_age}); refresh " + f"benchmarks/baselines/model-prices.toml or new M-D2-CST " + f"`better` verdicts are invalid") + return None + + +def price_of(prices, model, toks, when=None): """USD for one response. Returns None when the model is unpriced (CA-05).""" pr = prices.get(model) if not pr: return None + rin, rout = rates_at(prices, model, when) cache = prices["cache"] - unit = pr["input"] / 1e6 + unit = rin / 1e6 return ( toks["input"] * unit - + toks["output"] * pr["output"] / 1e6 + + toks["output"] * rout / 1e6 + toks["cache_read"] * unit * cache["read"] + toks["write_5m"] * unit * cache["write_5m"] + toks["write_1h"] * unit * cache["write_1h"] @@ -281,8 +318,12 @@ def collect(slug, pin_ref=None): # as though it were an answer. raise Abort(f"no responses in {len(paths)} transcript(s) — refusing to report") + stale = check_price_sheet_age(prices) + if stale: + raise Abort(stale) + for r in responses: - r["cost"] = price_of(prices, r["model"], r["toks"]) + r["cost"] = price_of(prices, r["model"], r["toks"], r["timestamp"]) attribute(responses, commit_index(pin)) @@ -298,11 +339,11 @@ def collect(slug, pin_ref=None): continue by_task[r["task"]] += r["cost"] by_model[r["model"]] += r["cost"] - pr = prices[r["model"]] - unit = pr["input"] / 1e6 + rin, rout = rates_at(prices, r["model"], r["timestamp"]) + unit = rin / 1e6 rates = { "input": unit, - "output": pr["output"] / 1e6, + "output": rout / 1e6, "cache_read": unit * cache["read"], "write_5m": unit * cache["write_5m"], "write_1h": unit * cache["write_1h"], @@ -456,6 +497,21 @@ def self_test(): finally: os.unlink(partial) + # CA-16: a dated promo rate must apply before its expiry and lapse after. + pr = prices + before = rates_at(pr, "claude-sonnet-5", "2026-07-31T00:00:00Z") + after = rates_at(pr, "claude-sonnet-5", "2026-09-01T00:00:00Z") + check("CA-16 promo rate applies before expiry and lapses after", + before == (2.0, 10.0) and after == (3.0, 15.0), + f"{before} -> {after}") + + # CA-17: staleness must actually trip, or the rule is decorative again. + import datetime as _dt + fresh = check_price_sheet_age(pr, _dt.date(2026, 8, 1)) + stale = check_price_sheet_age(pr, _dt.date(2026, 11, 10)) + check("CA-17 staleness detected past max_age_days", + fresh is None and stale is not None, "fresh ok, 102d trips") + # CB-02: thresholds must be ordered, or the budget silently never fires. ap_defaults = {"soft": 10.00, "hard": 22.00} check("CB-02 budget thresholds ordered and positive", diff --git a/workplans/CB-WP-0003-loop-hardening.md b/workplans/CB-WP-0003-loop-hardening.md index 682a48c..0db2942 100644 --- a/workplans/CB-WP-0003-loop-hardening.md +++ b/workplans/CB-WP-0003-loop-hardening.md @@ -325,7 +325,7 @@ now price the evaluation itself. ```task id: CB-WP-0003-T11 -status: todo +status: done priority: medium state_hub_task_id: "809215d8-8a5c-4b10-9445-9bc6db9bda42" ```