CB-WP-0004 T02: make task-done — close a task on measured numbers
Replaces the three hand-done steps of a task close (46 turns, $11.52 per CB-RES-0003): the heredoc flipping status in the workplan file, the hand-written hub call, and the hand-typed token counts. The third is the reason this task exists. Every update_task_status this repo produced carried estimated tokens_in/tokens_out — in a project whose central finding is that estimated token counts are worthless. task-done reads the measured figure from the transcripts, or refuses; there is no path through it that emits an estimate. cb-cost gains by_task_detail: cost, response count, model histogram and token components per task. task-done imports cb-cost rather than parsing its printed table, so the hub figure is not a copy that can drift from its source. The positive control found a real defect before the tool ran once. Attribution keyed on a bare T\d\d from the commit subject, so CB-WP-0002 T01, CB-WP-0003 T01 and CB-WP-0004 T01 shared a bucket: the self-test reported $12.10 for "T01" where the qualified figure is $2.33. That 5.2x overstatement would have been pushed to the hub as a *measured* number — the same fiction in a new form. task_label() now keys qualified subjects on the full id and leaves unqualified ones bare rather than retro-assigning them to a workplan. The pinned $93.15 benchmark is unchanged, so historical attribution was not disturbed. Fourth instance of trusted arithmetic: a number believed because a program produced it rather than a hand. Refusals, all exercised by --self-test: unknown id, typo'd id, already-done task, missing state_hub_task_id, no measured spend, and a status flip that produced no change. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
parent
3f1dbac164
commit
b52a9ec88a
5 changed files with 419 additions and 3 deletions
|
|
@ -44,7 +44,26 @@ from repo import ROOT as REPO # noqa: E402 (single source of fact, T01)
|
|||
|
||||
PRICES = os.path.join(REPO, "benchmarks", "baselines", "model-prices.toml")
|
||||
COMPONENTS = ("input", "output", "cache_read", "write_5m", "write_1h")
|
||||
# CA-08 attribution key. Qualified first: a bare `T\d\d` collides across
|
||||
# workplans — CB-WP-0002 T01, CB-WP-0003 T01 and CB-WP-0004 T01 all matched
|
||||
# the same bucket, so a per-task figure summed three unrelated tasks. Found
|
||||
# by T02's positive control, which reported $12.10 for "T01".
|
||||
TASK_QUALIFIED_RE = re.compile(r"\b(CB-WP-\d+)[ -](T\d\d)\b")
|
||||
TASK_RE = re.compile(r"\bT\d\d\b")
|
||||
|
||||
|
||||
def task_label(subject):
|
||||
"""Attribution label for a commit subject, or None.
|
||||
|
||||
Qualified ids (`CB-WP-0004 T01`) become `CB-WP-0004-T01`. Bare ids
|
||||
(`T01: ...`) stay bare — commits predating this convention did not name
|
||||
their workplan and must not be retroactively assigned to one.
|
||||
"""
|
||||
m = TASK_QUALIFIED_RE.search(subject)
|
||||
if m:
|
||||
return f"{m.group(1)}-{m.group(2)}"
|
||||
m = TASK_RE.search(subject)
|
||||
return m.group(0) if m else None
|
||||
UNATTRIBUTED = "UNATTRIBUTED"
|
||||
OPEN_REMAINDER = "OPEN (uncommitted)"
|
||||
|
||||
|
|
@ -323,8 +342,7 @@ def attribute(responses, commits):
|
|||
label_for = []
|
||||
prev = ""
|
||||
for ts, subject in commits:
|
||||
m = TASK_RE.search(subject)
|
||||
label_for.append((prev, ts, m.group(0) if m else UNATTRIBUTED))
|
||||
label_for.append((prev, ts, task_label(subject) or UNATTRIBUTED))
|
||||
prev = ts
|
||||
last = commits[-1][0] if commits else ""
|
||||
|
||||
|
|
@ -369,6 +387,13 @@ def collect(slug, pin_ref=None):
|
|||
attribute(responses, commit_index(pin))
|
||||
|
||||
by_task = collections.defaultdict(float)
|
||||
# T02: per-task token detail, so a task close can report *measured*
|
||||
# tokens. Every update_task_status in this project before T02 carried
|
||||
# hand-typed estimates — in a repo whose finding is that estimated
|
||||
# token counts are worthless.
|
||||
task_detail = collections.defaultdict(
|
||||
lambda: {"cost": 0.0, "responses": 0, "models": collections.Counter(),
|
||||
"toks": collections.Counter()})
|
||||
by_component = collections.Counter()
|
||||
by_component_cost = collections.defaultdict(float)
|
||||
by_model = collections.defaultdict(float)
|
||||
|
|
@ -379,6 +404,11 @@ def collect(slug, pin_ref=None):
|
|||
unpriced.append(r)
|
||||
continue
|
||||
by_task[r["task"]] += r["cost"]
|
||||
d = task_detail[r["task"]]
|
||||
d["cost"] += r["cost"]
|
||||
d["responses"] += 1
|
||||
d["models"][r["model"]] += 1
|
||||
d["toks"].update(r["toks"])
|
||||
by_model[r["model"]] += r["cost"]
|
||||
rin, rout = rates_at(prices, r["model"], r["timestamp"])
|
||||
unit = rin / 1e6
|
||||
|
|
@ -429,6 +459,27 @@ def collect(slug, pin_ref=None):
|
|||
"subagent_total": sub,
|
||||
"main_total": total - sub,
|
||||
"by_task": dict(by_task),
|
||||
"by_task_detail": {
|
||||
k: {
|
||||
"cost": v["cost"],
|
||||
"responses": v["responses"],
|
||||
# The model that ran the most responses for this task. The
|
||||
# hub schema has one model field per event; a task split
|
||||
# across models cannot be represented faithfully, so the
|
||||
# full count travels in the note.
|
||||
"model": v["models"].most_common(1)[0][0],
|
||||
"models": dict(v["models"]),
|
||||
"toks": dict(v["toks"]),
|
||||
# CA-03/CA-04: the hub has no cache fields. tokens_in is
|
||||
# the honest sum of everything billed on the input side.
|
||||
"tokens_in": (v["toks"].get("input", 0)
|
||||
+ v["toks"].get("cache_read", 0)
|
||||
+ v["toks"].get("write_5m", 0)
|
||||
+ v["toks"].get("write_1h", 0)),
|
||||
"tokens_out": v["toks"].get("output", 0),
|
||||
}
|
||||
for k, v in task_detail.items()
|
||||
},
|
||||
"by_model": dict(by_model),
|
||||
"tokens": dict(by_component),
|
||||
"cost_by_component": dict(by_component_cost),
|
||||
|
|
@ -566,6 +617,18 @@ def self_test():
|
|||
finally:
|
||||
os.unlink(partial)
|
||||
|
||||
# CA-08 label collision (found by T02's positive control): three
|
||||
# workplans each had a T01, and a bare-label bucket summed all three
|
||||
# into one figure — $12.10 for a task that cost $2.33. Qualified
|
||||
# subjects must produce distinct buckets.
|
||||
check("CA-08 qualified task ids do not collide across workplans",
|
||||
task_label("CB-WP-0004 T01: fix env") == "CB-WP-0004-T01"
|
||||
and task_label("CB-WP-0002 T01: survey") == "CB-WP-0002-T01"
|
||||
and task_label("CB-WP-0004-T01: hyphenated") == "CB-WP-0004-T01")
|
||||
check("CA-08 unqualified subjects stay bare, not retro-assigned",
|
||||
task_label("T01: cost-accounting survey") == "T01"
|
||||
and task_label("chore: no task here") is None)
|
||||
|
||||
# CA-16: a dated promo rate must apply before its expiry and lapse after.
|
||||
pr = prices
|
||||
before = rates_at(pr, "claude-sonnet-5", "2026-07-31T00:00:00Z")
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue