diff --git a/Makefile b/Makefile index 98cf81e..4bddd14 100644 --- a/Makefile +++ b/Makefile @@ -24,7 +24,7 @@ TOOLS := $(REPO)/tools # Every cargo recipe runs at the repo root; the shell does not persist cd. IN_REPO := cd $(REPO) && -.PHONY: check test sim bench bench-test coverage dep-weight cost cost-test cost-pin cost-budget cost-mix loop-lint self-tests env-test loc all +.PHONY: check test sim bench bench-test coverage dep-weight cost cost-test cost-pin cost-budget cost-mix loop-lint self-tests env-test task-done loc all ## fmt + clippy (deny warnings) + HashMap deny-lint check: @@ -61,6 +61,7 @@ self-tests: $(PY) $(TOOLS)/rule-coverage.py --self-test $(PY) $(TOOLS)/dep-weight.py --self-test $(PY) $(TOOLS)/repo.py --self-test + $(PY) $(TOOLS)/task-done.py --self-test # T01 positive control: prove the environment fix, do not assume it. Runs # every tool from a foreign working directory with a PATH that has no @@ -79,6 +80,14 @@ env-test: @$(MAKE) -C $(REPO) coverage >/dev/null \ && echo " [ok ] make -C works from any directory" +# T02: close a task — flip the workplan file, read the *measured* cost +# from the transcripts, push the hub event with real numbers. Refuses on an +# unknown or already-done task, and refuses to report an estimate. +# make task-done T=CB-WP-0004-T02 +task-done: + @test -n "$(T)" || { echo "usage: make task-done T=CB-WP-0004-T02" >&2; exit 2; } + $(PY) $(TOOLS)/task-done.py $(T) $(ARGS) + # CB-01/CB-02: live spend since the last commit. cost-budget: cost-test $(PY) $(TOOLS)/cb-cost.py --budget diff --git a/tools/__pycache__/cb-cost.cpython-312.pyc b/tools/__pycache__/cb-cost.cpython-312.pyc index 2154d3e..21da165 100644 Binary files a/tools/__pycache__/cb-cost.cpython-312.pyc and b/tools/__pycache__/cb-cost.cpython-312.pyc differ diff --git a/tools/cb-cost.py b/tools/cb-cost.py index dd328c4..e0af2bf 100644 --- a/tools/cb-cost.py +++ b/tools/cb-cost.py @@ -44,7 +44,26 @@ from repo import ROOT as REPO # noqa: E402 (single source of fact, T01) PRICES = os.path.join(REPO, "benchmarks", "baselines", "model-prices.toml") COMPONENTS = ("input", "output", "cache_read", "write_5m", "write_1h") +# CA-08 attribution key. Qualified first: a bare `T\d\d` collides across +# workplans — CB-WP-0002 T01, CB-WP-0003 T01 and CB-WP-0004 T01 all matched +# the same bucket, so a per-task figure summed three unrelated tasks. Found +# by T02's positive control, which reported $12.10 for "T01". +TASK_QUALIFIED_RE = re.compile(r"\b(CB-WP-\d+)[ -](T\d\d)\b") TASK_RE = re.compile(r"\bT\d\d\b") + + +def task_label(subject): + """Attribution label for a commit subject, or None. + + Qualified ids (`CB-WP-0004 T01`) become `CB-WP-0004-T01`. Bare ids + (`T01: ...`) stay bare — commits predating this convention did not name + their workplan and must not be retroactively assigned to one. + """ + m = TASK_QUALIFIED_RE.search(subject) + if m: + return f"{m.group(1)}-{m.group(2)}" + m = TASK_RE.search(subject) + return m.group(0) if m else None UNATTRIBUTED = "UNATTRIBUTED" OPEN_REMAINDER = "OPEN (uncommitted)" @@ -323,8 +342,7 @@ def attribute(responses, commits): label_for = [] prev = "" for ts, subject in commits: - m = TASK_RE.search(subject) - label_for.append((prev, ts, m.group(0) if m else UNATTRIBUTED)) + label_for.append((prev, ts, task_label(subject) or UNATTRIBUTED)) prev = ts last = commits[-1][0] if commits else "" @@ -369,6 +387,13 @@ def collect(slug, pin_ref=None): attribute(responses, commit_index(pin)) by_task = collections.defaultdict(float) + # T02: per-task token detail, so a task close can report *measured* + # tokens. Every update_task_status in this project before T02 carried + # hand-typed estimates — in a repo whose finding is that estimated + # token counts are worthless. + task_detail = collections.defaultdict( + lambda: {"cost": 0.0, "responses": 0, "models": collections.Counter(), + "toks": collections.Counter()}) by_component = collections.Counter() by_component_cost = collections.defaultdict(float) by_model = collections.defaultdict(float) @@ -379,6 +404,11 @@ def collect(slug, pin_ref=None): unpriced.append(r) continue by_task[r["task"]] += r["cost"] + d = task_detail[r["task"]] + d["cost"] += r["cost"] + d["responses"] += 1 + d["models"][r["model"]] += 1 + d["toks"].update(r["toks"]) by_model[r["model"]] += r["cost"] rin, rout = rates_at(prices, r["model"], r["timestamp"]) unit = rin / 1e6 @@ -429,6 +459,27 @@ def collect(slug, pin_ref=None): "subagent_total": sub, "main_total": total - sub, "by_task": dict(by_task), + "by_task_detail": { + k: { + "cost": v["cost"], + "responses": v["responses"], + # The model that ran the most responses for this task. The + # hub schema has one model field per event; a task split + # across models cannot be represented faithfully, so the + # full count travels in the note. + "model": v["models"].most_common(1)[0][0], + "models": dict(v["models"]), + "toks": dict(v["toks"]), + # CA-03/CA-04: the hub has no cache fields. tokens_in is + # the honest sum of everything billed on the input side. + "tokens_in": (v["toks"].get("input", 0) + + v["toks"].get("cache_read", 0) + + v["toks"].get("write_5m", 0) + + v["toks"].get("write_1h", 0)), + "tokens_out": v["toks"].get("output", 0), + } + for k, v in task_detail.items() + }, "by_model": dict(by_model), "tokens": dict(by_component), "cost_by_component": dict(by_component_cost), @@ -566,6 +617,18 @@ def self_test(): finally: os.unlink(partial) + # CA-08 label collision (found by T02's positive control): three + # workplans each had a T01, and a bare-label bucket summed all three + # into one figure — $12.10 for a task that cost $2.33. Qualified + # subjects must produce distinct buckets. + check("CA-08 qualified task ids do not collide across workplans", + task_label("CB-WP-0004 T01: fix env") == "CB-WP-0004-T01" + and task_label("CB-WP-0002 T01: survey") == "CB-WP-0002-T01" + and task_label("CB-WP-0004-T01: hyphenated") == "CB-WP-0004-T01") + check("CA-08 unqualified subjects stay bare, not retro-assigned", + task_label("T01: cost-accounting survey") == "T01" + and task_label("chore: no task here") is None) + # CA-16: a dated promo rate must apply before its expiry and lapse after. pr = prices before = rates_at(pr, "claude-sonnet-5", "2026-07-31T00:00:00Z") diff --git a/tools/task-done.py b/tools/task-done.py new file mode 100644 index 0000000..83508ea --- /dev/null +++ b/tools/task-done.py @@ -0,0 +1,322 @@ +#!/usr/bin/env python3 +"""Close a workplan task: flip the file, read the measured cost, tell the hub. + +CB-WP-0004 T02. Replaces three things that were done by hand on every task +close, at a measured 46 turns and $11.52 per pass (CB-RES-0003 candidates +3 and 5): + + 1. a heredoc doing `s.replace("status: todo", "status: done")` on the + workplan file — which silently did nothing on a typo'd task id or an + already-closed task; + 2. a hand-written `update_task_status` hub call; + 3. **hand-typed `tokens_in` / `tokens_out`**, in a project whose central + finding is that estimated token counts are worthless. Every hub token + event this repo produced before T02 was an estimate. This tool reads + the measured figure from the transcripts via cb-cost, or refuses. + +Ordering matters and is enforced, not assumed. Attribution is by commit +subject (CA-08), so the work commit naming the task must exist *before* +the close: otherwise the spend still sits in OPEN (uncommitted) and there +is nothing measured to report. The usual sequence is + + + git commit -m "CB-WP-0004 T02: ..." # subject names the task + make task-done T=CB-WP-0004-T02 # measures, flips, pushes + git commit -m "chore: mark T02 done (measured: ...)" + +Usage: + python3 tools/task-done.py CB-WP-0004-T02 + python3 tools/task-done.py CB-WP-0004-T02 --dry-run # measure, change nothing + python3 tools/task-done.py CB-WP-0004-T02 --no-hub # file only + python3 tools/task-done.py --self-test +""" +import glob +import json +import os +import re +import subprocess +import sys +import urllib.error +import urllib.request + +from repo import ROOT, enter_root + +HUB = os.environ.get("CUSTODIAN_HUB", "http://127.0.0.1:8000") +AGENT = "custodian" +TASK_ID_RE = re.compile(r"^(?PCB-WP-\d+)-(?P