diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml index 9185eb1..d9e71d0 100644 --- a/.forgejo/workflows/ci.yml +++ b/.forgejo/workflows/ci.yml @@ -36,7 +36,13 @@ jobs: # Positive control for the cost collector (AC-5..AC-9). Does not # gate on a dollar figure — transcripts are not present in CI — but # proves the collector still detects the failures it claims to. - - run: make cost-test + - run: make self-tests + + # InnerLoop rules made executable (CB-WP-0003 T01). Fails on an + # overlong artifact, a survey missing its tier/chaos declaration, an + # approved tier-L survey with no review trail, `unmeasured` in an + # evidence table, or a reporting tool with no --self-test. + - run: make loop-lint # InnerLoop v1.0 positive control, enforced rather than asserted in # prose: --test runs every benchmark once, so a workload that diff --git a/Makefile b/Makefile index 46e9e79..02f4e61 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ CARGO := cargo -.PHONY: check test sim bench bench-test coverage dep-weight cost cost-test cost-pin loc all +.PHONY: check test sim bench bench-test coverage dep-weight cost cost-test cost-pin cost-budget loop-lint self-tests loc all ## fmt + clippy (deny warnings) + HashMap deny-lint check: @@ -29,6 +29,21 @@ cost: cost-test cost-test: python3 tools/cb-cost.py --self-test +# InnerLoop rules that are mechanically checkable (CB-WP-0003 T01). +loop-lint: + python3 tools/loop-lint.py + +# Positive control for every reporting tool, per InnerLoop v1.1 Step 5. +self-tests: + python3 tools/cb-cost.py --self-test + python3 tools/loop-lint.py --self-test + python3 tools/rule-coverage.py --self-test + python3 tools/dep-weight.py --self-test + +# CB-01/CB-02: live spend since the last commit. +cost-budget: cost-test + python3 tools/cb-cost.py --budget + cost-pin: cost-test python3 tools/cb-cost.py --pin fc76445 --composition --by-task @@ -51,4 +66,4 @@ loc: printf '%-28s %s\n' $$d "$$(find $$d/src -name '*.rs' | xargs cat | grep -vcE '^\s*(//|$$)')"; \ done -all: check test sim coverage dep-weight cost-test bench-test +all: check test sim coverage dep-weight self-tests loop-lint bench-test diff --git a/WORK-RECORDS.md b/WORK-RECORDS.md index 418ce2e..8bf92e3 100644 --- a/WORK-RECORDS.md +++ b/WORK-RECORDS.md @@ -37,3 +37,4 @@ | task | CB-WP-0003-T08 | todo | — | workplans/CB-WP-0003-loop-hardening.md | | task | CB-WP-0003-T09 | todo | — | workplans/CB-WP-0003-loop-hardening.md | | task | CB-WP-0003-T10 | todo | — | workplans/CB-WP-0003-loop-hardening.md | +| task | CB-WP-0003-T11 | todo | — | workplans/CB-WP-0003-loop-hardening.md | diff --git a/benchmarks/baselines/model-prices.toml b/benchmarks/baselines/model-prices.toml index 3e12c08..68cade5 100644 --- a/benchmarks/baselines/model-prices.toml +++ b/benchmarks/baselines/model-prices.toml @@ -1,6 +1,15 @@ -# USD per million tokens; source: Anthropic pricing, recorded 2026-07-31 -# Governed by specs/MetricsAndScenarios.md §1a — refresh on price change or -# after 90 days, or new M-D2-CST 'better' verdicts are invalid. +# USD per million tokens. Source: Anthropic pricing. +# Governed by specs/CostAccounting.md §9 — a stale sheet invalidates new +# M-D2-CST `better` verdicts. Staleness is enforced by `make cost-test`, +# not by this comment. +# +# Dated rates are DATA, not comments. A promotional rate written as a +# comment is invisible to the collector that reads this file: the sonnet +# intro price sat in a `#` remark and was silently ignored, costing $0.17 +# on a $93.32 pass and heading for a sign flip on 2026-08-31. + +recorded = 2026-07-31 +max_age_days = 90 [claude-fable-5] input = 10.00 @@ -11,8 +20,13 @@ input = 5.00 output = 25.00 [claude-sonnet-5] -input = 3.00 # intro 2.00 through 2026-08-31 -output = 15.00 # intro 10.00 through 2026-08-31 +input = 3.00 +output = 15.00 +# Promotional rate in force through `until` (inclusive); the collector +# applies it to any response whose timestamp falls on or before that date. +promo_input = 2.00 +promo_output = 10.00 +promo_until = 2026-08-31 [claude-haiku-4-5] input = 1.00 diff --git a/crates/cb-game-runtime/src/scenario.rs b/crates/cb-game-runtime/src/scenario.rs index 1a6aa08..ed47391 100644 --- a/crates/cb-game-runtime/src/scenario.rs +++ b/crates/cb-game-runtime/src/scenario.rs @@ -21,6 +21,16 @@ pub struct ScenarioFile { /// scenario, not the kernel. #[serde(default)] pub provisional: bool, + /// Who must rule on the U-item this scenario encodes. Required when + /// `provisional` is true: a provisional default with no owner can + /// shape the kernel indefinitely while looking handled + /// (CB-WP-0003 T08). + #[serde(default)] + pub provisional_owner: String, + /// ISO date the provisional default was raised, so its age is + /// reportable by `make coverage`. + #[serde(default)] + pub provisional_raised: String, pub seed: u64, pub setup: Setup, pub commands: Vec, diff --git a/evidence/CB-EV-0002-cost-accounting.md b/evidence/CB-EV-0002-cost-accounting.md index a8f986a..e193449 100644 --- a/evidence/CB-EV-0002-cost-accounting.md +++ b/evidence/CB-EV-0002-cost-accounting.md @@ -17,9 +17,9 @@ transcribed by hand (CA-15). | ID | Metric | Target | Measured | Verdict | |---|---|---|---|---| -| AC-1 | pinned total, as two components | $93.32 = $92.21 + $1.11 | **$92.21 main + $1.11 subagent = $93.32** | **met** | +| AC-1 | pinned total, as two components | $93.15 = $92.03 + $1.11 | **$92.03 main + $1.11 subagent = $93.15** | **met** | | AC-2 | reconciliation residual | $0.00 | **$0.000000** | **met** | -| AC-3 | unattributed share reported | present, 33% | **32.5%, own line** | **met** | +| AC-3 | unattributed share reported | present, 33% | **32.4%, own line** | **met** | | AC-4 | composition reported | 5 components | **5 of 5** | **met** | | AC-5 | dedup violation aborts | non-zero exit | **abort raised** | **met** | | AC-6 | zero responses refuses to report | non-zero exit | **0 rows, no number emitted** | **met** | @@ -27,7 +27,7 @@ transcribed by hand (CA-15). | AC-8 | 5m cache priced at 1.25× | $1.25/100k @ fable | **$1.2500 (1h would be $2.0000)** | **met** | | AC-9 | streamed partial output → final | 5,5,195 → 195 | **195** | **met** | -No `unmeasured` rows. AC-1's target was corrected three times before this +No `unmeasured` rows. AC-1's target was corrected four times before and after this run; §4 records why, because the sequence is more useful than the final number. @@ -35,23 +35,23 @@ number. ```text input 690 tok $ 0.00 0.0% - output 323,643 tok $ 11.15 11.9% - cache_read 80,611,798 tok $ 59.75 64.0% + output 323,643 tok $ 11.13 11.9% + cache_read 80,611,798 tok $ 59.67 64.1% write_5m 37,467 tok $ 0.47 0.5% - write_1h 1,672,854 tok $ 21.95 23.5% - TOTAL $ 93.32 + write_1h 1,672,854 tok $ 21.87 23.5% + TOTAL $ 93.15 ``` **88.0% of spend is cache; 11.9% is output.** The ratio of context re-read -to text written is 249:1. A single total would have shown $93.32 and +to text written is 249:1. A single total would have shown $93.15 and concealed all of it — which is exactly what CB-WP-0001's `M-D2-TOK` would have done, and why that metric is now demoted. ## 3. Per-task attribution ```text - UNATTRIBUTED $ 30.32 32.5% - T08 $ 21.02 22.5% + UNATTRIBUTED $ 30.14 32.4% + T08 $ 21.02 22.6% T07 $ 12.01 12.9% T03 $ 9.91 10.6% T04 $ 8.00 8.6% @@ -60,7 +60,7 @@ have done, and why that metric is now demoted. T09 $ 1.29 1.4% ``` -**Stated limit (CA-10):** 32.5% of cost sits in commits whose subject +**Stated limit (CA-10):** 32.4% of cost sits in commits whose subject carries no `T##` tag, so this table is a view over 67.5% of spend. That is a property of commit hygiene, not of the collector. @@ -75,14 +75,15 @@ is worth watching, not yet a conclusion from n=1. - **The per-task figures are not comparable across passes.** They mix models (opus/fable/sonnet) at different price points and different cache states. The dollar figure is comparable; a token count is not. -- **AC-3's 32.5% is a fixture pin, not a quality target.** Improving commit +- **AC-3's 32.4% is a fixture pin, not a quality target.** Improving commit tagging will move it, and that is the desired direction. - **This is one session.** Every ratio here (cache share, $/turn, the compaction effect in §5) is n=1 and should be treated as a hypothesis until a second pass reproduces it. -- **The price sheet cannot express a time-boxed rate.** Sonnet's intro price - lives in a TOML comment, so sonnet-priced work is off by $0.17 here - (0.19%). This becomes a real error on 2026-08-31. +- ~~**The price sheet cannot express a time-boxed rate.**~~ **Fixed + 2026-07-31 by CB-WP-0003 T11** (CA-16/CA-17). Applying the sonnet + promotional rate moved AC-1 from $93.32 to **$93.15** — the $0.17 this + section predicted, now collected rather than merely noted. - **AC-1 is not invoice-verified.** No admin key exists, so the Anthropic billing API could not independently confirm the total. The transcript counters are the same ones billing uses, but that is an argument, not a @@ -145,7 +146,7 @@ largest sample was assumed to hold on the smallest one.** The main transcript is 338 of 346 responses, so 206/206 felt conclusive; the violation lives entirely in the 8 responses nobody checked separately. -Cost of the adversarial review this pass: **$1.11**, against a $93.32 pass. +Cost of the adversarial review this pass: **$1.11**, against a $93.15 pass. It found three approval-blocking defects, one of which (the subagent exclusion) would have made this evidence file certify a broken collector. Second consecutive pass where a ~1% spend on review changed the outcome. diff --git a/history/260731-inner-loop-rule-audit.md b/history/260731-inner-loop-rule-audit.md new file mode 100644 index 0000000..13999eb --- /dev/null +++ b/history/260731-inner-loop-rule-audit.md @@ -0,0 +1,157 @@ +# 2026-07-31 — InnerLoop v1.1 rule enforceability audit + +CB-WP-0003 T01. Every rule in `specs/InnerLoop.md` v1.1 classified as +**executable** (a command fails when it is violated), **checkable** (a +human or agent can verify it cheaply and objectively in review), or +**decorative** (neither) — and, per the revised task, tagged with the +**failure class it catches**. + +## Failure classes on record + +Seven error instances across three classes, from +`history/260731-inner-loop-retrospective.md` and +`history/260731-cost-accounting-retrospective.md`: + +| tag | class | instances | +|---|---|---| +| **HDN** | harness-does-nothing — a run succeeds while performing no work | 4 | +| **TA** | trusted arithmetic — a correct-looking sum over data that really exists | 2 | +| **SSB** | same-sample blind spot — a property verified on the large sample, assumed on the small | 1 | + +The reason for tagging: a rule that catches a class no observed error +belongs to is a candidate for deletion even when it is perfectly +executable, and a class with no rule covering it is where the next rule +should go. + +## The audit + +Status after this pass. `loop-lint` = `tools/loop-lint.py`, new here. + +| # | Rule | Class | Status | Enforced by | +|---|---|---|---|---| +| 1 | No implementation code before the ADR is committed | — | **checkable** | git history vs ADR date; not automated (see §Deferred) | +| 2 | Tier declared before work starts | — | **executable** | `loop-lint` tier-declared | +| 3 | Chaos roll recorded every time, even when it changes nothing | — | **executable** | `loop-lint` chaos-recorded | +| 4 | Invariants bind at every tier regardless of roll | — | executable (elsewhere) | `make all` — determinism tests, `disallowed_types` | +| 5 | Survey names a benchmark-to-beat per dimension | — | **checkable** | review; a regex cannot judge whether a number is a benchmark | +| 6 | Cited-only baselines cap the verdict at `parity` | TA | **decorative → checkable** | see §Changes | +| 7 | Tier-L survey gets one round of adversarial review | SSB | **executable** | `loop-lint` review-trail | +| 8 | Review trail: research/challenge/response in `history/` | SSB | **executable** | `loop-lint` review-trail | +| 9 | ADR states expected advantage per dimension | — | **checkable** | review | +| 10 | Spec carries an acceptance-metrics table | — | **checkable** | review | +| 11 | Every metric names its instrument | HDN | **checkable** | review; naming is textual, existence is not | +| 12 | **v1.1** The instrument must exist and emit its own target | TA | **checkable** | review; see §Deferred for why not yet executable | +| 13 | **v1.1** Inherited numbers are re-derived before use as a target | TA | **decorative** | nothing can detect a quoted number | +| 14 | A metric is checked against contracts in its own spec | — | **checkable** | review | +| 15 | Harness asserts it performed the work it reports | HDN | **executable** | `cargo bench -- --test`, cb-sim empty-run, `--self-test` | +| 16 | Fail loudly rather than report when the assertion fails | HDN | **executable** | same | +| 17 | Divisor pinned by a test | HDN | **executable** | `bench_shape` test module | +| 18 | A number that cannot prove its work is void | HDN | **executable** | same as 15 | +| 19 | **v1.1** Every reporting tool exposes `--self-test` | HDN, SSB | **executable** | `loop-lint` self-test | +| 20 | `--self-test` names a failure it detects, not the happy path | HDN | **checkable** | review; see §Deferred | +| 21 | Evidence names its disanalogies | — | **checkable** | review | +| 22 | `unmeasured` illegal in an evidence file | — | **executable** | `loop-lint` evidence-unmeasured | +| 23 | No silently-ignored input | HDN | **checkable** | review | +| 24 | Decisions get commands, not defaults | — | **checkable** | review | +| 25 | Scaffolds are exercised or marked | HDN | **checkable** | review | +| 26 | Coverage gates that count tags say so | — | **checkable** | review | +| 27 | Whole-file loadability (~400 lines) | — | **executable** | `loop-lint` loadability | +| 28 | Structured over prose | — | **decorative** | kept as guidance, marked | +| 29 | One command surface | — | **checkable** | review | +| 30 | Self-contained tasks | — | **checkable** | review | +| 31 | Evidence or it didn't happen | — | **checkable** | review | +| 32 | Token discipline per the global budget policy | — | **decorative — dead** | nothing; CB-WP-0003 T05 replaces it | +| 33–41 | Definition-of-done checklist (9 items) | mixed | **checkable** | review; each maps to an artifact whose existence is testable — see §Deferred | + +Counts: **11 executable**, **22 checkable**, **4 decorative** (one of +which is dead policy). + +## What the audit found by running + +`tools/loop-lint.py` was written to make rules 2, 3, 7, 8, 19, 22 and 27 +executable. On its first run it produced **three findings, all real, none +previously visible**: + +1. **`specs/ArchitectureBlueprint.md` is 543 lines** against a ~400-line + limit the loop has stated since v0.2. Nobody noticed because nothing + measured it. Disposition: split (§Changes). +2. **`tools/dep-weight.py` has no `--self-test`.** It has positive-control + logic — it refuses to report when a crate cannot be located — but + nothing verifies that control still works. +3. **`tools/rule-coverage.py` has no `--self-test`.** Same shape. + +Findings 2 and 3 are exactly the recursion this workplan is about: the +positive-control rule was applied to benchmarks and to the newest tool, +and not to the two older tools that report AM-1 and AM-4 numbers into +evidence files. + +## Changes made in this task + +- **Rule 27 made executable and the violation fixed.** + `ArchitectureBlueprint.md` split at its own section boundaries into + `ArchitectureBlueprint.md` (§1–8, the stack) and + `ArchitectureRuntime.md` (§9–15, runtime/tooling/process), linked both + ways. +- **Rules 2, 3, 7, 8, 19, 22 made executable** via `tools/loop-lint.py`, + wired as `make loop-lint` and into `make all` and CI. +- **Rule 6 (parity cap) demoted from decorative to checkable** by stating + the check explicitly: an evidence row citing a baseline whose provenance + is `cited` may not carry verdict `better`. That is mechanical against the + survey's provenance column and is queued for `loop-lint` once a second + evidence file exists to test it against — writing a matcher with one + sample is the SSB error this pass is trying to stop making. +- **Rule 28 (structured over prose) demoted to guidance**, with that + status stated in the spec. It is a style preference; nothing can judge + it, and leaving it phrased as a requirement is the false assurance this + audit exists to remove. +- **Rule 32 (token discipline) marked dead** in place, pointing at + CB-WP-0003 T05. It is not deleted yet because deleting it is T05's + decision, but it now reads as dead rather than as a live control. + +## Deferred, with reasons + +- **Rule 1 (ADR gate) is checkable but not automated.** A mechanical check + needs a map from capability → source paths, which does not exist. Cheap + version worth doing later: require every `specs/.md` to reference an + ADR, and every ADR to precede the first commit touching its capability's + directory. +- **Rule 12 (instrument emits its target) resists automation for now.** + Detecting "this number was typed rather than emitted" requires the tool's + output to be committed alongside the spec. The tractable form is to + require acceptance targets to appear verbatim in a committed tool output + file; deferred to T02 rather than guessed at here. +- **Rule 20 (self-test names a failure) is genuinely checkable only.** A + self-test that asserts `True` passes any structural check. Reviewing the + assertions is the control, and CB-WP-0002's AC-9 is the model: pin the + exact defect that occurred. +- **Rule 13 (re-derive inherited numbers) has no mechanical form at all** + and is left decorative *deliberately* — with its status stated. It is + the rule that caught the largest error in CB-WP-0002 ($248.46 → $92.21), + which is the counter-example to this workplan's own hypothesis: **an + unenforceable rule was the most valuable one in the pass.** The + hypothesis "a rule that cannot be executed is not a rule" is therefore + wrong as stated. The correct version is narrower: *a rule that cannot be + executed cannot be relied on to fire, so it must not be the only defence + for a class that matters.* + +## Open items raised by this audit + +- **AM-4a / AM-4b are unratified retargets.** Measured at 246,250 and set + at 250,000 in the same commit, by the implementer, with the + implementation changing in that commit too — a retarget under the test + added to InnerLoop §Step 4 by T07, not a correction. They must be + ratified by ADR or changed. Until then `make dep-weight` is enforcing a + target no reviewed decision stands behind. + +## Class coverage — where the gaps are + +| class | executable rules covering it | assessment | +|---|---|---| +| HDN | 15, 16, 17, 18, 19 | **well covered.** Five executable rules; the class that started this. | +| TA | none | **uncovered by any executable rule.** Rules 6, 12, 13 are checkable or decorative. This is the largest gap, and it is the class with the second-most instances. | +| SSB | 7, 8, 19 | **partly covered.** Rule 19 caught the one instance, by accident of running over all data rather than by design. Rules 7/8 enforce that a review *happened*, not that it sampled differently. T03 addresses the design gap. | + +The honest read: **the loop is hardened against the class it has already +suffered most from, and has no executable defence against the class that +produced its largest single error.** Trusted arithmetic is caught today +only by re-derivation, which is a discipline, not a gate. diff --git a/history/260731-loop-hardening-retrospective.md b/history/260731-loop-hardening-retrospective.md new file mode 100644 index 0000000..ce5b948 --- /dev/null +++ b/history/260731-loop-hardening-retrospective.md @@ -0,0 +1,134 @@ +# 2026-07-31 — retrospective: does hardening converge? + +CB-WP-0003 T10. Pass produced `tools/loop-lint.py`, +`specs/SessionShape.md`, the rule audit, the live cost budget, the +retarget test, dated price rates, M-D1-LNK, provisional expiry, and +InnerLoop **v1.2**. + +## The question this task was given + +> Is the set of mechanisms now complete, or is each new pass still finding +> a new class? If this pass produces an eighth instance in a fourth class, +> the honest conclusion is that class-by-class hardening does not +> converge, and the loop should optimize for *cheap detection and +> correction* rather than for prevention — which is a different design. + +**It produced both. The answer is: hardening does not converge.** + +## The ledger + +| # | class | instances | found by | +|---|---|---|---| +| 1 | **HDN** harness-does-nothing | **5** | executable assertions | +| 2 | **TA** trusted arithmetic | **3** | re-derivation | +| 3 | **SSB** same-sample blind spot | **1** | an assertion running over *all* data | +| 4 | **DFD** duplicated-fact drift *(new)* | **2** | reading the copy against its source | + +Ten instances, four classes. Three workplans in, **every pass has +produced at least one instance of a class the previous pass had not +seen.** + +### New this pass + +- **HDN #5 (latent).** `tools/rule-coverage.py` would have exited 0 + reporting `0/0` if its spec regex ever stopped matching — a silent pass + in the tool that reports our headline AM-1 number. Never fired; found + only because T02 required it to carry a `--self-test` and writing one + meant asking what failure it should detect. +- **TA #3.** *"0 of 330 tool calls were batched"* — 330 was the count of + single-call responses, not the total. Carried unverified from + CB-WP-0001's retrospective into CB-WP-0003 T04. Real figure: 7.8–8.6% + of responses batched. +- **DFD #1–2 (new class).** `specs/MetricsAndScenarios.md` §1a inlined a + copy of the price sheet; T11 changed the real sheet and the copy went + stale **within the same hour**. Separately, the acceptance figure + $92.87/$0.66 had to be chased across a survey, a workplan, and an + evidence file each time it moved. The defect is not arithmetic and not + a harness — it is **a fact stored in two places, where updating one + does not update the other**. + +DFD is genuinely distinct: no positive control catches it (both copies +are internally consistent), and re-derivation does not either (the copy +reproduces whatever it was copied from). It is caught only by reading a +copy against its source, which nothing in the loop required. + +## What this implies, and what changed because of it + +Prevention has not converged: four passes, four classes, no sign of +saturation. But the *correction* side has been consistently cheap and +fast: + +| correction | mechanism | cost | +|---|---|---| +| $248.46 → $92.21 | re-derivation | ~0 | +| $92.21 → $92.87 | adversarial review | $1.11 | +| $92.87 → $93.32 | positive control | ~0 | +| $93.32 → $93.15 | dated-rate fix | ~0 | + +Every error found in three passes was corrected inside the same session +for under ~1% of the pass. **So the loop should stop trying to enumerate +failure classes in advance and invest in the properties that make +correction cheap** — which it turns out already has, accidentally: + +1. **Numbers are re-derivable** because the raw data (transcripts, git, + the price sheet) is kept, not just the conclusions. +2. **Artifacts are small and committed**, so a wrong number is one grep + from every place that quotes it. +3. **Every reported number has a command**, so re-running is free. + +v1.2 makes this the stated design goal rather than an accident, and adds +the one rule the new class earns: + +> **Single source of fact.** A number, rate, or target lives in exactly +> one place. Everywhere else links to it. Where a copy is unavoidable, it +> is generated by a command, not typed. *(CA-15 said this for evidence +> tables; DFD shows it applies to specs quoting other specs.)* + +## What the gates did this pass + +Worth recording because it is the cheapest possible evidence that the +executable-rule thesis is at least partly right: + +- `loop-lint` found three real violations on its **first** run — + including a 543-line spec against a ~400-line rule the loop had stated + since v0.2 and never measured. +- `loop-lint` then **failed on its own author within the hour**, when a + T07 edit pushed `specs/InnerLoop.md` to 407 lines. The file was split + because a gate said so, not because anyone noticed. +- `cb-cost`'s dedup assertion fired on real data on its first run and + refused to print a number. + +Three gates, three catches, all on first contact. Against that: the two +most valuable corrections of the whole project ($248.46 → $92.21, and the +batching claim) came from a rule that is **decorative and cannot be made +executable** — *re-derive inherited numbers*. Both facts are true and the +audit records them side by side. + +## Revised hypothesis + +The workplan opened with: *a loop rule that cannot be executed is not a +rule.* Three passes of evidence say that is wrong as stated. The version +this pass supports: + +> An executable rule fires reliably and catches one class. A decorative +> rule fires unreliably and can catch any class, including ones not yet +> named. **Keep both, and never let a decorative rule be the only defence +> for a class that matters** — but do not delete it merely because it +> cannot be automated, which is what this workplan originally proposed. + +## Open, not closed + +- **AM-4a/AM-4b are unratified retargets** (T07). `make dep-weight` + enforces targets no reviewed decision stands behind. Maintainer + decision. +- **9 of 58 rules are claimed by a scenario and named nowhere in the + aggregate** (T09, M-D1-LNK). Reported, not failing, because tagging + them without confirming implementation is the overclaim the loop + already committed once. +- **5 provisional U-item defaults** now have an owner and an age, and + ground-game has not ruled (T08). +- **The chaos roll is in a 12-declaration calibration window** at d4 + (T06), expected to cost $20–30 and to be decided on evidence. +- **DFD has no gate.** The single-source-of-fact rule added in v1.2 is + prose. Making it executable — detecting a number duplicated across + artifacts — is the obvious next candidate and is not attempted here. diff --git a/scenarios/ground/gr-a04-bond-support.yaml b/scenarios/ground/gr-a04-bond-support.yaml index d70d957..f315220 100644 --- a/scenarios/ground/gr-a04-bond-support.yaml +++ b/scenarios/ground/gr-a04-bond-support.yaml @@ -5,6 +5,8 @@ description: > Bond, because GR-L02 requires the target's consent. covers: [GR-A04, GR-F04, GR-L02, GR-L05] provisional: true +provisional_owner: ground-game +provisional_raised: 2026-07-31 seed: 42 setup: players: 3 diff --git a/scenarios/ground/gr-d01-darvo-trigger.yaml b/scenarios/ground/gr-d01-darvo-trigger.yaml index 19ed4ab..fe99a86 100644 --- a/scenarios/ground/gr-d01-darvo-trigger.yaml +++ b/scenarios/ground/gr-d01-darvo-trigger.yaml @@ -5,6 +5,8 @@ description: > application (GR-F01 under the U2 default). covers: [GR-D01, GR-R08, GR-F01] provisional: true +provisional_owner: ground-game +provisional_raised: 2026-07-31 seed: 42 setup: players: 3 diff --git a/scenarios/ground/gr-d05-darvo-reverse.yaml b/scenarios/ground/gr-d05-darvo-reverse.yaml index 77efb18..f1e6d89 100644 --- a/scenarios/ground/gr-d05-darvo-reverse.yaml +++ b/scenarios/ground/gr-d05-darvo-reverse.yaml @@ -5,6 +5,8 @@ description: > owner takes −2 Stress and the sequence ends (GR-D05, GR-D07). covers: [GR-D05, GR-D07, GR-T01] provisional: true +provisional_owner: ground-game +provisional_raised: 2026-07-31 seed: 42 setup: players: 3 diff --git a/scenarios/ground/gr-e02-shared-ground.yaml b/scenarios/ground/gr-e02-shared-ground.yaml index 8182ef2..04e9aa7 100644 --- a/scenarios/ground/gr-e02-shared-ground.yaml +++ b/scenarios/ground/gr-e02-shared-ground.yaml @@ -5,6 +5,8 @@ description: > rating reduced by each Blame token and each Denied Problem. covers: [GR-R09, GR-E01, GR-E02, GR-P03] provisional: true +provisional_owner: ground-game +provisional_raised: 2026-07-31 seed: 42 setup: players: 3 diff --git a/scenarios/ground/gr-e04-coalitions.yaml b/scenarios/ground/gr-e04-coalitions.yaml index d9771ec..dd9f16d 100644 --- a/scenarios/ground/gr-e04-coalitions.yaml +++ b/scenarios/ground/gr-e04-coalitions.yaml @@ -5,6 +5,8 @@ description: > one. Personal scores subtract Blame (GR-E03, GR-T02). covers: [GR-E03, GR-E04, GR-O03] provisional: true +provisional_owner: ground-game +provisional_raised: 2026-07-31 seed: 42 setup: players: 3 diff --git a/specs/ArchitectureBlueprint.md b/specs/ArchitectureBlueprint.md index fbbdd1a..6d0ed3a 100644 --- a/specs/ArchitectureBlueprint.md +++ b/specs/ArchitectureBlueprint.md @@ -331,213 +331,14 @@ capabilities. --- -## 9. Rendering, input, and creation tools - -Rust-first stack: - -- `winit` — windows and platform input -- `wgpu` — GPU rendering (Vulkan / Metal / D3D / browser) -- `egui` — engine tools, inspectors, early editors -- Custom scene renderer for the game table -- **glTF** as the primary imported 3D format, wrapped in Clay-Borg asset - metadata and provenance - -Creator environment (grows over milestones): scene hierarchy, object -inspector, prototype browser, card-sheet importer, deck builder, zone and -snap-point editors, rule-state inspector, event timeline, player-view -switcher, hidden-information debugger, physics debugger, scenario recorder, -replay controls, package validator. - --- -## 10. Networking and sessions +**Continued in [ArchitectureRuntime.md](ArchitectureRuntime.md)** — §9 +rendering/input/creation tools, §10 networking and sessions, §11 the +agentic inner loop, §12 TargetRevenue integration, §13 repository +structure, §14 milestones, §15 governing design decisions. -Authoritative session host: - -```text -Client gesture -→ proposed command -→ session server validation -→ authoritative events -→ state update -→ player-specific projection -→ client animation -``` - -Capabilities: session discovery, auth and seat assignment, lobby/readiness, -command submission, commit/reveal windows, event-stream replication, -snapshot transfer, reconnection, state-hash verification, spectators, -player-specific redaction, host migration (later). - -Transport: **Quinn** (QUIC) for native; browser transport is a separate -adapter (WebTransport or WebSockets). The canonical protocol is defined -independently of any transport: - -```text -cb-session-protocol -├── CommandEnvelope -├── EventEnvelope -├── SnapshotEnvelope -├── CommitmentEnvelope -├── AssetRequest -└── CapabilityNegotiation -``` - ---- - -## 11. Agentic inner loop - -Agentic coding is a first-class product surface. Optimize for small -capability boundaries, executable specifications, controlled work areas, and -replayable failures. - -### Work packet (every agent task) - -```yaml -task_id: CB-PHYS-0042 -capability: tabletop.card-stacking -intent: Keep card stacks stable after drag release. -allowed_crates: - - cb-physics-api - - cb-physics-rapier - - cb-tabletop-physics -forbidden_changes: - - canonical game event schema -invariants: - - semantic card order must not depend on collider order -scenarios: - - scenarios/card-stack-20.yaml -benchmarks: - - benches/card-stack-stability.yaml -acceptance: - - all conformance tests pass - - no state divergence over 10,000 ticks - - benchmark regression below 3% -``` - -### CLI surface (`cb`) - -```bash -cb inspect capability tabletop.card -cb task prepare CB-PHYS-0042 -cb generate contracts -cb check --affected -cb test --affected # supports --format json -cb sim ground scenarios/mutual-attack.yaml -cb play ground --players 4 -cb replay artifacts/failure.cbreplay -cb compare physics-reference physics-rapier -cb bench --affected -cb evidence build CB-PHYS-0042 -cb release assess CB-PHYS-0042 -``` - -### Quality gates - -Formatting/linting, dependency-policy check, unit tests, capability -conformance tests, property tests, golden scenario tests, replay -determinism, snapshot migration, performance and memory budgets, rendering -comparison where relevant, security/sandbox tests, documentation and schema -consistency. - -Tooling: `cargo-nextest` (isolated parallel tests), Criterion -(regression-sensitive benchmarks), `sccache` (compile reuse), `tracing` -(structured diagnostics). - ---- - -## 12. TargetRevenue integration - -TargetRevenue governs **versioned capability improvements**, not the -monorepo as one indivisible target. - -```toml -improvement_id = "CB-GROUND-001" -capability = "game.ground.simultaneous-resolution" -classification = "10x" - -estimated_days = 4 -daily_rate = 1000 -target_revenue = 40000 - -phase = "commercial-recovery" -release_when_target_reached = "MIT" -trust_record = "required" -``` - -Components: improvement registry, workload ledger, cost model, revenue -attribution, dependency graph, phase license generator, revenue meter, -release gate, evidence bundle, trust service. - -Economic rule: - -> Optimized assimilations may be financed as independent improvements, while -> the canonical interface remains stable and reusable. - ---- - -## 13. Repository structure - -```text -clay-borg/ -├── INTENT.md -├── SCOPE.md -├── ARCHITECTURE.md # or specs/ArchitectureBlueprint.md (this file) -├── Cargo.toml -├── rust-toolchain.toml -│ -├── canon/ # entities, events, capabilities, schemas, terminology -├── crates/ -│ ├── cb-kernel/ cb-ids/ cb-time/ cb-rng/ cb-events/ -│ ├── cb-snapshot/ cb-capability/ -│ ├── cb-world/ cb-world-api/ cb-ecs-bevy/ -│ ├── cb-physics-api/ cb-physics-null/ cb-physics-reference/ cb-physics-rapier/ -│ ├── cb-render-api/ cb-render-null/ cb-render-wgpu/ -│ ├── cb-tabletop/ cb-tabletop-physics/ cb-tabletop-view/ -│ ├── cb-game-runtime/ cb-game-protocol/ cb-game-wasm/ -│ ├── cb-session/ cb-network-api/ cb-network-loopback/ cb-network-quic/ -│ └── cb-assets/ cb-ui/ cb-editor/ cb-observe/ cb-evidence/ -│ -├── games/ # ground/, fixture-cards/ -├── tools/ # cb-cli/, cb-agent/, cb-import/, cb-pack/ -├── scenarios/ -├── conformance/ -├── benchmarks/ -├── replays/ -├── examples/ -├── decisions/ -├── assimilation/ # assimilation manifests -└── target-revenue/ -``` - -Stay a monorepo during architectural formation. Extract a repository only -when a capability has a stable contract, an independent lifecycle, and a -genuine external consumer. - ---- - -## 14. Milestones - -| # | Milestone | Proves | -|---|---|---| -| 0 | Headless GROUND | Authoritative rules, commit/reveal, DARVO, replay — no rendering, no physics | -| 1 | Inspectable 2D table | Presentation bindings without 3D complexity | -| 2 | Physical 3D tabletop | wgpu + Rapier projection of semantic events | -| 3 | Networked sessions | Authoritative host, private projections, reconnection | -| 4 | Game creation framework | Editors, importers, Wasm game components | -| 5 | Second fixture game | Generality — abstractions promoted to Canon only after a second concrete use | - ---- - -## 15. Governing design decisions - -1. Build GROUND first, not a general engine first. -2. Keep rules independent from rendering and physics. -3. Use commands and events as the authoritative mutation mechanism. -4. Provide null, reference, and optimized implementations of important capabilities. -5. Never leak assimilated-library types into canonical interfaces. -6. Use server-authoritative physics and deterministic semantic rules. -7. Treat player visibility as a projection, not a UI afterthought. -8. Make every defect reproducible as a scenario and replay. -9. Give coding agents bounded work packets and stable commands. -10. Attach TargetRevenue phases to versioned improvements and evidence bundles. +*(Split 2026-07-31: this file was 543 lines against the loop's ~400-line +whole-file loadability rule. The rule had been stated since v0.2 and +nothing measured it until `tools/loop-lint.py` — see +`history/260731-inner-loop-rule-audit.md`.)* diff --git a/specs/ArchitectureRuntime.md b/specs/ArchitectureRuntime.md new file mode 100644 index 0000000..6d08b53 --- /dev/null +++ b/specs/ArchitectureRuntime.md @@ -0,0 +1,221 @@ +# Clay-Borg Architecture — Runtime, Tooling, and Process + +Second half of the architecture blueprint, split from +[ArchitectureBlueprint.md](ArchitectureBlueprint.md) on 2026-07-31 for +whole-file loadability. §1–8 (the layered stack, Clay Canon, runtime +substrate, simulation kernel, physics, world-building, tabletop domain +framework, game runtime) remain there; §9–15 are here. + +Section numbering is continuous with the first half and deliberately +unchanged, so existing references keep resolving. + +## 9. Rendering, input, and creation tools + +Rust-first stack: + +- `winit` — windows and platform input +- `wgpu` — GPU rendering (Vulkan / Metal / D3D / browser) +- `egui` — engine tools, inspectors, early editors +- Custom scene renderer for the game table +- **glTF** as the primary imported 3D format, wrapped in Clay-Borg asset + metadata and provenance + +Creator environment (grows over milestones): scene hierarchy, object +inspector, prototype browser, card-sheet importer, deck builder, zone and +snap-point editors, rule-state inspector, event timeline, player-view +switcher, hidden-information debugger, physics debugger, scenario recorder, +replay controls, package validator. + +--- + +## 10. Networking and sessions + +Authoritative session host: + +```text +Client gesture +→ proposed command +→ session server validation +→ authoritative events +→ state update +→ player-specific projection +→ client animation +``` + +Capabilities: session discovery, auth and seat assignment, lobby/readiness, +command submission, commit/reveal windows, event-stream replication, +snapshot transfer, reconnection, state-hash verification, spectators, +player-specific redaction, host migration (later). + +Transport: **Quinn** (QUIC) for native; browser transport is a separate +adapter (WebTransport or WebSockets). The canonical protocol is defined +independently of any transport: + +```text +cb-session-protocol +├── CommandEnvelope +├── EventEnvelope +├── SnapshotEnvelope +├── CommitmentEnvelope +├── AssetRequest +└── CapabilityNegotiation +``` + +--- + +## 11. Agentic inner loop + +Agentic coding is a first-class product surface. Optimize for small +capability boundaries, executable specifications, controlled work areas, and +replayable failures. + +### Work packet (every agent task) + +```yaml +task_id: CB-PHYS-0042 +capability: tabletop.card-stacking +intent: Keep card stacks stable after drag release. +allowed_crates: + - cb-physics-api + - cb-physics-rapier + - cb-tabletop-physics +forbidden_changes: + - canonical game event schema +invariants: + - semantic card order must not depend on collider order +scenarios: + - scenarios/card-stack-20.yaml +benchmarks: + - benches/card-stack-stability.yaml +acceptance: + - all conformance tests pass + - no state divergence over 10,000 ticks + - benchmark regression below 3% +``` + +### CLI surface (`cb`) + +```bash +cb inspect capability tabletop.card +cb task prepare CB-PHYS-0042 +cb generate contracts +cb check --affected +cb test --affected # supports --format json +cb sim ground scenarios/mutual-attack.yaml +cb play ground --players 4 +cb replay artifacts/failure.cbreplay +cb compare physics-reference physics-rapier +cb bench --affected +cb evidence build CB-PHYS-0042 +cb release assess CB-PHYS-0042 +``` + +### Quality gates + +Formatting/linting, dependency-policy check, unit tests, capability +conformance tests, property tests, golden scenario tests, replay +determinism, snapshot migration, performance and memory budgets, rendering +comparison where relevant, security/sandbox tests, documentation and schema +consistency. + +Tooling: `cargo-nextest` (isolated parallel tests), Criterion +(regression-sensitive benchmarks), `sccache` (compile reuse), `tracing` +(structured diagnostics). + +--- + +## 12. TargetRevenue integration + +TargetRevenue governs **versioned capability improvements**, not the +monorepo as one indivisible target. + +```toml +improvement_id = "CB-GROUND-001" +capability = "game.ground.simultaneous-resolution" +classification = "10x" + +estimated_days = 4 +daily_rate = 1000 +target_revenue = 40000 + +phase = "commercial-recovery" +release_when_target_reached = "MIT" +trust_record = "required" +``` + +Components: improvement registry, workload ledger, cost model, revenue +attribution, dependency graph, phase license generator, revenue meter, +release gate, evidence bundle, trust service. + +Economic rule: + +> Optimized assimilations may be financed as independent improvements, while +> the canonical interface remains stable and reusable. + +--- + +## 13. Repository structure + +```text +clay-borg/ +├── INTENT.md +├── SCOPE.md +├── ARCHITECTURE.md # or specs/ArchitectureBlueprint.md (this file) +├── Cargo.toml +├── rust-toolchain.toml +│ +├── canon/ # entities, events, capabilities, schemas, terminology +├── crates/ +│ ├── cb-kernel/ cb-ids/ cb-time/ cb-rng/ cb-events/ +│ ├── cb-snapshot/ cb-capability/ +│ ├── cb-world/ cb-world-api/ cb-ecs-bevy/ +│ ├── cb-physics-api/ cb-physics-null/ cb-physics-reference/ cb-physics-rapier/ +│ ├── cb-render-api/ cb-render-null/ cb-render-wgpu/ +│ ├── cb-tabletop/ cb-tabletop-physics/ cb-tabletop-view/ +│ ├── cb-game-runtime/ cb-game-protocol/ cb-game-wasm/ +│ ├── cb-session/ cb-network-api/ cb-network-loopback/ cb-network-quic/ +│ └── cb-assets/ cb-ui/ cb-editor/ cb-observe/ cb-evidence/ +│ +├── games/ # ground/, fixture-cards/ +├── tools/ # cb-cli/, cb-agent/, cb-import/, cb-pack/ +├── scenarios/ +├── conformance/ +├── benchmarks/ +├── replays/ +├── examples/ +├── decisions/ +├── assimilation/ # assimilation manifests +└── target-revenue/ +``` + +Stay a monorepo during architectural formation. Extract a repository only +when a capability has a stable contract, an independent lifecycle, and a +genuine external consumer. + +--- + +## 14. Milestones + +| # | Milestone | Proves | +|---|---|---| +| 0 | Headless GROUND | Authoritative rules, commit/reveal, DARVO, replay — no rendering, no physics | +| 1 | Inspectable 2D table | Presentation bindings without 3D complexity | +| 2 | Physical 3D tabletop | wgpu + Rapier projection of semantic events | +| 3 | Networked sessions | Authoritative host, private projections, reconnection | +| 4 | Game creation framework | Editors, importers, Wasm game components | +| 5 | Second fixture game | Generality — abstractions promoted to Canon only after a second concrete use | + +--- + +## 15. Governing design decisions + +1. Build GROUND first, not a general engine first. +2. Keep rules independent from rendering and physics. +3. Use commands and events as the authoritative mutation mechanism. +4. Provide null, reference, and optimized implementations of important capabilities. +5. Never leak assimilated-library types into canonical interfaces. +6. Use server-authoritative physics and deterministic semantic rules. +7. Treat player visibility as a projection, not a UI afterthought. +8. Make every defect reproducible as a scenario and replay. +9. Give coding agents bounded work packets and stable commands. +10. Attach TargetRevenue phases to versioned improvements and evidence bundles. diff --git a/specs/CostAccounting.md b/specs/CostAccounting.md index c22530c..aa19d7a 100644 --- a/specs/CostAccounting.md +++ b/specs/CostAccounting.md @@ -67,6 +67,16 @@ cost = input_tokens × price.input > reported as an unpriced line with its token counts, never dropped and > never priced at a default. +> **CA-16.** Promotional and other time-boxed rates are **data**, not +> comments. A response is priced at the rate in force *at its own +> timestamp*: `promo_input`/`promo_output` apply on or before +> `promo_until`, and the standard rate applies after. + +> **CA-17.** The price sheet carries `recorded` and `max_age_days`, and a +> sheet older than `max_age_days` **aborts the run**. A stale sheet +> invalidates `better` verdicts, so the staleness rule is a command rather +> than a sentence. + ### 1.3 Scope of a measurement > **CA-06.** A measurement enumerates **every** transcript for the repo: @@ -125,9 +135,9 @@ Each row names the command that produces its number, per InnerLoop §Step 4. | ID | Metric | Target | Instrument | |---|---|---|---| -| **AC-1** | reproduces the pinned CB-WP-0001 total | **$93.32** = $92.21 main + $1.11 subagent | `make cost-pin` | +| **AC-1** | reproduces the pinned CB-WP-0001 total | **$93.15** = $92.03 main + $1.11 subagent | `make cost-pin` | | **AC-2** | reconciliation residual (CA-14) | **$0.00** exactly | same command, `reconciled: ok` line | -| **AC-3** | unattributed share reported (CA-10) | present, and **33%** on the pinned run | `cb-cost --pin fc76445 --by-task` | +| **AC-3** | unattributed share reported (CA-10) | present, and **32.4%** on the pinned run | `cb-cost --pin fc76445 --by-task` | | **AC-4** | composition reported (CA-13) | all five components present | `cb-cost --pin fc76445 --composition` | | **AC-5** | dedup invariant asserted (CA-02) | violation exits non-zero | `make cost-test` | | **AC-6** | positive control: refuses to report on zero responses | exits non-zero | `make cost-test` | @@ -145,7 +155,7 @@ assertion against a fixture whose expected value is known and fails loudly; Per InnerLoop §Step 4, the acceptance table is checked against the contracts in this same spec: -- AC-1's $92.87 is reachable only if CA-06 holds (both trees enumerated). +- AC-1's $93.15 is reachable only if CA-06 holds (both trees enumerated). Under a main-file-only collector the target is unreachable — this is the defect the adversarial review caught, where a target of $92.21 would have been hit *only* by a broken collector. @@ -172,7 +182,88 @@ contracts in this same spec: necessarily shows a lossy projection. This is a limitation of the sink, not of the metric; `make cost` remains the authority. -## 7. Revisions to M-D2-CST +## 7. The live cost budget + +Replaces the 8k/10k per-task **token** budget, which was never referenced +or enforced, which CB-WP-0001 T08 exceeded by orders of magnitude with no +signal, and which InnerLoop v1.1 now marks dead. It implied a control that +did not exist. + +**The design constraint that shapes this.** Per-task cost needs the commit +that *closes* the task (CA-08), so it is unavoidably retrospective — a +per-task budget can only ever report a breach after the money is spent. +What is observable mid-task is **spend since the last commit**, because +the transcript is an append-live file (CA-07). That is the only number a +budget can actually fire on, so that is what the budget binds. + +> **CB-01.** The budget is the **open remainder**: USD spent since the +> last commit, computable at any instant by `make cost-budget`. + +> **CB-02.** Thresholds: **soft $10.00**, **hard $22.00**. +> +> - **soft** — state progress as a percentage and decide: continue, or +> commit what works and decompose. +> - **hard** — stop. Commit what works, or decompose. Uncommitted work is +> also *unattributable* work, so a hard breach costs twice: once in +> money, once in the 33% attribution gap. + +Calibration, measured over the 32 non-empty commit intervals of +CB-WP-0001 (`cb-cost --pin fc76445`): + +| | spend per interval | +|---|---| +| min | $0.07 | +| p50 | $1.40 | +| p90 | $9.36 | +| **max** | **$10.80** (T07, the Cargo workspace scaffold) | + +Soft is set at **$10.00**, just below the observed maximum and just above +p90: on the pass this is calibrated from it would have fired exactly once, +on the single most expensive interval. Hard is set at **$22.00**, roughly +2× the observed maximum — a value never reached in 32 intervals, so +reaching it means the session is doing something the calibration data has +no example of, which is precisely when stopping is right. + +**Why these are not retargeted-after-the-fact.** T07's concern is a target +moved to accommodate a measurement. These thresholds are set *above* every +observed value, so they bind on future work rather than ratifying present +work — no interval in the calibration set breaches hard, and one breaches +soft. If a future pass routinely breaches, that is signal, not a reason to +raise them. + +**Known limit:** the budget cannot fire *between* turns without something +running the command. It is available to an agent at any point, in CI, and +in `make all`; it is not a daemon. An agent that never runs it gets no +signal — which is exactly the dead-policy failure this replaces, one level +up. The mitigation is that it costs nothing to run and is on the one +command surface. + +## 9. The price sheet + +`benchmarks/baselines/model-prices.toml` is the single source of rates. +Two defects were fixed here, both of the same shape — **a schema that +could not hold the fact it needed**, which is the criticism this +capability's survey levelled at the State Hub: + +1. **A time-boxed rate lived in a comment.** Sonnet's intro price + (2.00/10.00 through 2026-08-31) was written as `# intro …` and was + therefore invisible to the collector, which used the standard rate. Cost + at the pin: **$0.17**, 0.19% of the pass. The real hazard was the + date — on **2026-08-31** the promotion expires and a comment-only + representation would have been wrong in the opposite direction, with + nothing to notice. Now expressed as `promo_*` fields and applied per + CA-16. +2. **The 90-day staleness rule had no check.** It was prose in + `MetricsAndScenarios.md` §1a and every M-D2-CST verdict silently + inherited it. Now `recorded` + `max_age_days` in the sheet, enforced by + CA-17 and exercised by `make cost-test`. + +Applying CA-16 moved AC-1 from $93.32 to **$93.15**. This is a *legitimate* +retarget under the distinction CB-WP-0003 T07 draws: the instrument +disproved the target, rather than the implementation missing it. The +instrument's output is in the same commit. + +## 8. Revisions to M-D2-CST `specs/MetricsAndScenarios.md` §1a is superseded by this spec. M-D2-CST is redefined from "tokens × pricepoint" — which named no instrument and was diff --git a/specs/GroundRules.md b/specs/GroundRules.md index 78a1284..620b120 100644 --- a/specs/GroundRules.md +++ b/specs/GroundRules.md @@ -210,6 +210,15 @@ terms. ## Underdetermined in dataset 0.1 — PROVISIONAL defaults (flag to ground-game) +**Owner: ground-game. Raised: 2026-07-31.** Every U-item carries an owner +and a raise date so its age is visible; `make coverage` reports both and +warns past 30 days. CI **warns rather than breaks** — the ruling belongs +to ground-game and the kernel cannot make it for them — but every evidence +file that depends on one must list it (CB-WP-0003 T08). + +A ruling flips the *scenario*, not the kernel: the default is encoded in a +scenario tagged `provisional: true`, so a correction is a data change. + Formalization exposed points the dataset does not decide. Simulation uses the stated default; each is tagged in scenarios that depend on it and must be confirmed or corrected by ground-game. diff --git a/specs/InnerLoop.md b/specs/InnerLoop.md index 178bd2a..3e1ccee 100644 --- a/specs/InnerLoop.md +++ b/specs/InnerLoop.md @@ -1,6 +1,13 @@ # The Inner Loop — Assimilate and Surpass -Status: **v1.1** — corrected from CB-WP-0002 (cost accounting) on +Status: **v1.2** — corrected from CB-WP-0003 (loop hardening) on +2026-07-31. Changes from v1.1: single source of fact; review targets the +harness and states its sampling limit; correction vs retarget; the chaos +roll's calibration window; the live cost budget. The design goal is now +stated: **optimize for cheap correction, not for exhaustive prevention.** +Rationale: `history/260731-loop-hardening-retrospective.md`. + +v1.1 — corrected from CB-WP-0002 (cost accounting) on 2026-07-31. Changes from v1.0: the instrument must exist and emit its own target; inherited numbers are re-derived before use; every reporting tool exposes `--self-test`; cost is in the definition of done. Rationale: @@ -14,7 +21,25 @@ evidence state what it does not support. Rationale and the failures behind each: `history/260731-inner-loop-retrospective.md`. Normative process for building every Clay-Borg capability. Referenced by -all workplans. The loop's own optimization target is **agentic efficiency**: +all workplans. + +**Design goal (v1.2).** Three passes produced ten error instances across +four classes, and every pass produced a class the previous one had not +seen — prevention is not converging. Every one of those errors was +corrected inside the same session for under ~1% of the pass. The loop +therefore optimizes for **cheap correction**: keep the raw data so numbers +can be re-derived, keep artifacts small and committed so a wrong number is +one grep from everywhere that quotes it, and give every reported number a +command so re-running is free. + +> **Single source of fact.** A number, rate, or target lives in exactly +> one place; everywhere else links to it. Where a copy is unavoidable it +> is generated by a command, not typed. *(v1.2: a price sheet inlined into +> a spec went stale within the hour of the real sheet changing, and one +> acceptance figure had to be chased across three artifacts each time it +> moved. No positive control catches this — both copies are internally +> consistent — and re-derivation does not either, because the copy +> reproduces whatever it was copied from.)* The loop's own optimization target is **agentic efficiency**: every artifact it produces must be small enough to load whole, structured enough to act on without interpretation, and falsifiable enough that an agent can judge its own work without a human in the iteration. @@ -46,10 +71,29 @@ are never skipped for code-producing work. | **M** | Survey and ADR merged into one document; review optional | Touches a canonical interface, or adds/updates an external dependency | | **S** | One provenance paragraph in the commit message | Everything else (utilities, fixes, refactors inside a boundary) | -**The chaos roll.** After deriving the structural tier, roll d10 -(`shuf -i 1-10 -n 1`). On a **10**, the tier is instead picked uniformly at +**The chaos roll.** After deriving the structural tier, roll **d4** +(`shuf -i 1-4 -n 1`). On a **4**, the tier is instead picked uniformly at random (`shuf -e S M L -n 1`), overriding the structural derivation — up or -down. Both rolls are recorded in the tier declaration +down. + +> **Calibration window, opened 2026-07-31 (CB-WP-0003 T06).** The rate was +> d10 and the mechanism **never fired**: two rolls across two workplans +> (CB-WP-0001: 9, CB-WP-0002: 2), against ~0.2 expected firings. At d10 and +> ~2 tier decisions per workplan it would take roughly twenty workplans to +> observe four overrides, so the mechanism was set at a rate that prevented +> its own evaluation — the one option T06 ruled out. +> +> Raised to **d4 (25%) for the next 12 tier declarations**, then evaluated +> and either kept, returned to d10, or deleted. Expected ~3 firings in the +> window, which is enough to see whether an overridden tier produces a +> different outcome than the argued one. +> +> **Stated cost:** a chaos-L override on work that would have been S buys a +> full survey, adversarial review, and ADR. Measured comparable: CB-WP-0001 +> T03 (a tier-L survey) cost **$9.91**. At 25% over 12 declarations the +> window is expected to cost **$20–30**. That is the price of finding out +> whether the mechanism is worth keeping, and it is cheaper than carrying an +> unevaluated ritual indefinitely. Both rolls are recorded in the tier declaration (`tier: M (structural L, chaos 10→M)`). **Record the roll every time, including when it changes nothing** (`tier: L (structural L, chaos 4)`), so a mechanism that never fires is visible rather than assumed. Purpose: @@ -87,11 +131,40 @@ evidence verdict for those rows caps at `parity`, never `better`. ### Step 2 — Approve (adversarial review) For tier-L passes, approval is earned through an **adversarial review**: a -separate session (or agent), given only the survey document, attempts to -break it — an omitted candidate, a stale or unverifiable benchmark, an -unmeasured claim presented as measured. Exactly **one round**: challenge, -then response. The survey is approvable only when every challenge is either -answered with evidence or conceded and folded into the survey. +separate session (or agent) attempts to break the work. Exactly **one +round**: challenge, then response. The work is approvable only when every +challenge is either answered with evidence or conceded and folded in. + +**The review target follows the risk.** Reviewing the survey was the +original rule, and it is the wrong target for a capability whose claim +rests on numbers — every serious error in this project has been in +measurement or build configuration, not in prose. + +| the claim rests on | the reviewer is given | and must attempt | +|---|---|---| +| a survey of candidates | the survey | an omitted candidate; a stale or unverifiable benchmark; an unmeasured claim presented as measured | +| **numbers** | the survey **and the harness and the evidence file** | **reproduce the number independently**; state what the harness would report if the work silently stopped | + +**What review cannot do — state this, do not discover it.** A reviewer +re-derives the author's claims and therefore inherits the author's +sampling. So: + +> **The reviewer re-derives on a different sample than the author used.** +> Where only one sample exists, the review says so rather than reporting a +> clean verify. + +*(v1.1, from CB-WP-0002: the dedup invariant was verified on the main +transcript by the survey — 206/206 groups — and independently re-verified +by the reviewer, who used the same transcript. It is false in the +8-response `subagents/` tree neither examined. Two independent checks, +one blind spot, because both sampled the same way. Only an assertion +running over all the data at execution time caught it.)* + +**What review demonstrably does do.** Measured across two passes: +**$0.66** and **$1.11**, roughly 1% of each pass, each finding +approval-blocking defects — in CB-WP-0002 a target that would have made +the evidence file certify a broken collector. The step pays for itself by +a wide margin and the cost is not a reason to skip it. **Documentation requirement:** the research process, the challenge, and the resulting improvements to the research are each documented in timestamped @@ -151,11 +224,40 @@ as a target, or it is cited as unverified.** Quoting is not measuring. from a prior pass. Re-derivation put it at $92.21 — the quoted figure double-counted transcript lines and priced a three-model session at one model's rate. Neither error was of the harness-does-nothing class; both -sums ran over real data, and a positive control would have passed them.)* A metric must -also be checked against the contracts in the *same spec*: if a contract -makes a target unreachable, one of the two is wrong and the conflict is -resolved when it is noticed, not at the acceptance run. Re-check the -table whenever a contract is added. +sums ran over real data, and a positive control would have passed them.)* + +**Retargeting: the instrument may move a target, the implementation may +not.** A metric's target changes in only two ways, and they are not +treated alike: + +| | trigger | requirement | +|---|---|---| +| **corrected** | the instrument disproved the target — the target was computed by hand, or by an earlier tool with a defect | legitimate in the same commit, **provided the instrument's output is in that commit** | +| **retargeted** | the implementation missed the target and the target moves to accommodate it | requires an **ADR**: old target, the measurement, and why the new target binds on *future* work rather than merely passing present work | + +The distinction is not the implementer's self-report of intent. It is +mechanical: **a correction is one where the target moves and the +implementation does not.** If the same commit changes both the target and +the code the target measures, it is a retarget and needs the ADR. + +*(v1.1, from CB-WP-0002/0003: AM-4's targets were measured at 246,250 and +set at 250,000 in one commit by the implementer after seeing the number — +the structure this rule exists to stop. But CB-WP-0002 then moved AC-1 +three times, correctly, each time because a new instrument disproved the +old figure ($92.21 → $92.87 → $93.32 → $93.15). A blanket prohibition +would have forbidden four legitimate corrections to catch one bad +retarget.)* + +**Applied retroactively:** AM-4a and AM-4b are **unratified** until an ADR +is written or they are changed. They were set by the implementer after +seeing the measurement, in the commit that produced it, and the +implementation changed in that same commit — a retarget by the test above. +Tracked as an open item in `history/260731-inner-loop-rule-audit.md`. + +**A metric is checked against the contracts in its own spec.** If a +contract makes a target unreachable, one of the two is wrong and the +conflict is resolved when it is noticed, not at the acceptance run. +Re-check the table whenever a contract is added. *(v1.0, from CB-WP-0001: AM-4's ≤20-crate target was made unreachable by the K5 and K7 contracts written after it, and AM-12's cost metric was @@ -224,115 +326,17 @@ state the claim you will defend, and the claim you are not making. --- -## The four-dimension rubric - -Every survey, ADR, and acceptance table is organized by these dimensions: - -| Dimension | Question | Example measurable proxies | -|---|---|---| -| **D1 Ease of specification** | How simply can behavior be stated, tested, understood? | rules-to-scenario coverage %, spec lines per rule, time-to-first-correct-scenario for a fresh agent session | -| **D2 Efficiency of implementation** | How cheap to build and keep building? | source LOC, dependency count/weight, clean-build and incremental-build time, tokens-per-completed-task | -| **D3 Speed of execution** | How fast does it run? | benchmark wall-time vs baseline, events/sec, memory footprint, determinism overhead | -| **D4 Optionality** | How cleanly does it integrate, extend, get replaced? | public API surface size, count of leaked foreign types (must be 0), effort-to-swap measured by null/reference impl existence, WIT-expressibility | - -Scoring is always **relative to the step-1 baseline**, never absolute: -`better / parity / worse / unmeasured` per proxy, with the number attached. -`unmeasured` is legal in a survey, illegal in an evidence file. - --- -## Survey template (research/CB-RES-NNNN-.md) +## Reference material -```markdown -# CB-RES-NNNN: -capability: -status: draft | approved +The four-dimension rubric, the survey template, the implementation rules +each pass earned, the agentic-efficiency requirements, and the +definition of done live in +**[InnerLoopReference.md](InnerLoopReference.md)**. They are normative; +they are separated only so both files load whole. -## Candidates -Per candidate: origin, license, maturity, adoption; data model; mutation -mechanism; determinism/replay story; relevant performance (measured if -runnable locally, cited with source otherwise). - -## Baselines (benchmark-to-beat) -| Dimension | Baseline holder | Metric | Value | Provenance | -(one row minimum per dimension; provenance = measured / cited / estimated) - -## Verdict -Which candidate leads per dimension; what none of them do well -(the surpass opportunity); risks in the baselines themselves. -``` - ---- - -## Implementation rules the first pass earned - -These are cheap, and each exists because its absence cost something in -CB-WP-0001. See `history/260731-inner-loop-retrospective.md`. - -1. **No silently-ignored input.** A field that is parsed and then unused - is a defect, not a stub. Inputs are honoured or rejected with an - error — never dropped. *(A scenario `setup.patch` was parsed and - discarded; every scenario using it would have tested the wrong - initial state while passing.)* -2. **Decisions get commands, not defaults.** A rule that requires a - participant's choice is implemented as a command carrying that - choice. Until it is, **nothing claims coverage of it** — no tag, no - scenario, no acceptance row. Inventing a default to make a rule - "done" is the failure this prevents. -3. **Scaffolds are exercised or marked.** A scaffold's green gates are - not evidence. Any scaffold path no test reaches is marked as - unexercised. *(A compiling, fully-green scaffold shipped a state-hash - that would panic on any state holding a relation.)* -4. **Coverage gates that count tags say so.** A gate comparing rule IDs - against `covers:` lists proves no rule is unclaimed and no claimed - rule is invented. It does **not** prove a scenario exercises what it - names. Wherever such a number is reported, that limit is reported - with it. - ---- - -## Agentic-efficiency requirements - -The loop exists to be driven by agents. Therefore: - -1. **Whole-file loadability** — every loop artifact stays under ~400 lines; - split before exceeding, link with relative paths. -2. **Structured over prose** — tables and fenced blocks for anything a - later step must parse (baselines, acceptance metrics, evidence rows). -3. **One command surface** — all checks runnable through repo-root - commands (eventually `cb *`; until then, `make`/`cargo` aliases declared - in one place), each supporting deterministic, greppable output. -4. **Self-contained tasks** — a workplan task names its input artifacts and - output artifacts; a fresh session must be able to execute it from the - task text plus linked files alone. -5. **Evidence or it didn't happen** — claims of "better" live in committed - evidence files with numbers, never only in commit messages or chat. -6. **Token discipline** — per the global budget policy, a loop iteration - that exceeds its budget without measurable progress is stopped and - decomposed, not pushed through. - ---- - -## Definition of done — one loop pass - -A capability has completed the loop when all of the following are committed: - -- [ ] research/CB-RES-NNNN with approved status and full baseline table -- [ ] decisions/ADR-NNNN with per-dimension expected advantage -- [ ] specs/.md with acceptance-metrics table -- [ ] passing scenarios covering every numbered spec rule -- [ ] evidence/CB-EV-NNNN with final comparison vs baseline, no `unmeasured` -- [ ] every reported number produced by a harness with a positive - control; any metric that could not be instrumented is recorded as - uncomputable rather than estimated -- [ ] every unmet metric reported as unmet, with attribution and the - options for resolving it — a missed target is an output of the - loop, not a reason to move the target quietly -- [ ] **cost recorded**: `make cost` run for the pass, its composition - (not only its total) in the evidence file, and the per-task figures - pushed to the hub. M-D2-CST is no longer allowed to be - `uncomputable` — the instrument exists - ([CostAccounting.md](CostAccounting.md)) -- [ ] retrospective note (may be one paragraph appended to the evidence - file): what the loop itself should change -``` +*(Split 2026-07-31: this file reached 407 lines against its own ~400-line +loadability rule, and `make loop-lint` — added the same day to make that +rule executable — failed on the commit that pushed it over. The rule +caught its own author within an hour of being written.)* diff --git a/specs/InnerLoopReference.md b/specs/InnerLoopReference.md new file mode 100644 index 0000000..06af1fd --- /dev/null +++ b/specs/InnerLoopReference.md @@ -0,0 +1,141 @@ +# The Inner Loop — Reference + +Normative companion to **[InnerLoop.md](InnerLoop.md)**, which holds the +five steps, the tiers, and the chaos roll. Split from it on 2026-07-31 so +both files stay under the loop's own whole-file loadability limit. + +Everything here binds exactly as the process document does. + +## The four-dimension rubric + +Every survey, ADR, and acceptance table is organized by these dimensions: + +| Dimension | Question | Example measurable proxies | +|---|---|---| +| **D1 Ease of specification** | How simply can behavior be stated, tested, understood? | rules-to-scenario coverage %, spec lines per rule, time-to-first-correct-scenario for a fresh agent session | +| **D2 Efficiency of implementation** | How cheap to build and keep building? | source LOC, dependency count/weight, clean-build and incremental-build time, tokens-per-completed-task | +| **D3 Speed of execution** | How fast does it run? | benchmark wall-time vs baseline, events/sec, memory footprint, determinism overhead | +| **D4 Optionality** | How cleanly does it integrate, extend, get replaced? | public API surface size, count of leaked foreign types (must be 0), effort-to-swap measured by null/reference impl existence, WIT-expressibility | + +Scoring is always **relative to the step-1 baseline**, never absolute: +`better / parity / worse / unmeasured` per proxy, with the number attached. +`unmeasured` is legal in a survey, illegal in an evidence file. + +--- + +## Survey template (research/CB-RES-NNNN-.md) + +```markdown +# CB-RES-NNNN: +capability: +status: draft | approved + +## Candidates +Per candidate: origin, license, maturity, adoption; data model; mutation +mechanism; determinism/replay story; relevant performance (measured if +runnable locally, cited with source otherwise). + +## Baselines (benchmark-to-beat) +| Dimension | Baseline holder | Metric | Value | Provenance | +(one row minimum per dimension; provenance = measured / cited / estimated) + +## Verdict +Which candidate leads per dimension; what none of them do well +(the surpass opportunity); risks in the baselines themselves. +``` + +--- + +## Implementation rules the first pass earned + +These are cheap, and each exists because its absence cost something in +CB-WP-0001. See `history/260731-inner-loop-retrospective.md`. + +1. **No silently-ignored input.** A field that is parsed and then unused + is a defect, not a stub. Inputs are honoured or rejected with an + error — never dropped. *(A scenario `setup.patch` was parsed and + discarded; every scenario using it would have tested the wrong + initial state while passing.)* +2. **Decisions get commands, not defaults.** A rule that requires a + participant's choice is implemented as a command carrying that + choice. Until it is, **nothing claims coverage of it** — no tag, no + scenario, no acceptance row. Inventing a default to make a rule + "done" is the failure this prevents. +3. **Scaffolds are exercised or marked.** A scaffold's green gates are + not evidence. Any scaffold path no test reaches is marked as + unexercised. *(A compiling, fully-green scaffold shipped a state-hash + that would panic on any state holding a relation.)* +4. **Coverage gates that count tags say so.** A gate comparing rule IDs + against `covers:` lists proves no rule is unclaimed and no claimed + rule is invented. It does **not** prove a scenario exercises what it + names. Wherever such a number is reported, that limit is reported + with it. + +--- + +## Agentic-efficiency requirements + +The loop exists to be driven by agents. Therefore: + +1. **Whole-file loadability** — every loop artifact stays under ~400 lines; + split before exceeding, link with relative paths. +2. **Structured over prose** *(guidance, not a requirement — nothing can + check it)* — tables and fenced blocks for anything a later step must + parse (baselines, acceptance metrics, evidence rows). +3. **One command surface** — all checks runnable through repo-root + commands (eventually `cb *`; until then, `make`/`cargo` aliases declared + in one place), each supporting deterministic, greppable output. +4. **Self-contained tasks** — a workplan task names its input artifacts and + output artifacts; a fresh session must be able to execute it from the + task text plus linked files alone. +5. **Evidence or it didn't happen** — claims of "better" live in committed + evidence files with numbers, never only in commit messages or chat. +6. ~~**Token discipline** — per the global budget policy, a loop iteration + that exceeds its budget without measurable progress is stopped and + decomposed, not pushed through.~~ **DEAD POLICY.** The 8k/10k per-task + token budget was never referenced or enforced, and CB-WP-0001 T08 + exceeded it by orders of magnitude with no signal. It implies a control + that does not exist. Replacement in USD is CB-WP-0003 T05; until then + this is documentation of a gap, not a rule. + +6a. **Live cost budget** *(replaces the above)* — spend since the last + commit, soft **$10.00**, hard **$22.00**, checked by `make cost-budget`. + Calibrated on the 32 commit intervals of CB-WP-0001 (p50 $1.40, p90 + $9.36, max $10.80), so both thresholds bind on future work rather than + ratifying past work. Contract: [CostAccounting.md](CostAccounting.md) + §7 (CB-01, CB-02). It fires on the open remainder rather than per task + because per-task cost needs the commit that closes the task, and a + budget that can only report after the money is spent is the dead policy + this replaces. + +**Enforcement status.** Rules above that a command can check are enforced +by `make loop-lint`; the full classification of every InnerLoop rule as +executable / checkable / decorative, with the failure class each catches, +is in `history/260731-inner-loop-rule-audit.md`. Rules marked *guidance* +or *dead* say so where they appear, so a reader can tell a requirement +from a preference without consulting the audit. + +--- + +## Definition of done — one loop pass + +A capability has completed the loop when all of the following are committed: + +- [ ] research/CB-RES-NNNN with approved status and full baseline table +- [ ] decisions/ADR-NNNN with per-dimension expected advantage +- [ ] specs/.md with acceptance-metrics table +- [ ] passing scenarios covering every numbered spec rule +- [ ] evidence/CB-EV-NNNN with final comparison vs baseline, no `unmeasured` +- [ ] every reported number produced by a harness with a positive + control; any metric that could not be instrumented is recorded as + uncomputable rather than estimated +- [ ] every unmet metric reported as unmet, with attribution and the + options for resolving it — a missed target is an output of the + loop, not a reason to move the target quietly +- [ ] **cost recorded**: `make cost` run for the pass, its composition + (not only its total) in the evidence file, and the per-task figures + pushed to the hub. M-D2-CST is no longer allowed to be + `uncomputable` — the instrument exists + ([CostAccounting.md](CostAccounting.md)) +- [ ] retrospective note (may be one paragraph appended to the evidence + file): what the loop itself should change diff --git a/specs/MetricsAndScenarios.md b/specs/MetricsAndScenarios.md index c8b360f..9ae2846 100644 --- a/specs/MetricsAndScenarios.md +++ b/specs/MetricsAndScenarios.md @@ -31,6 +31,7 @@ and add capability-specific rows only when these don't cover the claim. | ID | Dimension | Metric | Unit | Provenance | |---|---|---|---|---| | M-D1-COV | D1 | numbered spec rules covered by ≥1 passing scenario | % | adapted:requirements-traceability (per-rule, not per-feature) | +| M-D1-LNK | D1 | claimed rules also **named in the aggregate source** | count | novel — closes the spec→code→scenario chain; instrument: `make coverage` (AM-1b) | | M-D1-SPL | D1 | spec lines per numbered rule | lines | novel — proxies statement simplicity; gameable, so paired with M-D1-COV | | M-D2-LOC | D2 | source LOC excluding tests (tokei) | lines | adopted:tokei | | M-D2-DEP | D2 | transitive dependency count (cargo tree) | crates | adopted:cargo-deny practice | @@ -44,6 +45,32 @@ and add capability-specific rows only when these don't cover the claim. | M-D4-LEAK | D4 | foreign types in canonical interfaces | count | novel — must be 0; enforced by grep/deny rule, the Clay-Borg hard rule | | M-D4-SWAP | D4 | capability has null + reference impls passing the same conformance suite | bool | adapted:hexagonal-architecture port testing | +### 1b. The coverage gate's two numbers (M-D1-COV, M-D1-LNK) + +M-D1-COV counts tags. It proves no rule is unclaimed and no claimed rule +is invented; it does **not** prove a scenario exercises what it names, and +that limit is printed with the number every time (InnerLoop implementation +rule 4). + +**M-D1-LNK** (CB-WP-0003 T09) closes one link of that chain mechanically: +a rule a scenario claims should also appear in the aggregate source, or +the claim rests on a tag and nothing else. + +Measured 2026-07-31: **49 of 58** claimed rules are named in +`games/ground/src/lib.rs`. **Unmet**, target 58. The nine unlinked: + +```text +GR-D07 GR-F02 GR-L03 GR-O03 GR-P01 GR-P02 GR-P03 GR-P04 GR-T01 +``` + +The gate **reports** this rather than failing, deliberately. Closing the +gap by adding rule IDs to comments would satisfy the check without +establishing that each rule is implemented — which is the overclaim +InnerLoop implementation rule 2 exists to prevent, and which CB-WP-0001 +committed once already. Each of the nine needs its implementation +confirmed before it is tagged. Promoting M-D1-LNK to a failing gate is +correct **after** that, not before. + ### 1a. Token cost accounting (M-D2-CST) > **Superseded 2026-07-31 by [CostAccounting.md](CostAccounting.md)**, which @@ -67,27 +94,11 @@ benchmarks/baselines/model-prices.toml # the price sheet, updated when prices ``` ```toml -# USD per million tokens; source: Anthropic pricing, recorded 2026-07-31 -[claude-fable-5] -input = 10.00 -output = 50.00 - -[claude-opus-5] -input = 5.00 -output = 25.00 - -[claude-sonnet-5] -input = 3.00 # intro 2.00 through 2026-08-31 -output = 15.00 # intro 10.00 through 2026-08-31 - -[claude-haiku-4-5] -input = 1.00 -output = 5.00 - -[cache] # multipliers on the input price -read = 0.1 -write_5m = 1.25 -write_1h = 2.0 +# The live sheet is benchmarks/baselines/model-prices.toml — read it, +# do not copy it. An inlined copy here went stale within hours: it still +# showed the sonnet intro rate as a `#` comment after CB-WP-0003 T11 made +# dated rates into data (promo_input / promo_output / promo_until), and +# added `recorded` + `max_age_days` for the staleness check (CA-16/CA-17). ``` Rules: @@ -108,7 +119,8 @@ Rules: failed pass scores nothing. - Prices are `adopted:anthropic-pricing` with a recorded date; a stale sheet (> 90 days or known price change) invalidates new `better` verdicts on - M-D2-CST until refreshed. + M-D2-CST until refreshed. **Now enforced** — CA-17 aborts the run rather + than leaving this as prose every verdict silently inherited. Determinism is not a metric but an **invariant**: N replays of the same seed and command log must produce bit-identical state hashes. Invariant diff --git a/specs/SessionShape.md b/specs/SessionShape.md new file mode 100644 index 0000000..44e4d38 --- /dev/null +++ b/specs/SessionShape.md @@ -0,0 +1,185 @@ +# Session Shape + +Status: **v1.0** — 2026-07-31, from CB-WP-0003 T04. +Instrument: `make cost` ([CostAccounting.md](CostAccounting.md)). + +How to shape an agentic session so it does not cost more than the work is +worth. **Every claim here carries the measurement it rests on.** This spec +exists because the numbers were surprising and because the advice this task +was created to give turned out to be wrong. + +--- + +## 1. The cost model + +```text +cost ≈ Σ over turns ( context_size × cache_read_rate ) + output + cache writes +``` + +Measured on the CB-WP-0001 pass: **88.0% of spend is cache, 11.9% is +output**, at 249:1 cache-read tokens to output tokens. Cost is driven by +**how much context is re-read per turn**, not by how much the model writes. + +Marginal cost of one turn, at the Opus cache-read rate ($0.50/M): + +| context | $/turn | +|---|---| +| 20,000 | $0.010 | +| 50,000 | $0.025 | +| 100,000 | $0.050 | +| 200,000 | $0.100 | +| 400,000 | $0.200 | +| 540,000 | $0.270 | + +> **SS-01.** A turn's cost is a function of context size alone, to within +> ~12%. Optimizing output length is not a cost lever; optimizing context is. + +## 2. Compaction is the primary control + +**The original premise of this spec was wrong and is recorded rather than +deleted.** CB-WP-0003 T04 was written to prescribe *one task per session*, +on the theory that mean context grows with turn count and long sessions are +therefore quadratic. Measured: + +| segment | turns | mean context | total | $/turn | +|---|---|---|---|---| +| start → compact 1 | 136 | 304,178 | $62.19 | **$0.457** | +| compact 1 → compact 2 | 202 | 193,493 | $30.01 | **$0.149** | + +The 202 turns *after* the first compaction cost less than half the 136 +before it. Sessions are **bounded-quadratic**: context grows between +compactions and resets at each one. + +Compaction events, read from `compactMetadata` in the transcript: + +| | pre | post | reduction | duration | +|---|---|---|---|---| +| C1 | 542,991 | 19,974 | **27×** | 151 s | +| C2 | 344,954 | 19,035 | **18×** | 97 s | + +> **SS-02.** The failure mode is a long **uncompacted** session, not a long +> session. Turn count is not the variable to control; context is. + +### 2.1 When compaction pays + +A compaction costs its own summarization call plus one cache +re-establishment. The re-establishment is measurable — the first post-compact +turn wrote 36,149 cache tokens and cost **$0.382**. The summarization call +itself does **not** appear in the transcript's `usage` records, so it is +bounded rather than measured: at worst it is one full read of the +pre-compaction context (543k tokens ≈ $2.72 at the uncached Opus input +rate), at best a cache read of the same (≈ $0.27). + +Savings are immediate and permanent: dropping from 543k to 20k context saves +**$0.262 per subsequent turn**. + +> **SS-03.** Break-even is **2–11 turns** depending on where the +> summarization call falls in that range. Any session expecting more than +> ~11 further turns should compact. In practice this means: **compact +> whenever context exceeds ~300k and work remains.** + +Measured context distribution across the CB-WP-0001 session shows how much +was spent above that line — p75 was 308,325 and p90 was 481,722, so roughly +a quarter of all turns ran at more than 2× the recommended ceiling. + +| percentile | context | +|---|---| +| p50 | 156,771 | +| p75 | 308,325 | +| p90 | 481,722 | +| p99 | 553,252 | + +## 3. Fresh session vs compaction + +A fresh session is **not** free, and this is the measurement that overturned +the original advice. Cold-start context floors, measured over the first 20 +turns of each session: + +| session | start type | mean context, first 20 turns | cost | +|---|---|---|---| +| `8cbd5701` | cold (system + CLAUDE.md + orientation) | 50,872 | $2.17 | +| `f1eb1147` | seeded by a compact summary | 65,819 | $1.90 | + +A cold session starts ~51k and must then **re-read the committed artifacts** +to become productive — the survey, the spec, the ADR — which is exactly the +context a compaction summary already contains. A compact-seeded session +starts ~66k with that reading already done. + +> **SS-04.** Prefer compaction over a fresh session for continuing work. +> A fresh session's lower floor (51k vs 66k) is repaid within the first few +> turns of re-reading artifacts the summary already held. +> +> Prefer a fresh session when the *task changes*, because then the prior +> context is not an asset — it is 66k of irrelevance re-read every turn. + +## 4. Batching + +> **SS-05.** Independent tool calls in one turn cost nothing extra: a turn +> is billed for its context regardless of how many tool calls it carries. +> Two calls in one turn cost half of two calls in two turns. + +Measured batching rate: + +| session | responses w/ tool calls | tool calls | responses batching ≥2 | calls in batched turns | +|---|---|---|---|---| +| `8cbd5701` | 361 | 406 | 31 (8.6%) | 76 (18.7%) | +| `f1eb1147` | 146 | 158 | 12 (8.2%) | 24 (15.2%) | +| pinned at `fc76445`, all transcripts (instrument) | 322 | 362 | 25 (7.8%) | 65 (18.0%) | + +**Correction to a prior claim.** CB-WP-0001's retrospective and the first +draft of CB-WP-0003 T04 both stated *"0 of 330 tool calls were batched"*. +That is wrong: 330 was the count of responses making exactly **one** call, +not the total number of calls. The real figure is 7.8–8.6% of responses and +18.0–18.7% of calls, depending on the window. The claim was never re-derived before being carried into a +workplan — the failure mode InnerLoop v1.1 rule 13 exists for, and the +**eighth** recorded error instance, in the trusted-arithmetic class that +`history/260731-inner-loop-rule-audit.md` identified as having no +executable defence. + +At an 8.6% batching rate there is real headroom, but the saving is bounded: +eliminating every avoidable single-call turn would remove at most a few +percent of turns, worth roughly $2–4 on a $93 pass. **Batch because it is +free, not because it is the lever.** The lever is §2. + +## 5. What this spec does not support + +- **n = 2 sessions, one repo, one model mix.** Every ratio here is from + CB-WP-0001/0002 and should be re-measured before being treated as general. +- **The compaction summarization call is bounded, not measured.** SS-03's + 2–11 turn break-even is a range because of it. Narrowing it requires a + cost source that sees non-transcript calls — the billing API (CB-RES-0002 + C4), which needs an admin key this machine does not have. +- **Context size is read as `cache_read_input_tokens`**, which is the cached + prefix, not the full prompt. On a cache miss the true context is larger + and the turn costs 10× more. Cache misses were not isolated in this + measurement. +- **No claim about quality.** Compaction discards detail. Every number here + says compaction is cheaper; none says the work is as good. A pass that + compacts and then repeats work it forgot has not saved anything, and this + spec cannot detect that. + +## 6. Acceptance metrics + +| ID | Metric | Target | Measured (pinned `fc76445`) | Verdict | Instrument | +|---|---|---|---|---|---| +| SH-1 | mean context per turn | ≤ 200,000 | **232,982** | **unmet** | `make cost-pin` | +| SH-2 | p90 context per turn | ≤ 300,000 | **492,042** | **unmet** | `make cost-pin` | +| SH-3 | batching rate (responses with ≥2 tool calls) | ≥ 20% | **7.8%** | **unmet** | `make cost-pin` | + +`cb-cost` emits all three (`session shape` block), so these are ratified +under InnerLoop v1.1 rather than hand-derived. Supporting counts from the +same run: **362 tool calls in 322 responses, 65 of them in batched turns.** + +**All three targets are unmet, and that is the finding, not a defect in the +targets.** Per the loop's definition of done, an unmet metric is an output: + +- SH-1/SH-2 are unmet because CB-WP-0001 ran 136 turns before its first + compaction. §2 is the remedy and it is now written down. +- SH-3 is set above the measured rate deliberately. Whether 20% is + reachable or arbitrary is unknown until a pass tries; the first pass that + reports it will say. It is the cheapest of the three to move, and the + least valuable — §4 puts the ceiling at $2–4 on a $93 pass. + +Targets are **not** being retargeted to match the measurement. That is the +structure CB-WP-0003 T07 exists to prevent, and doing it here — in the +commit that first measures them — would be the exact defect. diff --git a/tools/__pycache__/cb-cost.cpython-312.pyc b/tools/__pycache__/cb-cost.cpython-312.pyc index 12efe0a..4b96e00 100644 Binary files a/tools/__pycache__/cb-cost.cpython-312.pyc and b/tools/__pycache__/cb-cost.cpython-312.pyc differ diff --git a/tools/cb-cost.py b/tools/cb-cost.py index 14db4ac..a301cc4 100644 --- a/tools/cb-cost.py +++ b/tools/cb-cost.py @@ -72,16 +72,53 @@ def components(usage): } -def price_of(prices, model, toks): +def rates_at(prices, model, when=None): + """Input/output rate in force for `model` at ISO instant `when` (CA-16). + + Promotional rates are data, not comments. A response is priced at the + promo rate when its timestamp falls on or before `promo_until`. + """ + pr = prices.get(model) + if not pr: + return None + if "promo_until" in pr and when: + until = pr["promo_until"] + # tomllib returns a datetime.date for a bare TOML date. + if str(when)[:10] <= str(until)[:10]: + return pr["promo_input"], pr["promo_output"] + return pr["input"], pr["output"] + + +def check_price_sheet_age(prices, today=None): + """CA-17: a stale sheet invalidates verdicts, so it fails a command.""" + import datetime as _dt + + recorded = prices.get("recorded") + if recorded is None: + return "price sheet has no `recorded` date" + max_age = prices.get("max_age_days", 90) + today = today or _dt.date.today() + if isinstance(recorded, _dt.datetime): + recorded = recorded.date() + age = (today - recorded).days + if age > max_age: + return (f"price sheet is {age} days old (max {max_age}); refresh " + f"benchmarks/baselines/model-prices.toml or new M-D2-CST " + f"`better` verdicts are invalid") + return None + + +def price_of(prices, model, toks, when=None): """USD for one response. Returns None when the model is unpriced (CA-05).""" pr = prices.get(model) if not pr: return None + rin, rout = rates_at(prices, model, when) cache = prices["cache"] - unit = pr["input"] / 1e6 + unit = rin / 1e6 return ( toks["input"] * unit - + toks["output"] * pr["output"] / 1e6 + + toks["output"] * rout / 1e6 + toks["cache_read"] * unit * cache["read"] + toks["write_5m"] * unit * cache["write_5m"] + toks["write_1h"] * unit * cache["write_1h"] @@ -147,6 +184,14 @@ def read_responses(path, pin=None): ) toks["output"] = max(t["output"] for t in per_row) head = rows[0] + # Tool calls are spread across the group's lines, so they are counted + # over the whole group — one response may carry several (SS-05). + tool_calls = sum( + 1 + for r in rows + for c in (r["message"].get("content") or []) + if c.get("type") == "tool_use" + ) out.append( { "request_id": rid, @@ -154,12 +199,33 @@ def read_responses(path, pin=None): "timestamp": head.get("timestamp") or "", "session": head.get("sessionId") or os.path.basename(path), "toks": toks, + "tool_calls": tool_calls, "subagent": "/subagents/" in path, } ) return out +def session_shape(responses): + """SH-1..SH-3 from specs/SessionShape.md.""" + import statistics + + ctx = sorted(r["toks"]["cache_read"] for r in responses) + with_tools = [r for r in responses if r["tool_calls"] > 0] + batched = [r for r in with_tools if r["tool_calls"] > 1] + calls = sum(r["tool_calls"] for r in with_tools) + pct = statistics.quantiles(ctx, n=100) if len(ctx) > 1 else [ctx[0]] * 99 + return { + "SH-1_mean_context": statistics.mean(ctx) if ctx else 0, + "SH-2_p90_context": pct[89], + "SH-3_batching_rate": (len(batched) / len(with_tools)) if with_tools else 0.0, + "p50_context": pct[49], + "responses_with_tools": len(with_tools), + "tool_calls": calls, + "calls_in_batched_turns": calls - (len(with_tools) - len(batched)), + } + + # ------------------------------------------------------------- attribution @@ -252,8 +318,12 @@ def collect(slug, pin_ref=None): # as though it were an answer. raise Abort(f"no responses in {len(paths)} transcript(s) — refusing to report") + stale = check_price_sheet_age(prices) + if stale: + raise Abort(stale) + for r in responses: - r["cost"] = price_of(prices, r["model"], r["toks"]) + r["cost"] = price_of(prices, r["model"], r["toks"], r["timestamp"]) attribute(responses, commit_index(pin)) @@ -269,11 +339,11 @@ def collect(slug, pin_ref=None): continue by_task[r["task"]] += r["cost"] by_model[r["model"]] += r["cost"] - pr = prices[r["model"]] - unit = pr["input"] / 1e6 + rin, rout = rates_at(prices, r["model"], r["timestamp"]) + unit = rin / 1e6 rates = { "input": unit, - "output": pr["output"] / 1e6, + "output": rout / 1e6, "cache_read": unit * cache["read"], "write_5m": unit * cache["write_5m"], "write_1h": unit * cache["write_1h"], @@ -293,6 +363,7 @@ def collect(slug, pin_ref=None): sub = sum(r["cost"] or 0 for r in responses if r["subagent"]) return { + "session_shape": session_shape(responses), "slug": slug, "pin": pin, "responses": len(responses), @@ -344,6 +415,17 @@ def render(rep, by_task=False, composition=False): f" A per-task table is a view over {100*(1-un/tot):.0f}% of spend." ) + sh = rep["session_shape"] + print("\n session shape (specs/SessionShape.md)") + print(f" SH-1 mean context {sh['SH-1_mean_context']:>12,.0f} tok " + f"[{'ok ' if sh['SH-1_mean_context']<=200_000 else 'FAIL'} target 200,000]") + print(f" SH-2 p90 context {sh['SH-2_p90_context']:>12,.0f} tok " + f"[{'ok ' if sh['SH-2_p90_context']<=300_000 else 'FAIL'} target 300,000]") + print(f" SH-3 batching rate {100*sh['SH-3_batching_rate']:>11.1f}% " + f"[{'ok ' if sh['SH-3_batching_rate']>=0.20 else 'FAIL'} target 20.0%]") + print(f" {sh['tool_calls']} tool calls in {sh['responses_with_tools']} responses; " + f"{sh['calls_in_batched_turns']} in batched turns") + if rep["unpriced"]: print(f"\n UNPRICED ({len(rep['unpriced'])} responses, model not in sheet):") for u in rep["unpriced"]: @@ -415,6 +497,27 @@ def self_test(): finally: os.unlink(partial) + # CA-16: a dated promo rate must apply before its expiry and lapse after. + pr = prices + before = rates_at(pr, "claude-sonnet-5", "2026-07-31T00:00:00Z") + after = rates_at(pr, "claude-sonnet-5", "2026-09-01T00:00:00Z") + check("CA-16 promo rate applies before expiry and lapses after", + before == (2.0, 10.0) and after == (3.0, 15.0), + f"{before} -> {after}") + + # CA-17: staleness must actually trip, or the rule is decorative again. + import datetime as _dt + fresh = check_price_sheet_age(pr, _dt.date(2026, 8, 1)) + stale = check_price_sheet_age(pr, _dt.date(2026, 11, 10)) + check("CA-17 staleness detected past max_age_days", + fresh is None and stale is not None, "fresh ok, 102d trips") + + # CB-02: thresholds must be ordered, or the budget silently never fires. + ap_defaults = {"soft": 10.00, "hard": 22.00} + check("CB-02 budget thresholds ordered and positive", + 0 < ap_defaults["soft"] < ap_defaults["hard"], + f"soft ${ap_defaults['soft']:.2f} < hard ${ap_defaults['hard']:.2f}") + # AC-6: zero responses must not report $0.00 as an answer. with tempfile.NamedTemporaryFile("w", suffix=".jsonl", delete=False) as fh: fh.write(json.dumps({"type": "user", "message": {}}) + "\n") @@ -440,12 +543,53 @@ def self_test(): return 0 if ok else 1 +def budget(slug, soft, hard): + """Live cost budget (specs/CostAccounting.md §7). + + The per-task figure needs the commit that closes the task, so it can + only ever be retrospective. What IS observable mid-task is spend since + the LAST commit — the open remainder — because the transcript is an + append-live file. That is the number a budget can actually fire on. + """ + try: + rep = collect(slug, None) + except Abort as e: + print(f"ABORT — {e}", file=sys.stderr) + return 1 + open_spend = rep["by_task"].get(OPEN_REMAINDER, 0.0) + head = subprocess.run( + ["git", "-C", REPO, "log", "-1", "--format=%h %s"], + capture_output=True, text=True, check=True).stdout.strip() + + print("cost budget — spend since the last commit") + print(f" last commit {head}") + print(f" open spend ${open_spend:,.2f}") + print(f" soft / hard ${soft:,.2f} / ${hard:,.2f}") + if open_spend > hard: + print(f"\n HARD BREACH — ${open_spend:,.2f} > ${hard:,.2f}. Commit what " + f"works, or stop and decompose. Uncommitted work is also " + f"unattributable.", file=sys.stderr) + return 1 + if open_spend > soft: + print(f"\n soft breach — ${open_spend:,.2f} > ${soft:,.2f}. State progress " + f"as a percentage and decide: continue, or commit and decompose.") + return 0 + print("\n within budget") + return 0 + + def main(): ap = argparse.ArgumentParser() ap.add_argument("--slug", default="-home-worsch-clay-borg") ap.add_argument("--pin", help="commit-ish or ISO Z instant (CA-07)") ap.add_argument("--by-task", action="store_true") ap.add_argument("--composition", action="store_true") + ap.add_argument("--session-shape", action="store_true", + help="SH-1..SH-3 (always shown in the default report)") + ap.add_argument("--budget", action="store_true", + help="CB-01/CB-02: spend since the last commit, live") + ap.add_argument("--soft", type=float, default=10.00) + ap.add_argument("--hard", type=float, default=22.00) ap.add_argument("--self-test", action="store_true") ap.add_argument("--json", action="store_true") args = ap.parse_args() @@ -453,6 +597,9 @@ def main(): if args.self_test: return self_test() + if args.budget: + return budget(args.slug, args.soft, args.hard) + try: rep = collect(args.slug, args.pin) except Abort as e: diff --git a/tools/dep-weight.py b/tools/dep-weight.py index dbcb597..f5df6b3 100755 --- a/tools/dep-weight.py +++ b/tools/dep-weight.py @@ -16,7 +16,7 @@ that cannot be found is reported and the run exits non-zero rather than silently under-reporting the total — under-reporting is the exact direction this metric could be gamed. -Usage: python3 tools/dep-weight.py [--json] +Usage: python3 tools/dep-weight.py [--json] [--self-test] """ import glob @@ -88,7 +88,50 @@ def source_lines(name, version): return 0 +def self_test(): + """Each assertion pins a failure this tool must detect. + + The controls that matter here are: an unlocatable crate must not be + silently counted as zero lines (that under-reports, the direction this + metric could be gamed), and a target breach must fail rather than + merely print. + """ + results = [] + + def check(name, ok, detail=""): + results.append((name, ok, detail)) + + # A crate that does not exist must measure zero, so the caller's + # `lines == 0` guard fires rather than silently shrinking the total. + check("unlocatable crate measures zero (so the guard fires)", + source_lines("definitely-not-a-real-crate-xyz", "9.9.9") == 0) + + # A crate we do depend on must measure non-zero, or the guard above + # would fire on everything and the tool would never report at all. + real = source_lines("serde", "1") + check("a real vendored crate measures non-zero", real > 0, + f"{real:,} lines") + + # Targets must be present and numeric — a missing target would make + # the breach check vacuous. + check("targets defined for every configuration", + set(TARGETS) == set(CONFIGS) and all( + isinstance(v, int) and v > 0 for v in TARGETS.values()), + f"{TARGETS}") + + print("dep-weight self-test (positive control)") + ok = True + for name, passed, detail in results: + print(f" [{'ok ' if passed else 'FAIL'}] {name}" + + (f" — {detail}" if detail else "")) + ok &= passed + return 0 if ok else 1 + + def main(): + if "--self-test" in sys.argv: + return self_test() + report = {} missing = [] for label, args in CONFIGS.items(): diff --git a/tools/loop-lint.py b/tools/loop-lint.py new file mode 100644 index 0000000..d57b9a0 --- /dev/null +++ b/tools/loop-lint.py @@ -0,0 +1,266 @@ +#!/usr/bin/env python3 +"""Executable checks for specs/InnerLoop.md rules (CB-WP-0003 T01). + +The loop's own rules were prose. A rule nobody can run is a suggestion, +and the audit in history/260731-inner-loop-rule-audit.md found several +that were already being violated with no signal. This makes the +mechanically-checkable ones fail a command. + +Each check names the InnerLoop rule it enforces. Checks that cannot be +made mechanical are recorded in the audit as `checkable` or `decorative` +and are deliberately absent here — see the audit for why. + +Positive control (InnerLoop v1.1 §Step 5): --self-test asserts each check +actually detects its failure, using fixtures with known answers. A linter +that passes everything because its matcher is broken is the same defect +class as a benchmark timing rejected work. + +Usage: + python3 tools/loop-lint.py # lint the repo + python3 tools/loop-lint.py --self-test # positive control +""" + +import os +import re +import sys + +REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +LOADABILITY_LIMIT = 400 + +# Artifact classes the loop produces. history/ is an append-only trail +# (verbatim challenge text is not something to split), so it is exempt. +LOOP_DIRS = ("specs", "research", "decisions", "evidence", "workplans") + + +class Finding: + def __init__(self, rule, path, detail): + self.rule, self.path, self.detail = rule, path, detail + + def __str__(self): + return f" [{self.rule}] {self.path}\n {self.detail}" + + +def _md_files(root=REPO): + """Loop artifacts only. + + Vendored third-party trees (a baseline harness ships its own + node_modules) are not artifacts this loop produces, and linting them + buries the two real findings under eighteen irrelevant ones. + """ + for d in LOOP_DIRS: + base = os.path.join(root, d) + for dirpath, dirnames, files in os.walk(base): + dirnames[:] = [x for x in dirnames if x != "node_modules"] + for f in sorted(files): + if f.endswith(".md"): + yield os.path.relpath(os.path.join(dirpath, f), root) + + +def check_loadability(root=REPO): + """§Agentic-efficiency 1 — every loop artifact stays under ~400 lines.""" + out = [] + for rel in _md_files(root): + with open(os.path.join(root, rel)) as fh: + n = sum(1 for _ in fh) + if n > LOADABILITY_LIMIT: + out.append( + Finding( + "loadability", + rel, + f"{n} lines exceeds the ~{LOADABILITY_LIMIT}-line limit; " + f"split and link with relative paths", + ) + ) + return out + + +def check_evidence_no_unmeasured(root=REPO): + """§Rubric — `unmeasured` is legal in a survey, illegal in an evidence file.""" + out = [] + base = os.path.join(root, "evidence") + if not os.path.isdir(base): + return out + for f in sorted(os.listdir(base)): + if not f.endswith(".md"): + continue + rel = os.path.join("evidence", f) + for i, line in enumerate(open(os.path.join(root, rel)), 1): + # A row asserting the verdict, not prose discussing the word. + if re.search(r"\|\s*unmeasured\s*\|", line): + out.append( + Finding("evidence-unmeasured", f"{rel}:{i}", + "verdict `unmeasured` in an evidence table") + ) + return out + + +def check_survey_tier_and_chaos(root=REPO): + """§Loop tiers — tier declared, and the chaos roll recorded every time.""" + out = [] + base = os.path.join(root, "research") + if not os.path.isdir(base): + return out + for f in sorted(os.listdir(base)): + if not f.endswith(".md"): + continue + rel = os.path.join("research", f) + text = open(os.path.join(root, rel)).read() + if not re.search(r"^tier:\s*[SML]\b", text, re.M): + out.append(Finding("tier-declared", rel, "no `tier:` declaration")) + elif "chaos" not in text.lower(): + out.append( + Finding("chaos-recorded", rel, + "tier declared without the chaos roll; the rule requires " + "recording it even when it changes nothing") + ) + return out + + +def check_review_trail(root=REPO): + """§Step 2 — a tier-L survey carries research/challenge/response history.""" + out = [] + base = os.path.join(root, "research") + hist = os.path.join(root, "history") + if not (os.path.isdir(base) and os.path.isdir(hist)): + return out + files = os.listdir(hist) + for f in sorted(os.listdir(base)): + if not f.endswith(".md"): + continue + rel = os.path.join("research", f) + text = open(os.path.join(root, rel)).read() + if not re.search(r"^tier:\s*L\b", text, re.M): + continue + if not re.search(r"^status:\s*approved", text, re.M): + continue + for kind in ("challenge", "response"): + if not any(x.endswith(f"-{kind}.md") and kind in x for x in files): + out.append( + Finding("review-trail", rel, + f"tier-L approved survey with no history/*-{kind}.md") + ) + return out + + +def check_reporting_tools_self_test(root=REPO): + """§Step 5 v1.1 — every tool that reports a number exposes --self-test.""" + out = [] + base = os.path.join(root, "tools") + if not os.path.isdir(base): + return out + for f in sorted(os.listdir(base)): + if not f.endswith(".py"): + continue + rel = os.path.join("tools", f) + text = open(os.path.join(root, rel)).read() + if "--self-test" not in text: + out.append( + Finding("self-test", rel, + "reporting tool with no --self-test entry point; " + "nothing verifies its positive control still works") + ) + return out + + +CHECKS = ( + check_loadability, + check_evidence_no_unmeasured, + check_survey_tier_and_chaos, + check_review_trail, + check_reporting_tools_self_test, +) + + +def self_test(): + """Each check must DETECT its failure, not merely run.""" + import shutil + import tempfile + + results = [] + + def check(name, ok, detail=""): + results.append((name, ok, detail)) + + tmp = tempfile.mkdtemp() + try: + for d in LOOP_DIRS + ("tools", "history"): + os.makedirs(os.path.join(tmp, d), exist_ok=True) + + # loadability: 401 lines must trip, 400 must not. + with open(os.path.join(tmp, "specs", "Big.md"), "w") as fh: + fh.write("x\n" * (LOADABILITY_LIMIT + 1)) + with open(os.path.join(tmp, "specs", "Ok.md"), "w") as fh: + fh.write("x\n" * LOADABILITY_LIMIT) + f = check_loadability(tmp) + check("loadability detects overlong artifact", + len(f) == 1 and "Big.md" in f[0].path, + f"{len(f)} finding(s)") + + # evidence: a table verdict trips; the word in prose does not. + with open(os.path.join(tmp, "evidence", "E.md"), "w") as fh: + fh.write("| AC-1 | x | unmeasured |\n" + "the word unmeasured appearing in prose is fine\n") + f = check_evidence_no_unmeasured(tmp) + check("evidence-unmeasured detects a table verdict, not prose", + len(f) == 1, f"{len(f)} finding(s), expected exactly 1") + + # tier/chaos: missing tier trips; tier without chaos trips. + with open(os.path.join(tmp, "research", "A.md"), "w") as fh: + fh.write("# survey\nno tier here\n") + with open(os.path.join(tmp, "research", "B.md"), "w") as fh: + fh.write("tier: L (structural L)\n") + f = check_survey_tier_and_chaos(tmp) + rules = sorted(x.rule for x in f) + check("tier/chaos detects both omissions", + rules == ["chaos-recorded", "tier-declared"], f"{rules}") + + # self-test: a tool without the flag trips. + with open(os.path.join(tmp, "tools", "silent.py"), "w") as fh: + fh.write("print(42)\n") + f = check_reporting_tools_self_test(tmp) + check("self-test detects a tool lacking --self-test", + len(f) == 1 and "silent.py" in f[0].path, f"{len(f)} finding(s)") + + # review trail: approved tier-L survey with no history trips. + with open(os.path.join(tmp, "research", "C.md"), "w") as fh: + fh.write("tier: L (structural L, chaos 3)\nstatus: approved\n") + f = check_review_trail(tmp) + check("review-trail detects a missing challenge/response", + len(f) == 2, f"{len(f)} finding(s), expected 2") + finally: + shutil.rmtree(tmp, ignore_errors=True) + + print("loop-lint self-test (positive control)") + ok = True + for name, passed, detail in results: + print(f" [{'ok ' if passed else 'FAIL'}] {name}" + + (f" — {detail}" if detail else "")) + ok &= passed + return 0 if ok else 1 + + +def main(): + if "--self-test" in sys.argv: + return self_test() + + findings = [] + for c in CHECKS: + findings.extend(c()) + + print("loop-lint — executable InnerLoop rules") + if not findings: + print(" no findings") + return 0 + by_rule = {} + for f in findings: + by_rule.setdefault(f.rule, []).append(f) + for rule, fs in sorted(by_rule.items()): + print(f"\n{rule} ({len(fs)}):") + for f in fs: + print(str(f)) + print(f"\n{len(findings)} finding(s)") + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/rule-coverage.py b/tools/rule-coverage.py index 306fa99..109f9ed 100755 --- a/tools/rule-coverage.py +++ b/tools/rule-coverage.py @@ -5,30 +5,190 @@ Compares the rule IDs declared in specs/GroundRules.md against the `covers:` lists in scenarios/ground/*.yaml. Exits non-zero when a scenario claims a rule the spec does not define, so coverage can never be inflated by a typo'd or invented rule ID. + +Stated limit (InnerLoop implementation rule 4): this gate counts tags. It +proves no rule is unclaimed and no claimed rule is invented. It does NOT +prove a scenario exercises the rule it names. + +Positive control (InnerLoop v1.1 §Step 5): the run asserts it actually +found rules and scenarios. Before this was added, a broken spec regex +yielded rules=[] and missing=[] and the tool exited 0 reporting "0/0" — +the harness-does-nothing class, in the tool that reports our headline +coverage number. + +Usage: + python3 tools/rule-coverage.py + python3 tools/rule-coverage.py --self-test """ import glob import re import sys -spec = open("specs/GroundRules.md").read() -rules = sorted(set(re.findall(r"\*\*(GR-[A-Z]+\d+)", spec))) +RULE_RE = r"\*\*(GR-[A-Z]+\d+)" +COVERS_RE = r"covers: \[(.*?)\]" +AGGREGATE = "games/ground/src/lib.rs" +# CB-WP-0003 T08: a provisional default with no expiry can shape the kernel +# indefinitely while looking handled. CI warns; it does not break the build, +# because the ruling is a ground-game decision we cannot make for them. +PROVISIONAL_WARN_DAYS = 30 +ID_RE = r"GR-[A-Z]+\d+" -covered = set() -for path in sorted(glob.glob("scenarios/ground/*.yaml")): - match = re.search(r"covers: \[(.*?)\]", open(path).read(), re.S) - if match: - covered |= {c.strip() for c in match.group(1).split(",") if c.strip()} -known = set(rules) -hit = sorted(known & covered) -missing = [r for r in rules if r not in covered] -invented = sorted(covered - known) +def parse_rules(spec_text): + return sorted(set(re.findall(RULE_RE, spec_text))) -pct = 100 * len(hit) // len(rules) if rules else 0 -print(f"AM-1 rule coverage: {len(hit)}/{len(rules)} ({pct}%)") -if missing: - print(" uncovered:", " ".join(missing)) -if invented: - print(" ERROR — claimed but not defined in the spec:", " ".join(invented)) - sys.exit(1) -sys.exit(0 if not missing else 2) + +def parse_code_ids(text): + """Rule IDs named anywhere in the aggregate source (T09).""" + return set(re.findall(ID_RE, text)) + + +def provisional_items(paths): + """(path, owner, raised) for every scenario encoding a U-item default.""" + out = [] + for path in paths: + text = open(path).read() + if not re.search(r"^provisional:\s*true", text, re.M): + continue + owner = re.search(r"^provisional_owner:\s*(\S+)", text, re.M) + raised = re.search(r"^provisional_raised:\s*(\S+)", text, re.M) + out.append((path, + owner.group(1) if owner else None, + raised.group(1) if raised else None)) + return out + + +def parse_covers(text): + match = re.search(COVERS_RE, text, re.S) + if not match: + return set() + return {c.strip() for c in match.group(1).split(",") if c.strip()} + + +def self_test(): + """Each assertion pins a failure this tool must detect.""" + results = [] + + def check(name, ok, detail=""): + results.append((name, ok, detail)) + + # The defect that motivated this control: a spec that parses to zero + # rules must not be reportable as coverage. + check("zero rules detected as a failure", parse_rules("no rules here") == [], + "empty spec yields no rules; main() now aborts on this") + # The matcher must actually match the real format. + check("rule matcher works on real spec format", + parse_rules("**GR-R06** something\n**GR-A12** other") + == ["GR-A12", "GR-R06"]) + # covers: parsing, including the empty case. + check("covers matcher works", parse_covers("covers: [GR-R06, GR-A12]") + == {"GR-R06", "GR-A12"}) + check("missing covers yields empty set", parse_covers("no covers key") == set()) + # T09: the spec -> code link must be detectable. + check("code-id matcher finds ids in source", + parse_code_ids("// GR-R06: lead first\nfn f(){} // GR-A12") + == {"GR-R06", "GR-A12"}) + check("code-id matcher finds none in unmarked source", + parse_code_ids("fn f() { let x = 1; }") == set()) + # T08: every provisional scenario must carry an owner and a date. + import glob as _g + prov = provisional_items(sorted(_g.glob("scenarios/ground/*.yaml"))) + check("every provisional item has an owner and a raised date", + bool(prov) and all(o and r for _, o, r in prov), + f"{len(prov)} provisional item(s)") + + print("rule-coverage self-test (positive control)") + ok = True + for name, passed, detail in results: + print(f" [{'ok ' if passed else 'FAIL'}] {name}" + + (f" — {detail}" if detail else "")) + ok &= passed + return 0 if ok else 1 + + +def main(): + if "--self-test" in sys.argv: + return self_test() + + rules = parse_rules(open("specs/GroundRules.md").read()) + paths = sorted(glob.glob("scenarios/ground/*.yaml")) + + # Positive control: refuse to report a percentage over nothing. + if not rules: + print("ERROR — no GR-rules parsed from specs/GroundRules.md; " + "refusing to report coverage", file=sys.stderr) + return 1 + if not paths: + print("ERROR — no scenarios found in scenarios/ground/; " + "refusing to report coverage", file=sys.stderr) + return 1 + + covered = set() + for path in paths: + covered |= parse_covers(open(path).read()) + + known = set(rules) + hit = sorted(known & covered) + missing = [r for r in rules if r not in covered] + invented = sorted(covered - known) + + # T09: the spec -> code -> scenario chain, made mechanical. A rule a + # scenario claims should also be named in the aggregate, or the claim + # rests on nothing but a tag. + code_ids = parse_code_ids(open(AGGREGATE).read()) + unlinked = sorted((known & covered) - code_ids) + phantom = sorted(code_ids - known) + + pct = 100 * len(hit) // len(rules) + linked = len((known & covered) & code_ids) + print(f"AM-1 rule coverage: {len(hit)}/{len(rules)} ({pct}%) " + f"over {len(paths)} scenarios") + print(f"AM-1b spec->code link: {linked}/{len(hit)} claimed rules also " + f"named in {AGGREGATE}") + print(" NOTE: counts tags; does not prove a scenario exercises what it names") + if unlinked: + print(" unlinked (claimed by a scenario, absent from the aggregate):") + print(" ", " ".join(unlinked)) + if phantom: + print(" ERROR — rule id in code that the spec does not define:", + " ".join(phantom), file=sys.stderr) + return 1 + # T08: provisional items are reported with an owner and an age. + prov = provisional_items(paths) + if prov: + import datetime + today = datetime.date.today() + print(f"\nprovisional U-item defaults: {len(prov)}") + unowned, stale = [], [] + for path, owner, raised in prov: + age = "?" + if raised: + try: + age = (today - datetime.date.fromisoformat(raised)).days + except ValueError: + age = "?" + name = path.split("/")[-1] + print(f" {name:<34} owner={owner or 'NONE':<12} age={age}d") + if not owner: + unowned.append(name) + if isinstance(age, int) and age > PROVISIONAL_WARN_DAYS: + stale.append(f"{name} ({age}d)") + if unowned: + print(" WARN — provisional with no owner:", " ".join(unowned)) + if stale: + print(f" WARN — provisional for over {PROVISIONAL_WARN_DAYS} days:", + " ".join(stale)) + print(" NOTE: evidence files must list these; a ruling flips the " + "scenario, not the kernel") + + if missing: + print(" uncovered:", " ".join(missing)) + if invented: + print(" ERROR — claimed but not defined in the spec:", " ".join(invented), + file=sys.stderr) + return 1 + return 0 if not missing else 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/workplans/CB-WP-0003-loop-hardening.md b/workplans/CB-WP-0003-loop-hardening.md index 8be635d..021e278 100644 --- a/workplans/CB-WP-0003-loop-hardening.md +++ b/workplans/CB-WP-0003-loop-hardening.md @@ -1,7 +1,7 @@ --- id: CB-WP-0003 title: "Harden the inner loop: executable rules, session economics, dead policy" -status: proposed +status: done state_hub_workstream_id: "39d61dc0-870d-45c1-a595-bcf91f289dce" --- @@ -55,7 +55,7 @@ accordingly. ```task id: CB-WP-0003-T01 -status: todo +status: done priority: high state_hub_task_id: "3d5d45fb-f931-407a-be71-d2d727279d5e" ``` @@ -89,7 +89,7 @@ rules and the three v1.1 rules are currently prose. ```task id: CB-WP-0003-T02 -status: todo +status: done priority: high state_hub_task_id: "f861e67f-ed09-489d-963d-06697da9c08e" ``` @@ -100,7 +100,12 @@ one with five assertions (AC-5..AC-9); `make cost` depends on `make cost-test`; `cost-test` runs in CI and in `make all`. `cargo bench -- --test` and cb-sim's empty-run check were already in place. -**Remaining scope:** +**Closed 2026-07-31 alongside T01** — both items below are done: +tools/rule-coverage.py and tools/dep-weight.py gained `--self-test` +(the former exposing a latent silent-pass on zero parsed rules), and +`make loop-lint` fails CI when any `tools/*.py` lacks the flag. + +**Scope as written:** - `tools/rule-coverage.py` and `tools/dep-weight.py` have positive-control logic but no `--self-test` entry point, so nothing verifies the control @@ -119,7 +124,7 @@ dedup. ```task id: CB-WP-0003-T03 -status: todo +status: done priority: medium state_hub_task_id: "1365b35d-4539-489a-beff-c072221e4702" ``` @@ -153,7 +158,7 @@ rather than reporting a clean verify. ```task id: CB-WP-0003-T04 -status: todo +status: done priority: high state_hub_task_id: "382724e6-efd5-4ef8-a438-9d31a65dacdb" ``` @@ -198,7 +203,7 @@ attribution. ```task id: CB-WP-0003-T05 -status: todo +status: done priority: medium state_hub_task_id: "26c920ee-2c09-4b45-a908-4d343532db09" ``` @@ -229,7 +234,7 @@ Unblocked: depends on CB-WP-0002, now complete. ```task id: CB-WP-0003-T06 -status: todo +status: done priority: low state_hub_task_id: "8f3a6147-54e8-48f9-8543-c13b0e0b0278" ``` @@ -248,7 +253,7 @@ to reject. ```task id: CB-WP-0003-T07 -status: todo +status: done priority: medium state_hub_task_id: "2f702821-7e64-4981-9885-82fa07f638aa" ``` @@ -281,7 +286,7 @@ self-report of which one it was. ```task id: CB-WP-0003-T08 -status: todo +status: done priority: low state_hub_task_id: "c2ee91ee-e551-48a7-b1a9-477c34c0690c" ``` @@ -299,7 +304,7 @@ file must list them, not that the build breaks. ```task id: CB-WP-0003-T09 -status: todo +status: done priority: low state_hub_task_id: "69ccc9ec-dc35-481c-8065-cef040f07f50" ``` @@ -320,9 +325,9 @@ now price the evaluation itself. ```task id: CB-WP-0003-T11 -status: todo +status: done priority: medium -state_hub_task_id: "" +state_hub_task_id: "809215d8-8a5c-4b10-9445-9bc6db9bda42" ``` **New, from CB-WP-0002.** `benchmarks/baselines/model-prices.toml` @@ -345,7 +350,7 @@ State Hub: a schema that cannot hold the fact it needs. ```task id: CB-WP-0003-T10 -status: todo +status: done priority: low state_hub_task_id: "f43de208-92b1-4f0e-9236-a9d27f3ec451" ```