diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml index d9e71d0..9185eb1 100644 --- a/.forgejo/workflows/ci.yml +++ b/.forgejo/workflows/ci.yml @@ -36,13 +36,7 @@ jobs: # Positive control for the cost collector (AC-5..AC-9). Does not # gate on a dollar figure — transcripts are not present in CI — but # proves the collector still detects the failures it claims to. - - run: make self-tests - - # InnerLoop rules made executable (CB-WP-0003 T01). Fails on an - # overlong artifact, a survey missing its tier/chaos declaration, an - # approved tier-L survey with no review trail, `unmeasured` in an - # evidence table, or a reporting tool with no --self-test. - - run: make loop-lint + - run: make cost-test # InnerLoop v1.0 positive control, enforced rather than asserted in # prose: --test runs every benchmark once, so a workload that diff --git a/Makefile b/Makefile index 02f4e61..46e9e79 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ CARGO := cargo -.PHONY: check test sim bench bench-test coverage dep-weight cost cost-test cost-pin cost-budget loop-lint self-tests loc all +.PHONY: check test sim bench bench-test coverage dep-weight cost cost-test cost-pin loc all ## fmt + clippy (deny warnings) + HashMap deny-lint check: @@ -29,21 +29,6 @@ cost: cost-test cost-test: python3 tools/cb-cost.py --self-test -# InnerLoop rules that are mechanically checkable (CB-WP-0003 T01). -loop-lint: - python3 tools/loop-lint.py - -# Positive control for every reporting tool, per InnerLoop v1.1 Step 5. -self-tests: - python3 tools/cb-cost.py --self-test - python3 tools/loop-lint.py --self-test - python3 tools/rule-coverage.py --self-test - python3 tools/dep-weight.py --self-test - -# CB-01/CB-02: live spend since the last commit. -cost-budget: cost-test - python3 tools/cb-cost.py --budget - cost-pin: cost-test python3 tools/cb-cost.py --pin fc76445 --composition --by-task @@ -66,4 +51,4 @@ loc: printf '%-28s %s\n' $$d "$$(find $$d/src -name '*.rs' | xargs cat | grep -vcE '^\s*(//|$$)')"; \ done -all: check test sim coverage dep-weight self-tests loop-lint bench-test +all: check test sim coverage dep-weight cost-test bench-test diff --git a/WORK-RECORDS.md b/WORK-RECORDS.md index 8bf92e3..418ce2e 100644 --- a/WORK-RECORDS.md +++ b/WORK-RECORDS.md @@ -37,4 +37,3 @@ | task | CB-WP-0003-T08 | todo | — | workplans/CB-WP-0003-loop-hardening.md | | task | CB-WP-0003-T09 | todo | — | workplans/CB-WP-0003-loop-hardening.md | | task | CB-WP-0003-T10 | todo | — | workplans/CB-WP-0003-loop-hardening.md | -| task | CB-WP-0003-T11 | todo | — | workplans/CB-WP-0003-loop-hardening.md | diff --git a/benchmarks/baselines/model-prices.toml b/benchmarks/baselines/model-prices.toml index 68cade5..3e12c08 100644 --- a/benchmarks/baselines/model-prices.toml +++ b/benchmarks/baselines/model-prices.toml @@ -1,15 +1,6 @@ -# USD per million tokens. Source: Anthropic pricing. -# Governed by specs/CostAccounting.md §9 — a stale sheet invalidates new -# M-D2-CST `better` verdicts. Staleness is enforced by `make cost-test`, -# not by this comment. -# -# Dated rates are DATA, not comments. A promotional rate written as a -# comment is invisible to the collector that reads this file: the sonnet -# intro price sat in a `#` remark and was silently ignored, costing $0.17 -# on a $93.32 pass and heading for a sign flip on 2026-08-31. - -recorded = 2026-07-31 -max_age_days = 90 +# USD per million tokens; source: Anthropic pricing, recorded 2026-07-31 +# Governed by specs/MetricsAndScenarios.md §1a — refresh on price change or +# after 90 days, or new M-D2-CST 'better' verdicts are invalid. [claude-fable-5] input = 10.00 @@ -20,13 +11,8 @@ input = 5.00 output = 25.00 [claude-sonnet-5] -input = 3.00 -output = 15.00 -# Promotional rate in force through `until` (inclusive); the collector -# applies it to any response whose timestamp falls on or before that date. -promo_input = 2.00 -promo_output = 10.00 -promo_until = 2026-08-31 +input = 3.00 # intro 2.00 through 2026-08-31 +output = 15.00 # intro 10.00 through 2026-08-31 [claude-haiku-4-5] input = 1.00 diff --git a/crates/cb-game-runtime/src/scenario.rs b/crates/cb-game-runtime/src/scenario.rs index ed47391..1a6aa08 100644 --- a/crates/cb-game-runtime/src/scenario.rs +++ b/crates/cb-game-runtime/src/scenario.rs @@ -21,16 +21,6 @@ pub struct ScenarioFile { /// scenario, not the kernel. #[serde(default)] pub provisional: bool, - /// Who must rule on the U-item this scenario encodes. Required when - /// `provisional` is true: a provisional default with no owner can - /// shape the kernel indefinitely while looking handled - /// (CB-WP-0003 T08). - #[serde(default)] - pub provisional_owner: String, - /// ISO date the provisional default was raised, so its age is - /// reportable by `make coverage`. - #[serde(default)] - pub provisional_raised: String, pub seed: u64, pub setup: Setup, pub commands: Vec, diff --git a/evidence/CB-EV-0002-cost-accounting.md b/evidence/CB-EV-0002-cost-accounting.md index e193449..a8f986a 100644 --- a/evidence/CB-EV-0002-cost-accounting.md +++ b/evidence/CB-EV-0002-cost-accounting.md @@ -17,9 +17,9 @@ transcribed by hand (CA-15). | ID | Metric | Target | Measured | Verdict | |---|---|---|---|---| -| AC-1 | pinned total, as two components | $93.15 = $92.03 + $1.11 | **$92.03 main + $1.11 subagent = $93.15** | **met** | +| AC-1 | pinned total, as two components | $93.32 = $92.21 + $1.11 | **$92.21 main + $1.11 subagent = $93.32** | **met** | | AC-2 | reconciliation residual | $0.00 | **$0.000000** | **met** | -| AC-3 | unattributed share reported | present, 33% | **32.4%, own line** | **met** | +| AC-3 | unattributed share reported | present, 33% | **32.5%, own line** | **met** | | AC-4 | composition reported | 5 components | **5 of 5** | **met** | | AC-5 | dedup violation aborts | non-zero exit | **abort raised** | **met** | | AC-6 | zero responses refuses to report | non-zero exit | **0 rows, no number emitted** | **met** | @@ -27,7 +27,7 @@ transcribed by hand (CA-15). | AC-8 | 5m cache priced at 1.25× | $1.25/100k @ fable | **$1.2500 (1h would be $2.0000)** | **met** | | AC-9 | streamed partial output → final | 5,5,195 → 195 | **195** | **met** | -No `unmeasured` rows. AC-1's target was corrected four times before and after this +No `unmeasured` rows. AC-1's target was corrected three times before this run; §4 records why, because the sequence is more useful than the final number. @@ -35,23 +35,23 @@ number. ```text input 690 tok $ 0.00 0.0% - output 323,643 tok $ 11.13 11.9% - cache_read 80,611,798 tok $ 59.67 64.1% + output 323,643 tok $ 11.15 11.9% + cache_read 80,611,798 tok $ 59.75 64.0% write_5m 37,467 tok $ 0.47 0.5% - write_1h 1,672,854 tok $ 21.87 23.5% - TOTAL $ 93.15 + write_1h 1,672,854 tok $ 21.95 23.5% + TOTAL $ 93.32 ``` **88.0% of spend is cache; 11.9% is output.** The ratio of context re-read -to text written is 249:1. A single total would have shown $93.15 and +to text written is 249:1. A single total would have shown $93.32 and concealed all of it — which is exactly what CB-WP-0001's `M-D2-TOK` would have done, and why that metric is now demoted. ## 3. Per-task attribution ```text - UNATTRIBUTED $ 30.14 32.4% - T08 $ 21.02 22.6% + UNATTRIBUTED $ 30.32 32.5% + T08 $ 21.02 22.5% T07 $ 12.01 12.9% T03 $ 9.91 10.6% T04 $ 8.00 8.6% @@ -60,7 +60,7 @@ have done, and why that metric is now demoted. T09 $ 1.29 1.4% ``` -**Stated limit (CA-10):** 32.4% of cost sits in commits whose subject +**Stated limit (CA-10):** 32.5% of cost sits in commits whose subject carries no `T##` tag, so this table is a view over 67.5% of spend. That is a property of commit hygiene, not of the collector. @@ -75,15 +75,14 @@ is worth watching, not yet a conclusion from n=1. - **The per-task figures are not comparable across passes.** They mix models (opus/fable/sonnet) at different price points and different cache states. The dollar figure is comparable; a token count is not. -- **AC-3's 32.4% is a fixture pin, not a quality target.** Improving commit +- **AC-3's 32.5% is a fixture pin, not a quality target.** Improving commit tagging will move it, and that is the desired direction. - **This is one session.** Every ratio here (cache share, $/turn, the compaction effect in §5) is n=1 and should be treated as a hypothesis until a second pass reproduces it. -- ~~**The price sheet cannot express a time-boxed rate.**~~ **Fixed - 2026-07-31 by CB-WP-0003 T11** (CA-16/CA-17). Applying the sonnet - promotional rate moved AC-1 from $93.32 to **$93.15** — the $0.17 this - section predicted, now collected rather than merely noted. +- **The price sheet cannot express a time-boxed rate.** Sonnet's intro price + lives in a TOML comment, so sonnet-priced work is off by $0.17 here + (0.19%). This becomes a real error on 2026-08-31. - **AC-1 is not invoice-verified.** No admin key exists, so the Anthropic billing API could not independently confirm the total. The transcript counters are the same ones billing uses, but that is an argument, not a @@ -146,7 +145,7 @@ largest sample was assumed to hold on the smallest one.** The main transcript is 338 of 346 responses, so 206/206 felt conclusive; the violation lives entirely in the 8 responses nobody checked separately. -Cost of the adversarial review this pass: **$1.11**, against a $93.15 pass. +Cost of the adversarial review this pass: **$1.11**, against a $93.32 pass. It found three approval-blocking defects, one of which (the subagent exclusion) would have made this evidence file certify a broken collector. Second consecutive pass where a ~1% spend on review changed the outcome. diff --git a/history/260731-inner-loop-rule-audit.md b/history/260731-inner-loop-rule-audit.md deleted file mode 100644 index 13999eb..0000000 --- a/history/260731-inner-loop-rule-audit.md +++ /dev/null @@ -1,157 +0,0 @@ -# 2026-07-31 — InnerLoop v1.1 rule enforceability audit - -CB-WP-0003 T01. Every rule in `specs/InnerLoop.md` v1.1 classified as -**executable** (a command fails when it is violated), **checkable** (a -human or agent can verify it cheaply and objectively in review), or -**decorative** (neither) — and, per the revised task, tagged with the -**failure class it catches**. - -## Failure classes on record - -Seven error instances across three classes, from -`history/260731-inner-loop-retrospective.md` and -`history/260731-cost-accounting-retrospective.md`: - -| tag | class | instances | -|---|---|---| -| **HDN** | harness-does-nothing — a run succeeds while performing no work | 4 | -| **TA** | trusted arithmetic — a correct-looking sum over data that really exists | 2 | -| **SSB** | same-sample blind spot — a property verified on the large sample, assumed on the small | 1 | - -The reason for tagging: a rule that catches a class no observed error -belongs to is a candidate for deletion even when it is perfectly -executable, and a class with no rule covering it is where the next rule -should go. - -## The audit - -Status after this pass. `loop-lint` = `tools/loop-lint.py`, new here. - -| # | Rule | Class | Status | Enforced by | -|---|---|---|---|---| -| 1 | No implementation code before the ADR is committed | — | **checkable** | git history vs ADR date; not automated (see §Deferred) | -| 2 | Tier declared before work starts | — | **executable** | `loop-lint` tier-declared | -| 3 | Chaos roll recorded every time, even when it changes nothing | — | **executable** | `loop-lint` chaos-recorded | -| 4 | Invariants bind at every tier regardless of roll | — | executable (elsewhere) | `make all` — determinism tests, `disallowed_types` | -| 5 | Survey names a benchmark-to-beat per dimension | — | **checkable** | review; a regex cannot judge whether a number is a benchmark | -| 6 | Cited-only baselines cap the verdict at `parity` | TA | **decorative → checkable** | see §Changes | -| 7 | Tier-L survey gets one round of adversarial review | SSB | **executable** | `loop-lint` review-trail | -| 8 | Review trail: research/challenge/response in `history/` | SSB | **executable** | `loop-lint` review-trail | -| 9 | ADR states expected advantage per dimension | — | **checkable** | review | -| 10 | Spec carries an acceptance-metrics table | — | **checkable** | review | -| 11 | Every metric names its instrument | HDN | **checkable** | review; naming is textual, existence is not | -| 12 | **v1.1** The instrument must exist and emit its own target | TA | **checkable** | review; see §Deferred for why not yet executable | -| 13 | **v1.1** Inherited numbers are re-derived before use as a target | TA | **decorative** | nothing can detect a quoted number | -| 14 | A metric is checked against contracts in its own spec | — | **checkable** | review | -| 15 | Harness asserts it performed the work it reports | HDN | **executable** | `cargo bench -- --test`, cb-sim empty-run, `--self-test` | -| 16 | Fail loudly rather than report when the assertion fails | HDN | **executable** | same | -| 17 | Divisor pinned by a test | HDN | **executable** | `bench_shape` test module | -| 18 | A number that cannot prove its work is void | HDN | **executable** | same as 15 | -| 19 | **v1.1** Every reporting tool exposes `--self-test` | HDN, SSB | **executable** | `loop-lint` self-test | -| 20 | `--self-test` names a failure it detects, not the happy path | HDN | **checkable** | review; see §Deferred | -| 21 | Evidence names its disanalogies | — | **checkable** | review | -| 22 | `unmeasured` illegal in an evidence file | — | **executable** | `loop-lint` evidence-unmeasured | -| 23 | No silently-ignored input | HDN | **checkable** | review | -| 24 | Decisions get commands, not defaults | — | **checkable** | review | -| 25 | Scaffolds are exercised or marked | HDN | **checkable** | review | -| 26 | Coverage gates that count tags say so | — | **checkable** | review | -| 27 | Whole-file loadability (~400 lines) | — | **executable** | `loop-lint` loadability | -| 28 | Structured over prose | — | **decorative** | kept as guidance, marked | -| 29 | One command surface | — | **checkable** | review | -| 30 | Self-contained tasks | — | **checkable** | review | -| 31 | Evidence or it didn't happen | — | **checkable** | review | -| 32 | Token discipline per the global budget policy | — | **decorative — dead** | nothing; CB-WP-0003 T05 replaces it | -| 33–41 | Definition-of-done checklist (9 items) | mixed | **checkable** | review; each maps to an artifact whose existence is testable — see §Deferred | - -Counts: **11 executable**, **22 checkable**, **4 decorative** (one of -which is dead policy). - -## What the audit found by running - -`tools/loop-lint.py` was written to make rules 2, 3, 7, 8, 19, 22 and 27 -executable. On its first run it produced **three findings, all real, none -previously visible**: - -1. **`specs/ArchitectureBlueprint.md` is 543 lines** against a ~400-line - limit the loop has stated since v0.2. Nobody noticed because nothing - measured it. Disposition: split (§Changes). -2. **`tools/dep-weight.py` has no `--self-test`.** It has positive-control - logic — it refuses to report when a crate cannot be located — but - nothing verifies that control still works. -3. **`tools/rule-coverage.py` has no `--self-test`.** Same shape. - -Findings 2 and 3 are exactly the recursion this workplan is about: the -positive-control rule was applied to benchmarks and to the newest tool, -and not to the two older tools that report AM-1 and AM-4 numbers into -evidence files. - -## Changes made in this task - -- **Rule 27 made executable and the violation fixed.** - `ArchitectureBlueprint.md` split at its own section boundaries into - `ArchitectureBlueprint.md` (§1–8, the stack) and - `ArchitectureRuntime.md` (§9–15, runtime/tooling/process), linked both - ways. -- **Rules 2, 3, 7, 8, 19, 22 made executable** via `tools/loop-lint.py`, - wired as `make loop-lint` and into `make all` and CI. -- **Rule 6 (parity cap) demoted from decorative to checkable** by stating - the check explicitly: an evidence row citing a baseline whose provenance - is `cited` may not carry verdict `better`. That is mechanical against the - survey's provenance column and is queued for `loop-lint` once a second - evidence file exists to test it against — writing a matcher with one - sample is the SSB error this pass is trying to stop making. -- **Rule 28 (structured over prose) demoted to guidance**, with that - status stated in the spec. It is a style preference; nothing can judge - it, and leaving it phrased as a requirement is the false assurance this - audit exists to remove. -- **Rule 32 (token discipline) marked dead** in place, pointing at - CB-WP-0003 T05. It is not deleted yet because deleting it is T05's - decision, but it now reads as dead rather than as a live control. - -## Deferred, with reasons - -- **Rule 1 (ADR gate) is checkable but not automated.** A mechanical check - needs a map from capability → source paths, which does not exist. Cheap - version worth doing later: require every `specs/.md` to reference an - ADR, and every ADR to precede the first commit touching its capability's - directory. -- **Rule 12 (instrument emits its target) resists automation for now.** - Detecting "this number was typed rather than emitted" requires the tool's - output to be committed alongside the spec. The tractable form is to - require acceptance targets to appear verbatim in a committed tool output - file; deferred to T02 rather than guessed at here. -- **Rule 20 (self-test names a failure) is genuinely checkable only.** A - self-test that asserts `True` passes any structural check. Reviewing the - assertions is the control, and CB-WP-0002's AC-9 is the model: pin the - exact defect that occurred. -- **Rule 13 (re-derive inherited numbers) has no mechanical form at all** - and is left decorative *deliberately* — with its status stated. It is - the rule that caught the largest error in CB-WP-0002 ($248.46 → $92.21), - which is the counter-example to this workplan's own hypothesis: **an - unenforceable rule was the most valuable one in the pass.** The - hypothesis "a rule that cannot be executed is not a rule" is therefore - wrong as stated. The correct version is narrower: *a rule that cannot be - executed cannot be relied on to fire, so it must not be the only defence - for a class that matters.* - -## Open items raised by this audit - -- **AM-4a / AM-4b are unratified retargets.** Measured at 246,250 and set - at 250,000 in the same commit, by the implementer, with the - implementation changing in that commit too — a retarget under the test - added to InnerLoop §Step 4 by T07, not a correction. They must be - ratified by ADR or changed. Until then `make dep-weight` is enforcing a - target no reviewed decision stands behind. - -## Class coverage — where the gaps are - -| class | executable rules covering it | assessment | -|---|---|---| -| HDN | 15, 16, 17, 18, 19 | **well covered.** Five executable rules; the class that started this. | -| TA | none | **uncovered by any executable rule.** Rules 6, 12, 13 are checkable or decorative. This is the largest gap, and it is the class with the second-most instances. | -| SSB | 7, 8, 19 | **partly covered.** Rule 19 caught the one instance, by accident of running over all data rather than by design. Rules 7/8 enforce that a review *happened*, not that it sampled differently. T03 addresses the design gap. | - -The honest read: **the loop is hardened against the class it has already -suffered most from, and has no executable defence against the class that -produced its largest single error.** Trusted arithmetic is caught today -only by re-derivation, which is a discipline, not a gate. diff --git a/history/260731-loop-hardening-retrospective.md b/history/260731-loop-hardening-retrospective.md deleted file mode 100644 index ce5b948..0000000 --- a/history/260731-loop-hardening-retrospective.md +++ /dev/null @@ -1,134 +0,0 @@ -# 2026-07-31 — retrospective: does hardening converge? - -CB-WP-0003 T10. Pass produced `tools/loop-lint.py`, -`specs/SessionShape.md`, the rule audit, the live cost budget, the -retarget test, dated price rates, M-D1-LNK, provisional expiry, and -InnerLoop **v1.2**. - -## The question this task was given - -> Is the set of mechanisms now complete, or is each new pass still finding -> a new class? If this pass produces an eighth instance in a fourth class, -> the honest conclusion is that class-by-class hardening does not -> converge, and the loop should optimize for *cheap detection and -> correction* rather than for prevention — which is a different design. - -**It produced both. The answer is: hardening does not converge.** - -## The ledger - -| # | class | instances | found by | -|---|---|---|---| -| 1 | **HDN** harness-does-nothing | **5** | executable assertions | -| 2 | **TA** trusted arithmetic | **3** | re-derivation | -| 3 | **SSB** same-sample blind spot | **1** | an assertion running over *all* data | -| 4 | **DFD** duplicated-fact drift *(new)* | **2** | reading the copy against its source | - -Ten instances, four classes. Three workplans in, **every pass has -produced at least one instance of a class the previous pass had not -seen.** - -### New this pass - -- **HDN #5 (latent).** `tools/rule-coverage.py` would have exited 0 - reporting `0/0` if its spec regex ever stopped matching — a silent pass - in the tool that reports our headline AM-1 number. Never fired; found - only because T02 required it to carry a `--self-test` and writing one - meant asking what failure it should detect. -- **TA #3.** *"0 of 330 tool calls were batched"* — 330 was the count of - single-call responses, not the total. Carried unverified from - CB-WP-0001's retrospective into CB-WP-0003 T04. Real figure: 7.8–8.6% - of responses batched. -- **DFD #1–2 (new class).** `specs/MetricsAndScenarios.md` §1a inlined a - copy of the price sheet; T11 changed the real sheet and the copy went - stale **within the same hour**. Separately, the acceptance figure - $92.87/$0.66 had to be chased across a survey, a workplan, and an - evidence file each time it moved. The defect is not arithmetic and not - a harness — it is **a fact stored in two places, where updating one - does not update the other**. - -DFD is genuinely distinct: no positive control catches it (both copies -are internally consistent), and re-derivation does not either (the copy -reproduces whatever it was copied from). It is caught only by reading a -copy against its source, which nothing in the loop required. - -## What this implies, and what changed because of it - -Prevention has not converged: four passes, four classes, no sign of -saturation. But the *correction* side has been consistently cheap and -fast: - -| correction | mechanism | cost | -|---|---|---| -| $248.46 → $92.21 | re-derivation | ~0 | -| $92.21 → $92.87 | adversarial review | $1.11 | -| $92.87 → $93.32 | positive control | ~0 | -| $93.32 → $93.15 | dated-rate fix | ~0 | - -Every error found in three passes was corrected inside the same session -for under ~1% of the pass. **So the loop should stop trying to enumerate -failure classes in advance and invest in the properties that make -correction cheap** — which it turns out already has, accidentally: - -1. **Numbers are re-derivable** because the raw data (transcripts, git, - the price sheet) is kept, not just the conclusions. -2. **Artifacts are small and committed**, so a wrong number is one grep - from every place that quotes it. -3. **Every reported number has a command**, so re-running is free. - -v1.2 makes this the stated design goal rather than an accident, and adds -the one rule the new class earns: - -> **Single source of fact.** A number, rate, or target lives in exactly -> one place. Everywhere else links to it. Where a copy is unavoidable, it -> is generated by a command, not typed. *(CA-15 said this for evidence -> tables; DFD shows it applies to specs quoting other specs.)* - -## What the gates did this pass - -Worth recording because it is the cheapest possible evidence that the -executable-rule thesis is at least partly right: - -- `loop-lint` found three real violations on its **first** run — - including a 543-line spec against a ~400-line rule the loop had stated - since v0.2 and never measured. -- `loop-lint` then **failed on its own author within the hour**, when a - T07 edit pushed `specs/InnerLoop.md` to 407 lines. The file was split - because a gate said so, not because anyone noticed. -- `cb-cost`'s dedup assertion fired on real data on its first run and - refused to print a number. - -Three gates, three catches, all on first contact. Against that: the two -most valuable corrections of the whole project ($248.46 → $92.21, and the -batching claim) came from a rule that is **decorative and cannot be made -executable** — *re-derive inherited numbers*. Both facts are true and the -audit records them side by side. - -## Revised hypothesis - -The workplan opened with: *a loop rule that cannot be executed is not a -rule.* Three passes of evidence say that is wrong as stated. The version -this pass supports: - -> An executable rule fires reliably and catches one class. A decorative -> rule fires unreliably and can catch any class, including ones not yet -> named. **Keep both, and never let a decorative rule be the only defence -> for a class that matters** — but do not delete it merely because it -> cannot be automated, which is what this workplan originally proposed. - -## Open, not closed - -- **AM-4a/AM-4b are unratified retargets** (T07). `make dep-weight` - enforces targets no reviewed decision stands behind. Maintainer - decision. -- **9 of 58 rules are claimed by a scenario and named nowhere in the - aggregate** (T09, M-D1-LNK). Reported, not failing, because tagging - them without confirming implementation is the overclaim the loop - already committed once. -- **5 provisional U-item defaults** now have an owner and an age, and - ground-game has not ruled (T08). -- **The chaos roll is in a 12-declaration calibration window** at d4 - (T06), expected to cost $20–30 and to be decided on evidence. -- **DFD has no gate.** The single-source-of-fact rule added in v1.2 is - prose. Making it executable — detecting a number duplicated across - artifacts — is the obvious next candidate and is not attempted here. diff --git a/scenarios/ground/gr-a04-bond-support.yaml b/scenarios/ground/gr-a04-bond-support.yaml index f315220..d70d957 100644 --- a/scenarios/ground/gr-a04-bond-support.yaml +++ b/scenarios/ground/gr-a04-bond-support.yaml @@ -5,8 +5,6 @@ description: > Bond, because GR-L02 requires the target's consent. covers: [GR-A04, GR-F04, GR-L02, GR-L05] provisional: true -provisional_owner: ground-game -provisional_raised: 2026-07-31 seed: 42 setup: players: 3 diff --git a/scenarios/ground/gr-d01-darvo-trigger.yaml b/scenarios/ground/gr-d01-darvo-trigger.yaml index fe99a86..19ed4ab 100644 --- a/scenarios/ground/gr-d01-darvo-trigger.yaml +++ b/scenarios/ground/gr-d01-darvo-trigger.yaml @@ -5,8 +5,6 @@ description: > application (GR-F01 under the U2 default). covers: [GR-D01, GR-R08, GR-F01] provisional: true -provisional_owner: ground-game -provisional_raised: 2026-07-31 seed: 42 setup: players: 3 diff --git a/scenarios/ground/gr-d05-darvo-reverse.yaml b/scenarios/ground/gr-d05-darvo-reverse.yaml index f1e6d89..77efb18 100644 --- a/scenarios/ground/gr-d05-darvo-reverse.yaml +++ b/scenarios/ground/gr-d05-darvo-reverse.yaml @@ -5,8 +5,6 @@ description: > owner takes −2 Stress and the sequence ends (GR-D05, GR-D07). covers: [GR-D05, GR-D07, GR-T01] provisional: true -provisional_owner: ground-game -provisional_raised: 2026-07-31 seed: 42 setup: players: 3 diff --git a/scenarios/ground/gr-e02-shared-ground.yaml b/scenarios/ground/gr-e02-shared-ground.yaml index 04e9aa7..8182ef2 100644 --- a/scenarios/ground/gr-e02-shared-ground.yaml +++ b/scenarios/ground/gr-e02-shared-ground.yaml @@ -5,8 +5,6 @@ description: > rating reduced by each Blame token and each Denied Problem. covers: [GR-R09, GR-E01, GR-E02, GR-P03] provisional: true -provisional_owner: ground-game -provisional_raised: 2026-07-31 seed: 42 setup: players: 3 diff --git a/scenarios/ground/gr-e04-coalitions.yaml b/scenarios/ground/gr-e04-coalitions.yaml index dd9f16d..d9771ec 100644 --- a/scenarios/ground/gr-e04-coalitions.yaml +++ b/scenarios/ground/gr-e04-coalitions.yaml @@ -5,8 +5,6 @@ description: > one. Personal scores subtract Blame (GR-E03, GR-T02). covers: [GR-E03, GR-E04, GR-O03] provisional: true -provisional_owner: ground-game -provisional_raised: 2026-07-31 seed: 42 setup: players: 3 diff --git a/specs/ArchitectureBlueprint.md b/specs/ArchitectureBlueprint.md index 6d0ed3a..fbbdd1a 100644 --- a/specs/ArchitectureBlueprint.md +++ b/specs/ArchitectureBlueprint.md @@ -331,14 +331,213 @@ capabilities. --- +## 9. Rendering, input, and creation tools + +Rust-first stack: + +- `winit` — windows and platform input +- `wgpu` — GPU rendering (Vulkan / Metal / D3D / browser) +- `egui` — engine tools, inspectors, early editors +- Custom scene renderer for the game table +- **glTF** as the primary imported 3D format, wrapped in Clay-Borg asset + metadata and provenance + +Creator environment (grows over milestones): scene hierarchy, object +inspector, prototype browser, card-sheet importer, deck builder, zone and +snap-point editors, rule-state inspector, event timeline, player-view +switcher, hidden-information debugger, physics debugger, scenario recorder, +replay controls, package validator. + --- -**Continued in [ArchitectureRuntime.md](ArchitectureRuntime.md)** — §9 -rendering/input/creation tools, §10 networking and sessions, §11 the -agentic inner loop, §12 TargetRevenue integration, §13 repository -structure, §14 milestones, §15 governing design decisions. +## 10. Networking and sessions -*(Split 2026-07-31: this file was 543 lines against the loop's ~400-line -whole-file loadability rule. The rule had been stated since v0.2 and -nothing measured it until `tools/loop-lint.py` — see -`history/260731-inner-loop-rule-audit.md`.)* +Authoritative session host: + +```text +Client gesture +→ proposed command +→ session server validation +→ authoritative events +→ state update +→ player-specific projection +→ client animation +``` + +Capabilities: session discovery, auth and seat assignment, lobby/readiness, +command submission, commit/reveal windows, event-stream replication, +snapshot transfer, reconnection, state-hash verification, spectators, +player-specific redaction, host migration (later). + +Transport: **Quinn** (QUIC) for native; browser transport is a separate +adapter (WebTransport or WebSockets). The canonical protocol is defined +independently of any transport: + +```text +cb-session-protocol +├── CommandEnvelope +├── EventEnvelope +├── SnapshotEnvelope +├── CommitmentEnvelope +├── AssetRequest +└── CapabilityNegotiation +``` + +--- + +## 11. Agentic inner loop + +Agentic coding is a first-class product surface. Optimize for small +capability boundaries, executable specifications, controlled work areas, and +replayable failures. + +### Work packet (every agent task) + +```yaml +task_id: CB-PHYS-0042 +capability: tabletop.card-stacking +intent: Keep card stacks stable after drag release. +allowed_crates: + - cb-physics-api + - cb-physics-rapier + - cb-tabletop-physics +forbidden_changes: + - canonical game event schema +invariants: + - semantic card order must not depend on collider order +scenarios: + - scenarios/card-stack-20.yaml +benchmarks: + - benches/card-stack-stability.yaml +acceptance: + - all conformance tests pass + - no state divergence over 10,000 ticks + - benchmark regression below 3% +``` + +### CLI surface (`cb`) + +```bash +cb inspect capability tabletop.card +cb task prepare CB-PHYS-0042 +cb generate contracts +cb check --affected +cb test --affected # supports --format json +cb sim ground scenarios/mutual-attack.yaml +cb play ground --players 4 +cb replay artifacts/failure.cbreplay +cb compare physics-reference physics-rapier +cb bench --affected +cb evidence build CB-PHYS-0042 +cb release assess CB-PHYS-0042 +``` + +### Quality gates + +Formatting/linting, dependency-policy check, unit tests, capability +conformance tests, property tests, golden scenario tests, replay +determinism, snapshot migration, performance and memory budgets, rendering +comparison where relevant, security/sandbox tests, documentation and schema +consistency. + +Tooling: `cargo-nextest` (isolated parallel tests), Criterion +(regression-sensitive benchmarks), `sccache` (compile reuse), `tracing` +(structured diagnostics). + +--- + +## 12. TargetRevenue integration + +TargetRevenue governs **versioned capability improvements**, not the +monorepo as one indivisible target. + +```toml +improvement_id = "CB-GROUND-001" +capability = "game.ground.simultaneous-resolution" +classification = "10x" + +estimated_days = 4 +daily_rate = 1000 +target_revenue = 40000 + +phase = "commercial-recovery" +release_when_target_reached = "MIT" +trust_record = "required" +``` + +Components: improvement registry, workload ledger, cost model, revenue +attribution, dependency graph, phase license generator, revenue meter, +release gate, evidence bundle, trust service. + +Economic rule: + +> Optimized assimilations may be financed as independent improvements, while +> the canonical interface remains stable and reusable. + +--- + +## 13. Repository structure + +```text +clay-borg/ +├── INTENT.md +├── SCOPE.md +├── ARCHITECTURE.md # or specs/ArchitectureBlueprint.md (this file) +├── Cargo.toml +├── rust-toolchain.toml +│ +├── canon/ # entities, events, capabilities, schemas, terminology +├── crates/ +│ ├── cb-kernel/ cb-ids/ cb-time/ cb-rng/ cb-events/ +│ ├── cb-snapshot/ cb-capability/ +│ ├── cb-world/ cb-world-api/ cb-ecs-bevy/ +│ ├── cb-physics-api/ cb-physics-null/ cb-physics-reference/ cb-physics-rapier/ +│ ├── cb-render-api/ cb-render-null/ cb-render-wgpu/ +│ ├── cb-tabletop/ cb-tabletop-physics/ cb-tabletop-view/ +│ ├── cb-game-runtime/ cb-game-protocol/ cb-game-wasm/ +│ ├── cb-session/ cb-network-api/ cb-network-loopback/ cb-network-quic/ +│ └── cb-assets/ cb-ui/ cb-editor/ cb-observe/ cb-evidence/ +│ +├── games/ # ground/, fixture-cards/ +├── tools/ # cb-cli/, cb-agent/, cb-import/, cb-pack/ +├── scenarios/ +├── conformance/ +├── benchmarks/ +├── replays/ +├── examples/ +├── decisions/ +├── assimilation/ # assimilation manifests +└── target-revenue/ +``` + +Stay a monorepo during architectural formation. Extract a repository only +when a capability has a stable contract, an independent lifecycle, and a +genuine external consumer. + +--- + +## 14. Milestones + +| # | Milestone | Proves | +|---|---|---| +| 0 | Headless GROUND | Authoritative rules, commit/reveal, DARVO, replay — no rendering, no physics | +| 1 | Inspectable 2D table | Presentation bindings without 3D complexity | +| 2 | Physical 3D tabletop | wgpu + Rapier projection of semantic events | +| 3 | Networked sessions | Authoritative host, private projections, reconnection | +| 4 | Game creation framework | Editors, importers, Wasm game components | +| 5 | Second fixture game | Generality — abstractions promoted to Canon only after a second concrete use | + +--- + +## 15. Governing design decisions + +1. Build GROUND first, not a general engine first. +2. Keep rules independent from rendering and physics. +3. Use commands and events as the authoritative mutation mechanism. +4. Provide null, reference, and optimized implementations of important capabilities. +5. Never leak assimilated-library types into canonical interfaces. +6. Use server-authoritative physics and deterministic semantic rules. +7. Treat player visibility as a projection, not a UI afterthought. +8. Make every defect reproducible as a scenario and replay. +9. Give coding agents bounded work packets and stable commands. +10. Attach TargetRevenue phases to versioned improvements and evidence bundles. diff --git a/specs/ArchitectureRuntime.md b/specs/ArchitectureRuntime.md deleted file mode 100644 index 6d08b53..0000000 --- a/specs/ArchitectureRuntime.md +++ /dev/null @@ -1,221 +0,0 @@ -# Clay-Borg Architecture — Runtime, Tooling, and Process - -Second half of the architecture blueprint, split from -[ArchitectureBlueprint.md](ArchitectureBlueprint.md) on 2026-07-31 for -whole-file loadability. §1–8 (the layered stack, Clay Canon, runtime -substrate, simulation kernel, physics, world-building, tabletop domain -framework, game runtime) remain there; §9–15 are here. - -Section numbering is continuous with the first half and deliberately -unchanged, so existing references keep resolving. - -## 9. Rendering, input, and creation tools - -Rust-first stack: - -- `winit` — windows and platform input -- `wgpu` — GPU rendering (Vulkan / Metal / D3D / browser) -- `egui` — engine tools, inspectors, early editors -- Custom scene renderer for the game table -- **glTF** as the primary imported 3D format, wrapped in Clay-Borg asset - metadata and provenance - -Creator environment (grows over milestones): scene hierarchy, object -inspector, prototype browser, card-sheet importer, deck builder, zone and -snap-point editors, rule-state inspector, event timeline, player-view -switcher, hidden-information debugger, physics debugger, scenario recorder, -replay controls, package validator. - ---- - -## 10. Networking and sessions - -Authoritative session host: - -```text -Client gesture -→ proposed command -→ session server validation -→ authoritative events -→ state update -→ player-specific projection -→ client animation -``` - -Capabilities: session discovery, auth and seat assignment, lobby/readiness, -command submission, commit/reveal windows, event-stream replication, -snapshot transfer, reconnection, state-hash verification, spectators, -player-specific redaction, host migration (later). - -Transport: **Quinn** (QUIC) for native; browser transport is a separate -adapter (WebTransport or WebSockets). The canonical protocol is defined -independently of any transport: - -```text -cb-session-protocol -├── CommandEnvelope -├── EventEnvelope -├── SnapshotEnvelope -├── CommitmentEnvelope -├── AssetRequest -└── CapabilityNegotiation -``` - ---- - -## 11. Agentic inner loop - -Agentic coding is a first-class product surface. Optimize for small -capability boundaries, executable specifications, controlled work areas, and -replayable failures. - -### Work packet (every agent task) - -```yaml -task_id: CB-PHYS-0042 -capability: tabletop.card-stacking -intent: Keep card stacks stable after drag release. -allowed_crates: - - cb-physics-api - - cb-physics-rapier - - cb-tabletop-physics -forbidden_changes: - - canonical game event schema -invariants: - - semantic card order must not depend on collider order -scenarios: - - scenarios/card-stack-20.yaml -benchmarks: - - benches/card-stack-stability.yaml -acceptance: - - all conformance tests pass - - no state divergence over 10,000 ticks - - benchmark regression below 3% -``` - -### CLI surface (`cb`) - -```bash -cb inspect capability tabletop.card -cb task prepare CB-PHYS-0042 -cb generate contracts -cb check --affected -cb test --affected # supports --format json -cb sim ground scenarios/mutual-attack.yaml -cb play ground --players 4 -cb replay artifacts/failure.cbreplay -cb compare physics-reference physics-rapier -cb bench --affected -cb evidence build CB-PHYS-0042 -cb release assess CB-PHYS-0042 -``` - -### Quality gates - -Formatting/linting, dependency-policy check, unit tests, capability -conformance tests, property tests, golden scenario tests, replay -determinism, snapshot migration, performance and memory budgets, rendering -comparison where relevant, security/sandbox tests, documentation and schema -consistency. - -Tooling: `cargo-nextest` (isolated parallel tests), Criterion -(regression-sensitive benchmarks), `sccache` (compile reuse), `tracing` -(structured diagnostics). - ---- - -## 12. TargetRevenue integration - -TargetRevenue governs **versioned capability improvements**, not the -monorepo as one indivisible target. - -```toml -improvement_id = "CB-GROUND-001" -capability = "game.ground.simultaneous-resolution" -classification = "10x" - -estimated_days = 4 -daily_rate = 1000 -target_revenue = 40000 - -phase = "commercial-recovery" -release_when_target_reached = "MIT" -trust_record = "required" -``` - -Components: improvement registry, workload ledger, cost model, revenue -attribution, dependency graph, phase license generator, revenue meter, -release gate, evidence bundle, trust service. - -Economic rule: - -> Optimized assimilations may be financed as independent improvements, while -> the canonical interface remains stable and reusable. - ---- - -## 13. Repository structure - -```text -clay-borg/ -├── INTENT.md -├── SCOPE.md -├── ARCHITECTURE.md # or specs/ArchitectureBlueprint.md (this file) -├── Cargo.toml -├── rust-toolchain.toml -│ -├── canon/ # entities, events, capabilities, schemas, terminology -├── crates/ -│ ├── cb-kernel/ cb-ids/ cb-time/ cb-rng/ cb-events/ -│ ├── cb-snapshot/ cb-capability/ -│ ├── cb-world/ cb-world-api/ cb-ecs-bevy/ -│ ├── cb-physics-api/ cb-physics-null/ cb-physics-reference/ cb-physics-rapier/ -│ ├── cb-render-api/ cb-render-null/ cb-render-wgpu/ -│ ├── cb-tabletop/ cb-tabletop-physics/ cb-tabletop-view/ -│ ├── cb-game-runtime/ cb-game-protocol/ cb-game-wasm/ -│ ├── cb-session/ cb-network-api/ cb-network-loopback/ cb-network-quic/ -│ └── cb-assets/ cb-ui/ cb-editor/ cb-observe/ cb-evidence/ -│ -├── games/ # ground/, fixture-cards/ -├── tools/ # cb-cli/, cb-agent/, cb-import/, cb-pack/ -├── scenarios/ -├── conformance/ -├── benchmarks/ -├── replays/ -├── examples/ -├── decisions/ -├── assimilation/ # assimilation manifests -└── target-revenue/ -``` - -Stay a monorepo during architectural formation. Extract a repository only -when a capability has a stable contract, an independent lifecycle, and a -genuine external consumer. - ---- - -## 14. Milestones - -| # | Milestone | Proves | -|---|---|---| -| 0 | Headless GROUND | Authoritative rules, commit/reveal, DARVO, replay — no rendering, no physics | -| 1 | Inspectable 2D table | Presentation bindings without 3D complexity | -| 2 | Physical 3D tabletop | wgpu + Rapier projection of semantic events | -| 3 | Networked sessions | Authoritative host, private projections, reconnection | -| 4 | Game creation framework | Editors, importers, Wasm game components | -| 5 | Second fixture game | Generality — abstractions promoted to Canon only after a second concrete use | - ---- - -## 15. Governing design decisions - -1. Build GROUND first, not a general engine first. -2. Keep rules independent from rendering and physics. -3. Use commands and events as the authoritative mutation mechanism. -4. Provide null, reference, and optimized implementations of important capabilities. -5. Never leak assimilated-library types into canonical interfaces. -6. Use server-authoritative physics and deterministic semantic rules. -7. Treat player visibility as a projection, not a UI afterthought. -8. Make every defect reproducible as a scenario and replay. -9. Give coding agents bounded work packets and stable commands. -10. Attach TargetRevenue phases to versioned improvements and evidence bundles. diff --git a/specs/CostAccounting.md b/specs/CostAccounting.md index aa19d7a..c22530c 100644 --- a/specs/CostAccounting.md +++ b/specs/CostAccounting.md @@ -67,16 +67,6 @@ cost = input_tokens × price.input > reported as an unpriced line with its token counts, never dropped and > never priced at a default. -> **CA-16.** Promotional and other time-boxed rates are **data**, not -> comments. A response is priced at the rate in force *at its own -> timestamp*: `promo_input`/`promo_output` apply on or before -> `promo_until`, and the standard rate applies after. - -> **CA-17.** The price sheet carries `recorded` and `max_age_days`, and a -> sheet older than `max_age_days` **aborts the run**. A stale sheet -> invalidates `better` verdicts, so the staleness rule is a command rather -> than a sentence. - ### 1.3 Scope of a measurement > **CA-06.** A measurement enumerates **every** transcript for the repo: @@ -135,9 +125,9 @@ Each row names the command that produces its number, per InnerLoop §Step 4. | ID | Metric | Target | Instrument | |---|---|---|---| -| **AC-1** | reproduces the pinned CB-WP-0001 total | **$93.15** = $92.03 main + $1.11 subagent | `make cost-pin` | +| **AC-1** | reproduces the pinned CB-WP-0001 total | **$93.32** = $92.21 main + $1.11 subagent | `make cost-pin` | | **AC-2** | reconciliation residual (CA-14) | **$0.00** exactly | same command, `reconciled: ok` line | -| **AC-3** | unattributed share reported (CA-10) | present, and **32.4%** on the pinned run | `cb-cost --pin fc76445 --by-task` | +| **AC-3** | unattributed share reported (CA-10) | present, and **33%** on the pinned run | `cb-cost --pin fc76445 --by-task` | | **AC-4** | composition reported (CA-13) | all five components present | `cb-cost --pin fc76445 --composition` | | **AC-5** | dedup invariant asserted (CA-02) | violation exits non-zero | `make cost-test` | | **AC-6** | positive control: refuses to report on zero responses | exits non-zero | `make cost-test` | @@ -155,7 +145,7 @@ assertion against a fixture whose expected value is known and fails loudly; Per InnerLoop §Step 4, the acceptance table is checked against the contracts in this same spec: -- AC-1's $93.15 is reachable only if CA-06 holds (both trees enumerated). +- AC-1's $92.87 is reachable only if CA-06 holds (both trees enumerated). Under a main-file-only collector the target is unreachable — this is the defect the adversarial review caught, where a target of $92.21 would have been hit *only* by a broken collector. @@ -182,88 +172,7 @@ contracts in this same spec: necessarily shows a lossy projection. This is a limitation of the sink, not of the metric; `make cost` remains the authority. -## 7. The live cost budget - -Replaces the 8k/10k per-task **token** budget, which was never referenced -or enforced, which CB-WP-0001 T08 exceeded by orders of magnitude with no -signal, and which InnerLoop v1.1 now marks dead. It implied a control that -did not exist. - -**The design constraint that shapes this.** Per-task cost needs the commit -that *closes* the task (CA-08), so it is unavoidably retrospective — a -per-task budget can only ever report a breach after the money is spent. -What is observable mid-task is **spend since the last commit**, because -the transcript is an append-live file (CA-07). That is the only number a -budget can actually fire on, so that is what the budget binds. - -> **CB-01.** The budget is the **open remainder**: USD spent since the -> last commit, computable at any instant by `make cost-budget`. - -> **CB-02.** Thresholds: **soft $10.00**, **hard $22.00**. -> -> - **soft** — state progress as a percentage and decide: continue, or -> commit what works and decompose. -> - **hard** — stop. Commit what works, or decompose. Uncommitted work is -> also *unattributable* work, so a hard breach costs twice: once in -> money, once in the 33% attribution gap. - -Calibration, measured over the 32 non-empty commit intervals of -CB-WP-0001 (`cb-cost --pin fc76445`): - -| | spend per interval | -|---|---| -| min | $0.07 | -| p50 | $1.40 | -| p90 | $9.36 | -| **max** | **$10.80** (T07, the Cargo workspace scaffold) | - -Soft is set at **$10.00**, just below the observed maximum and just above -p90: on the pass this is calibrated from it would have fired exactly once, -on the single most expensive interval. Hard is set at **$22.00**, roughly -2× the observed maximum — a value never reached in 32 intervals, so -reaching it means the session is doing something the calibration data has -no example of, which is precisely when stopping is right. - -**Why these are not retargeted-after-the-fact.** T07's concern is a target -moved to accommodate a measurement. These thresholds are set *above* every -observed value, so they bind on future work rather than ratifying present -work — no interval in the calibration set breaches hard, and one breaches -soft. If a future pass routinely breaches, that is signal, not a reason to -raise them. - -**Known limit:** the budget cannot fire *between* turns without something -running the command. It is available to an agent at any point, in CI, and -in `make all`; it is not a daemon. An agent that never runs it gets no -signal — which is exactly the dead-policy failure this replaces, one level -up. The mitigation is that it costs nothing to run and is on the one -command surface. - -## 9. The price sheet - -`benchmarks/baselines/model-prices.toml` is the single source of rates. -Two defects were fixed here, both of the same shape — **a schema that -could not hold the fact it needed**, which is the criticism this -capability's survey levelled at the State Hub: - -1. **A time-boxed rate lived in a comment.** Sonnet's intro price - (2.00/10.00 through 2026-08-31) was written as `# intro …` and was - therefore invisible to the collector, which used the standard rate. Cost - at the pin: **$0.17**, 0.19% of the pass. The real hazard was the - date — on **2026-08-31** the promotion expires and a comment-only - representation would have been wrong in the opposite direction, with - nothing to notice. Now expressed as `promo_*` fields and applied per - CA-16. -2. **The 90-day staleness rule had no check.** It was prose in - `MetricsAndScenarios.md` §1a and every M-D2-CST verdict silently - inherited it. Now `recorded` + `max_age_days` in the sheet, enforced by - CA-17 and exercised by `make cost-test`. - -Applying CA-16 moved AC-1 from $93.32 to **$93.15**. This is a *legitimate* -retarget under the distinction CB-WP-0003 T07 draws: the instrument -disproved the target, rather than the implementation missing it. The -instrument's output is in the same commit. - -## 8. Revisions to M-D2-CST +## 7. Revisions to M-D2-CST `specs/MetricsAndScenarios.md` §1a is superseded by this spec. M-D2-CST is redefined from "tokens × pricepoint" — which named no instrument and was diff --git a/specs/GroundRules.md b/specs/GroundRules.md index 620b120..78a1284 100644 --- a/specs/GroundRules.md +++ b/specs/GroundRules.md @@ -210,15 +210,6 @@ terms. ## Underdetermined in dataset 0.1 — PROVISIONAL defaults (flag to ground-game) -**Owner: ground-game. Raised: 2026-07-31.** Every U-item carries an owner -and a raise date so its age is visible; `make coverage` reports both and -warns past 30 days. CI **warns rather than breaks** — the ruling belongs -to ground-game and the kernel cannot make it for them — but every evidence -file that depends on one must list it (CB-WP-0003 T08). - -A ruling flips the *scenario*, not the kernel: the default is encoded in a -scenario tagged `provisional: true`, so a correction is a data change. - Formalization exposed points the dataset does not decide. Simulation uses the stated default; each is tagged in scenarios that depend on it and must be confirmed or corrected by ground-game. diff --git a/specs/InnerLoop.md b/specs/InnerLoop.md index 3e1ccee..178bd2a 100644 --- a/specs/InnerLoop.md +++ b/specs/InnerLoop.md @@ -1,13 +1,6 @@ # The Inner Loop — Assimilate and Surpass -Status: **v1.2** — corrected from CB-WP-0003 (loop hardening) on -2026-07-31. Changes from v1.1: single source of fact; review targets the -harness and states its sampling limit; correction vs retarget; the chaos -roll's calibration window; the live cost budget. The design goal is now -stated: **optimize for cheap correction, not for exhaustive prevention.** -Rationale: `history/260731-loop-hardening-retrospective.md`. - -v1.1 — corrected from CB-WP-0002 (cost accounting) on +Status: **v1.1** — corrected from CB-WP-0002 (cost accounting) on 2026-07-31. Changes from v1.0: the instrument must exist and emit its own target; inherited numbers are re-derived before use; every reporting tool exposes `--self-test`; cost is in the definition of done. Rationale: @@ -21,25 +14,7 @@ evidence state what it does not support. Rationale and the failures behind each: `history/260731-inner-loop-retrospective.md`. Normative process for building every Clay-Borg capability. Referenced by -all workplans. - -**Design goal (v1.2).** Three passes produced ten error instances across -four classes, and every pass produced a class the previous one had not -seen — prevention is not converging. Every one of those errors was -corrected inside the same session for under ~1% of the pass. The loop -therefore optimizes for **cheap correction**: keep the raw data so numbers -can be re-derived, keep artifacts small and committed so a wrong number is -one grep from everywhere that quotes it, and give every reported number a -command so re-running is free. - -> **Single source of fact.** A number, rate, or target lives in exactly -> one place; everywhere else links to it. Where a copy is unavoidable it -> is generated by a command, not typed. *(v1.2: a price sheet inlined into -> a spec went stale within the hour of the real sheet changing, and one -> acceptance figure had to be chased across three artifacts each time it -> moved. No positive control catches this — both copies are internally -> consistent — and re-derivation does not either, because the copy -> reproduces whatever it was copied from.)* The loop's own optimization target is **agentic efficiency**: +all workplans. The loop's own optimization target is **agentic efficiency**: every artifact it produces must be small enough to load whole, structured enough to act on without interpretation, and falsifiable enough that an agent can judge its own work without a human in the iteration. @@ -71,29 +46,10 @@ are never skipped for code-producing work. | **M** | Survey and ADR merged into one document; review optional | Touches a canonical interface, or adds/updates an external dependency | | **S** | One provenance paragraph in the commit message | Everything else (utilities, fixes, refactors inside a boundary) | -**The chaos roll.** After deriving the structural tier, roll **d4** -(`shuf -i 1-4 -n 1`). On a **4**, the tier is instead picked uniformly at +**The chaos roll.** After deriving the structural tier, roll d10 +(`shuf -i 1-10 -n 1`). On a **10**, the tier is instead picked uniformly at random (`shuf -e S M L -n 1`), overriding the structural derivation — up or -down. - -> **Calibration window, opened 2026-07-31 (CB-WP-0003 T06).** The rate was -> d10 and the mechanism **never fired**: two rolls across two workplans -> (CB-WP-0001: 9, CB-WP-0002: 2), against ~0.2 expected firings. At d10 and -> ~2 tier decisions per workplan it would take roughly twenty workplans to -> observe four overrides, so the mechanism was set at a rate that prevented -> its own evaluation — the one option T06 ruled out. -> -> Raised to **d4 (25%) for the next 12 tier declarations**, then evaluated -> and either kept, returned to d10, or deleted. Expected ~3 firings in the -> window, which is enough to see whether an overridden tier produces a -> different outcome than the argued one. -> -> **Stated cost:** a chaos-L override on work that would have been S buys a -> full survey, adversarial review, and ADR. Measured comparable: CB-WP-0001 -> T03 (a tier-L survey) cost **$9.91**. At 25% over 12 declarations the -> window is expected to cost **$20–30**. That is the price of finding out -> whether the mechanism is worth keeping, and it is cheaper than carrying an -> unevaluated ritual indefinitely. Both rolls are recorded in the tier declaration +down. Both rolls are recorded in the tier declaration (`tier: M (structural L, chaos 10→M)`). **Record the roll every time, including when it changes nothing** (`tier: L (structural L, chaos 4)`), so a mechanism that never fires is visible rather than assumed. Purpose: @@ -131,40 +87,11 @@ evidence verdict for those rows caps at `parity`, never `better`. ### Step 2 — Approve (adversarial review) For tier-L passes, approval is earned through an **adversarial review**: a -separate session (or agent) attempts to break the work. Exactly **one -round**: challenge, then response. The work is approvable only when every -challenge is either answered with evidence or conceded and folded in. - -**The review target follows the risk.** Reviewing the survey was the -original rule, and it is the wrong target for a capability whose claim -rests on numbers — every serious error in this project has been in -measurement or build configuration, not in prose. - -| the claim rests on | the reviewer is given | and must attempt | -|---|---|---| -| a survey of candidates | the survey | an omitted candidate; a stale or unverifiable benchmark; an unmeasured claim presented as measured | -| **numbers** | the survey **and the harness and the evidence file** | **reproduce the number independently**; state what the harness would report if the work silently stopped | - -**What review cannot do — state this, do not discover it.** A reviewer -re-derives the author's claims and therefore inherits the author's -sampling. So: - -> **The reviewer re-derives on a different sample than the author used.** -> Where only one sample exists, the review says so rather than reporting a -> clean verify. - -*(v1.1, from CB-WP-0002: the dedup invariant was verified on the main -transcript by the survey — 206/206 groups — and independently re-verified -by the reviewer, who used the same transcript. It is false in the -8-response `subagents/` tree neither examined. Two independent checks, -one blind spot, because both sampled the same way. Only an assertion -running over all the data at execution time caught it.)* - -**What review demonstrably does do.** Measured across two passes: -**$0.66** and **$1.11**, roughly 1% of each pass, each finding -approval-blocking defects — in CB-WP-0002 a target that would have made -the evidence file certify a broken collector. The step pays for itself by -a wide margin and the cost is not a reason to skip it. +separate session (or agent), given only the survey document, attempts to +break it — an omitted candidate, a stale or unverifiable benchmark, an +unmeasured claim presented as measured. Exactly **one round**: challenge, +then response. The survey is approvable only when every challenge is either +answered with evidence or conceded and folded into the survey. **Documentation requirement:** the research process, the challenge, and the resulting improvements to the research are each documented in timestamped @@ -224,40 +151,11 @@ as a target, or it is cited as unverified.** Quoting is not measuring. from a prior pass. Re-derivation put it at $92.21 — the quoted figure double-counted transcript lines and priced a three-model session at one model's rate. Neither error was of the harness-does-nothing class; both -sums ran over real data, and a positive control would have passed them.)* - -**Retargeting: the instrument may move a target, the implementation may -not.** A metric's target changes in only two ways, and they are not -treated alike: - -| | trigger | requirement | -|---|---|---| -| **corrected** | the instrument disproved the target — the target was computed by hand, or by an earlier tool with a defect | legitimate in the same commit, **provided the instrument's output is in that commit** | -| **retargeted** | the implementation missed the target and the target moves to accommodate it | requires an **ADR**: old target, the measurement, and why the new target binds on *future* work rather than merely passing present work | - -The distinction is not the implementer's self-report of intent. It is -mechanical: **a correction is one where the target moves and the -implementation does not.** If the same commit changes both the target and -the code the target measures, it is a retarget and needs the ADR. - -*(v1.1, from CB-WP-0002/0003: AM-4's targets were measured at 246,250 and -set at 250,000 in one commit by the implementer after seeing the number — -the structure this rule exists to stop. But CB-WP-0002 then moved AC-1 -three times, correctly, each time because a new instrument disproved the -old figure ($92.21 → $92.87 → $93.32 → $93.15). A blanket prohibition -would have forbidden four legitimate corrections to catch one bad -retarget.)* - -**Applied retroactively:** AM-4a and AM-4b are **unratified** until an ADR -is written or they are changed. They were set by the implementer after -seeing the measurement, in the commit that produced it, and the -implementation changed in that same commit — a retarget by the test above. -Tracked as an open item in `history/260731-inner-loop-rule-audit.md`. - -**A metric is checked against the contracts in its own spec.** If a -contract makes a target unreachable, one of the two is wrong and the -conflict is resolved when it is noticed, not at the acceptance run. -Re-check the table whenever a contract is added. +sums ran over real data, and a positive control would have passed them.)* A metric must +also be checked against the contracts in the *same spec*: if a contract +makes a target unreachable, one of the two is wrong and the conflict is +resolved when it is noticed, not at the acceptance run. Re-check the +table whenever a contract is added. *(v1.0, from CB-WP-0001: AM-4's ≤20-crate target was made unreachable by the K5 and K7 contracts written after it, and AM-12's cost metric was @@ -326,17 +224,115 @@ state the claim you will defend, and the claim you are not making. --- +## The four-dimension rubric + +Every survey, ADR, and acceptance table is organized by these dimensions: + +| Dimension | Question | Example measurable proxies | +|---|---|---| +| **D1 Ease of specification** | How simply can behavior be stated, tested, understood? | rules-to-scenario coverage %, spec lines per rule, time-to-first-correct-scenario for a fresh agent session | +| **D2 Efficiency of implementation** | How cheap to build and keep building? | source LOC, dependency count/weight, clean-build and incremental-build time, tokens-per-completed-task | +| **D3 Speed of execution** | How fast does it run? | benchmark wall-time vs baseline, events/sec, memory footprint, determinism overhead | +| **D4 Optionality** | How cleanly does it integrate, extend, get replaced? | public API surface size, count of leaked foreign types (must be 0), effort-to-swap measured by null/reference impl existence, WIT-expressibility | + +Scoring is always **relative to the step-1 baseline**, never absolute: +`better / parity / worse / unmeasured` per proxy, with the number attached. +`unmeasured` is legal in a survey, illegal in an evidence file. + --- -## Reference material +## Survey template (research/CB-RES-NNNN-.md) -The four-dimension rubric, the survey template, the implementation rules -each pass earned, the agentic-efficiency requirements, and the -definition of done live in -**[InnerLoopReference.md](InnerLoopReference.md)**. They are normative; -they are separated only so both files load whole. +```markdown +# CB-RES-NNNN: +capability: +status: draft | approved -*(Split 2026-07-31: this file reached 407 lines against its own ~400-line -loadability rule, and `make loop-lint` — added the same day to make that -rule executable — failed on the commit that pushed it over. The rule -caught its own author within an hour of being written.)* +## Candidates +Per candidate: origin, license, maturity, adoption; data model; mutation +mechanism; determinism/replay story; relevant performance (measured if +runnable locally, cited with source otherwise). + +## Baselines (benchmark-to-beat) +| Dimension | Baseline holder | Metric | Value | Provenance | +(one row minimum per dimension; provenance = measured / cited / estimated) + +## Verdict +Which candidate leads per dimension; what none of them do well +(the surpass opportunity); risks in the baselines themselves. +``` + +--- + +## Implementation rules the first pass earned + +These are cheap, and each exists because its absence cost something in +CB-WP-0001. See `history/260731-inner-loop-retrospective.md`. + +1. **No silently-ignored input.** A field that is parsed and then unused + is a defect, not a stub. Inputs are honoured or rejected with an + error — never dropped. *(A scenario `setup.patch` was parsed and + discarded; every scenario using it would have tested the wrong + initial state while passing.)* +2. **Decisions get commands, not defaults.** A rule that requires a + participant's choice is implemented as a command carrying that + choice. Until it is, **nothing claims coverage of it** — no tag, no + scenario, no acceptance row. Inventing a default to make a rule + "done" is the failure this prevents. +3. **Scaffolds are exercised or marked.** A scaffold's green gates are + not evidence. Any scaffold path no test reaches is marked as + unexercised. *(A compiling, fully-green scaffold shipped a state-hash + that would panic on any state holding a relation.)* +4. **Coverage gates that count tags say so.** A gate comparing rule IDs + against `covers:` lists proves no rule is unclaimed and no claimed + rule is invented. It does **not** prove a scenario exercises what it + names. Wherever such a number is reported, that limit is reported + with it. + +--- + +## Agentic-efficiency requirements + +The loop exists to be driven by agents. Therefore: + +1. **Whole-file loadability** — every loop artifact stays under ~400 lines; + split before exceeding, link with relative paths. +2. **Structured over prose** — tables and fenced blocks for anything a + later step must parse (baselines, acceptance metrics, evidence rows). +3. **One command surface** — all checks runnable through repo-root + commands (eventually `cb *`; until then, `make`/`cargo` aliases declared + in one place), each supporting deterministic, greppable output. +4. **Self-contained tasks** — a workplan task names its input artifacts and + output artifacts; a fresh session must be able to execute it from the + task text plus linked files alone. +5. **Evidence or it didn't happen** — claims of "better" live in committed + evidence files with numbers, never only in commit messages or chat. +6. **Token discipline** — per the global budget policy, a loop iteration + that exceeds its budget without measurable progress is stopped and + decomposed, not pushed through. + +--- + +## Definition of done — one loop pass + +A capability has completed the loop when all of the following are committed: + +- [ ] research/CB-RES-NNNN with approved status and full baseline table +- [ ] decisions/ADR-NNNN with per-dimension expected advantage +- [ ] specs/.md with acceptance-metrics table +- [ ] passing scenarios covering every numbered spec rule +- [ ] evidence/CB-EV-NNNN with final comparison vs baseline, no `unmeasured` +- [ ] every reported number produced by a harness with a positive + control; any metric that could not be instrumented is recorded as + uncomputable rather than estimated +- [ ] every unmet metric reported as unmet, with attribution and the + options for resolving it — a missed target is an output of the + loop, not a reason to move the target quietly +- [ ] **cost recorded**: `make cost` run for the pass, its composition + (not only its total) in the evidence file, and the per-task figures + pushed to the hub. M-D2-CST is no longer allowed to be + `uncomputable` — the instrument exists + ([CostAccounting.md](CostAccounting.md)) +- [ ] retrospective note (may be one paragraph appended to the evidence + file): what the loop itself should change +``` diff --git a/specs/InnerLoopReference.md b/specs/InnerLoopReference.md deleted file mode 100644 index 06af1fd..0000000 --- a/specs/InnerLoopReference.md +++ /dev/null @@ -1,141 +0,0 @@ -# The Inner Loop — Reference - -Normative companion to **[InnerLoop.md](InnerLoop.md)**, which holds the -five steps, the tiers, and the chaos roll. Split from it on 2026-07-31 so -both files stay under the loop's own whole-file loadability limit. - -Everything here binds exactly as the process document does. - -## The four-dimension rubric - -Every survey, ADR, and acceptance table is organized by these dimensions: - -| Dimension | Question | Example measurable proxies | -|---|---|---| -| **D1 Ease of specification** | How simply can behavior be stated, tested, understood? | rules-to-scenario coverage %, spec lines per rule, time-to-first-correct-scenario for a fresh agent session | -| **D2 Efficiency of implementation** | How cheap to build and keep building? | source LOC, dependency count/weight, clean-build and incremental-build time, tokens-per-completed-task | -| **D3 Speed of execution** | How fast does it run? | benchmark wall-time vs baseline, events/sec, memory footprint, determinism overhead | -| **D4 Optionality** | How cleanly does it integrate, extend, get replaced? | public API surface size, count of leaked foreign types (must be 0), effort-to-swap measured by null/reference impl existence, WIT-expressibility | - -Scoring is always **relative to the step-1 baseline**, never absolute: -`better / parity / worse / unmeasured` per proxy, with the number attached. -`unmeasured` is legal in a survey, illegal in an evidence file. - ---- - -## Survey template (research/CB-RES-NNNN-.md) - -```markdown -# CB-RES-NNNN: -capability: -status: draft | approved - -## Candidates -Per candidate: origin, license, maturity, adoption; data model; mutation -mechanism; determinism/replay story; relevant performance (measured if -runnable locally, cited with source otherwise). - -## Baselines (benchmark-to-beat) -| Dimension | Baseline holder | Metric | Value | Provenance | -(one row minimum per dimension; provenance = measured / cited / estimated) - -## Verdict -Which candidate leads per dimension; what none of them do well -(the surpass opportunity); risks in the baselines themselves. -``` - ---- - -## Implementation rules the first pass earned - -These are cheap, and each exists because its absence cost something in -CB-WP-0001. See `history/260731-inner-loop-retrospective.md`. - -1. **No silently-ignored input.** A field that is parsed and then unused - is a defect, not a stub. Inputs are honoured or rejected with an - error — never dropped. *(A scenario `setup.patch` was parsed and - discarded; every scenario using it would have tested the wrong - initial state while passing.)* -2. **Decisions get commands, not defaults.** A rule that requires a - participant's choice is implemented as a command carrying that - choice. Until it is, **nothing claims coverage of it** — no tag, no - scenario, no acceptance row. Inventing a default to make a rule - "done" is the failure this prevents. -3. **Scaffolds are exercised or marked.** A scaffold's green gates are - not evidence. Any scaffold path no test reaches is marked as - unexercised. *(A compiling, fully-green scaffold shipped a state-hash - that would panic on any state holding a relation.)* -4. **Coverage gates that count tags say so.** A gate comparing rule IDs - against `covers:` lists proves no rule is unclaimed and no claimed - rule is invented. It does **not** prove a scenario exercises what it - names. Wherever such a number is reported, that limit is reported - with it. - ---- - -## Agentic-efficiency requirements - -The loop exists to be driven by agents. Therefore: - -1. **Whole-file loadability** — every loop artifact stays under ~400 lines; - split before exceeding, link with relative paths. -2. **Structured over prose** *(guidance, not a requirement — nothing can - check it)* — tables and fenced blocks for anything a later step must - parse (baselines, acceptance metrics, evidence rows). -3. **One command surface** — all checks runnable through repo-root - commands (eventually `cb *`; until then, `make`/`cargo` aliases declared - in one place), each supporting deterministic, greppable output. -4. **Self-contained tasks** — a workplan task names its input artifacts and - output artifacts; a fresh session must be able to execute it from the - task text plus linked files alone. -5. **Evidence or it didn't happen** — claims of "better" live in committed - evidence files with numbers, never only in commit messages or chat. -6. ~~**Token discipline** — per the global budget policy, a loop iteration - that exceeds its budget without measurable progress is stopped and - decomposed, not pushed through.~~ **DEAD POLICY.** The 8k/10k per-task - token budget was never referenced or enforced, and CB-WP-0001 T08 - exceeded it by orders of magnitude with no signal. It implies a control - that does not exist. Replacement in USD is CB-WP-0003 T05; until then - this is documentation of a gap, not a rule. - -6a. **Live cost budget** *(replaces the above)* — spend since the last - commit, soft **$10.00**, hard **$22.00**, checked by `make cost-budget`. - Calibrated on the 32 commit intervals of CB-WP-0001 (p50 $1.40, p90 - $9.36, max $10.80), so both thresholds bind on future work rather than - ratifying past work. Contract: [CostAccounting.md](CostAccounting.md) - §7 (CB-01, CB-02). It fires on the open remainder rather than per task - because per-task cost needs the commit that closes the task, and a - budget that can only report after the money is spent is the dead policy - this replaces. - -**Enforcement status.** Rules above that a command can check are enforced -by `make loop-lint`; the full classification of every InnerLoop rule as -executable / checkable / decorative, with the failure class each catches, -is in `history/260731-inner-loop-rule-audit.md`. Rules marked *guidance* -or *dead* say so where they appear, so a reader can tell a requirement -from a preference without consulting the audit. - ---- - -## Definition of done — one loop pass - -A capability has completed the loop when all of the following are committed: - -- [ ] research/CB-RES-NNNN with approved status and full baseline table -- [ ] decisions/ADR-NNNN with per-dimension expected advantage -- [ ] specs/.md with acceptance-metrics table -- [ ] passing scenarios covering every numbered spec rule -- [ ] evidence/CB-EV-NNNN with final comparison vs baseline, no `unmeasured` -- [ ] every reported number produced by a harness with a positive - control; any metric that could not be instrumented is recorded as - uncomputable rather than estimated -- [ ] every unmet metric reported as unmet, with attribution and the - options for resolving it — a missed target is an output of the - loop, not a reason to move the target quietly -- [ ] **cost recorded**: `make cost` run for the pass, its composition - (not only its total) in the evidence file, and the per-task figures - pushed to the hub. M-D2-CST is no longer allowed to be - `uncomputable` — the instrument exists - ([CostAccounting.md](CostAccounting.md)) -- [ ] retrospective note (may be one paragraph appended to the evidence - file): what the loop itself should change diff --git a/specs/MetricsAndScenarios.md b/specs/MetricsAndScenarios.md index 9ae2846..c8b360f 100644 --- a/specs/MetricsAndScenarios.md +++ b/specs/MetricsAndScenarios.md @@ -31,7 +31,6 @@ and add capability-specific rows only when these don't cover the claim. | ID | Dimension | Metric | Unit | Provenance | |---|---|---|---|---| | M-D1-COV | D1 | numbered spec rules covered by ≥1 passing scenario | % | adapted:requirements-traceability (per-rule, not per-feature) | -| M-D1-LNK | D1 | claimed rules also **named in the aggregate source** | count | novel — closes the spec→code→scenario chain; instrument: `make coverage` (AM-1b) | | M-D1-SPL | D1 | spec lines per numbered rule | lines | novel — proxies statement simplicity; gameable, so paired with M-D1-COV | | M-D2-LOC | D2 | source LOC excluding tests (tokei) | lines | adopted:tokei | | M-D2-DEP | D2 | transitive dependency count (cargo tree) | crates | adopted:cargo-deny practice | @@ -45,32 +44,6 @@ and add capability-specific rows only when these don't cover the claim. | M-D4-LEAK | D4 | foreign types in canonical interfaces | count | novel — must be 0; enforced by grep/deny rule, the Clay-Borg hard rule | | M-D4-SWAP | D4 | capability has null + reference impls passing the same conformance suite | bool | adapted:hexagonal-architecture port testing | -### 1b. The coverage gate's two numbers (M-D1-COV, M-D1-LNK) - -M-D1-COV counts tags. It proves no rule is unclaimed and no claimed rule -is invented; it does **not** prove a scenario exercises what it names, and -that limit is printed with the number every time (InnerLoop implementation -rule 4). - -**M-D1-LNK** (CB-WP-0003 T09) closes one link of that chain mechanically: -a rule a scenario claims should also appear in the aggregate source, or -the claim rests on a tag and nothing else. - -Measured 2026-07-31: **49 of 58** claimed rules are named in -`games/ground/src/lib.rs`. **Unmet**, target 58. The nine unlinked: - -```text -GR-D07 GR-F02 GR-L03 GR-O03 GR-P01 GR-P02 GR-P03 GR-P04 GR-T01 -``` - -The gate **reports** this rather than failing, deliberately. Closing the -gap by adding rule IDs to comments would satisfy the check without -establishing that each rule is implemented — which is the overclaim -InnerLoop implementation rule 2 exists to prevent, and which CB-WP-0001 -committed once already. Each of the nine needs its implementation -confirmed before it is tagged. Promoting M-D1-LNK to a failing gate is -correct **after** that, not before. - ### 1a. Token cost accounting (M-D2-CST) > **Superseded 2026-07-31 by [CostAccounting.md](CostAccounting.md)**, which @@ -94,11 +67,27 @@ benchmarks/baselines/model-prices.toml # the price sheet, updated when prices ``` ```toml -# The live sheet is benchmarks/baselines/model-prices.toml — read it, -# do not copy it. An inlined copy here went stale within hours: it still -# showed the sonnet intro rate as a `#` comment after CB-WP-0003 T11 made -# dated rates into data (promo_input / promo_output / promo_until), and -# added `recorded` + `max_age_days` for the staleness check (CA-16/CA-17). +# USD per million tokens; source: Anthropic pricing, recorded 2026-07-31 +[claude-fable-5] +input = 10.00 +output = 50.00 + +[claude-opus-5] +input = 5.00 +output = 25.00 + +[claude-sonnet-5] +input = 3.00 # intro 2.00 through 2026-08-31 +output = 15.00 # intro 10.00 through 2026-08-31 + +[claude-haiku-4-5] +input = 1.00 +output = 5.00 + +[cache] # multipliers on the input price +read = 0.1 +write_5m = 1.25 +write_1h = 2.0 ``` Rules: @@ -119,8 +108,7 @@ Rules: failed pass scores nothing. - Prices are `adopted:anthropic-pricing` with a recorded date; a stale sheet (> 90 days or known price change) invalidates new `better` verdicts on - M-D2-CST until refreshed. **Now enforced** — CA-17 aborts the run rather - than leaving this as prose every verdict silently inherited. + M-D2-CST until refreshed. Determinism is not a metric but an **invariant**: N replays of the same seed and command log must produce bit-identical state hashes. Invariant diff --git a/specs/SessionShape.md b/specs/SessionShape.md deleted file mode 100644 index 44e4d38..0000000 --- a/specs/SessionShape.md +++ /dev/null @@ -1,185 +0,0 @@ -# Session Shape - -Status: **v1.0** — 2026-07-31, from CB-WP-0003 T04. -Instrument: `make cost` ([CostAccounting.md](CostAccounting.md)). - -How to shape an agentic session so it does not cost more than the work is -worth. **Every claim here carries the measurement it rests on.** This spec -exists because the numbers were surprising and because the advice this task -was created to give turned out to be wrong. - ---- - -## 1. The cost model - -```text -cost ≈ Σ over turns ( context_size × cache_read_rate ) + output + cache writes -``` - -Measured on the CB-WP-0001 pass: **88.0% of spend is cache, 11.9% is -output**, at 249:1 cache-read tokens to output tokens. Cost is driven by -**how much context is re-read per turn**, not by how much the model writes. - -Marginal cost of one turn, at the Opus cache-read rate ($0.50/M): - -| context | $/turn | -|---|---| -| 20,000 | $0.010 | -| 50,000 | $0.025 | -| 100,000 | $0.050 | -| 200,000 | $0.100 | -| 400,000 | $0.200 | -| 540,000 | $0.270 | - -> **SS-01.** A turn's cost is a function of context size alone, to within -> ~12%. Optimizing output length is not a cost lever; optimizing context is. - -## 2. Compaction is the primary control - -**The original premise of this spec was wrong and is recorded rather than -deleted.** CB-WP-0003 T04 was written to prescribe *one task per session*, -on the theory that mean context grows with turn count and long sessions are -therefore quadratic. Measured: - -| segment | turns | mean context | total | $/turn | -|---|---|---|---|---| -| start → compact 1 | 136 | 304,178 | $62.19 | **$0.457** | -| compact 1 → compact 2 | 202 | 193,493 | $30.01 | **$0.149** | - -The 202 turns *after* the first compaction cost less than half the 136 -before it. Sessions are **bounded-quadratic**: context grows between -compactions and resets at each one. - -Compaction events, read from `compactMetadata` in the transcript: - -| | pre | post | reduction | duration | -|---|---|---|---|---| -| C1 | 542,991 | 19,974 | **27×** | 151 s | -| C2 | 344,954 | 19,035 | **18×** | 97 s | - -> **SS-02.** The failure mode is a long **uncompacted** session, not a long -> session. Turn count is not the variable to control; context is. - -### 2.1 When compaction pays - -A compaction costs its own summarization call plus one cache -re-establishment. The re-establishment is measurable — the first post-compact -turn wrote 36,149 cache tokens and cost **$0.382**. The summarization call -itself does **not** appear in the transcript's `usage` records, so it is -bounded rather than measured: at worst it is one full read of the -pre-compaction context (543k tokens ≈ $2.72 at the uncached Opus input -rate), at best a cache read of the same (≈ $0.27). - -Savings are immediate and permanent: dropping from 543k to 20k context saves -**$0.262 per subsequent turn**. - -> **SS-03.** Break-even is **2–11 turns** depending on where the -> summarization call falls in that range. Any session expecting more than -> ~11 further turns should compact. In practice this means: **compact -> whenever context exceeds ~300k and work remains.** - -Measured context distribution across the CB-WP-0001 session shows how much -was spent above that line — p75 was 308,325 and p90 was 481,722, so roughly -a quarter of all turns ran at more than 2× the recommended ceiling. - -| percentile | context | -|---|---| -| p50 | 156,771 | -| p75 | 308,325 | -| p90 | 481,722 | -| p99 | 553,252 | - -## 3. Fresh session vs compaction - -A fresh session is **not** free, and this is the measurement that overturned -the original advice. Cold-start context floors, measured over the first 20 -turns of each session: - -| session | start type | mean context, first 20 turns | cost | -|---|---|---|---| -| `8cbd5701` | cold (system + CLAUDE.md + orientation) | 50,872 | $2.17 | -| `f1eb1147` | seeded by a compact summary | 65,819 | $1.90 | - -A cold session starts ~51k and must then **re-read the committed artifacts** -to become productive — the survey, the spec, the ADR — which is exactly the -context a compaction summary already contains. A compact-seeded session -starts ~66k with that reading already done. - -> **SS-04.** Prefer compaction over a fresh session for continuing work. -> A fresh session's lower floor (51k vs 66k) is repaid within the first few -> turns of re-reading artifacts the summary already held. -> -> Prefer a fresh session when the *task changes*, because then the prior -> context is not an asset — it is 66k of irrelevance re-read every turn. - -## 4. Batching - -> **SS-05.** Independent tool calls in one turn cost nothing extra: a turn -> is billed for its context regardless of how many tool calls it carries. -> Two calls in one turn cost half of two calls in two turns. - -Measured batching rate: - -| session | responses w/ tool calls | tool calls | responses batching ≥2 | calls in batched turns | -|---|---|---|---|---| -| `8cbd5701` | 361 | 406 | 31 (8.6%) | 76 (18.7%) | -| `f1eb1147` | 146 | 158 | 12 (8.2%) | 24 (15.2%) | -| pinned at `fc76445`, all transcripts (instrument) | 322 | 362 | 25 (7.8%) | 65 (18.0%) | - -**Correction to a prior claim.** CB-WP-0001's retrospective and the first -draft of CB-WP-0003 T04 both stated *"0 of 330 tool calls were batched"*. -That is wrong: 330 was the count of responses making exactly **one** call, -not the total number of calls. The real figure is 7.8–8.6% of responses and -18.0–18.7% of calls, depending on the window. The claim was never re-derived before being carried into a -workplan — the failure mode InnerLoop v1.1 rule 13 exists for, and the -**eighth** recorded error instance, in the trusted-arithmetic class that -`history/260731-inner-loop-rule-audit.md` identified as having no -executable defence. - -At an 8.6% batching rate there is real headroom, but the saving is bounded: -eliminating every avoidable single-call turn would remove at most a few -percent of turns, worth roughly $2–4 on a $93 pass. **Batch because it is -free, not because it is the lever.** The lever is §2. - -## 5. What this spec does not support - -- **n = 2 sessions, one repo, one model mix.** Every ratio here is from - CB-WP-0001/0002 and should be re-measured before being treated as general. -- **The compaction summarization call is bounded, not measured.** SS-03's - 2–11 turn break-even is a range because of it. Narrowing it requires a - cost source that sees non-transcript calls — the billing API (CB-RES-0002 - C4), which needs an admin key this machine does not have. -- **Context size is read as `cache_read_input_tokens`**, which is the cached - prefix, not the full prompt. On a cache miss the true context is larger - and the turn costs 10× more. Cache misses were not isolated in this - measurement. -- **No claim about quality.** Compaction discards detail. Every number here - says compaction is cheaper; none says the work is as good. A pass that - compacts and then repeats work it forgot has not saved anything, and this - spec cannot detect that. - -## 6. Acceptance metrics - -| ID | Metric | Target | Measured (pinned `fc76445`) | Verdict | Instrument | -|---|---|---|---|---|---| -| SH-1 | mean context per turn | ≤ 200,000 | **232,982** | **unmet** | `make cost-pin` | -| SH-2 | p90 context per turn | ≤ 300,000 | **492,042** | **unmet** | `make cost-pin` | -| SH-3 | batching rate (responses with ≥2 tool calls) | ≥ 20% | **7.8%** | **unmet** | `make cost-pin` | - -`cb-cost` emits all three (`session shape` block), so these are ratified -under InnerLoop v1.1 rather than hand-derived. Supporting counts from the -same run: **362 tool calls in 322 responses, 65 of them in batched turns.** - -**All three targets are unmet, and that is the finding, not a defect in the -targets.** Per the loop's definition of done, an unmet metric is an output: - -- SH-1/SH-2 are unmet because CB-WP-0001 ran 136 turns before its first - compaction. §2 is the remedy and it is now written down. -- SH-3 is set above the measured rate deliberately. Whether 20% is - reachable or arbitrary is unknown until a pass tries; the first pass that - reports it will say. It is the cheapest of the three to move, and the - least valuable — §4 puts the ceiling at $2–4 on a $93 pass. - -Targets are **not** being retargeted to match the measurement. That is the -structure CB-WP-0003 T07 exists to prevent, and doing it here — in the -commit that first measures them — would be the exact defect. diff --git a/tools/__pycache__/cb-cost.cpython-312.pyc b/tools/__pycache__/cb-cost.cpython-312.pyc index 4b96e00..12efe0a 100644 Binary files a/tools/__pycache__/cb-cost.cpython-312.pyc and b/tools/__pycache__/cb-cost.cpython-312.pyc differ diff --git a/tools/cb-cost.py b/tools/cb-cost.py index a301cc4..14db4ac 100644 --- a/tools/cb-cost.py +++ b/tools/cb-cost.py @@ -72,53 +72,16 @@ def components(usage): } -def rates_at(prices, model, when=None): - """Input/output rate in force for `model` at ISO instant `when` (CA-16). - - Promotional rates are data, not comments. A response is priced at the - promo rate when its timestamp falls on or before `promo_until`. - """ - pr = prices.get(model) - if not pr: - return None - if "promo_until" in pr and when: - until = pr["promo_until"] - # tomllib returns a datetime.date for a bare TOML date. - if str(when)[:10] <= str(until)[:10]: - return pr["promo_input"], pr["promo_output"] - return pr["input"], pr["output"] - - -def check_price_sheet_age(prices, today=None): - """CA-17: a stale sheet invalidates verdicts, so it fails a command.""" - import datetime as _dt - - recorded = prices.get("recorded") - if recorded is None: - return "price sheet has no `recorded` date" - max_age = prices.get("max_age_days", 90) - today = today or _dt.date.today() - if isinstance(recorded, _dt.datetime): - recorded = recorded.date() - age = (today - recorded).days - if age > max_age: - return (f"price sheet is {age} days old (max {max_age}); refresh " - f"benchmarks/baselines/model-prices.toml or new M-D2-CST " - f"`better` verdicts are invalid") - return None - - -def price_of(prices, model, toks, when=None): +def price_of(prices, model, toks): """USD for one response. Returns None when the model is unpriced (CA-05).""" pr = prices.get(model) if not pr: return None - rin, rout = rates_at(prices, model, when) cache = prices["cache"] - unit = rin / 1e6 + unit = pr["input"] / 1e6 return ( toks["input"] * unit - + toks["output"] * rout / 1e6 + + toks["output"] * pr["output"] / 1e6 + toks["cache_read"] * unit * cache["read"] + toks["write_5m"] * unit * cache["write_5m"] + toks["write_1h"] * unit * cache["write_1h"] @@ -184,14 +147,6 @@ def read_responses(path, pin=None): ) toks["output"] = max(t["output"] for t in per_row) head = rows[0] - # Tool calls are spread across the group's lines, so they are counted - # over the whole group — one response may carry several (SS-05). - tool_calls = sum( - 1 - for r in rows - for c in (r["message"].get("content") or []) - if c.get("type") == "tool_use" - ) out.append( { "request_id": rid, @@ -199,33 +154,12 @@ def read_responses(path, pin=None): "timestamp": head.get("timestamp") or "", "session": head.get("sessionId") or os.path.basename(path), "toks": toks, - "tool_calls": tool_calls, "subagent": "/subagents/" in path, } ) return out -def session_shape(responses): - """SH-1..SH-3 from specs/SessionShape.md.""" - import statistics - - ctx = sorted(r["toks"]["cache_read"] for r in responses) - with_tools = [r for r in responses if r["tool_calls"] > 0] - batched = [r for r in with_tools if r["tool_calls"] > 1] - calls = sum(r["tool_calls"] for r in with_tools) - pct = statistics.quantiles(ctx, n=100) if len(ctx) > 1 else [ctx[0]] * 99 - return { - "SH-1_mean_context": statistics.mean(ctx) if ctx else 0, - "SH-2_p90_context": pct[89], - "SH-3_batching_rate": (len(batched) / len(with_tools)) if with_tools else 0.0, - "p50_context": pct[49], - "responses_with_tools": len(with_tools), - "tool_calls": calls, - "calls_in_batched_turns": calls - (len(with_tools) - len(batched)), - } - - # ------------------------------------------------------------- attribution @@ -318,12 +252,8 @@ def collect(slug, pin_ref=None): # as though it were an answer. raise Abort(f"no responses in {len(paths)} transcript(s) — refusing to report") - stale = check_price_sheet_age(prices) - if stale: - raise Abort(stale) - for r in responses: - r["cost"] = price_of(prices, r["model"], r["toks"], r["timestamp"]) + r["cost"] = price_of(prices, r["model"], r["toks"]) attribute(responses, commit_index(pin)) @@ -339,11 +269,11 @@ def collect(slug, pin_ref=None): continue by_task[r["task"]] += r["cost"] by_model[r["model"]] += r["cost"] - rin, rout = rates_at(prices, r["model"], r["timestamp"]) - unit = rin / 1e6 + pr = prices[r["model"]] + unit = pr["input"] / 1e6 rates = { "input": unit, - "output": rout / 1e6, + "output": pr["output"] / 1e6, "cache_read": unit * cache["read"], "write_5m": unit * cache["write_5m"], "write_1h": unit * cache["write_1h"], @@ -363,7 +293,6 @@ def collect(slug, pin_ref=None): sub = sum(r["cost"] or 0 for r in responses if r["subagent"]) return { - "session_shape": session_shape(responses), "slug": slug, "pin": pin, "responses": len(responses), @@ -415,17 +344,6 @@ def render(rep, by_task=False, composition=False): f" A per-task table is a view over {100*(1-un/tot):.0f}% of spend." ) - sh = rep["session_shape"] - print("\n session shape (specs/SessionShape.md)") - print(f" SH-1 mean context {sh['SH-1_mean_context']:>12,.0f} tok " - f"[{'ok ' if sh['SH-1_mean_context']<=200_000 else 'FAIL'} target 200,000]") - print(f" SH-2 p90 context {sh['SH-2_p90_context']:>12,.0f} tok " - f"[{'ok ' if sh['SH-2_p90_context']<=300_000 else 'FAIL'} target 300,000]") - print(f" SH-3 batching rate {100*sh['SH-3_batching_rate']:>11.1f}% " - f"[{'ok ' if sh['SH-3_batching_rate']>=0.20 else 'FAIL'} target 20.0%]") - print(f" {sh['tool_calls']} tool calls in {sh['responses_with_tools']} responses; " - f"{sh['calls_in_batched_turns']} in batched turns") - if rep["unpriced"]: print(f"\n UNPRICED ({len(rep['unpriced'])} responses, model not in sheet):") for u in rep["unpriced"]: @@ -497,27 +415,6 @@ def self_test(): finally: os.unlink(partial) - # CA-16: a dated promo rate must apply before its expiry and lapse after. - pr = prices - before = rates_at(pr, "claude-sonnet-5", "2026-07-31T00:00:00Z") - after = rates_at(pr, "claude-sonnet-5", "2026-09-01T00:00:00Z") - check("CA-16 promo rate applies before expiry and lapses after", - before == (2.0, 10.0) and after == (3.0, 15.0), - f"{before} -> {after}") - - # CA-17: staleness must actually trip, or the rule is decorative again. - import datetime as _dt - fresh = check_price_sheet_age(pr, _dt.date(2026, 8, 1)) - stale = check_price_sheet_age(pr, _dt.date(2026, 11, 10)) - check("CA-17 staleness detected past max_age_days", - fresh is None and stale is not None, "fresh ok, 102d trips") - - # CB-02: thresholds must be ordered, or the budget silently never fires. - ap_defaults = {"soft": 10.00, "hard": 22.00} - check("CB-02 budget thresholds ordered and positive", - 0 < ap_defaults["soft"] < ap_defaults["hard"], - f"soft ${ap_defaults['soft']:.2f} < hard ${ap_defaults['hard']:.2f}") - # AC-6: zero responses must not report $0.00 as an answer. with tempfile.NamedTemporaryFile("w", suffix=".jsonl", delete=False) as fh: fh.write(json.dumps({"type": "user", "message": {}}) + "\n") @@ -543,53 +440,12 @@ def self_test(): return 0 if ok else 1 -def budget(slug, soft, hard): - """Live cost budget (specs/CostAccounting.md §7). - - The per-task figure needs the commit that closes the task, so it can - only ever be retrospective. What IS observable mid-task is spend since - the LAST commit — the open remainder — because the transcript is an - append-live file. That is the number a budget can actually fire on. - """ - try: - rep = collect(slug, None) - except Abort as e: - print(f"ABORT — {e}", file=sys.stderr) - return 1 - open_spend = rep["by_task"].get(OPEN_REMAINDER, 0.0) - head = subprocess.run( - ["git", "-C", REPO, "log", "-1", "--format=%h %s"], - capture_output=True, text=True, check=True).stdout.strip() - - print("cost budget — spend since the last commit") - print(f" last commit {head}") - print(f" open spend ${open_spend:,.2f}") - print(f" soft / hard ${soft:,.2f} / ${hard:,.2f}") - if open_spend > hard: - print(f"\n HARD BREACH — ${open_spend:,.2f} > ${hard:,.2f}. Commit what " - f"works, or stop and decompose. Uncommitted work is also " - f"unattributable.", file=sys.stderr) - return 1 - if open_spend > soft: - print(f"\n soft breach — ${open_spend:,.2f} > ${soft:,.2f}. State progress " - f"as a percentage and decide: continue, or commit and decompose.") - return 0 - print("\n within budget") - return 0 - - def main(): ap = argparse.ArgumentParser() ap.add_argument("--slug", default="-home-worsch-clay-borg") ap.add_argument("--pin", help="commit-ish or ISO Z instant (CA-07)") ap.add_argument("--by-task", action="store_true") ap.add_argument("--composition", action="store_true") - ap.add_argument("--session-shape", action="store_true", - help="SH-1..SH-3 (always shown in the default report)") - ap.add_argument("--budget", action="store_true", - help="CB-01/CB-02: spend since the last commit, live") - ap.add_argument("--soft", type=float, default=10.00) - ap.add_argument("--hard", type=float, default=22.00) ap.add_argument("--self-test", action="store_true") ap.add_argument("--json", action="store_true") args = ap.parse_args() @@ -597,9 +453,6 @@ def main(): if args.self_test: return self_test() - if args.budget: - return budget(args.slug, args.soft, args.hard) - try: rep = collect(args.slug, args.pin) except Abort as e: diff --git a/tools/dep-weight.py b/tools/dep-weight.py index f5df6b3..dbcb597 100755 --- a/tools/dep-weight.py +++ b/tools/dep-weight.py @@ -16,7 +16,7 @@ that cannot be found is reported and the run exits non-zero rather than silently under-reporting the total — under-reporting is the exact direction this metric could be gamed. -Usage: python3 tools/dep-weight.py [--json] [--self-test] +Usage: python3 tools/dep-weight.py [--json] """ import glob @@ -88,50 +88,7 @@ def source_lines(name, version): return 0 -def self_test(): - """Each assertion pins a failure this tool must detect. - - The controls that matter here are: an unlocatable crate must not be - silently counted as zero lines (that under-reports, the direction this - metric could be gamed), and a target breach must fail rather than - merely print. - """ - results = [] - - def check(name, ok, detail=""): - results.append((name, ok, detail)) - - # A crate that does not exist must measure zero, so the caller's - # `lines == 0` guard fires rather than silently shrinking the total. - check("unlocatable crate measures zero (so the guard fires)", - source_lines("definitely-not-a-real-crate-xyz", "9.9.9") == 0) - - # A crate we do depend on must measure non-zero, or the guard above - # would fire on everything and the tool would never report at all. - real = source_lines("serde", "1") - check("a real vendored crate measures non-zero", real > 0, - f"{real:,} lines") - - # Targets must be present and numeric — a missing target would make - # the breach check vacuous. - check("targets defined for every configuration", - set(TARGETS) == set(CONFIGS) and all( - isinstance(v, int) and v > 0 for v in TARGETS.values()), - f"{TARGETS}") - - print("dep-weight self-test (positive control)") - ok = True - for name, passed, detail in results: - print(f" [{'ok ' if passed else 'FAIL'}] {name}" - + (f" — {detail}" if detail else "")) - ok &= passed - return 0 if ok else 1 - - def main(): - if "--self-test" in sys.argv: - return self_test() - report = {} missing = [] for label, args in CONFIGS.items(): diff --git a/tools/loop-lint.py b/tools/loop-lint.py deleted file mode 100644 index d57b9a0..0000000 --- a/tools/loop-lint.py +++ /dev/null @@ -1,266 +0,0 @@ -#!/usr/bin/env python3 -"""Executable checks for specs/InnerLoop.md rules (CB-WP-0003 T01). - -The loop's own rules were prose. A rule nobody can run is a suggestion, -and the audit in history/260731-inner-loop-rule-audit.md found several -that were already being violated with no signal. This makes the -mechanically-checkable ones fail a command. - -Each check names the InnerLoop rule it enforces. Checks that cannot be -made mechanical are recorded in the audit as `checkable` or `decorative` -and are deliberately absent here — see the audit for why. - -Positive control (InnerLoop v1.1 §Step 5): --self-test asserts each check -actually detects its failure, using fixtures with known answers. A linter -that passes everything because its matcher is broken is the same defect -class as a benchmark timing rejected work. - -Usage: - python3 tools/loop-lint.py # lint the repo - python3 tools/loop-lint.py --self-test # positive control -""" - -import os -import re -import sys - -REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -LOADABILITY_LIMIT = 400 - -# Artifact classes the loop produces. history/ is an append-only trail -# (verbatim challenge text is not something to split), so it is exempt. -LOOP_DIRS = ("specs", "research", "decisions", "evidence", "workplans") - - -class Finding: - def __init__(self, rule, path, detail): - self.rule, self.path, self.detail = rule, path, detail - - def __str__(self): - return f" [{self.rule}] {self.path}\n {self.detail}" - - -def _md_files(root=REPO): - """Loop artifacts only. - - Vendored third-party trees (a baseline harness ships its own - node_modules) are not artifacts this loop produces, and linting them - buries the two real findings under eighteen irrelevant ones. - """ - for d in LOOP_DIRS: - base = os.path.join(root, d) - for dirpath, dirnames, files in os.walk(base): - dirnames[:] = [x for x in dirnames if x != "node_modules"] - for f in sorted(files): - if f.endswith(".md"): - yield os.path.relpath(os.path.join(dirpath, f), root) - - -def check_loadability(root=REPO): - """§Agentic-efficiency 1 — every loop artifact stays under ~400 lines.""" - out = [] - for rel in _md_files(root): - with open(os.path.join(root, rel)) as fh: - n = sum(1 for _ in fh) - if n > LOADABILITY_LIMIT: - out.append( - Finding( - "loadability", - rel, - f"{n} lines exceeds the ~{LOADABILITY_LIMIT}-line limit; " - f"split and link with relative paths", - ) - ) - return out - - -def check_evidence_no_unmeasured(root=REPO): - """§Rubric — `unmeasured` is legal in a survey, illegal in an evidence file.""" - out = [] - base = os.path.join(root, "evidence") - if not os.path.isdir(base): - return out - for f in sorted(os.listdir(base)): - if not f.endswith(".md"): - continue - rel = os.path.join("evidence", f) - for i, line in enumerate(open(os.path.join(root, rel)), 1): - # A row asserting the verdict, not prose discussing the word. - if re.search(r"\|\s*unmeasured\s*\|", line): - out.append( - Finding("evidence-unmeasured", f"{rel}:{i}", - "verdict `unmeasured` in an evidence table") - ) - return out - - -def check_survey_tier_and_chaos(root=REPO): - """§Loop tiers — tier declared, and the chaos roll recorded every time.""" - out = [] - base = os.path.join(root, "research") - if not os.path.isdir(base): - return out - for f in sorted(os.listdir(base)): - if not f.endswith(".md"): - continue - rel = os.path.join("research", f) - text = open(os.path.join(root, rel)).read() - if not re.search(r"^tier:\s*[SML]\b", text, re.M): - out.append(Finding("tier-declared", rel, "no `tier:` declaration")) - elif "chaos" not in text.lower(): - out.append( - Finding("chaos-recorded", rel, - "tier declared without the chaos roll; the rule requires " - "recording it even when it changes nothing") - ) - return out - - -def check_review_trail(root=REPO): - """§Step 2 — a tier-L survey carries research/challenge/response history.""" - out = [] - base = os.path.join(root, "research") - hist = os.path.join(root, "history") - if not (os.path.isdir(base) and os.path.isdir(hist)): - return out - files = os.listdir(hist) - for f in sorted(os.listdir(base)): - if not f.endswith(".md"): - continue - rel = os.path.join("research", f) - text = open(os.path.join(root, rel)).read() - if not re.search(r"^tier:\s*L\b", text, re.M): - continue - if not re.search(r"^status:\s*approved", text, re.M): - continue - for kind in ("challenge", "response"): - if not any(x.endswith(f"-{kind}.md") and kind in x for x in files): - out.append( - Finding("review-trail", rel, - f"tier-L approved survey with no history/*-{kind}.md") - ) - return out - - -def check_reporting_tools_self_test(root=REPO): - """§Step 5 v1.1 — every tool that reports a number exposes --self-test.""" - out = [] - base = os.path.join(root, "tools") - if not os.path.isdir(base): - return out - for f in sorted(os.listdir(base)): - if not f.endswith(".py"): - continue - rel = os.path.join("tools", f) - text = open(os.path.join(root, rel)).read() - if "--self-test" not in text: - out.append( - Finding("self-test", rel, - "reporting tool with no --self-test entry point; " - "nothing verifies its positive control still works") - ) - return out - - -CHECKS = ( - check_loadability, - check_evidence_no_unmeasured, - check_survey_tier_and_chaos, - check_review_trail, - check_reporting_tools_self_test, -) - - -def self_test(): - """Each check must DETECT its failure, not merely run.""" - import shutil - import tempfile - - results = [] - - def check(name, ok, detail=""): - results.append((name, ok, detail)) - - tmp = tempfile.mkdtemp() - try: - for d in LOOP_DIRS + ("tools", "history"): - os.makedirs(os.path.join(tmp, d), exist_ok=True) - - # loadability: 401 lines must trip, 400 must not. - with open(os.path.join(tmp, "specs", "Big.md"), "w") as fh: - fh.write("x\n" * (LOADABILITY_LIMIT + 1)) - with open(os.path.join(tmp, "specs", "Ok.md"), "w") as fh: - fh.write("x\n" * LOADABILITY_LIMIT) - f = check_loadability(tmp) - check("loadability detects overlong artifact", - len(f) == 1 and "Big.md" in f[0].path, - f"{len(f)} finding(s)") - - # evidence: a table verdict trips; the word in prose does not. - with open(os.path.join(tmp, "evidence", "E.md"), "w") as fh: - fh.write("| AC-1 | x | unmeasured |\n" - "the word unmeasured appearing in prose is fine\n") - f = check_evidence_no_unmeasured(tmp) - check("evidence-unmeasured detects a table verdict, not prose", - len(f) == 1, f"{len(f)} finding(s), expected exactly 1") - - # tier/chaos: missing tier trips; tier without chaos trips. - with open(os.path.join(tmp, "research", "A.md"), "w") as fh: - fh.write("# survey\nno tier here\n") - with open(os.path.join(tmp, "research", "B.md"), "w") as fh: - fh.write("tier: L (structural L)\n") - f = check_survey_tier_and_chaos(tmp) - rules = sorted(x.rule for x in f) - check("tier/chaos detects both omissions", - rules == ["chaos-recorded", "tier-declared"], f"{rules}") - - # self-test: a tool without the flag trips. - with open(os.path.join(tmp, "tools", "silent.py"), "w") as fh: - fh.write("print(42)\n") - f = check_reporting_tools_self_test(tmp) - check("self-test detects a tool lacking --self-test", - len(f) == 1 and "silent.py" in f[0].path, f"{len(f)} finding(s)") - - # review trail: approved tier-L survey with no history trips. - with open(os.path.join(tmp, "research", "C.md"), "w") as fh: - fh.write("tier: L (structural L, chaos 3)\nstatus: approved\n") - f = check_review_trail(tmp) - check("review-trail detects a missing challenge/response", - len(f) == 2, f"{len(f)} finding(s), expected 2") - finally: - shutil.rmtree(tmp, ignore_errors=True) - - print("loop-lint self-test (positive control)") - ok = True - for name, passed, detail in results: - print(f" [{'ok ' if passed else 'FAIL'}] {name}" - + (f" — {detail}" if detail else "")) - ok &= passed - return 0 if ok else 1 - - -def main(): - if "--self-test" in sys.argv: - return self_test() - - findings = [] - for c in CHECKS: - findings.extend(c()) - - print("loop-lint — executable InnerLoop rules") - if not findings: - print(" no findings") - return 0 - by_rule = {} - for f in findings: - by_rule.setdefault(f.rule, []).append(f) - for rule, fs in sorted(by_rule.items()): - print(f"\n{rule} ({len(fs)}):") - for f in fs: - print(str(f)) - print(f"\n{len(findings)} finding(s)") - return 1 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/tools/rule-coverage.py b/tools/rule-coverage.py index 109f9ed..306fa99 100755 --- a/tools/rule-coverage.py +++ b/tools/rule-coverage.py @@ -5,190 +5,30 @@ Compares the rule IDs declared in specs/GroundRules.md against the `covers:` lists in scenarios/ground/*.yaml. Exits non-zero when a scenario claims a rule the spec does not define, so coverage can never be inflated by a typo'd or invented rule ID. - -Stated limit (InnerLoop implementation rule 4): this gate counts tags. It -proves no rule is unclaimed and no claimed rule is invented. It does NOT -prove a scenario exercises the rule it names. - -Positive control (InnerLoop v1.1 §Step 5): the run asserts it actually -found rules and scenarios. Before this was added, a broken spec regex -yielded rules=[] and missing=[] and the tool exited 0 reporting "0/0" — -the harness-does-nothing class, in the tool that reports our headline -coverage number. - -Usage: - python3 tools/rule-coverage.py - python3 tools/rule-coverage.py --self-test """ import glob import re import sys -RULE_RE = r"\*\*(GR-[A-Z]+\d+)" -COVERS_RE = r"covers: \[(.*?)\]" -AGGREGATE = "games/ground/src/lib.rs" -# CB-WP-0003 T08: a provisional default with no expiry can shape the kernel -# indefinitely while looking handled. CI warns; it does not break the build, -# because the ruling is a ground-game decision we cannot make for them. -PROVISIONAL_WARN_DAYS = 30 -ID_RE = r"GR-[A-Z]+\d+" +spec = open("specs/GroundRules.md").read() +rules = sorted(set(re.findall(r"\*\*(GR-[A-Z]+\d+)", spec))) +covered = set() +for path in sorted(glob.glob("scenarios/ground/*.yaml")): + match = re.search(r"covers: \[(.*?)\]", open(path).read(), re.S) + if match: + covered |= {c.strip() for c in match.group(1).split(",") if c.strip()} -def parse_rules(spec_text): - return sorted(set(re.findall(RULE_RE, spec_text))) +known = set(rules) +hit = sorted(known & covered) +missing = [r for r in rules if r not in covered] +invented = sorted(covered - known) - -def parse_code_ids(text): - """Rule IDs named anywhere in the aggregate source (T09).""" - return set(re.findall(ID_RE, text)) - - -def provisional_items(paths): - """(path, owner, raised) for every scenario encoding a U-item default.""" - out = [] - for path in paths: - text = open(path).read() - if not re.search(r"^provisional:\s*true", text, re.M): - continue - owner = re.search(r"^provisional_owner:\s*(\S+)", text, re.M) - raised = re.search(r"^provisional_raised:\s*(\S+)", text, re.M) - out.append((path, - owner.group(1) if owner else None, - raised.group(1) if raised else None)) - return out - - -def parse_covers(text): - match = re.search(COVERS_RE, text, re.S) - if not match: - return set() - return {c.strip() for c in match.group(1).split(",") if c.strip()} - - -def self_test(): - """Each assertion pins a failure this tool must detect.""" - results = [] - - def check(name, ok, detail=""): - results.append((name, ok, detail)) - - # The defect that motivated this control: a spec that parses to zero - # rules must not be reportable as coverage. - check("zero rules detected as a failure", parse_rules("no rules here") == [], - "empty spec yields no rules; main() now aborts on this") - # The matcher must actually match the real format. - check("rule matcher works on real spec format", - parse_rules("**GR-R06** something\n**GR-A12** other") - == ["GR-A12", "GR-R06"]) - # covers: parsing, including the empty case. - check("covers matcher works", parse_covers("covers: [GR-R06, GR-A12]") - == {"GR-R06", "GR-A12"}) - check("missing covers yields empty set", parse_covers("no covers key") == set()) - # T09: the spec -> code link must be detectable. - check("code-id matcher finds ids in source", - parse_code_ids("// GR-R06: lead first\nfn f(){} // GR-A12") - == {"GR-R06", "GR-A12"}) - check("code-id matcher finds none in unmarked source", - parse_code_ids("fn f() { let x = 1; }") == set()) - # T08: every provisional scenario must carry an owner and a date. - import glob as _g - prov = provisional_items(sorted(_g.glob("scenarios/ground/*.yaml"))) - check("every provisional item has an owner and a raised date", - bool(prov) and all(o and r for _, o, r in prov), - f"{len(prov)} provisional item(s)") - - print("rule-coverage self-test (positive control)") - ok = True - for name, passed, detail in results: - print(f" [{'ok ' if passed else 'FAIL'}] {name}" - + (f" — {detail}" if detail else "")) - ok &= passed - return 0 if ok else 1 - - -def main(): - if "--self-test" in sys.argv: - return self_test() - - rules = parse_rules(open("specs/GroundRules.md").read()) - paths = sorted(glob.glob("scenarios/ground/*.yaml")) - - # Positive control: refuse to report a percentage over nothing. - if not rules: - print("ERROR — no GR-rules parsed from specs/GroundRules.md; " - "refusing to report coverage", file=sys.stderr) - return 1 - if not paths: - print("ERROR — no scenarios found in scenarios/ground/; " - "refusing to report coverage", file=sys.stderr) - return 1 - - covered = set() - for path in paths: - covered |= parse_covers(open(path).read()) - - known = set(rules) - hit = sorted(known & covered) - missing = [r for r in rules if r not in covered] - invented = sorted(covered - known) - - # T09: the spec -> code -> scenario chain, made mechanical. A rule a - # scenario claims should also be named in the aggregate, or the claim - # rests on nothing but a tag. - code_ids = parse_code_ids(open(AGGREGATE).read()) - unlinked = sorted((known & covered) - code_ids) - phantom = sorted(code_ids - known) - - pct = 100 * len(hit) // len(rules) - linked = len((known & covered) & code_ids) - print(f"AM-1 rule coverage: {len(hit)}/{len(rules)} ({pct}%) " - f"over {len(paths)} scenarios") - print(f"AM-1b spec->code link: {linked}/{len(hit)} claimed rules also " - f"named in {AGGREGATE}") - print(" NOTE: counts tags; does not prove a scenario exercises what it names") - if unlinked: - print(" unlinked (claimed by a scenario, absent from the aggregate):") - print(" ", " ".join(unlinked)) - if phantom: - print(" ERROR — rule id in code that the spec does not define:", - " ".join(phantom), file=sys.stderr) - return 1 - # T08: provisional items are reported with an owner and an age. - prov = provisional_items(paths) - if prov: - import datetime - today = datetime.date.today() - print(f"\nprovisional U-item defaults: {len(prov)}") - unowned, stale = [], [] - for path, owner, raised in prov: - age = "?" - if raised: - try: - age = (today - datetime.date.fromisoformat(raised)).days - except ValueError: - age = "?" - name = path.split("/")[-1] - print(f" {name:<34} owner={owner or 'NONE':<12} age={age}d") - if not owner: - unowned.append(name) - if isinstance(age, int) and age > PROVISIONAL_WARN_DAYS: - stale.append(f"{name} ({age}d)") - if unowned: - print(" WARN — provisional with no owner:", " ".join(unowned)) - if stale: - print(f" WARN — provisional for over {PROVISIONAL_WARN_DAYS} days:", - " ".join(stale)) - print(" NOTE: evidence files must list these; a ruling flips the " - "scenario, not the kernel") - - if missing: - print(" uncovered:", " ".join(missing)) - if invented: - print(" ERROR — claimed but not defined in the spec:", " ".join(invented), - file=sys.stderr) - return 1 - return 0 if not missing else 2 - - -if __name__ == "__main__": - sys.exit(main()) +pct = 100 * len(hit) // len(rules) if rules else 0 +print(f"AM-1 rule coverage: {len(hit)}/{len(rules)} ({pct}%)") +if missing: + print(" uncovered:", " ".join(missing)) +if invented: + print(" ERROR — claimed but not defined in the spec:", " ".join(invented)) + sys.exit(1) +sys.exit(0 if not missing else 2) diff --git a/workplans/CB-WP-0003-loop-hardening.md b/workplans/CB-WP-0003-loop-hardening.md index 021e278..8be635d 100644 --- a/workplans/CB-WP-0003-loop-hardening.md +++ b/workplans/CB-WP-0003-loop-hardening.md @@ -1,7 +1,7 @@ --- id: CB-WP-0003 title: "Harden the inner loop: executable rules, session economics, dead policy" -status: done +status: proposed state_hub_workstream_id: "39d61dc0-870d-45c1-a595-bcf91f289dce" --- @@ -55,7 +55,7 @@ accordingly. ```task id: CB-WP-0003-T01 -status: done +status: todo priority: high state_hub_task_id: "3d5d45fb-f931-407a-be71-d2d727279d5e" ``` @@ -89,7 +89,7 @@ rules and the three v1.1 rules are currently prose. ```task id: CB-WP-0003-T02 -status: done +status: todo priority: high state_hub_task_id: "f861e67f-ed09-489d-963d-06697da9c08e" ``` @@ -100,12 +100,7 @@ one with five assertions (AC-5..AC-9); `make cost` depends on `make cost-test`; `cost-test` runs in CI and in `make all`. `cargo bench -- --test` and cb-sim's empty-run check were already in place. -**Closed 2026-07-31 alongside T01** — both items below are done: -tools/rule-coverage.py and tools/dep-weight.py gained `--self-test` -(the former exposing a latent silent-pass on zero parsed rules), and -`make loop-lint` fails CI when any `tools/*.py` lacks the flag. - -**Scope as written:** +**Remaining scope:** - `tools/rule-coverage.py` and `tools/dep-weight.py` have positive-control logic but no `--self-test` entry point, so nothing verifies the control @@ -124,7 +119,7 @@ dedup. ```task id: CB-WP-0003-T03 -status: done +status: todo priority: medium state_hub_task_id: "1365b35d-4539-489a-beff-c072221e4702" ``` @@ -158,7 +153,7 @@ rather than reporting a clean verify. ```task id: CB-WP-0003-T04 -status: done +status: todo priority: high state_hub_task_id: "382724e6-efd5-4ef8-a438-9d31a65dacdb" ``` @@ -203,7 +198,7 @@ attribution. ```task id: CB-WP-0003-T05 -status: done +status: todo priority: medium state_hub_task_id: "26c920ee-2c09-4b45-a908-4d343532db09" ``` @@ -234,7 +229,7 @@ Unblocked: depends on CB-WP-0002, now complete. ```task id: CB-WP-0003-T06 -status: done +status: todo priority: low state_hub_task_id: "8f3a6147-54e8-48f9-8543-c13b0e0b0278" ``` @@ -253,7 +248,7 @@ to reject. ```task id: CB-WP-0003-T07 -status: done +status: todo priority: medium state_hub_task_id: "2f702821-7e64-4981-9885-82fa07f638aa" ``` @@ -286,7 +281,7 @@ self-report of which one it was. ```task id: CB-WP-0003-T08 -status: done +status: todo priority: low state_hub_task_id: "c2ee91ee-e551-48a7-b1a9-477c34c0690c" ``` @@ -304,7 +299,7 @@ file must list them, not that the build breaks. ```task id: CB-WP-0003-T09 -status: done +status: todo priority: low state_hub_task_id: "69ccc9ec-dc35-481c-8065-cef040f07f50" ``` @@ -325,9 +320,9 @@ now price the evaluation itself. ```task id: CB-WP-0003-T11 -status: done +status: todo priority: medium -state_hub_task_id: "809215d8-8a5c-4b10-9445-9bc6db9bda42" +state_hub_task_id: "" ``` **New, from CB-WP-0002.** `benchmarks/baselines/model-prices.toml` @@ -350,7 +345,7 @@ State Hub: a schema that cannot hold the fact it needs. ```task id: CB-WP-0003-T10 -status: done +status: todo priority: low state_hub_task_id: "f43de208-92b1-4f0e-9236-a9d27f3ec451" ```