Compare commits
No commits in common. "0da376f7df0ade22e24f57a919c11f13216e44bf" and "06b3df6ecae632ae6fa0691a4f5fad5e28a29618" have entirely different histories.
0da376f7df
...
06b3df6eca
27 changed files with 421 additions and 1854 deletions
|
|
@ -36,13 +36,7 @@ jobs:
|
|||
# Positive control for the cost collector (AC-5..AC-9). Does not
|
||||
# gate on a dollar figure — transcripts are not present in CI — but
|
||||
# proves the collector still detects the failures it claims to.
|
||||
- run: make self-tests
|
||||
|
||||
# InnerLoop rules made executable (CB-WP-0003 T01). Fails on an
|
||||
# overlong artifact, a survey missing its tier/chaos declaration, an
|
||||
# approved tier-L survey with no review trail, `unmeasured` in an
|
||||
# evidence table, or a reporting tool with no --self-test.
|
||||
- run: make loop-lint
|
||||
- run: make cost-test
|
||||
|
||||
# InnerLoop v1.0 positive control, enforced rather than asserted in
|
||||
# prose: --test runs every benchmark once, so a workload that
|
||||
|
|
|
|||
19
Makefile
19
Makefile
|
|
@ -3,7 +3,7 @@
|
|||
|
||||
CARGO := cargo
|
||||
|
||||
.PHONY: check test sim bench bench-test coverage dep-weight cost cost-test cost-pin cost-budget loop-lint self-tests loc all
|
||||
.PHONY: check test sim bench bench-test coverage dep-weight cost cost-test cost-pin loc all
|
||||
|
||||
## fmt + clippy (deny warnings) + HashMap deny-lint
|
||||
check:
|
||||
|
|
@ -29,21 +29,6 @@ cost: cost-test
|
|||
cost-test:
|
||||
python3 tools/cb-cost.py --self-test
|
||||
|
||||
# InnerLoop rules that are mechanically checkable (CB-WP-0003 T01).
|
||||
loop-lint:
|
||||
python3 tools/loop-lint.py
|
||||
|
||||
# Positive control for every reporting tool, per InnerLoop v1.1 Step 5.
|
||||
self-tests:
|
||||
python3 tools/cb-cost.py --self-test
|
||||
python3 tools/loop-lint.py --self-test
|
||||
python3 tools/rule-coverage.py --self-test
|
||||
python3 tools/dep-weight.py --self-test
|
||||
|
||||
# CB-01/CB-02: live spend since the last commit.
|
||||
cost-budget: cost-test
|
||||
python3 tools/cb-cost.py --budget
|
||||
|
||||
cost-pin: cost-test
|
||||
python3 tools/cb-cost.py --pin fc76445 --composition --by-task
|
||||
|
||||
|
|
@ -66,4 +51,4 @@ loc:
|
|||
printf '%-28s %s\n' $$d "$$(find $$d/src -name '*.rs' | xargs cat | grep -vcE '^\s*(//|$$)')"; \
|
||||
done
|
||||
|
||||
all: check test sim coverage dep-weight self-tests loop-lint bench-test
|
||||
all: check test sim coverage dep-weight cost-test bench-test
|
||||
|
|
|
|||
|
|
@ -37,4 +37,3 @@
|
|||
| task | CB-WP-0003-T08 | todo | — | workplans/CB-WP-0003-loop-hardening.md |
|
||||
| task | CB-WP-0003-T09 | todo | — | workplans/CB-WP-0003-loop-hardening.md |
|
||||
| task | CB-WP-0003-T10 | todo | — | workplans/CB-WP-0003-loop-hardening.md |
|
||||
| task | CB-WP-0003-T11 | todo | — | workplans/CB-WP-0003-loop-hardening.md |
|
||||
|
|
|
|||
|
|
@ -1,15 +1,6 @@
|
|||
# USD per million tokens. Source: Anthropic pricing.
|
||||
# Governed by specs/CostAccounting.md §9 — a stale sheet invalidates new
|
||||
# M-D2-CST `better` verdicts. Staleness is enforced by `make cost-test`,
|
||||
# not by this comment.
|
||||
#
|
||||
# Dated rates are DATA, not comments. A promotional rate written as a
|
||||
# comment is invisible to the collector that reads this file: the sonnet
|
||||
# intro price sat in a `#` remark and was silently ignored, costing $0.17
|
||||
# on a $93.32 pass and heading for a sign flip on 2026-08-31.
|
||||
|
||||
recorded = 2026-07-31
|
||||
max_age_days = 90
|
||||
# USD per million tokens; source: Anthropic pricing, recorded 2026-07-31
|
||||
# Governed by specs/MetricsAndScenarios.md §1a — refresh on price change or
|
||||
# after 90 days, or new M-D2-CST 'better' verdicts are invalid.
|
||||
|
||||
[claude-fable-5]
|
||||
input = 10.00
|
||||
|
|
@ -20,13 +11,8 @@ input = 5.00
|
|||
output = 25.00
|
||||
|
||||
[claude-sonnet-5]
|
||||
input = 3.00
|
||||
output = 15.00
|
||||
# Promotional rate in force through `until` (inclusive); the collector
|
||||
# applies it to any response whose timestamp falls on or before that date.
|
||||
promo_input = 2.00
|
||||
promo_output = 10.00
|
||||
promo_until = 2026-08-31
|
||||
input = 3.00 # intro 2.00 through 2026-08-31
|
||||
output = 15.00 # intro 10.00 through 2026-08-31
|
||||
|
||||
[claude-haiku-4-5]
|
||||
input = 1.00
|
||||
|
|
|
|||
|
|
@ -21,16 +21,6 @@ pub struct ScenarioFile {
|
|||
/// scenario, not the kernel.
|
||||
#[serde(default)]
|
||||
pub provisional: bool,
|
||||
/// Who must rule on the U-item this scenario encodes. Required when
|
||||
/// `provisional` is true: a provisional default with no owner can
|
||||
/// shape the kernel indefinitely while looking handled
|
||||
/// (CB-WP-0003 T08).
|
||||
#[serde(default)]
|
||||
pub provisional_owner: String,
|
||||
/// ISO date the provisional default was raised, so its age is
|
||||
/// reportable by `make coverage`.
|
||||
#[serde(default)]
|
||||
pub provisional_raised: String,
|
||||
pub seed: u64,
|
||||
pub setup: Setup,
|
||||
pub commands: Vec<CommandStep>,
|
||||
|
|
|
|||
|
|
@ -17,9 +17,9 @@ transcribed by hand (CA-15).
|
|||
|
||||
| ID | Metric | Target | Measured | Verdict |
|
||||
|---|---|---|---|---|
|
||||
| AC-1 | pinned total, as two components | $93.15 = $92.03 + $1.11 | **$92.03 main + $1.11 subagent = $93.15** | **met** |
|
||||
| AC-1 | pinned total, as two components | $93.32 = $92.21 + $1.11 | **$92.21 main + $1.11 subagent = $93.32** | **met** |
|
||||
| AC-2 | reconciliation residual | $0.00 | **$0.000000** | **met** |
|
||||
| AC-3 | unattributed share reported | present, 33% | **32.4%, own line** | **met** |
|
||||
| AC-3 | unattributed share reported | present, 33% | **32.5%, own line** | **met** |
|
||||
| AC-4 | composition reported | 5 components | **5 of 5** | **met** |
|
||||
| AC-5 | dedup violation aborts | non-zero exit | **abort raised** | **met** |
|
||||
| AC-6 | zero responses refuses to report | non-zero exit | **0 rows, no number emitted** | **met** |
|
||||
|
|
@ -27,7 +27,7 @@ transcribed by hand (CA-15).
|
|||
| AC-8 | 5m cache priced at 1.25× | $1.25/100k @ fable | **$1.2500 (1h would be $2.0000)** | **met** |
|
||||
| AC-9 | streamed partial output → final | 5,5,195 → 195 | **195** | **met** |
|
||||
|
||||
No `unmeasured` rows. AC-1's target was corrected four times before and after this
|
||||
No `unmeasured` rows. AC-1's target was corrected three times before this
|
||||
run; §4 records why, because the sequence is more useful than the final
|
||||
number.
|
||||
|
||||
|
|
@ -35,23 +35,23 @@ number.
|
|||
|
||||
```text
|
||||
input 690 tok $ 0.00 0.0%
|
||||
output 323,643 tok $ 11.13 11.9%
|
||||
cache_read 80,611,798 tok $ 59.67 64.1%
|
||||
output 323,643 tok $ 11.15 11.9%
|
||||
cache_read 80,611,798 tok $ 59.75 64.0%
|
||||
write_5m 37,467 tok $ 0.47 0.5%
|
||||
write_1h 1,672,854 tok $ 21.87 23.5%
|
||||
TOTAL $ 93.15
|
||||
write_1h 1,672,854 tok $ 21.95 23.5%
|
||||
TOTAL $ 93.32
|
||||
```
|
||||
|
||||
**88.0% of spend is cache; 11.9% is output.** The ratio of context re-read
|
||||
to text written is 249:1. A single total would have shown $93.15 and
|
||||
to text written is 249:1. A single total would have shown $93.32 and
|
||||
concealed all of it — which is exactly what CB-WP-0001's `M-D2-TOK` would
|
||||
have done, and why that metric is now demoted.
|
||||
|
||||
## 3. Per-task attribution
|
||||
|
||||
```text
|
||||
UNATTRIBUTED $ 30.14 32.4%
|
||||
T08 $ 21.02 22.6%
|
||||
UNATTRIBUTED $ 30.32 32.5%
|
||||
T08 $ 21.02 22.5%
|
||||
T07 $ 12.01 12.9%
|
||||
T03 $ 9.91 10.6%
|
||||
T04 $ 8.00 8.6%
|
||||
|
|
@ -60,7 +60,7 @@ have done, and why that metric is now demoted.
|
|||
T09 $ 1.29 1.4%
|
||||
```
|
||||
|
||||
**Stated limit (CA-10):** 32.4% of cost sits in commits whose subject
|
||||
**Stated limit (CA-10):** 32.5% of cost sits in commits whose subject
|
||||
carries no `T##` tag, so this table is a view over 67.5% of spend. That is
|
||||
a property of commit hygiene, not of the collector.
|
||||
|
||||
|
|
@ -75,15 +75,14 @@ is worth watching, not yet a conclusion from n=1.
|
|||
- **The per-task figures are not comparable across passes.** They mix
|
||||
models (opus/fable/sonnet) at different price points and different cache
|
||||
states. The dollar figure is comparable; a token count is not.
|
||||
- **AC-3's 32.4% is a fixture pin, not a quality target.** Improving commit
|
||||
- **AC-3's 32.5% is a fixture pin, not a quality target.** Improving commit
|
||||
tagging will move it, and that is the desired direction.
|
||||
- **This is one session.** Every ratio here (cache share, $/turn, the
|
||||
compaction effect in §5) is n=1 and should be treated as a hypothesis
|
||||
until a second pass reproduces it.
|
||||
- ~~**The price sheet cannot express a time-boxed rate.**~~ **Fixed
|
||||
2026-07-31 by CB-WP-0003 T11** (CA-16/CA-17). Applying the sonnet
|
||||
promotional rate moved AC-1 from $93.32 to **$93.15** — the $0.17 this
|
||||
section predicted, now collected rather than merely noted.
|
||||
- **The price sheet cannot express a time-boxed rate.** Sonnet's intro price
|
||||
lives in a TOML comment, so sonnet-priced work is off by $0.17 here
|
||||
(0.19%). This becomes a real error on 2026-08-31.
|
||||
- **AC-1 is not invoice-verified.** No admin key exists, so the Anthropic
|
||||
billing API could not independently confirm the total. The transcript
|
||||
counters are the same ones billing uses, but that is an argument, not a
|
||||
|
|
@ -146,7 +145,7 @@ largest sample was assumed to hold on the smallest one.** The main
|
|||
transcript is 338 of 346 responses, so 206/206 felt conclusive; the
|
||||
violation lives entirely in the 8 responses nobody checked separately.
|
||||
|
||||
Cost of the adversarial review this pass: **$1.11**, against a $93.15 pass.
|
||||
Cost of the adversarial review this pass: **$1.11**, against a $93.32 pass.
|
||||
It found three approval-blocking defects, one of which (the subagent
|
||||
exclusion) would have made this evidence file certify a broken collector.
|
||||
Second consecutive pass where a ~1% spend on review changed the outcome.
|
||||
|
|
|
|||
|
|
@ -1,157 +0,0 @@
|
|||
# 2026-07-31 — InnerLoop v1.1 rule enforceability audit
|
||||
|
||||
CB-WP-0003 T01. Every rule in `specs/InnerLoop.md` v1.1 classified as
|
||||
**executable** (a command fails when it is violated), **checkable** (a
|
||||
human or agent can verify it cheaply and objectively in review), or
|
||||
**decorative** (neither) — and, per the revised task, tagged with the
|
||||
**failure class it catches**.
|
||||
|
||||
## Failure classes on record
|
||||
|
||||
Seven error instances across three classes, from
|
||||
`history/260731-inner-loop-retrospective.md` and
|
||||
`history/260731-cost-accounting-retrospective.md`:
|
||||
|
||||
| tag | class | instances |
|
||||
|---|---|---|
|
||||
| **HDN** | harness-does-nothing — a run succeeds while performing no work | 4 |
|
||||
| **TA** | trusted arithmetic — a correct-looking sum over data that really exists | 2 |
|
||||
| **SSB** | same-sample blind spot — a property verified on the large sample, assumed on the small | 1 |
|
||||
|
||||
The reason for tagging: a rule that catches a class no observed error
|
||||
belongs to is a candidate for deletion even when it is perfectly
|
||||
executable, and a class with no rule covering it is where the next rule
|
||||
should go.
|
||||
|
||||
## The audit
|
||||
|
||||
Status after this pass. `loop-lint` = `tools/loop-lint.py`, new here.
|
||||
|
||||
| # | Rule | Class | Status | Enforced by |
|
||||
|---|---|---|---|---|
|
||||
| 1 | No implementation code before the ADR is committed | — | **checkable** | git history vs ADR date; not automated (see §Deferred) |
|
||||
| 2 | Tier declared before work starts | — | **executable** | `loop-lint` tier-declared |
|
||||
| 3 | Chaos roll recorded every time, even when it changes nothing | — | **executable** | `loop-lint` chaos-recorded |
|
||||
| 4 | Invariants bind at every tier regardless of roll | — | executable (elsewhere) | `make all` — determinism tests, `disallowed_types` |
|
||||
| 5 | Survey names a benchmark-to-beat per dimension | — | **checkable** | review; a regex cannot judge whether a number is a benchmark |
|
||||
| 6 | Cited-only baselines cap the verdict at `parity` | TA | **decorative → checkable** | see §Changes |
|
||||
| 7 | Tier-L survey gets one round of adversarial review | SSB | **executable** | `loop-lint` review-trail |
|
||||
| 8 | Review trail: research/challenge/response in `history/` | SSB | **executable** | `loop-lint` review-trail |
|
||||
| 9 | ADR states expected advantage per dimension | — | **checkable** | review |
|
||||
| 10 | Spec carries an acceptance-metrics table | — | **checkable** | review |
|
||||
| 11 | Every metric names its instrument | HDN | **checkable** | review; naming is textual, existence is not |
|
||||
| 12 | **v1.1** The instrument must exist and emit its own target | TA | **checkable** | review; see §Deferred for why not yet executable |
|
||||
| 13 | **v1.1** Inherited numbers are re-derived before use as a target | TA | **decorative** | nothing can detect a quoted number |
|
||||
| 14 | A metric is checked against contracts in its own spec | — | **checkable** | review |
|
||||
| 15 | Harness asserts it performed the work it reports | HDN | **executable** | `cargo bench -- --test`, cb-sim empty-run, `--self-test` |
|
||||
| 16 | Fail loudly rather than report when the assertion fails | HDN | **executable** | same |
|
||||
| 17 | Divisor pinned by a test | HDN | **executable** | `bench_shape` test module |
|
||||
| 18 | A number that cannot prove its work is void | HDN | **executable** | same as 15 |
|
||||
| 19 | **v1.1** Every reporting tool exposes `--self-test` | HDN, SSB | **executable** | `loop-lint` self-test |
|
||||
| 20 | `--self-test` names a failure it detects, not the happy path | HDN | **checkable** | review; see §Deferred |
|
||||
| 21 | Evidence names its disanalogies | — | **checkable** | review |
|
||||
| 22 | `unmeasured` illegal in an evidence file | — | **executable** | `loop-lint` evidence-unmeasured |
|
||||
| 23 | No silently-ignored input | HDN | **checkable** | review |
|
||||
| 24 | Decisions get commands, not defaults | — | **checkable** | review |
|
||||
| 25 | Scaffolds are exercised or marked | HDN | **checkable** | review |
|
||||
| 26 | Coverage gates that count tags say so | — | **checkable** | review |
|
||||
| 27 | Whole-file loadability (~400 lines) | — | **executable** | `loop-lint` loadability |
|
||||
| 28 | Structured over prose | — | **decorative** | kept as guidance, marked |
|
||||
| 29 | One command surface | — | **checkable** | review |
|
||||
| 30 | Self-contained tasks | — | **checkable** | review |
|
||||
| 31 | Evidence or it didn't happen | — | **checkable** | review |
|
||||
| 32 | Token discipline per the global budget policy | — | **decorative — dead** | nothing; CB-WP-0003 T05 replaces it |
|
||||
| 33–41 | Definition-of-done checklist (9 items) | mixed | **checkable** | review; each maps to an artifact whose existence is testable — see §Deferred |
|
||||
|
||||
Counts: **11 executable**, **22 checkable**, **4 decorative** (one of
|
||||
which is dead policy).
|
||||
|
||||
## What the audit found by running
|
||||
|
||||
`tools/loop-lint.py` was written to make rules 2, 3, 7, 8, 19, 22 and 27
|
||||
executable. On its first run it produced **three findings, all real, none
|
||||
previously visible**:
|
||||
|
||||
1. **`specs/ArchitectureBlueprint.md` is 543 lines** against a ~400-line
|
||||
limit the loop has stated since v0.2. Nobody noticed because nothing
|
||||
measured it. Disposition: split (§Changes).
|
||||
2. **`tools/dep-weight.py` has no `--self-test`.** It has positive-control
|
||||
logic — it refuses to report when a crate cannot be located — but
|
||||
nothing verifies that control still works.
|
||||
3. **`tools/rule-coverage.py` has no `--self-test`.** Same shape.
|
||||
|
||||
Findings 2 and 3 are exactly the recursion this workplan is about: the
|
||||
positive-control rule was applied to benchmarks and to the newest tool,
|
||||
and not to the two older tools that report AM-1 and AM-4 numbers into
|
||||
evidence files.
|
||||
|
||||
## Changes made in this task
|
||||
|
||||
- **Rule 27 made executable and the violation fixed.**
|
||||
`ArchitectureBlueprint.md` split at its own section boundaries into
|
||||
`ArchitectureBlueprint.md` (§1–8, the stack) and
|
||||
`ArchitectureRuntime.md` (§9–15, runtime/tooling/process), linked both
|
||||
ways.
|
||||
- **Rules 2, 3, 7, 8, 19, 22 made executable** via `tools/loop-lint.py`,
|
||||
wired as `make loop-lint` and into `make all` and CI.
|
||||
- **Rule 6 (parity cap) demoted from decorative to checkable** by stating
|
||||
the check explicitly: an evidence row citing a baseline whose provenance
|
||||
is `cited` may not carry verdict `better`. That is mechanical against the
|
||||
survey's provenance column and is queued for `loop-lint` once a second
|
||||
evidence file exists to test it against — writing a matcher with one
|
||||
sample is the SSB error this pass is trying to stop making.
|
||||
- **Rule 28 (structured over prose) demoted to guidance**, with that
|
||||
status stated in the spec. It is a style preference; nothing can judge
|
||||
it, and leaving it phrased as a requirement is the false assurance this
|
||||
audit exists to remove.
|
||||
- **Rule 32 (token discipline) marked dead** in place, pointing at
|
||||
CB-WP-0003 T05. It is not deleted yet because deleting it is T05's
|
||||
decision, but it now reads as dead rather than as a live control.
|
||||
|
||||
## Deferred, with reasons
|
||||
|
||||
- **Rule 1 (ADR gate) is checkable but not automated.** A mechanical check
|
||||
needs a map from capability → source paths, which does not exist. Cheap
|
||||
version worth doing later: require every `specs/<X>.md` to reference an
|
||||
ADR, and every ADR to precede the first commit touching its capability's
|
||||
directory.
|
||||
- **Rule 12 (instrument emits its target) resists automation for now.**
|
||||
Detecting "this number was typed rather than emitted" requires the tool's
|
||||
output to be committed alongside the spec. The tractable form is to
|
||||
require acceptance targets to appear verbatim in a committed tool output
|
||||
file; deferred to T02 rather than guessed at here.
|
||||
- **Rule 20 (self-test names a failure) is genuinely checkable only.** A
|
||||
self-test that asserts `True` passes any structural check. Reviewing the
|
||||
assertions is the control, and CB-WP-0002's AC-9 is the model: pin the
|
||||
exact defect that occurred.
|
||||
- **Rule 13 (re-derive inherited numbers) has no mechanical form at all**
|
||||
and is left decorative *deliberately* — with its status stated. It is
|
||||
the rule that caught the largest error in CB-WP-0002 ($248.46 → $92.21),
|
||||
which is the counter-example to this workplan's own hypothesis: **an
|
||||
unenforceable rule was the most valuable one in the pass.** The
|
||||
hypothesis "a rule that cannot be executed is not a rule" is therefore
|
||||
wrong as stated. The correct version is narrower: *a rule that cannot be
|
||||
executed cannot be relied on to fire, so it must not be the only defence
|
||||
for a class that matters.*
|
||||
|
||||
## Open items raised by this audit
|
||||
|
||||
- **AM-4a / AM-4b are unratified retargets.** Measured at 246,250 and set
|
||||
at 250,000 in the same commit, by the implementer, with the
|
||||
implementation changing in that commit too — a retarget under the test
|
||||
added to InnerLoop §Step 4 by T07, not a correction. They must be
|
||||
ratified by ADR or changed. Until then `make dep-weight` is enforcing a
|
||||
target no reviewed decision stands behind.
|
||||
|
||||
## Class coverage — where the gaps are
|
||||
|
||||
| class | executable rules covering it | assessment |
|
||||
|---|---|---|
|
||||
| HDN | 15, 16, 17, 18, 19 | **well covered.** Five executable rules; the class that started this. |
|
||||
| TA | none | **uncovered by any executable rule.** Rules 6, 12, 13 are checkable or decorative. This is the largest gap, and it is the class with the second-most instances. |
|
||||
| SSB | 7, 8, 19 | **partly covered.** Rule 19 caught the one instance, by accident of running over all data rather than by design. Rules 7/8 enforce that a review *happened*, not that it sampled differently. T03 addresses the design gap. |
|
||||
|
||||
The honest read: **the loop is hardened against the class it has already
|
||||
suffered most from, and has no executable defence against the class that
|
||||
produced its largest single error.** Trusted arithmetic is caught today
|
||||
only by re-derivation, which is a discipline, not a gate.
|
||||
|
|
@ -1,134 +0,0 @@
|
|||
# 2026-07-31 — retrospective: does hardening converge?
|
||||
|
||||
CB-WP-0003 T10. Pass produced `tools/loop-lint.py`,
|
||||
`specs/SessionShape.md`, the rule audit, the live cost budget, the
|
||||
retarget test, dated price rates, M-D1-LNK, provisional expiry, and
|
||||
InnerLoop **v1.2**.
|
||||
|
||||
## The question this task was given
|
||||
|
||||
> Is the set of mechanisms now complete, or is each new pass still finding
|
||||
> a new class? If this pass produces an eighth instance in a fourth class,
|
||||
> the honest conclusion is that class-by-class hardening does not
|
||||
> converge, and the loop should optimize for *cheap detection and
|
||||
> correction* rather than for prevention — which is a different design.
|
||||
|
||||
**It produced both. The answer is: hardening does not converge.**
|
||||
|
||||
## The ledger
|
||||
|
||||
| # | class | instances | found by |
|
||||
|---|---|---|---|
|
||||
| 1 | **HDN** harness-does-nothing | **5** | executable assertions |
|
||||
| 2 | **TA** trusted arithmetic | **3** | re-derivation |
|
||||
| 3 | **SSB** same-sample blind spot | **1** | an assertion running over *all* data |
|
||||
| 4 | **DFD** duplicated-fact drift *(new)* | **2** | reading the copy against its source |
|
||||
|
||||
Ten instances, four classes. Three workplans in, **every pass has
|
||||
produced at least one instance of a class the previous pass had not
|
||||
seen.**
|
||||
|
||||
### New this pass
|
||||
|
||||
- **HDN #5 (latent).** `tools/rule-coverage.py` would have exited 0
|
||||
reporting `0/0` if its spec regex ever stopped matching — a silent pass
|
||||
in the tool that reports our headline AM-1 number. Never fired; found
|
||||
only because T02 required it to carry a `--self-test` and writing one
|
||||
meant asking what failure it should detect.
|
||||
- **TA #3.** *"0 of 330 tool calls were batched"* — 330 was the count of
|
||||
single-call responses, not the total. Carried unverified from
|
||||
CB-WP-0001's retrospective into CB-WP-0003 T04. Real figure: 7.8–8.6%
|
||||
of responses batched.
|
||||
- **DFD #1–2 (new class).** `specs/MetricsAndScenarios.md` §1a inlined a
|
||||
copy of the price sheet; T11 changed the real sheet and the copy went
|
||||
stale **within the same hour**. Separately, the acceptance figure
|
||||
$92.87/$0.66 had to be chased across a survey, a workplan, and an
|
||||
evidence file each time it moved. The defect is not arithmetic and not
|
||||
a harness — it is **a fact stored in two places, where updating one
|
||||
does not update the other**.
|
||||
|
||||
DFD is genuinely distinct: no positive control catches it (both copies
|
||||
are internally consistent), and re-derivation does not either (the copy
|
||||
reproduces whatever it was copied from). It is caught only by reading a
|
||||
copy against its source, which nothing in the loop required.
|
||||
|
||||
## What this implies, and what changed because of it
|
||||
|
||||
Prevention has not converged: four passes, four classes, no sign of
|
||||
saturation. But the *correction* side has been consistently cheap and
|
||||
fast:
|
||||
|
||||
| correction | mechanism | cost |
|
||||
|---|---|---|
|
||||
| $248.46 → $92.21 | re-derivation | ~0 |
|
||||
| $92.21 → $92.87 | adversarial review | $1.11 |
|
||||
| $92.87 → $93.32 | positive control | ~0 |
|
||||
| $93.32 → $93.15 | dated-rate fix | ~0 |
|
||||
|
||||
Every error found in three passes was corrected inside the same session
|
||||
for under ~1% of the pass. **So the loop should stop trying to enumerate
|
||||
failure classes in advance and invest in the properties that make
|
||||
correction cheap** — which it turns out already has, accidentally:
|
||||
|
||||
1. **Numbers are re-derivable** because the raw data (transcripts, git,
|
||||
the price sheet) is kept, not just the conclusions.
|
||||
2. **Artifacts are small and committed**, so a wrong number is one grep
|
||||
from every place that quotes it.
|
||||
3. **Every reported number has a command**, so re-running is free.
|
||||
|
||||
v1.2 makes this the stated design goal rather than an accident, and adds
|
||||
the one rule the new class earns:
|
||||
|
||||
> **Single source of fact.** A number, rate, or target lives in exactly
|
||||
> one place. Everywhere else links to it. Where a copy is unavoidable, it
|
||||
> is generated by a command, not typed. *(CA-15 said this for evidence
|
||||
> tables; DFD shows it applies to specs quoting other specs.)*
|
||||
|
||||
## What the gates did this pass
|
||||
|
||||
Worth recording because it is the cheapest possible evidence that the
|
||||
executable-rule thesis is at least partly right:
|
||||
|
||||
- `loop-lint` found three real violations on its **first** run —
|
||||
including a 543-line spec against a ~400-line rule the loop had stated
|
||||
since v0.2 and never measured.
|
||||
- `loop-lint` then **failed on its own author within the hour**, when a
|
||||
T07 edit pushed `specs/InnerLoop.md` to 407 lines. The file was split
|
||||
because a gate said so, not because anyone noticed.
|
||||
- `cb-cost`'s dedup assertion fired on real data on its first run and
|
||||
refused to print a number.
|
||||
|
||||
Three gates, three catches, all on first contact. Against that: the two
|
||||
most valuable corrections of the whole project ($248.46 → $92.21, and the
|
||||
batching claim) came from a rule that is **decorative and cannot be made
|
||||
executable** — *re-derive inherited numbers*. Both facts are true and the
|
||||
audit records them side by side.
|
||||
|
||||
## Revised hypothesis
|
||||
|
||||
The workplan opened with: *a loop rule that cannot be executed is not a
|
||||
rule.* Three passes of evidence say that is wrong as stated. The version
|
||||
this pass supports:
|
||||
|
||||
> An executable rule fires reliably and catches one class. A decorative
|
||||
> rule fires unreliably and can catch any class, including ones not yet
|
||||
> named. **Keep both, and never let a decorative rule be the only defence
|
||||
> for a class that matters** — but do not delete it merely because it
|
||||
> cannot be automated, which is what this workplan originally proposed.
|
||||
|
||||
## Open, not closed
|
||||
|
||||
- **AM-4a/AM-4b are unratified retargets** (T07). `make dep-weight`
|
||||
enforces targets no reviewed decision stands behind. Maintainer
|
||||
decision.
|
||||
- **9 of 58 rules are claimed by a scenario and named nowhere in the
|
||||
aggregate** (T09, M-D1-LNK). Reported, not failing, because tagging
|
||||
them without confirming implementation is the overclaim the loop
|
||||
already committed once.
|
||||
- **5 provisional U-item defaults** now have an owner and an age, and
|
||||
ground-game has not ruled (T08).
|
||||
- **The chaos roll is in a 12-declaration calibration window** at d4
|
||||
(T06), expected to cost $20–30 and to be decided on evidence.
|
||||
- **DFD has no gate.** The single-source-of-fact rule added in v1.2 is
|
||||
prose. Making it executable — detecting a number duplicated across
|
||||
artifacts — is the obvious next candidate and is not attempted here.
|
||||
|
|
@ -5,8 +5,6 @@ description: >
|
|||
Bond, because GR-L02 requires the target's consent.
|
||||
covers: [GR-A04, GR-F04, GR-L02, GR-L05]
|
||||
provisional: true
|
||||
provisional_owner: ground-game
|
||||
provisional_raised: 2026-07-31
|
||||
seed: 42
|
||||
setup:
|
||||
players: 3
|
||||
|
|
|
|||
|
|
@ -5,8 +5,6 @@ description: >
|
|||
application (GR-F01 under the U2 default).
|
||||
covers: [GR-D01, GR-R08, GR-F01]
|
||||
provisional: true
|
||||
provisional_owner: ground-game
|
||||
provisional_raised: 2026-07-31
|
||||
seed: 42
|
||||
setup:
|
||||
players: 3
|
||||
|
|
|
|||
|
|
@ -5,8 +5,6 @@ description: >
|
|||
owner takes −2 Stress and the sequence ends (GR-D05, GR-D07).
|
||||
covers: [GR-D05, GR-D07, GR-T01]
|
||||
provisional: true
|
||||
provisional_owner: ground-game
|
||||
provisional_raised: 2026-07-31
|
||||
seed: 42
|
||||
setup:
|
||||
players: 3
|
||||
|
|
|
|||
|
|
@ -5,8 +5,6 @@ description: >
|
|||
rating reduced by each Blame token and each Denied Problem.
|
||||
covers: [GR-R09, GR-E01, GR-E02, GR-P03]
|
||||
provisional: true
|
||||
provisional_owner: ground-game
|
||||
provisional_raised: 2026-07-31
|
||||
seed: 42
|
||||
setup:
|
||||
players: 3
|
||||
|
|
|
|||
|
|
@ -5,8 +5,6 @@ description: >
|
|||
one. Personal scores subtract Blame (GR-E03, GR-T02).
|
||||
covers: [GR-E03, GR-E04, GR-O03]
|
||||
provisional: true
|
||||
provisional_owner: ground-game
|
||||
provisional_raised: 2026-07-31
|
||||
seed: 42
|
||||
setup:
|
||||
players: 3
|
||||
|
|
|
|||
|
|
@ -331,14 +331,213 @@ capabilities.
|
|||
|
||||
---
|
||||
|
||||
## 9. Rendering, input, and creation tools
|
||||
|
||||
Rust-first stack:
|
||||
|
||||
- `winit` — windows and platform input
|
||||
- `wgpu` — GPU rendering (Vulkan / Metal / D3D / browser)
|
||||
- `egui` — engine tools, inspectors, early editors
|
||||
- Custom scene renderer for the game table
|
||||
- **glTF** as the primary imported 3D format, wrapped in Clay-Borg asset
|
||||
metadata and provenance
|
||||
|
||||
Creator environment (grows over milestones): scene hierarchy, object
|
||||
inspector, prototype browser, card-sheet importer, deck builder, zone and
|
||||
snap-point editors, rule-state inspector, event timeline, player-view
|
||||
switcher, hidden-information debugger, physics debugger, scenario recorder,
|
||||
replay controls, package validator.
|
||||
|
||||
---
|
||||
|
||||
**Continued in [ArchitectureRuntime.md](ArchitectureRuntime.md)** — §9
|
||||
rendering/input/creation tools, §10 networking and sessions, §11 the
|
||||
agentic inner loop, §12 TargetRevenue integration, §13 repository
|
||||
structure, §14 milestones, §15 governing design decisions.
|
||||
## 10. Networking and sessions
|
||||
|
||||
*(Split 2026-07-31: this file was 543 lines against the loop's ~400-line
|
||||
whole-file loadability rule. The rule had been stated since v0.2 and
|
||||
nothing measured it until `tools/loop-lint.py` — see
|
||||
`history/260731-inner-loop-rule-audit.md`.)*
|
||||
Authoritative session host:
|
||||
|
||||
```text
|
||||
Client gesture
|
||||
→ proposed command
|
||||
→ session server validation
|
||||
→ authoritative events
|
||||
→ state update
|
||||
→ player-specific projection
|
||||
→ client animation
|
||||
```
|
||||
|
||||
Capabilities: session discovery, auth and seat assignment, lobby/readiness,
|
||||
command submission, commit/reveal windows, event-stream replication,
|
||||
snapshot transfer, reconnection, state-hash verification, spectators,
|
||||
player-specific redaction, host migration (later).
|
||||
|
||||
Transport: **Quinn** (QUIC) for native; browser transport is a separate
|
||||
adapter (WebTransport or WebSockets). The canonical protocol is defined
|
||||
independently of any transport:
|
||||
|
||||
```text
|
||||
cb-session-protocol
|
||||
├── CommandEnvelope
|
||||
├── EventEnvelope
|
||||
├── SnapshotEnvelope
|
||||
├── CommitmentEnvelope
|
||||
├── AssetRequest
|
||||
└── CapabilityNegotiation
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 11. Agentic inner loop
|
||||
|
||||
Agentic coding is a first-class product surface. Optimize for small
|
||||
capability boundaries, executable specifications, controlled work areas, and
|
||||
replayable failures.
|
||||
|
||||
### Work packet (every agent task)
|
||||
|
||||
```yaml
|
||||
task_id: CB-PHYS-0042
|
||||
capability: tabletop.card-stacking
|
||||
intent: Keep card stacks stable after drag release.
|
||||
allowed_crates:
|
||||
- cb-physics-api
|
||||
- cb-physics-rapier
|
||||
- cb-tabletop-physics
|
||||
forbidden_changes:
|
||||
- canonical game event schema
|
||||
invariants:
|
||||
- semantic card order must not depend on collider order
|
||||
scenarios:
|
||||
- scenarios/card-stack-20.yaml
|
||||
benchmarks:
|
||||
- benches/card-stack-stability.yaml
|
||||
acceptance:
|
||||
- all conformance tests pass
|
||||
- no state divergence over 10,000 ticks
|
||||
- benchmark regression below 3%
|
||||
```
|
||||
|
||||
### CLI surface (`cb`)
|
||||
|
||||
```bash
|
||||
cb inspect capability tabletop.card
|
||||
cb task prepare CB-PHYS-0042
|
||||
cb generate contracts
|
||||
cb check --affected
|
||||
cb test --affected # supports --format json
|
||||
cb sim ground scenarios/mutual-attack.yaml
|
||||
cb play ground --players 4
|
||||
cb replay artifacts/failure.cbreplay
|
||||
cb compare physics-reference physics-rapier
|
||||
cb bench --affected
|
||||
cb evidence build CB-PHYS-0042
|
||||
cb release assess CB-PHYS-0042
|
||||
```
|
||||
|
||||
### Quality gates
|
||||
|
||||
Formatting/linting, dependency-policy check, unit tests, capability
|
||||
conformance tests, property tests, golden scenario tests, replay
|
||||
determinism, snapshot migration, performance and memory budgets, rendering
|
||||
comparison where relevant, security/sandbox tests, documentation and schema
|
||||
consistency.
|
||||
|
||||
Tooling: `cargo-nextest` (isolated parallel tests), Criterion
|
||||
(regression-sensitive benchmarks), `sccache` (compile reuse), `tracing`
|
||||
(structured diagnostics).
|
||||
|
||||
---
|
||||
|
||||
## 12. TargetRevenue integration
|
||||
|
||||
TargetRevenue governs **versioned capability improvements**, not the
|
||||
monorepo as one indivisible target.
|
||||
|
||||
```toml
|
||||
improvement_id = "CB-GROUND-001"
|
||||
capability = "game.ground.simultaneous-resolution"
|
||||
classification = "10x"
|
||||
|
||||
estimated_days = 4
|
||||
daily_rate = 1000
|
||||
target_revenue = 40000
|
||||
|
||||
phase = "commercial-recovery"
|
||||
release_when_target_reached = "MIT"
|
||||
trust_record = "required"
|
||||
```
|
||||
|
||||
Components: improvement registry, workload ledger, cost model, revenue
|
||||
attribution, dependency graph, phase license generator, revenue meter,
|
||||
release gate, evidence bundle, trust service.
|
||||
|
||||
Economic rule:
|
||||
|
||||
> Optimized assimilations may be financed as independent improvements, while
|
||||
> the canonical interface remains stable and reusable.
|
||||
|
||||
---
|
||||
|
||||
## 13. Repository structure
|
||||
|
||||
```text
|
||||
clay-borg/
|
||||
├── INTENT.md
|
||||
├── SCOPE.md
|
||||
├── ARCHITECTURE.md # or specs/ArchitectureBlueprint.md (this file)
|
||||
├── Cargo.toml
|
||||
├── rust-toolchain.toml
|
||||
│
|
||||
├── canon/ # entities, events, capabilities, schemas, terminology
|
||||
├── crates/
|
||||
│ ├── cb-kernel/ cb-ids/ cb-time/ cb-rng/ cb-events/
|
||||
│ ├── cb-snapshot/ cb-capability/
|
||||
│ ├── cb-world/ cb-world-api/ cb-ecs-bevy/
|
||||
│ ├── cb-physics-api/ cb-physics-null/ cb-physics-reference/ cb-physics-rapier/
|
||||
│ ├── cb-render-api/ cb-render-null/ cb-render-wgpu/
|
||||
│ ├── cb-tabletop/ cb-tabletop-physics/ cb-tabletop-view/
|
||||
│ ├── cb-game-runtime/ cb-game-protocol/ cb-game-wasm/
|
||||
│ ├── cb-session/ cb-network-api/ cb-network-loopback/ cb-network-quic/
|
||||
│ └── cb-assets/ cb-ui/ cb-editor/ cb-observe/ cb-evidence/
|
||||
│
|
||||
├── games/ # ground/, fixture-cards/
|
||||
├── tools/ # cb-cli/, cb-agent/, cb-import/, cb-pack/
|
||||
├── scenarios/
|
||||
├── conformance/
|
||||
├── benchmarks/
|
||||
├── replays/
|
||||
├── examples/
|
||||
├── decisions/
|
||||
├── assimilation/ # assimilation manifests
|
||||
└── target-revenue/
|
||||
```
|
||||
|
||||
Stay a monorepo during architectural formation. Extract a repository only
|
||||
when a capability has a stable contract, an independent lifecycle, and a
|
||||
genuine external consumer.
|
||||
|
||||
---
|
||||
|
||||
## 14. Milestones
|
||||
|
||||
| # | Milestone | Proves |
|
||||
|---|---|---|
|
||||
| 0 | Headless GROUND | Authoritative rules, commit/reveal, DARVO, replay — no rendering, no physics |
|
||||
| 1 | Inspectable 2D table | Presentation bindings without 3D complexity |
|
||||
| 2 | Physical 3D tabletop | wgpu + Rapier projection of semantic events |
|
||||
| 3 | Networked sessions | Authoritative host, private projections, reconnection |
|
||||
| 4 | Game creation framework | Editors, importers, Wasm game components |
|
||||
| 5 | Second fixture game | Generality — abstractions promoted to Canon only after a second concrete use |
|
||||
|
||||
---
|
||||
|
||||
## 15. Governing design decisions
|
||||
|
||||
1. Build GROUND first, not a general engine first.
|
||||
2. Keep rules independent from rendering and physics.
|
||||
3. Use commands and events as the authoritative mutation mechanism.
|
||||
4. Provide null, reference, and optimized implementations of important capabilities.
|
||||
5. Never leak assimilated-library types into canonical interfaces.
|
||||
6. Use server-authoritative physics and deterministic semantic rules.
|
||||
7. Treat player visibility as a projection, not a UI afterthought.
|
||||
8. Make every defect reproducible as a scenario and replay.
|
||||
9. Give coding agents bounded work packets and stable commands.
|
||||
10. Attach TargetRevenue phases to versioned improvements and evidence bundles.
|
||||
|
|
|
|||
|
|
@ -1,221 +0,0 @@
|
|||
# Clay-Borg Architecture — Runtime, Tooling, and Process
|
||||
|
||||
Second half of the architecture blueprint, split from
|
||||
[ArchitectureBlueprint.md](ArchitectureBlueprint.md) on 2026-07-31 for
|
||||
whole-file loadability. §1–8 (the layered stack, Clay Canon, runtime
|
||||
substrate, simulation kernel, physics, world-building, tabletop domain
|
||||
framework, game runtime) remain there; §9–15 are here.
|
||||
|
||||
Section numbering is continuous with the first half and deliberately
|
||||
unchanged, so existing references keep resolving.
|
||||
|
||||
## 9. Rendering, input, and creation tools
|
||||
|
||||
Rust-first stack:
|
||||
|
||||
- `winit` — windows and platform input
|
||||
- `wgpu` — GPU rendering (Vulkan / Metal / D3D / browser)
|
||||
- `egui` — engine tools, inspectors, early editors
|
||||
- Custom scene renderer for the game table
|
||||
- **glTF** as the primary imported 3D format, wrapped in Clay-Borg asset
|
||||
metadata and provenance
|
||||
|
||||
Creator environment (grows over milestones): scene hierarchy, object
|
||||
inspector, prototype browser, card-sheet importer, deck builder, zone and
|
||||
snap-point editors, rule-state inspector, event timeline, player-view
|
||||
switcher, hidden-information debugger, physics debugger, scenario recorder,
|
||||
replay controls, package validator.
|
||||
|
||||
---
|
||||
|
||||
## 10. Networking and sessions
|
||||
|
||||
Authoritative session host:
|
||||
|
||||
```text
|
||||
Client gesture
|
||||
→ proposed command
|
||||
→ session server validation
|
||||
→ authoritative events
|
||||
→ state update
|
||||
→ player-specific projection
|
||||
→ client animation
|
||||
```
|
||||
|
||||
Capabilities: session discovery, auth and seat assignment, lobby/readiness,
|
||||
command submission, commit/reveal windows, event-stream replication,
|
||||
snapshot transfer, reconnection, state-hash verification, spectators,
|
||||
player-specific redaction, host migration (later).
|
||||
|
||||
Transport: **Quinn** (QUIC) for native; browser transport is a separate
|
||||
adapter (WebTransport or WebSockets). The canonical protocol is defined
|
||||
independently of any transport:
|
||||
|
||||
```text
|
||||
cb-session-protocol
|
||||
├── CommandEnvelope
|
||||
├── EventEnvelope
|
||||
├── SnapshotEnvelope
|
||||
├── CommitmentEnvelope
|
||||
├── AssetRequest
|
||||
└── CapabilityNegotiation
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 11. Agentic inner loop
|
||||
|
||||
Agentic coding is a first-class product surface. Optimize for small
|
||||
capability boundaries, executable specifications, controlled work areas, and
|
||||
replayable failures.
|
||||
|
||||
### Work packet (every agent task)
|
||||
|
||||
```yaml
|
||||
task_id: CB-PHYS-0042
|
||||
capability: tabletop.card-stacking
|
||||
intent: Keep card stacks stable after drag release.
|
||||
allowed_crates:
|
||||
- cb-physics-api
|
||||
- cb-physics-rapier
|
||||
- cb-tabletop-physics
|
||||
forbidden_changes:
|
||||
- canonical game event schema
|
||||
invariants:
|
||||
- semantic card order must not depend on collider order
|
||||
scenarios:
|
||||
- scenarios/card-stack-20.yaml
|
||||
benchmarks:
|
||||
- benches/card-stack-stability.yaml
|
||||
acceptance:
|
||||
- all conformance tests pass
|
||||
- no state divergence over 10,000 ticks
|
||||
- benchmark regression below 3%
|
||||
```
|
||||
|
||||
### CLI surface (`cb`)
|
||||
|
||||
```bash
|
||||
cb inspect capability tabletop.card
|
||||
cb task prepare CB-PHYS-0042
|
||||
cb generate contracts
|
||||
cb check --affected
|
||||
cb test --affected # supports --format json
|
||||
cb sim ground scenarios/mutual-attack.yaml
|
||||
cb play ground --players 4
|
||||
cb replay artifacts/failure.cbreplay
|
||||
cb compare physics-reference physics-rapier
|
||||
cb bench --affected
|
||||
cb evidence build CB-PHYS-0042
|
||||
cb release assess CB-PHYS-0042
|
||||
```
|
||||
|
||||
### Quality gates
|
||||
|
||||
Formatting/linting, dependency-policy check, unit tests, capability
|
||||
conformance tests, property tests, golden scenario tests, replay
|
||||
determinism, snapshot migration, performance and memory budgets, rendering
|
||||
comparison where relevant, security/sandbox tests, documentation and schema
|
||||
consistency.
|
||||
|
||||
Tooling: `cargo-nextest` (isolated parallel tests), Criterion
|
||||
(regression-sensitive benchmarks), `sccache` (compile reuse), `tracing`
|
||||
(structured diagnostics).
|
||||
|
||||
---
|
||||
|
||||
## 12. TargetRevenue integration
|
||||
|
||||
TargetRevenue governs **versioned capability improvements**, not the
|
||||
monorepo as one indivisible target.
|
||||
|
||||
```toml
|
||||
improvement_id = "CB-GROUND-001"
|
||||
capability = "game.ground.simultaneous-resolution"
|
||||
classification = "10x"
|
||||
|
||||
estimated_days = 4
|
||||
daily_rate = 1000
|
||||
target_revenue = 40000
|
||||
|
||||
phase = "commercial-recovery"
|
||||
release_when_target_reached = "MIT"
|
||||
trust_record = "required"
|
||||
```
|
||||
|
||||
Components: improvement registry, workload ledger, cost model, revenue
|
||||
attribution, dependency graph, phase license generator, revenue meter,
|
||||
release gate, evidence bundle, trust service.
|
||||
|
||||
Economic rule:
|
||||
|
||||
> Optimized assimilations may be financed as independent improvements, while
|
||||
> the canonical interface remains stable and reusable.
|
||||
|
||||
---
|
||||
|
||||
## 13. Repository structure
|
||||
|
||||
```text
|
||||
clay-borg/
|
||||
├── INTENT.md
|
||||
├── SCOPE.md
|
||||
├── ARCHITECTURE.md # or specs/ArchitectureBlueprint.md (this file)
|
||||
├── Cargo.toml
|
||||
├── rust-toolchain.toml
|
||||
│
|
||||
├── canon/ # entities, events, capabilities, schemas, terminology
|
||||
├── crates/
|
||||
│ ├── cb-kernel/ cb-ids/ cb-time/ cb-rng/ cb-events/
|
||||
│ ├── cb-snapshot/ cb-capability/
|
||||
│ ├── cb-world/ cb-world-api/ cb-ecs-bevy/
|
||||
│ ├── cb-physics-api/ cb-physics-null/ cb-physics-reference/ cb-physics-rapier/
|
||||
│ ├── cb-render-api/ cb-render-null/ cb-render-wgpu/
|
||||
│ ├── cb-tabletop/ cb-tabletop-physics/ cb-tabletop-view/
|
||||
│ ├── cb-game-runtime/ cb-game-protocol/ cb-game-wasm/
|
||||
│ ├── cb-session/ cb-network-api/ cb-network-loopback/ cb-network-quic/
|
||||
│ └── cb-assets/ cb-ui/ cb-editor/ cb-observe/ cb-evidence/
|
||||
│
|
||||
├── games/ # ground/, fixture-cards/
|
||||
├── tools/ # cb-cli/, cb-agent/, cb-import/, cb-pack/
|
||||
├── scenarios/
|
||||
├── conformance/
|
||||
├── benchmarks/
|
||||
├── replays/
|
||||
├── examples/
|
||||
├── decisions/
|
||||
├── assimilation/ # assimilation manifests
|
||||
└── target-revenue/
|
||||
```
|
||||
|
||||
Stay a monorepo during architectural formation. Extract a repository only
|
||||
when a capability has a stable contract, an independent lifecycle, and a
|
||||
genuine external consumer.
|
||||
|
||||
---
|
||||
|
||||
## 14. Milestones
|
||||
|
||||
| # | Milestone | Proves |
|
||||
|---|---|---|
|
||||
| 0 | Headless GROUND | Authoritative rules, commit/reveal, DARVO, replay — no rendering, no physics |
|
||||
| 1 | Inspectable 2D table | Presentation bindings without 3D complexity |
|
||||
| 2 | Physical 3D tabletop | wgpu + Rapier projection of semantic events |
|
||||
| 3 | Networked sessions | Authoritative host, private projections, reconnection |
|
||||
| 4 | Game creation framework | Editors, importers, Wasm game components |
|
||||
| 5 | Second fixture game | Generality — abstractions promoted to Canon only after a second concrete use |
|
||||
|
||||
---
|
||||
|
||||
## 15. Governing design decisions
|
||||
|
||||
1. Build GROUND first, not a general engine first.
|
||||
2. Keep rules independent from rendering and physics.
|
||||
3. Use commands and events as the authoritative mutation mechanism.
|
||||
4. Provide null, reference, and optimized implementations of important capabilities.
|
||||
5. Never leak assimilated-library types into canonical interfaces.
|
||||
6. Use server-authoritative physics and deterministic semantic rules.
|
||||
7. Treat player visibility as a projection, not a UI afterthought.
|
||||
8. Make every defect reproducible as a scenario and replay.
|
||||
9. Give coding agents bounded work packets and stable commands.
|
||||
10. Attach TargetRevenue phases to versioned improvements and evidence bundles.
|
||||
|
|
@ -67,16 +67,6 @@ cost = input_tokens × price.input
|
|||
> reported as an unpriced line with its token counts, never dropped and
|
||||
> never priced at a default.
|
||||
|
||||
> **CA-16.** Promotional and other time-boxed rates are **data**, not
|
||||
> comments. A response is priced at the rate in force *at its own
|
||||
> timestamp*: `promo_input`/`promo_output` apply on or before
|
||||
> `promo_until`, and the standard rate applies after.
|
||||
|
||||
> **CA-17.** The price sheet carries `recorded` and `max_age_days`, and a
|
||||
> sheet older than `max_age_days` **aborts the run**. A stale sheet
|
||||
> invalidates `better` verdicts, so the staleness rule is a command rather
|
||||
> than a sentence.
|
||||
|
||||
### 1.3 Scope of a measurement
|
||||
|
||||
> **CA-06.** A measurement enumerates **every** transcript for the repo:
|
||||
|
|
@ -135,9 +125,9 @@ Each row names the command that produces its number, per InnerLoop §Step 4.
|
|||
|
||||
| ID | Metric | Target | Instrument |
|
||||
|---|---|---|---|
|
||||
| **AC-1** | reproduces the pinned CB-WP-0001 total | **$93.15** = $92.03 main + $1.11 subagent | `make cost-pin` |
|
||||
| **AC-1** | reproduces the pinned CB-WP-0001 total | **$93.32** = $92.21 main + $1.11 subagent | `make cost-pin` |
|
||||
| **AC-2** | reconciliation residual (CA-14) | **$0.00** exactly | same command, `reconciled: ok` line |
|
||||
| **AC-3** | unattributed share reported (CA-10) | present, and **32.4%** on the pinned run | `cb-cost --pin fc76445 --by-task` |
|
||||
| **AC-3** | unattributed share reported (CA-10) | present, and **33%** on the pinned run | `cb-cost --pin fc76445 --by-task` |
|
||||
| **AC-4** | composition reported (CA-13) | all five components present | `cb-cost --pin fc76445 --composition` |
|
||||
| **AC-5** | dedup invariant asserted (CA-02) | violation exits non-zero | `make cost-test` |
|
||||
| **AC-6** | positive control: refuses to report on zero responses | exits non-zero | `make cost-test` |
|
||||
|
|
@ -155,7 +145,7 @@ assertion against a fixture whose expected value is known and fails loudly;
|
|||
Per InnerLoop §Step 4, the acceptance table is checked against the
|
||||
contracts in this same spec:
|
||||
|
||||
- AC-1's $93.15 is reachable only if CA-06 holds (both trees enumerated).
|
||||
- AC-1's $92.87 is reachable only if CA-06 holds (both trees enumerated).
|
||||
Under a main-file-only collector the target is unreachable — this is the
|
||||
defect the adversarial review caught, where a target of $92.21 would have
|
||||
been hit *only* by a broken collector.
|
||||
|
|
@ -182,88 +172,7 @@ contracts in this same spec:
|
|||
necessarily shows a lossy projection. This is a limitation of the sink,
|
||||
not of the metric; `make cost` remains the authority.
|
||||
|
||||
## 7. The live cost budget
|
||||
|
||||
Replaces the 8k/10k per-task **token** budget, which was never referenced
|
||||
or enforced, which CB-WP-0001 T08 exceeded by orders of magnitude with no
|
||||
signal, and which InnerLoop v1.1 now marks dead. It implied a control that
|
||||
did not exist.
|
||||
|
||||
**The design constraint that shapes this.** Per-task cost needs the commit
|
||||
that *closes* the task (CA-08), so it is unavoidably retrospective — a
|
||||
per-task budget can only ever report a breach after the money is spent.
|
||||
What is observable mid-task is **spend since the last commit**, because
|
||||
the transcript is an append-live file (CA-07). That is the only number a
|
||||
budget can actually fire on, so that is what the budget binds.
|
||||
|
||||
> **CB-01.** The budget is the **open remainder**: USD spent since the
|
||||
> last commit, computable at any instant by `make cost-budget`.
|
||||
|
||||
> **CB-02.** Thresholds: **soft $10.00**, **hard $22.00**.
|
||||
>
|
||||
> - **soft** — state progress as a percentage and decide: continue, or
|
||||
> commit what works and decompose.
|
||||
> - **hard** — stop. Commit what works, or decompose. Uncommitted work is
|
||||
> also *unattributable* work, so a hard breach costs twice: once in
|
||||
> money, once in the 33% attribution gap.
|
||||
|
||||
Calibration, measured over the 32 non-empty commit intervals of
|
||||
CB-WP-0001 (`cb-cost --pin fc76445`):
|
||||
|
||||
| | spend per interval |
|
||||
|---|---|
|
||||
| min | $0.07 |
|
||||
| p50 | $1.40 |
|
||||
| p90 | $9.36 |
|
||||
| **max** | **$10.80** (T07, the Cargo workspace scaffold) |
|
||||
|
||||
Soft is set at **$10.00**, just below the observed maximum and just above
|
||||
p90: on the pass this is calibrated from it would have fired exactly once,
|
||||
on the single most expensive interval. Hard is set at **$22.00**, roughly
|
||||
2× the observed maximum — a value never reached in 32 intervals, so
|
||||
reaching it means the session is doing something the calibration data has
|
||||
no example of, which is precisely when stopping is right.
|
||||
|
||||
**Why these are not retargeted-after-the-fact.** T07's concern is a target
|
||||
moved to accommodate a measurement. These thresholds are set *above* every
|
||||
observed value, so they bind on future work rather than ratifying present
|
||||
work — no interval in the calibration set breaches hard, and one breaches
|
||||
soft. If a future pass routinely breaches, that is signal, not a reason to
|
||||
raise them.
|
||||
|
||||
**Known limit:** the budget cannot fire *between* turns without something
|
||||
running the command. It is available to an agent at any point, in CI, and
|
||||
in `make all`; it is not a daemon. An agent that never runs it gets no
|
||||
signal — which is exactly the dead-policy failure this replaces, one level
|
||||
up. The mitigation is that it costs nothing to run and is on the one
|
||||
command surface.
|
||||
|
||||
## 9. The price sheet
|
||||
|
||||
`benchmarks/baselines/model-prices.toml` is the single source of rates.
|
||||
Two defects were fixed here, both of the same shape — **a schema that
|
||||
could not hold the fact it needed**, which is the criticism this
|
||||
capability's survey levelled at the State Hub:
|
||||
|
||||
1. **A time-boxed rate lived in a comment.** Sonnet's intro price
|
||||
(2.00/10.00 through 2026-08-31) was written as `# intro …` and was
|
||||
therefore invisible to the collector, which used the standard rate. Cost
|
||||
at the pin: **$0.17**, 0.19% of the pass. The real hazard was the
|
||||
date — on **2026-08-31** the promotion expires and a comment-only
|
||||
representation would have been wrong in the opposite direction, with
|
||||
nothing to notice. Now expressed as `promo_*` fields and applied per
|
||||
CA-16.
|
||||
2. **The 90-day staleness rule had no check.** It was prose in
|
||||
`MetricsAndScenarios.md` §1a and every M-D2-CST verdict silently
|
||||
inherited it. Now `recorded` + `max_age_days` in the sheet, enforced by
|
||||
CA-17 and exercised by `make cost-test`.
|
||||
|
||||
Applying CA-16 moved AC-1 from $93.32 to **$93.15**. This is a *legitimate*
|
||||
retarget under the distinction CB-WP-0003 T07 draws: the instrument
|
||||
disproved the target, rather than the implementation missing it. The
|
||||
instrument's output is in the same commit.
|
||||
|
||||
## 8. Revisions to M-D2-CST
|
||||
## 7. Revisions to M-D2-CST
|
||||
|
||||
`specs/MetricsAndScenarios.md` §1a is superseded by this spec. M-D2-CST is
|
||||
redefined from "tokens × pricepoint" — which named no instrument and was
|
||||
|
|
|
|||
|
|
@ -210,15 +210,6 @@ terms.
|
|||
|
||||
## Underdetermined in dataset 0.1 — PROVISIONAL defaults (flag to ground-game)
|
||||
|
||||
**Owner: ground-game. Raised: 2026-07-31.** Every U-item carries an owner
|
||||
and a raise date so its age is visible; `make coverage` reports both and
|
||||
warns past 30 days. CI **warns rather than breaks** — the ruling belongs
|
||||
to ground-game and the kernel cannot make it for them — but every evidence
|
||||
file that depends on one must list it (CB-WP-0003 T08).
|
||||
|
||||
A ruling flips the *scenario*, not the kernel: the default is encoded in a
|
||||
scenario tagged `provisional: true`, so a correction is a data change.
|
||||
|
||||
Formalization exposed points the dataset does not decide. Simulation uses
|
||||
the stated default; each is tagged in scenarios that depend on it and must
|
||||
be confirmed or corrected by ground-game.
|
||||
|
|
|
|||
|
|
@ -1,13 +1,6 @@
|
|||
# The Inner Loop — Assimilate and Surpass
|
||||
|
||||
Status: **v1.2** — corrected from CB-WP-0003 (loop hardening) on
|
||||
2026-07-31. Changes from v1.1: single source of fact; review targets the
|
||||
harness and states its sampling limit; correction vs retarget; the chaos
|
||||
roll's calibration window; the live cost budget. The design goal is now
|
||||
stated: **optimize for cheap correction, not for exhaustive prevention.**
|
||||
Rationale: `history/260731-loop-hardening-retrospective.md`.
|
||||
|
||||
v1.1 — corrected from CB-WP-0002 (cost accounting) on
|
||||
Status: **v1.1** — corrected from CB-WP-0002 (cost accounting) on
|
||||
2026-07-31. Changes from v1.0: the instrument must exist and emit its own
|
||||
target; inherited numbers are re-derived before use; every reporting tool
|
||||
exposes `--self-test`; cost is in the definition of done. Rationale:
|
||||
|
|
@ -21,25 +14,7 @@ evidence state what it does not support. Rationale and the failures behind
|
|||
each: `history/260731-inner-loop-retrospective.md`.
|
||||
|
||||
Normative process for building every Clay-Borg capability. Referenced by
|
||||
all workplans.
|
||||
|
||||
**Design goal (v1.2).** Three passes produced ten error instances across
|
||||
four classes, and every pass produced a class the previous one had not
|
||||
seen — prevention is not converging. Every one of those errors was
|
||||
corrected inside the same session for under ~1% of the pass. The loop
|
||||
therefore optimizes for **cheap correction**: keep the raw data so numbers
|
||||
can be re-derived, keep artifacts small and committed so a wrong number is
|
||||
one grep from everywhere that quotes it, and give every reported number a
|
||||
command so re-running is free.
|
||||
|
||||
> **Single source of fact.** A number, rate, or target lives in exactly
|
||||
> one place; everywhere else links to it. Where a copy is unavoidable it
|
||||
> is generated by a command, not typed. *(v1.2: a price sheet inlined into
|
||||
> a spec went stale within the hour of the real sheet changing, and one
|
||||
> acceptance figure had to be chased across three artifacts each time it
|
||||
> moved. No positive control catches this — both copies are internally
|
||||
> consistent — and re-derivation does not either, because the copy
|
||||
> reproduces whatever it was copied from.)* The loop's own optimization target is **agentic efficiency**:
|
||||
all workplans. The loop's own optimization target is **agentic efficiency**:
|
||||
every artifact it produces must be small enough to load whole, structured
|
||||
enough to act on without interpretation, and falsifiable enough that an
|
||||
agent can judge its own work without a human in the iteration.
|
||||
|
|
@ -71,29 +46,10 @@ are never skipped for code-producing work.
|
|||
| **M** | Survey and ADR merged into one document; review optional | Touches a canonical interface, or adds/updates an external dependency |
|
||||
| **S** | One provenance paragraph in the commit message | Everything else (utilities, fixes, refactors inside a boundary) |
|
||||
|
||||
**The chaos roll.** After deriving the structural tier, roll **d4**
|
||||
(`shuf -i 1-4 -n 1`). On a **4**, the tier is instead picked uniformly at
|
||||
**The chaos roll.** After deriving the structural tier, roll d10
|
||||
(`shuf -i 1-10 -n 1`). On a **10**, the tier is instead picked uniformly at
|
||||
random (`shuf -e S M L -n 1`), overriding the structural derivation — up or
|
||||
down.
|
||||
|
||||
> **Calibration window, opened 2026-07-31 (CB-WP-0003 T06).** The rate was
|
||||
> d10 and the mechanism **never fired**: two rolls across two workplans
|
||||
> (CB-WP-0001: 9, CB-WP-0002: 2), against ~0.2 expected firings. At d10 and
|
||||
> ~2 tier decisions per workplan it would take roughly twenty workplans to
|
||||
> observe four overrides, so the mechanism was set at a rate that prevented
|
||||
> its own evaluation — the one option T06 ruled out.
|
||||
>
|
||||
> Raised to **d4 (25%) for the next 12 tier declarations**, then evaluated
|
||||
> and either kept, returned to d10, or deleted. Expected ~3 firings in the
|
||||
> window, which is enough to see whether an overridden tier produces a
|
||||
> different outcome than the argued one.
|
||||
>
|
||||
> **Stated cost:** a chaos-L override on work that would have been S buys a
|
||||
> full survey, adversarial review, and ADR. Measured comparable: CB-WP-0001
|
||||
> T03 (a tier-L survey) cost **$9.91**. At 25% over 12 declarations the
|
||||
> window is expected to cost **$20–30**. That is the price of finding out
|
||||
> whether the mechanism is worth keeping, and it is cheaper than carrying an
|
||||
> unevaluated ritual indefinitely. Both rolls are recorded in the tier declaration
|
||||
down. Both rolls are recorded in the tier declaration
|
||||
(`tier: M (structural L, chaos 10→M)`). **Record the roll every time,
|
||||
including when it changes nothing** (`tier: L (structural L, chaos 4)`),
|
||||
so a mechanism that never fires is visible rather than assumed. Purpose:
|
||||
|
|
@ -131,40 +87,11 @@ evidence verdict for those rows caps at `parity`, never `better`.
|
|||
### Step 2 — Approve (adversarial review)
|
||||
|
||||
For tier-L passes, approval is earned through an **adversarial review**: a
|
||||
separate session (or agent) attempts to break the work. Exactly **one
|
||||
round**: challenge, then response. The work is approvable only when every
|
||||
challenge is either answered with evidence or conceded and folded in.
|
||||
|
||||
**The review target follows the risk.** Reviewing the survey was the
|
||||
original rule, and it is the wrong target for a capability whose claim
|
||||
rests on numbers — every serious error in this project has been in
|
||||
measurement or build configuration, not in prose.
|
||||
|
||||
| the claim rests on | the reviewer is given | and must attempt |
|
||||
|---|---|---|
|
||||
| a survey of candidates | the survey | an omitted candidate; a stale or unverifiable benchmark; an unmeasured claim presented as measured |
|
||||
| **numbers** | the survey **and the harness and the evidence file** | **reproduce the number independently**; state what the harness would report if the work silently stopped |
|
||||
|
||||
**What review cannot do — state this, do not discover it.** A reviewer
|
||||
re-derives the author's claims and therefore inherits the author's
|
||||
sampling. So:
|
||||
|
||||
> **The reviewer re-derives on a different sample than the author used.**
|
||||
> Where only one sample exists, the review says so rather than reporting a
|
||||
> clean verify.
|
||||
|
||||
*(v1.1, from CB-WP-0002: the dedup invariant was verified on the main
|
||||
transcript by the survey — 206/206 groups — and independently re-verified
|
||||
by the reviewer, who used the same transcript. It is false in the
|
||||
8-response `subagents/` tree neither examined. Two independent checks,
|
||||
one blind spot, because both sampled the same way. Only an assertion
|
||||
running over all the data at execution time caught it.)*
|
||||
|
||||
**What review demonstrably does do.** Measured across two passes:
|
||||
**$0.66** and **$1.11**, roughly 1% of each pass, each finding
|
||||
approval-blocking defects — in CB-WP-0002 a target that would have made
|
||||
the evidence file certify a broken collector. The step pays for itself by
|
||||
a wide margin and the cost is not a reason to skip it.
|
||||
separate session (or agent), given only the survey document, attempts to
|
||||
break it — an omitted candidate, a stale or unverifiable benchmark, an
|
||||
unmeasured claim presented as measured. Exactly **one round**: challenge,
|
||||
then response. The survey is approvable only when every challenge is either
|
||||
answered with evidence or conceded and folded into the survey.
|
||||
|
||||
**Documentation requirement:** the research process, the challenge, and the
|
||||
resulting improvements to the research are each documented in timestamped
|
||||
|
|
@ -224,40 +151,11 @@ as a target, or it is cited as unverified.** Quoting is not measuring.
|
|||
from a prior pass. Re-derivation put it at $92.21 — the quoted figure
|
||||
double-counted transcript lines and priced a three-model session at one
|
||||
model's rate. Neither error was of the harness-does-nothing class; both
|
||||
sums ran over real data, and a positive control would have passed them.)*
|
||||
|
||||
**Retargeting: the instrument may move a target, the implementation may
|
||||
not.** A metric's target changes in only two ways, and they are not
|
||||
treated alike:
|
||||
|
||||
| | trigger | requirement |
|
||||
|---|---|---|
|
||||
| **corrected** | the instrument disproved the target — the target was computed by hand, or by an earlier tool with a defect | legitimate in the same commit, **provided the instrument's output is in that commit** |
|
||||
| **retargeted** | the implementation missed the target and the target moves to accommodate it | requires an **ADR**: old target, the measurement, and why the new target binds on *future* work rather than merely passing present work |
|
||||
|
||||
The distinction is not the implementer's self-report of intent. It is
|
||||
mechanical: **a correction is one where the target moves and the
|
||||
implementation does not.** If the same commit changes both the target and
|
||||
the code the target measures, it is a retarget and needs the ADR.
|
||||
|
||||
*(v1.1, from CB-WP-0002/0003: AM-4's targets were measured at 246,250 and
|
||||
set at 250,000 in one commit by the implementer after seeing the number —
|
||||
the structure this rule exists to stop. But CB-WP-0002 then moved AC-1
|
||||
three times, correctly, each time because a new instrument disproved the
|
||||
old figure ($92.21 → $92.87 → $93.32 → $93.15). A blanket prohibition
|
||||
would have forbidden four legitimate corrections to catch one bad
|
||||
retarget.)*
|
||||
|
||||
**Applied retroactively:** AM-4a and AM-4b are **unratified** until an ADR
|
||||
is written or they are changed. They were set by the implementer after
|
||||
seeing the measurement, in the commit that produced it, and the
|
||||
implementation changed in that same commit — a retarget by the test above.
|
||||
Tracked as an open item in `history/260731-inner-loop-rule-audit.md`.
|
||||
|
||||
**A metric is checked against the contracts in its own spec.** If a
|
||||
contract makes a target unreachable, one of the two is wrong and the
|
||||
conflict is resolved when it is noticed, not at the acceptance run.
|
||||
Re-check the table whenever a contract is added.
|
||||
sums ran over real data, and a positive control would have passed them.)* A metric must
|
||||
also be checked against the contracts in the *same spec*: if a contract
|
||||
makes a target unreachable, one of the two is wrong and the conflict is
|
||||
resolved when it is noticed, not at the acceptance run. Re-check the
|
||||
table whenever a contract is added.
|
||||
|
||||
*(v1.0, from CB-WP-0001: AM-4's ≤20-crate target was made unreachable by
|
||||
the K5 and K7 contracts written after it, and AM-12's cost metric was
|
||||
|
|
@ -326,17 +224,115 @@ state the claim you will defend, and the claim you are not making.
|
|||
|
||||
---
|
||||
|
||||
## The four-dimension rubric
|
||||
|
||||
Every survey, ADR, and acceptance table is organized by these dimensions:
|
||||
|
||||
| Dimension | Question | Example measurable proxies |
|
||||
|---|---|---|
|
||||
| **D1 Ease of specification** | How simply can behavior be stated, tested, understood? | rules-to-scenario coverage %, spec lines per rule, time-to-first-correct-scenario for a fresh agent session |
|
||||
| **D2 Efficiency of implementation** | How cheap to build and keep building? | source LOC, dependency count/weight, clean-build and incremental-build time, tokens-per-completed-task |
|
||||
| **D3 Speed of execution** | How fast does it run? | benchmark wall-time vs baseline, events/sec, memory footprint, determinism overhead |
|
||||
| **D4 Optionality** | How cleanly does it integrate, extend, get replaced? | public API surface size, count of leaked foreign types (must be 0), effort-to-swap measured by null/reference impl existence, WIT-expressibility |
|
||||
|
||||
Scoring is always **relative to the step-1 baseline**, never absolute:
|
||||
`better / parity / worse / unmeasured` per proxy, with the number attached.
|
||||
`unmeasured` is legal in a survey, illegal in an evidence file.
|
||||
|
||||
---
|
||||
|
||||
## Reference material
|
||||
## Survey template (research/CB-RES-NNNN-<slug>.md)
|
||||
|
||||
The four-dimension rubric, the survey template, the implementation rules
|
||||
each pass earned, the agentic-efficiency requirements, and the
|
||||
definition of done live in
|
||||
**[InnerLoopReference.md](InnerLoopReference.md)**. They are normative;
|
||||
they are separated only so both files load whole.
|
||||
```markdown
|
||||
# CB-RES-NNNN: <capability>
|
||||
capability: <canonical.capability.id>
|
||||
status: draft | approved
|
||||
|
||||
*(Split 2026-07-31: this file reached 407 lines against its own ~400-line
|
||||
loadability rule, and `make loop-lint` — added the same day to make that
|
||||
rule executable — failed on the commit that pushed it over. The rule
|
||||
caught its own author within an hour of being written.)*
|
||||
## Candidates
|
||||
Per candidate: origin, license, maturity, adoption; data model; mutation
|
||||
mechanism; determinism/replay story; relevant performance (measured if
|
||||
runnable locally, cited with source otherwise).
|
||||
|
||||
## Baselines (benchmark-to-beat)
|
||||
| Dimension | Baseline holder | Metric | Value | Provenance |
|
||||
(one row minimum per dimension; provenance = measured / cited / estimated)
|
||||
|
||||
## Verdict
|
||||
Which candidate leads per dimension; what none of them do well
|
||||
(the surpass opportunity); risks in the baselines themselves.
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Implementation rules the first pass earned
|
||||
|
||||
These are cheap, and each exists because its absence cost something in
|
||||
CB-WP-0001. See `history/260731-inner-loop-retrospective.md`.
|
||||
|
||||
1. **No silently-ignored input.** A field that is parsed and then unused
|
||||
is a defect, not a stub. Inputs are honoured or rejected with an
|
||||
error — never dropped. *(A scenario `setup.patch` was parsed and
|
||||
discarded; every scenario using it would have tested the wrong
|
||||
initial state while passing.)*
|
||||
2. **Decisions get commands, not defaults.** A rule that requires a
|
||||
participant's choice is implemented as a command carrying that
|
||||
choice. Until it is, **nothing claims coverage of it** — no tag, no
|
||||
scenario, no acceptance row. Inventing a default to make a rule
|
||||
"done" is the failure this prevents.
|
||||
3. **Scaffolds are exercised or marked.** A scaffold's green gates are
|
||||
not evidence. Any scaffold path no test reaches is marked as
|
||||
unexercised. *(A compiling, fully-green scaffold shipped a state-hash
|
||||
that would panic on any state holding a relation.)*
|
||||
4. **Coverage gates that count tags say so.** A gate comparing rule IDs
|
||||
against `covers:` lists proves no rule is unclaimed and no claimed
|
||||
rule is invented. It does **not** prove a scenario exercises what it
|
||||
names. Wherever such a number is reported, that limit is reported
|
||||
with it.
|
||||
|
||||
---
|
||||
|
||||
## Agentic-efficiency requirements
|
||||
|
||||
The loop exists to be driven by agents. Therefore:
|
||||
|
||||
1. **Whole-file loadability** — every loop artifact stays under ~400 lines;
|
||||
split before exceeding, link with relative paths.
|
||||
2. **Structured over prose** — tables and fenced blocks for anything a
|
||||
later step must parse (baselines, acceptance metrics, evidence rows).
|
||||
3. **One command surface** — all checks runnable through repo-root
|
||||
commands (eventually `cb *`; until then, `make`/`cargo` aliases declared
|
||||
in one place), each supporting deterministic, greppable output.
|
||||
4. **Self-contained tasks** — a workplan task names its input artifacts and
|
||||
output artifacts; a fresh session must be able to execute it from the
|
||||
task text plus linked files alone.
|
||||
5. **Evidence or it didn't happen** — claims of "better" live in committed
|
||||
evidence files with numbers, never only in commit messages or chat.
|
||||
6. **Token discipline** — per the global budget policy, a loop iteration
|
||||
that exceeds its budget without measurable progress is stopped and
|
||||
decomposed, not pushed through.
|
||||
|
||||
---
|
||||
|
||||
## Definition of done — one loop pass
|
||||
|
||||
A capability has completed the loop when all of the following are committed:
|
||||
|
||||
- [ ] research/CB-RES-NNNN with approved status and full baseline table
|
||||
- [ ] decisions/ADR-NNNN with per-dimension expected advantage
|
||||
- [ ] specs/<Capability>.md with acceptance-metrics table
|
||||
- [ ] passing scenarios covering every numbered spec rule
|
||||
- [ ] evidence/CB-EV-NNNN with final comparison vs baseline, no `unmeasured`
|
||||
- [ ] every reported number produced by a harness with a positive
|
||||
control; any metric that could not be instrumented is recorded as
|
||||
uncomputable rather than estimated
|
||||
- [ ] every unmet metric reported as unmet, with attribution and the
|
||||
options for resolving it — a missed target is an output of the
|
||||
loop, not a reason to move the target quietly
|
||||
- [ ] **cost recorded**: `make cost` run for the pass, its composition
|
||||
(not only its total) in the evidence file, and the per-task figures
|
||||
pushed to the hub. M-D2-CST is no longer allowed to be
|
||||
`uncomputable` — the instrument exists
|
||||
([CostAccounting.md](CostAccounting.md))
|
||||
- [ ] retrospective note (may be one paragraph appended to the evidence
|
||||
file): what the loop itself should change
|
||||
```
|
||||
|
|
|
|||
|
|
@ -1,141 +0,0 @@
|
|||
# The Inner Loop — Reference
|
||||
|
||||
Normative companion to **[InnerLoop.md](InnerLoop.md)**, which holds the
|
||||
five steps, the tiers, and the chaos roll. Split from it on 2026-07-31 so
|
||||
both files stay under the loop's own whole-file loadability limit.
|
||||
|
||||
Everything here binds exactly as the process document does.
|
||||
|
||||
## The four-dimension rubric
|
||||
|
||||
Every survey, ADR, and acceptance table is organized by these dimensions:
|
||||
|
||||
| Dimension | Question | Example measurable proxies |
|
||||
|---|---|---|
|
||||
| **D1 Ease of specification** | How simply can behavior be stated, tested, understood? | rules-to-scenario coverage %, spec lines per rule, time-to-first-correct-scenario for a fresh agent session |
|
||||
| **D2 Efficiency of implementation** | How cheap to build and keep building? | source LOC, dependency count/weight, clean-build and incremental-build time, tokens-per-completed-task |
|
||||
| **D3 Speed of execution** | How fast does it run? | benchmark wall-time vs baseline, events/sec, memory footprint, determinism overhead |
|
||||
| **D4 Optionality** | How cleanly does it integrate, extend, get replaced? | public API surface size, count of leaked foreign types (must be 0), effort-to-swap measured by null/reference impl existence, WIT-expressibility |
|
||||
|
||||
Scoring is always **relative to the step-1 baseline**, never absolute:
|
||||
`better / parity / worse / unmeasured` per proxy, with the number attached.
|
||||
`unmeasured` is legal in a survey, illegal in an evidence file.
|
||||
|
||||
---
|
||||
|
||||
## Survey template (research/CB-RES-NNNN-<slug>.md)
|
||||
|
||||
```markdown
|
||||
# CB-RES-NNNN: <capability>
|
||||
capability: <canonical.capability.id>
|
||||
status: draft | approved
|
||||
|
||||
## Candidates
|
||||
Per candidate: origin, license, maturity, adoption; data model; mutation
|
||||
mechanism; determinism/replay story; relevant performance (measured if
|
||||
runnable locally, cited with source otherwise).
|
||||
|
||||
## Baselines (benchmark-to-beat)
|
||||
| Dimension | Baseline holder | Metric | Value | Provenance |
|
||||
(one row minimum per dimension; provenance = measured / cited / estimated)
|
||||
|
||||
## Verdict
|
||||
Which candidate leads per dimension; what none of them do well
|
||||
(the surpass opportunity); risks in the baselines themselves.
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Implementation rules the first pass earned
|
||||
|
||||
These are cheap, and each exists because its absence cost something in
|
||||
CB-WP-0001. See `history/260731-inner-loop-retrospective.md`.
|
||||
|
||||
1. **No silently-ignored input.** A field that is parsed and then unused
|
||||
is a defect, not a stub. Inputs are honoured or rejected with an
|
||||
error — never dropped. *(A scenario `setup.patch` was parsed and
|
||||
discarded; every scenario using it would have tested the wrong
|
||||
initial state while passing.)*
|
||||
2. **Decisions get commands, not defaults.** A rule that requires a
|
||||
participant's choice is implemented as a command carrying that
|
||||
choice. Until it is, **nothing claims coverage of it** — no tag, no
|
||||
scenario, no acceptance row. Inventing a default to make a rule
|
||||
"done" is the failure this prevents.
|
||||
3. **Scaffolds are exercised or marked.** A scaffold's green gates are
|
||||
not evidence. Any scaffold path no test reaches is marked as
|
||||
unexercised. *(A compiling, fully-green scaffold shipped a state-hash
|
||||
that would panic on any state holding a relation.)*
|
||||
4. **Coverage gates that count tags say so.** A gate comparing rule IDs
|
||||
against `covers:` lists proves no rule is unclaimed and no claimed
|
||||
rule is invented. It does **not** prove a scenario exercises what it
|
||||
names. Wherever such a number is reported, that limit is reported
|
||||
with it.
|
||||
|
||||
---
|
||||
|
||||
## Agentic-efficiency requirements
|
||||
|
||||
The loop exists to be driven by agents. Therefore:
|
||||
|
||||
1. **Whole-file loadability** — every loop artifact stays under ~400 lines;
|
||||
split before exceeding, link with relative paths.
|
||||
2. **Structured over prose** *(guidance, not a requirement — nothing can
|
||||
check it)* — tables and fenced blocks for anything a later step must
|
||||
parse (baselines, acceptance metrics, evidence rows).
|
||||
3. **One command surface** — all checks runnable through repo-root
|
||||
commands (eventually `cb *`; until then, `make`/`cargo` aliases declared
|
||||
in one place), each supporting deterministic, greppable output.
|
||||
4. **Self-contained tasks** — a workplan task names its input artifacts and
|
||||
output artifacts; a fresh session must be able to execute it from the
|
||||
task text plus linked files alone.
|
||||
5. **Evidence or it didn't happen** — claims of "better" live in committed
|
||||
evidence files with numbers, never only in commit messages or chat.
|
||||
6. ~~**Token discipline** — per the global budget policy, a loop iteration
|
||||
that exceeds its budget without measurable progress is stopped and
|
||||
decomposed, not pushed through.~~ **DEAD POLICY.** The 8k/10k per-task
|
||||
token budget was never referenced or enforced, and CB-WP-0001 T08
|
||||
exceeded it by orders of magnitude with no signal. It implies a control
|
||||
that does not exist. Replacement in USD is CB-WP-0003 T05; until then
|
||||
this is documentation of a gap, not a rule.
|
||||
|
||||
6a. **Live cost budget** *(replaces the above)* — spend since the last
|
||||
commit, soft **$10.00**, hard **$22.00**, checked by `make cost-budget`.
|
||||
Calibrated on the 32 commit intervals of CB-WP-0001 (p50 $1.40, p90
|
||||
$9.36, max $10.80), so both thresholds bind on future work rather than
|
||||
ratifying past work. Contract: [CostAccounting.md](CostAccounting.md)
|
||||
§7 (CB-01, CB-02). It fires on the open remainder rather than per task
|
||||
because per-task cost needs the commit that closes the task, and a
|
||||
budget that can only report after the money is spent is the dead policy
|
||||
this replaces.
|
||||
|
||||
**Enforcement status.** Rules above that a command can check are enforced
|
||||
by `make loop-lint`; the full classification of every InnerLoop rule as
|
||||
executable / checkable / decorative, with the failure class each catches,
|
||||
is in `history/260731-inner-loop-rule-audit.md`. Rules marked *guidance*
|
||||
or *dead* say so where they appear, so a reader can tell a requirement
|
||||
from a preference without consulting the audit.
|
||||
|
||||
---
|
||||
|
||||
## Definition of done — one loop pass
|
||||
|
||||
A capability has completed the loop when all of the following are committed:
|
||||
|
||||
- [ ] research/CB-RES-NNNN with approved status and full baseline table
|
||||
- [ ] decisions/ADR-NNNN with per-dimension expected advantage
|
||||
- [ ] specs/<Capability>.md with acceptance-metrics table
|
||||
- [ ] passing scenarios covering every numbered spec rule
|
||||
- [ ] evidence/CB-EV-NNNN with final comparison vs baseline, no `unmeasured`
|
||||
- [ ] every reported number produced by a harness with a positive
|
||||
control; any metric that could not be instrumented is recorded as
|
||||
uncomputable rather than estimated
|
||||
- [ ] every unmet metric reported as unmet, with attribution and the
|
||||
options for resolving it — a missed target is an output of the
|
||||
loop, not a reason to move the target quietly
|
||||
- [ ] **cost recorded**: `make cost` run for the pass, its composition
|
||||
(not only its total) in the evidence file, and the per-task figures
|
||||
pushed to the hub. M-D2-CST is no longer allowed to be
|
||||
`uncomputable` — the instrument exists
|
||||
([CostAccounting.md](CostAccounting.md))
|
||||
- [ ] retrospective note (may be one paragraph appended to the evidence
|
||||
file): what the loop itself should change
|
||||
|
|
@ -31,7 +31,6 @@ and add capability-specific rows only when these don't cover the claim.
|
|||
| ID | Dimension | Metric | Unit | Provenance |
|
||||
|---|---|---|---|---|
|
||||
| M-D1-COV | D1 | numbered spec rules covered by ≥1 passing scenario | % | adapted:requirements-traceability (per-rule, not per-feature) |
|
||||
| M-D1-LNK | D1 | claimed rules also **named in the aggregate source** | count | novel — closes the spec→code→scenario chain; instrument: `make coverage` (AM-1b) |
|
||||
| M-D1-SPL | D1 | spec lines per numbered rule | lines | novel — proxies statement simplicity; gameable, so paired with M-D1-COV |
|
||||
| M-D2-LOC | D2 | source LOC excluding tests (tokei) | lines | adopted:tokei |
|
||||
| M-D2-DEP | D2 | transitive dependency count (cargo tree) | crates | adopted:cargo-deny practice |
|
||||
|
|
@ -45,32 +44,6 @@ and add capability-specific rows only when these don't cover the claim.
|
|||
| M-D4-LEAK | D4 | foreign types in canonical interfaces | count | novel — must be 0; enforced by grep/deny rule, the Clay-Borg hard rule |
|
||||
| M-D4-SWAP | D4 | capability has null + reference impls passing the same conformance suite | bool | adapted:hexagonal-architecture port testing |
|
||||
|
||||
### 1b. The coverage gate's two numbers (M-D1-COV, M-D1-LNK)
|
||||
|
||||
M-D1-COV counts tags. It proves no rule is unclaimed and no claimed rule
|
||||
is invented; it does **not** prove a scenario exercises what it names, and
|
||||
that limit is printed with the number every time (InnerLoop implementation
|
||||
rule 4).
|
||||
|
||||
**M-D1-LNK** (CB-WP-0003 T09) closes one link of that chain mechanically:
|
||||
a rule a scenario claims should also appear in the aggregate source, or
|
||||
the claim rests on a tag and nothing else.
|
||||
|
||||
Measured 2026-07-31: **49 of 58** claimed rules are named in
|
||||
`games/ground/src/lib.rs`. **Unmet**, target 58. The nine unlinked:
|
||||
|
||||
```text
|
||||
GR-D07 GR-F02 GR-L03 GR-O03 GR-P01 GR-P02 GR-P03 GR-P04 GR-T01
|
||||
```
|
||||
|
||||
The gate **reports** this rather than failing, deliberately. Closing the
|
||||
gap by adding rule IDs to comments would satisfy the check without
|
||||
establishing that each rule is implemented — which is the overclaim
|
||||
InnerLoop implementation rule 2 exists to prevent, and which CB-WP-0001
|
||||
committed once already. Each of the nine needs its implementation
|
||||
confirmed before it is tagged. Promoting M-D1-LNK to a failing gate is
|
||||
correct **after** that, not before.
|
||||
|
||||
### 1a. Token cost accounting (M-D2-CST)
|
||||
|
||||
> **Superseded 2026-07-31 by [CostAccounting.md](CostAccounting.md)**, which
|
||||
|
|
@ -94,11 +67,27 @@ benchmarks/baselines/model-prices.toml # the price sheet, updated when prices
|
|||
```
|
||||
|
||||
```toml
|
||||
# The live sheet is benchmarks/baselines/model-prices.toml — read it,
|
||||
# do not copy it. An inlined copy here went stale within hours: it still
|
||||
# showed the sonnet intro rate as a `#` comment after CB-WP-0003 T11 made
|
||||
# dated rates into data (promo_input / promo_output / promo_until), and
|
||||
# added `recorded` + `max_age_days` for the staleness check (CA-16/CA-17).
|
||||
# USD per million tokens; source: Anthropic pricing, recorded 2026-07-31
|
||||
[claude-fable-5]
|
||||
input = 10.00
|
||||
output = 50.00
|
||||
|
||||
[claude-opus-5]
|
||||
input = 5.00
|
||||
output = 25.00
|
||||
|
||||
[claude-sonnet-5]
|
||||
input = 3.00 # intro 2.00 through 2026-08-31
|
||||
output = 15.00 # intro 10.00 through 2026-08-31
|
||||
|
||||
[claude-haiku-4-5]
|
||||
input = 1.00
|
||||
output = 5.00
|
||||
|
||||
[cache] # multipliers on the input price
|
||||
read = 0.1
|
||||
write_5m = 1.25
|
||||
write_1h = 2.0
|
||||
```
|
||||
|
||||
Rules:
|
||||
|
|
@ -119,8 +108,7 @@ Rules:
|
|||
failed pass scores nothing.
|
||||
- Prices are `adopted:anthropic-pricing` with a recorded date; a stale sheet
|
||||
(> 90 days or known price change) invalidates new `better` verdicts on
|
||||
M-D2-CST until refreshed. **Now enforced** — CA-17 aborts the run rather
|
||||
than leaving this as prose every verdict silently inherited.
|
||||
M-D2-CST until refreshed.
|
||||
|
||||
Determinism is not a metric but an **invariant**: N replays of the same
|
||||
seed and command log must produce bit-identical state hashes. Invariant
|
||||
|
|
|
|||
|
|
@ -1,185 +0,0 @@
|
|||
# Session Shape
|
||||
|
||||
Status: **v1.0** — 2026-07-31, from CB-WP-0003 T04.
|
||||
Instrument: `make cost` ([CostAccounting.md](CostAccounting.md)).
|
||||
|
||||
How to shape an agentic session so it does not cost more than the work is
|
||||
worth. **Every claim here carries the measurement it rests on.** This spec
|
||||
exists because the numbers were surprising and because the advice this task
|
||||
was created to give turned out to be wrong.
|
||||
|
||||
---
|
||||
|
||||
## 1. The cost model
|
||||
|
||||
```text
|
||||
cost ≈ Σ over turns ( context_size × cache_read_rate ) + output + cache writes
|
||||
```
|
||||
|
||||
Measured on the CB-WP-0001 pass: **88.0% of spend is cache, 11.9% is
|
||||
output**, at 249:1 cache-read tokens to output tokens. Cost is driven by
|
||||
**how much context is re-read per turn**, not by how much the model writes.
|
||||
|
||||
Marginal cost of one turn, at the Opus cache-read rate ($0.50/M):
|
||||
|
||||
| context | $/turn |
|
||||
|---|---|
|
||||
| 20,000 | $0.010 |
|
||||
| 50,000 | $0.025 |
|
||||
| 100,000 | $0.050 |
|
||||
| 200,000 | $0.100 |
|
||||
| 400,000 | $0.200 |
|
||||
| 540,000 | $0.270 |
|
||||
|
||||
> **SS-01.** A turn's cost is a function of context size alone, to within
|
||||
> ~12%. Optimizing output length is not a cost lever; optimizing context is.
|
||||
|
||||
## 2. Compaction is the primary control
|
||||
|
||||
**The original premise of this spec was wrong and is recorded rather than
|
||||
deleted.** CB-WP-0003 T04 was written to prescribe *one task per session*,
|
||||
on the theory that mean context grows with turn count and long sessions are
|
||||
therefore quadratic. Measured:
|
||||
|
||||
| segment | turns | mean context | total | $/turn |
|
||||
|---|---|---|---|---|
|
||||
| start → compact 1 | 136 | 304,178 | $62.19 | **$0.457** |
|
||||
| compact 1 → compact 2 | 202 | 193,493 | $30.01 | **$0.149** |
|
||||
|
||||
The 202 turns *after* the first compaction cost less than half the 136
|
||||
before it. Sessions are **bounded-quadratic**: context grows between
|
||||
compactions and resets at each one.
|
||||
|
||||
Compaction events, read from `compactMetadata` in the transcript:
|
||||
|
||||
| | pre | post | reduction | duration |
|
||||
|---|---|---|---|---|
|
||||
| C1 | 542,991 | 19,974 | **27×** | 151 s |
|
||||
| C2 | 344,954 | 19,035 | **18×** | 97 s |
|
||||
|
||||
> **SS-02.** The failure mode is a long **uncompacted** session, not a long
|
||||
> session. Turn count is not the variable to control; context is.
|
||||
|
||||
### 2.1 When compaction pays
|
||||
|
||||
A compaction costs its own summarization call plus one cache
|
||||
re-establishment. The re-establishment is measurable — the first post-compact
|
||||
turn wrote 36,149 cache tokens and cost **$0.382**. The summarization call
|
||||
itself does **not** appear in the transcript's `usage` records, so it is
|
||||
bounded rather than measured: at worst it is one full read of the
|
||||
pre-compaction context (543k tokens ≈ $2.72 at the uncached Opus input
|
||||
rate), at best a cache read of the same (≈ $0.27).
|
||||
|
||||
Savings are immediate and permanent: dropping from 543k to 20k context saves
|
||||
**$0.262 per subsequent turn**.
|
||||
|
||||
> **SS-03.** Break-even is **2–11 turns** depending on where the
|
||||
> summarization call falls in that range. Any session expecting more than
|
||||
> ~11 further turns should compact. In practice this means: **compact
|
||||
> whenever context exceeds ~300k and work remains.**
|
||||
|
||||
Measured context distribution across the CB-WP-0001 session shows how much
|
||||
was spent above that line — p75 was 308,325 and p90 was 481,722, so roughly
|
||||
a quarter of all turns ran at more than 2× the recommended ceiling.
|
||||
|
||||
| percentile | context |
|
||||
|---|---|
|
||||
| p50 | 156,771 |
|
||||
| p75 | 308,325 |
|
||||
| p90 | 481,722 |
|
||||
| p99 | 553,252 |
|
||||
|
||||
## 3. Fresh session vs compaction
|
||||
|
||||
A fresh session is **not** free, and this is the measurement that overturned
|
||||
the original advice. Cold-start context floors, measured over the first 20
|
||||
turns of each session:
|
||||
|
||||
| session | start type | mean context, first 20 turns | cost |
|
||||
|---|---|---|---|
|
||||
| `8cbd5701` | cold (system + CLAUDE.md + orientation) | 50,872 | $2.17 |
|
||||
| `f1eb1147` | seeded by a compact summary | 65,819 | $1.90 |
|
||||
|
||||
A cold session starts ~51k and must then **re-read the committed artifacts**
|
||||
to become productive — the survey, the spec, the ADR — which is exactly the
|
||||
context a compaction summary already contains. A compact-seeded session
|
||||
starts ~66k with that reading already done.
|
||||
|
||||
> **SS-04.** Prefer compaction over a fresh session for continuing work.
|
||||
> A fresh session's lower floor (51k vs 66k) is repaid within the first few
|
||||
> turns of re-reading artifacts the summary already held.
|
||||
>
|
||||
> Prefer a fresh session when the *task changes*, because then the prior
|
||||
> context is not an asset — it is 66k of irrelevance re-read every turn.
|
||||
|
||||
## 4. Batching
|
||||
|
||||
> **SS-05.** Independent tool calls in one turn cost nothing extra: a turn
|
||||
> is billed for its context regardless of how many tool calls it carries.
|
||||
> Two calls in one turn cost half of two calls in two turns.
|
||||
|
||||
Measured batching rate:
|
||||
|
||||
| session | responses w/ tool calls | tool calls | responses batching ≥2 | calls in batched turns |
|
||||
|---|---|---|---|---|
|
||||
| `8cbd5701` | 361 | 406 | 31 (8.6%) | 76 (18.7%) |
|
||||
| `f1eb1147` | 146 | 158 | 12 (8.2%) | 24 (15.2%) |
|
||||
| pinned at `fc76445`, all transcripts (instrument) | 322 | 362 | 25 (7.8%) | 65 (18.0%) |
|
||||
|
||||
**Correction to a prior claim.** CB-WP-0001's retrospective and the first
|
||||
draft of CB-WP-0003 T04 both stated *"0 of 330 tool calls were batched"*.
|
||||
That is wrong: 330 was the count of responses making exactly **one** call,
|
||||
not the total number of calls. The real figure is 7.8–8.6% of responses and
|
||||
18.0–18.7% of calls, depending on the window. The claim was never re-derived before being carried into a
|
||||
workplan — the failure mode InnerLoop v1.1 rule 13 exists for, and the
|
||||
**eighth** recorded error instance, in the trusted-arithmetic class that
|
||||
`history/260731-inner-loop-rule-audit.md` identified as having no
|
||||
executable defence.
|
||||
|
||||
At an 8.6% batching rate there is real headroom, but the saving is bounded:
|
||||
eliminating every avoidable single-call turn would remove at most a few
|
||||
percent of turns, worth roughly $2–4 on a $93 pass. **Batch because it is
|
||||
free, not because it is the lever.** The lever is §2.
|
||||
|
||||
## 5. What this spec does not support
|
||||
|
||||
- **n = 2 sessions, one repo, one model mix.** Every ratio here is from
|
||||
CB-WP-0001/0002 and should be re-measured before being treated as general.
|
||||
- **The compaction summarization call is bounded, not measured.** SS-03's
|
||||
2–11 turn break-even is a range because of it. Narrowing it requires a
|
||||
cost source that sees non-transcript calls — the billing API (CB-RES-0002
|
||||
C4), which needs an admin key this machine does not have.
|
||||
- **Context size is read as `cache_read_input_tokens`**, which is the cached
|
||||
prefix, not the full prompt. On a cache miss the true context is larger
|
||||
and the turn costs 10× more. Cache misses were not isolated in this
|
||||
measurement.
|
||||
- **No claim about quality.** Compaction discards detail. Every number here
|
||||
says compaction is cheaper; none says the work is as good. A pass that
|
||||
compacts and then repeats work it forgot has not saved anything, and this
|
||||
spec cannot detect that.
|
||||
|
||||
## 6. Acceptance metrics
|
||||
|
||||
| ID | Metric | Target | Measured (pinned `fc76445`) | Verdict | Instrument |
|
||||
|---|---|---|---|---|---|
|
||||
| SH-1 | mean context per turn | ≤ 200,000 | **232,982** | **unmet** | `make cost-pin` |
|
||||
| SH-2 | p90 context per turn | ≤ 300,000 | **492,042** | **unmet** | `make cost-pin` |
|
||||
| SH-3 | batching rate (responses with ≥2 tool calls) | ≥ 20% | **7.8%** | **unmet** | `make cost-pin` |
|
||||
|
||||
`cb-cost` emits all three (`session shape` block), so these are ratified
|
||||
under InnerLoop v1.1 rather than hand-derived. Supporting counts from the
|
||||
same run: **362 tool calls in 322 responses, 65 of them in batched turns.**
|
||||
|
||||
**All three targets are unmet, and that is the finding, not a defect in the
|
||||
targets.** Per the loop's definition of done, an unmet metric is an output:
|
||||
|
||||
- SH-1/SH-2 are unmet because CB-WP-0001 ran 136 turns before its first
|
||||
compaction. §2 is the remedy and it is now written down.
|
||||
- SH-3 is set above the measured rate deliberately. Whether 20% is
|
||||
reachable or arbitrary is unknown until a pass tries; the first pass that
|
||||
reports it will say. It is the cheapest of the three to move, and the
|
||||
least valuable — §4 puts the ceiling at $2–4 on a $93 pass.
|
||||
|
||||
Targets are **not** being retargeted to match the measurement. That is the
|
||||
structure CB-WP-0003 T07 exists to prevent, and doing it here — in the
|
||||
commit that first measures them — would be the exact defect.
|
||||
Binary file not shown.
161
tools/cb-cost.py
161
tools/cb-cost.py
|
|
@ -72,53 +72,16 @@ def components(usage):
|
|||
}
|
||||
|
||||
|
||||
def rates_at(prices, model, when=None):
|
||||
"""Input/output rate in force for `model` at ISO instant `when` (CA-16).
|
||||
|
||||
Promotional rates are data, not comments. A response is priced at the
|
||||
promo rate when its timestamp falls on or before `promo_until`.
|
||||
"""
|
||||
pr = prices.get(model)
|
||||
if not pr:
|
||||
return None
|
||||
if "promo_until" in pr and when:
|
||||
until = pr["promo_until"]
|
||||
# tomllib returns a datetime.date for a bare TOML date.
|
||||
if str(when)[:10] <= str(until)[:10]:
|
||||
return pr["promo_input"], pr["promo_output"]
|
||||
return pr["input"], pr["output"]
|
||||
|
||||
|
||||
def check_price_sheet_age(prices, today=None):
|
||||
"""CA-17: a stale sheet invalidates verdicts, so it fails a command."""
|
||||
import datetime as _dt
|
||||
|
||||
recorded = prices.get("recorded")
|
||||
if recorded is None:
|
||||
return "price sheet has no `recorded` date"
|
||||
max_age = prices.get("max_age_days", 90)
|
||||
today = today or _dt.date.today()
|
||||
if isinstance(recorded, _dt.datetime):
|
||||
recorded = recorded.date()
|
||||
age = (today - recorded).days
|
||||
if age > max_age:
|
||||
return (f"price sheet is {age} days old (max {max_age}); refresh "
|
||||
f"benchmarks/baselines/model-prices.toml or new M-D2-CST "
|
||||
f"`better` verdicts are invalid")
|
||||
return None
|
||||
|
||||
|
||||
def price_of(prices, model, toks, when=None):
|
||||
def price_of(prices, model, toks):
|
||||
"""USD for one response. Returns None when the model is unpriced (CA-05)."""
|
||||
pr = prices.get(model)
|
||||
if not pr:
|
||||
return None
|
||||
rin, rout = rates_at(prices, model, when)
|
||||
cache = prices["cache"]
|
||||
unit = rin / 1e6
|
||||
unit = pr["input"] / 1e6
|
||||
return (
|
||||
toks["input"] * unit
|
||||
+ toks["output"] * rout / 1e6
|
||||
+ toks["output"] * pr["output"] / 1e6
|
||||
+ toks["cache_read"] * unit * cache["read"]
|
||||
+ toks["write_5m"] * unit * cache["write_5m"]
|
||||
+ toks["write_1h"] * unit * cache["write_1h"]
|
||||
|
|
@ -184,14 +147,6 @@ def read_responses(path, pin=None):
|
|||
)
|
||||
toks["output"] = max(t["output"] for t in per_row)
|
||||
head = rows[0]
|
||||
# Tool calls are spread across the group's lines, so they are counted
|
||||
# over the whole group — one response may carry several (SS-05).
|
||||
tool_calls = sum(
|
||||
1
|
||||
for r in rows
|
||||
for c in (r["message"].get("content") or [])
|
||||
if c.get("type") == "tool_use"
|
||||
)
|
||||
out.append(
|
||||
{
|
||||
"request_id": rid,
|
||||
|
|
@ -199,33 +154,12 @@ def read_responses(path, pin=None):
|
|||
"timestamp": head.get("timestamp") or "",
|
||||
"session": head.get("sessionId") or os.path.basename(path),
|
||||
"toks": toks,
|
||||
"tool_calls": tool_calls,
|
||||
"subagent": "/subagents/" in path,
|
||||
}
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def session_shape(responses):
|
||||
"""SH-1..SH-3 from specs/SessionShape.md."""
|
||||
import statistics
|
||||
|
||||
ctx = sorted(r["toks"]["cache_read"] for r in responses)
|
||||
with_tools = [r for r in responses if r["tool_calls"] > 0]
|
||||
batched = [r for r in with_tools if r["tool_calls"] > 1]
|
||||
calls = sum(r["tool_calls"] for r in with_tools)
|
||||
pct = statistics.quantiles(ctx, n=100) if len(ctx) > 1 else [ctx[0]] * 99
|
||||
return {
|
||||
"SH-1_mean_context": statistics.mean(ctx) if ctx else 0,
|
||||
"SH-2_p90_context": pct[89],
|
||||
"SH-3_batching_rate": (len(batched) / len(with_tools)) if with_tools else 0.0,
|
||||
"p50_context": pct[49],
|
||||
"responses_with_tools": len(with_tools),
|
||||
"tool_calls": calls,
|
||||
"calls_in_batched_turns": calls - (len(with_tools) - len(batched)),
|
||||
}
|
||||
|
||||
|
||||
# ------------------------------------------------------------- attribution
|
||||
|
||||
|
||||
|
|
@ -318,12 +252,8 @@ def collect(slug, pin_ref=None):
|
|||
# as though it were an answer.
|
||||
raise Abort(f"no responses in {len(paths)} transcript(s) — refusing to report")
|
||||
|
||||
stale = check_price_sheet_age(prices)
|
||||
if stale:
|
||||
raise Abort(stale)
|
||||
|
||||
for r in responses:
|
||||
r["cost"] = price_of(prices, r["model"], r["toks"], r["timestamp"])
|
||||
r["cost"] = price_of(prices, r["model"], r["toks"])
|
||||
|
||||
attribute(responses, commit_index(pin))
|
||||
|
||||
|
|
@ -339,11 +269,11 @@ def collect(slug, pin_ref=None):
|
|||
continue
|
||||
by_task[r["task"]] += r["cost"]
|
||||
by_model[r["model"]] += r["cost"]
|
||||
rin, rout = rates_at(prices, r["model"], r["timestamp"])
|
||||
unit = rin / 1e6
|
||||
pr = prices[r["model"]]
|
||||
unit = pr["input"] / 1e6
|
||||
rates = {
|
||||
"input": unit,
|
||||
"output": rout / 1e6,
|
||||
"output": pr["output"] / 1e6,
|
||||
"cache_read": unit * cache["read"],
|
||||
"write_5m": unit * cache["write_5m"],
|
||||
"write_1h": unit * cache["write_1h"],
|
||||
|
|
@ -363,7 +293,6 @@ def collect(slug, pin_ref=None):
|
|||
|
||||
sub = sum(r["cost"] or 0 for r in responses if r["subagent"])
|
||||
return {
|
||||
"session_shape": session_shape(responses),
|
||||
"slug": slug,
|
||||
"pin": pin,
|
||||
"responses": len(responses),
|
||||
|
|
@ -415,17 +344,6 @@ def render(rep, by_task=False, composition=False):
|
|||
f" A per-task table is a view over {100*(1-un/tot):.0f}% of spend."
|
||||
)
|
||||
|
||||
sh = rep["session_shape"]
|
||||
print("\n session shape (specs/SessionShape.md)")
|
||||
print(f" SH-1 mean context {sh['SH-1_mean_context']:>12,.0f} tok "
|
||||
f"[{'ok ' if sh['SH-1_mean_context']<=200_000 else 'FAIL'} target 200,000]")
|
||||
print(f" SH-2 p90 context {sh['SH-2_p90_context']:>12,.0f} tok "
|
||||
f"[{'ok ' if sh['SH-2_p90_context']<=300_000 else 'FAIL'} target 300,000]")
|
||||
print(f" SH-3 batching rate {100*sh['SH-3_batching_rate']:>11.1f}% "
|
||||
f"[{'ok ' if sh['SH-3_batching_rate']>=0.20 else 'FAIL'} target 20.0%]")
|
||||
print(f" {sh['tool_calls']} tool calls in {sh['responses_with_tools']} responses; "
|
||||
f"{sh['calls_in_batched_turns']} in batched turns")
|
||||
|
||||
if rep["unpriced"]:
|
||||
print(f"\n UNPRICED ({len(rep['unpriced'])} responses, model not in sheet):")
|
||||
for u in rep["unpriced"]:
|
||||
|
|
@ -497,27 +415,6 @@ def self_test():
|
|||
finally:
|
||||
os.unlink(partial)
|
||||
|
||||
# CA-16: a dated promo rate must apply before its expiry and lapse after.
|
||||
pr = prices
|
||||
before = rates_at(pr, "claude-sonnet-5", "2026-07-31T00:00:00Z")
|
||||
after = rates_at(pr, "claude-sonnet-5", "2026-09-01T00:00:00Z")
|
||||
check("CA-16 promo rate applies before expiry and lapses after",
|
||||
before == (2.0, 10.0) and after == (3.0, 15.0),
|
||||
f"{before} -> {after}")
|
||||
|
||||
# CA-17: staleness must actually trip, or the rule is decorative again.
|
||||
import datetime as _dt
|
||||
fresh = check_price_sheet_age(pr, _dt.date(2026, 8, 1))
|
||||
stale = check_price_sheet_age(pr, _dt.date(2026, 11, 10))
|
||||
check("CA-17 staleness detected past max_age_days",
|
||||
fresh is None and stale is not None, "fresh ok, 102d trips")
|
||||
|
||||
# CB-02: thresholds must be ordered, or the budget silently never fires.
|
||||
ap_defaults = {"soft": 10.00, "hard": 22.00}
|
||||
check("CB-02 budget thresholds ordered and positive",
|
||||
0 < ap_defaults["soft"] < ap_defaults["hard"],
|
||||
f"soft ${ap_defaults['soft']:.2f} < hard ${ap_defaults['hard']:.2f}")
|
||||
|
||||
# AC-6: zero responses must not report $0.00 as an answer.
|
||||
with tempfile.NamedTemporaryFile("w", suffix=".jsonl", delete=False) as fh:
|
||||
fh.write(json.dumps({"type": "user", "message": {}}) + "\n")
|
||||
|
|
@ -543,53 +440,12 @@ def self_test():
|
|||
return 0 if ok else 1
|
||||
|
||||
|
||||
def budget(slug, soft, hard):
|
||||
"""Live cost budget (specs/CostAccounting.md §7).
|
||||
|
||||
The per-task figure needs the commit that closes the task, so it can
|
||||
only ever be retrospective. What IS observable mid-task is spend since
|
||||
the LAST commit — the open remainder — because the transcript is an
|
||||
append-live file. That is the number a budget can actually fire on.
|
||||
"""
|
||||
try:
|
||||
rep = collect(slug, None)
|
||||
except Abort as e:
|
||||
print(f"ABORT — {e}", file=sys.stderr)
|
||||
return 1
|
||||
open_spend = rep["by_task"].get(OPEN_REMAINDER, 0.0)
|
||||
head = subprocess.run(
|
||||
["git", "-C", REPO, "log", "-1", "--format=%h %s"],
|
||||
capture_output=True, text=True, check=True).stdout.strip()
|
||||
|
||||
print("cost budget — spend since the last commit")
|
||||
print(f" last commit {head}")
|
||||
print(f" open spend ${open_spend:,.2f}")
|
||||
print(f" soft / hard ${soft:,.2f} / ${hard:,.2f}")
|
||||
if open_spend > hard:
|
||||
print(f"\n HARD BREACH — ${open_spend:,.2f} > ${hard:,.2f}. Commit what "
|
||||
f"works, or stop and decompose. Uncommitted work is also "
|
||||
f"unattributable.", file=sys.stderr)
|
||||
return 1
|
||||
if open_spend > soft:
|
||||
print(f"\n soft breach — ${open_spend:,.2f} > ${soft:,.2f}. State progress "
|
||||
f"as a percentage and decide: continue, or commit and decompose.")
|
||||
return 0
|
||||
print("\n within budget")
|
||||
return 0
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--slug", default="-home-worsch-clay-borg")
|
||||
ap.add_argument("--pin", help="commit-ish or ISO Z instant (CA-07)")
|
||||
ap.add_argument("--by-task", action="store_true")
|
||||
ap.add_argument("--composition", action="store_true")
|
||||
ap.add_argument("--session-shape", action="store_true",
|
||||
help="SH-1..SH-3 (always shown in the default report)")
|
||||
ap.add_argument("--budget", action="store_true",
|
||||
help="CB-01/CB-02: spend since the last commit, live")
|
||||
ap.add_argument("--soft", type=float, default=10.00)
|
||||
ap.add_argument("--hard", type=float, default=22.00)
|
||||
ap.add_argument("--self-test", action="store_true")
|
||||
ap.add_argument("--json", action="store_true")
|
||||
args = ap.parse_args()
|
||||
|
|
@ -597,9 +453,6 @@ def main():
|
|||
if args.self_test:
|
||||
return self_test()
|
||||
|
||||
if args.budget:
|
||||
return budget(args.slug, args.soft, args.hard)
|
||||
|
||||
try:
|
||||
rep = collect(args.slug, args.pin)
|
||||
except Abort as e:
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ that cannot be found is reported and the run exits non-zero rather than
|
|||
silently under-reporting the total — under-reporting is the exact
|
||||
direction this metric could be gamed.
|
||||
|
||||
Usage: python3 tools/dep-weight.py [--json] [--self-test]
|
||||
Usage: python3 tools/dep-weight.py [--json]
|
||||
"""
|
||||
|
||||
import glob
|
||||
|
|
@ -88,50 +88,7 @@ def source_lines(name, version):
|
|||
return 0
|
||||
|
||||
|
||||
def self_test():
|
||||
"""Each assertion pins a failure this tool must detect.
|
||||
|
||||
The controls that matter here are: an unlocatable crate must not be
|
||||
silently counted as zero lines (that under-reports, the direction this
|
||||
metric could be gamed), and a target breach must fail rather than
|
||||
merely print.
|
||||
"""
|
||||
results = []
|
||||
|
||||
def check(name, ok, detail=""):
|
||||
results.append((name, ok, detail))
|
||||
|
||||
# A crate that does not exist must measure zero, so the caller's
|
||||
# `lines == 0` guard fires rather than silently shrinking the total.
|
||||
check("unlocatable crate measures zero (so the guard fires)",
|
||||
source_lines("definitely-not-a-real-crate-xyz", "9.9.9") == 0)
|
||||
|
||||
# A crate we do depend on must measure non-zero, or the guard above
|
||||
# would fire on everything and the tool would never report at all.
|
||||
real = source_lines("serde", "1")
|
||||
check("a real vendored crate measures non-zero", real > 0,
|
||||
f"{real:,} lines")
|
||||
|
||||
# Targets must be present and numeric — a missing target would make
|
||||
# the breach check vacuous.
|
||||
check("targets defined for every configuration",
|
||||
set(TARGETS) == set(CONFIGS) and all(
|
||||
isinstance(v, int) and v > 0 for v in TARGETS.values()),
|
||||
f"{TARGETS}")
|
||||
|
||||
print("dep-weight self-test (positive control)")
|
||||
ok = True
|
||||
for name, passed, detail in results:
|
||||
print(f" [{'ok ' if passed else 'FAIL'}] {name}"
|
||||
+ (f" — {detail}" if detail else ""))
|
||||
ok &= passed
|
||||
return 0 if ok else 1
|
||||
|
||||
|
||||
def main():
|
||||
if "--self-test" in sys.argv:
|
||||
return self_test()
|
||||
|
||||
report = {}
|
||||
missing = []
|
||||
for label, args in CONFIGS.items():
|
||||
|
|
|
|||
|
|
@ -1,266 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Executable checks for specs/InnerLoop.md rules (CB-WP-0003 T01).
|
||||
|
||||
The loop's own rules were prose. A rule nobody can run is a suggestion,
|
||||
and the audit in history/260731-inner-loop-rule-audit.md found several
|
||||
that were already being violated with no signal. This makes the
|
||||
mechanically-checkable ones fail a command.
|
||||
|
||||
Each check names the InnerLoop rule it enforces. Checks that cannot be
|
||||
made mechanical are recorded in the audit as `checkable` or `decorative`
|
||||
and are deliberately absent here — see the audit for why.
|
||||
|
||||
Positive control (InnerLoop v1.1 §Step 5): --self-test asserts each check
|
||||
actually detects its failure, using fixtures with known answers. A linter
|
||||
that passes everything because its matcher is broken is the same defect
|
||||
class as a benchmark timing rejected work.
|
||||
|
||||
Usage:
|
||||
python3 tools/loop-lint.py # lint the repo
|
||||
python3 tools/loop-lint.py --self-test # positive control
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
LOADABILITY_LIMIT = 400
|
||||
|
||||
# Artifact classes the loop produces. history/ is an append-only trail
|
||||
# (verbatim challenge text is not something to split), so it is exempt.
|
||||
LOOP_DIRS = ("specs", "research", "decisions", "evidence", "workplans")
|
||||
|
||||
|
||||
class Finding:
|
||||
def __init__(self, rule, path, detail):
|
||||
self.rule, self.path, self.detail = rule, path, detail
|
||||
|
||||
def __str__(self):
|
||||
return f" [{self.rule}] {self.path}\n {self.detail}"
|
||||
|
||||
|
||||
def _md_files(root=REPO):
|
||||
"""Loop artifacts only.
|
||||
|
||||
Vendored third-party trees (a baseline harness ships its own
|
||||
node_modules) are not artifacts this loop produces, and linting them
|
||||
buries the two real findings under eighteen irrelevant ones.
|
||||
"""
|
||||
for d in LOOP_DIRS:
|
||||
base = os.path.join(root, d)
|
||||
for dirpath, dirnames, files in os.walk(base):
|
||||
dirnames[:] = [x for x in dirnames if x != "node_modules"]
|
||||
for f in sorted(files):
|
||||
if f.endswith(".md"):
|
||||
yield os.path.relpath(os.path.join(dirpath, f), root)
|
||||
|
||||
|
||||
def check_loadability(root=REPO):
|
||||
"""§Agentic-efficiency 1 — every loop artifact stays under ~400 lines."""
|
||||
out = []
|
||||
for rel in _md_files(root):
|
||||
with open(os.path.join(root, rel)) as fh:
|
||||
n = sum(1 for _ in fh)
|
||||
if n > LOADABILITY_LIMIT:
|
||||
out.append(
|
||||
Finding(
|
||||
"loadability",
|
||||
rel,
|
||||
f"{n} lines exceeds the ~{LOADABILITY_LIMIT}-line limit; "
|
||||
f"split and link with relative paths",
|
||||
)
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def check_evidence_no_unmeasured(root=REPO):
|
||||
"""§Rubric — `unmeasured` is legal in a survey, illegal in an evidence file."""
|
||||
out = []
|
||||
base = os.path.join(root, "evidence")
|
||||
if not os.path.isdir(base):
|
||||
return out
|
||||
for f in sorted(os.listdir(base)):
|
||||
if not f.endswith(".md"):
|
||||
continue
|
||||
rel = os.path.join("evidence", f)
|
||||
for i, line in enumerate(open(os.path.join(root, rel)), 1):
|
||||
# A row asserting the verdict, not prose discussing the word.
|
||||
if re.search(r"\|\s*unmeasured\s*\|", line):
|
||||
out.append(
|
||||
Finding("evidence-unmeasured", f"{rel}:{i}",
|
||||
"verdict `unmeasured` in an evidence table")
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def check_survey_tier_and_chaos(root=REPO):
|
||||
"""§Loop tiers — tier declared, and the chaos roll recorded every time."""
|
||||
out = []
|
||||
base = os.path.join(root, "research")
|
||||
if not os.path.isdir(base):
|
||||
return out
|
||||
for f in sorted(os.listdir(base)):
|
||||
if not f.endswith(".md"):
|
||||
continue
|
||||
rel = os.path.join("research", f)
|
||||
text = open(os.path.join(root, rel)).read()
|
||||
if not re.search(r"^tier:\s*[SML]\b", text, re.M):
|
||||
out.append(Finding("tier-declared", rel, "no `tier:` declaration"))
|
||||
elif "chaos" not in text.lower():
|
||||
out.append(
|
||||
Finding("chaos-recorded", rel,
|
||||
"tier declared without the chaos roll; the rule requires "
|
||||
"recording it even when it changes nothing")
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def check_review_trail(root=REPO):
|
||||
"""§Step 2 — a tier-L survey carries research/challenge/response history."""
|
||||
out = []
|
||||
base = os.path.join(root, "research")
|
||||
hist = os.path.join(root, "history")
|
||||
if not (os.path.isdir(base) and os.path.isdir(hist)):
|
||||
return out
|
||||
files = os.listdir(hist)
|
||||
for f in sorted(os.listdir(base)):
|
||||
if not f.endswith(".md"):
|
||||
continue
|
||||
rel = os.path.join("research", f)
|
||||
text = open(os.path.join(root, rel)).read()
|
||||
if not re.search(r"^tier:\s*L\b", text, re.M):
|
||||
continue
|
||||
if not re.search(r"^status:\s*approved", text, re.M):
|
||||
continue
|
||||
for kind in ("challenge", "response"):
|
||||
if not any(x.endswith(f"-{kind}.md") and kind in x for x in files):
|
||||
out.append(
|
||||
Finding("review-trail", rel,
|
||||
f"tier-L approved survey with no history/*-{kind}.md")
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def check_reporting_tools_self_test(root=REPO):
|
||||
"""§Step 5 v1.1 — every tool that reports a number exposes --self-test."""
|
||||
out = []
|
||||
base = os.path.join(root, "tools")
|
||||
if not os.path.isdir(base):
|
||||
return out
|
||||
for f in sorted(os.listdir(base)):
|
||||
if not f.endswith(".py"):
|
||||
continue
|
||||
rel = os.path.join("tools", f)
|
||||
text = open(os.path.join(root, rel)).read()
|
||||
if "--self-test" not in text:
|
||||
out.append(
|
||||
Finding("self-test", rel,
|
||||
"reporting tool with no --self-test entry point; "
|
||||
"nothing verifies its positive control still works")
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
CHECKS = (
|
||||
check_loadability,
|
||||
check_evidence_no_unmeasured,
|
||||
check_survey_tier_and_chaos,
|
||||
check_review_trail,
|
||||
check_reporting_tools_self_test,
|
||||
)
|
||||
|
||||
|
||||
def self_test():
|
||||
"""Each check must DETECT its failure, not merely run."""
|
||||
import shutil
|
||||
import tempfile
|
||||
|
||||
results = []
|
||||
|
||||
def check(name, ok, detail=""):
|
||||
results.append((name, ok, detail))
|
||||
|
||||
tmp = tempfile.mkdtemp()
|
||||
try:
|
||||
for d in LOOP_DIRS + ("tools", "history"):
|
||||
os.makedirs(os.path.join(tmp, d), exist_ok=True)
|
||||
|
||||
# loadability: 401 lines must trip, 400 must not.
|
||||
with open(os.path.join(tmp, "specs", "Big.md"), "w") as fh:
|
||||
fh.write("x\n" * (LOADABILITY_LIMIT + 1))
|
||||
with open(os.path.join(tmp, "specs", "Ok.md"), "w") as fh:
|
||||
fh.write("x\n" * LOADABILITY_LIMIT)
|
||||
f = check_loadability(tmp)
|
||||
check("loadability detects overlong artifact",
|
||||
len(f) == 1 and "Big.md" in f[0].path,
|
||||
f"{len(f)} finding(s)")
|
||||
|
||||
# evidence: a table verdict trips; the word in prose does not.
|
||||
with open(os.path.join(tmp, "evidence", "E.md"), "w") as fh:
|
||||
fh.write("| AC-1 | x | unmeasured |\n"
|
||||
"the word unmeasured appearing in prose is fine\n")
|
||||
f = check_evidence_no_unmeasured(tmp)
|
||||
check("evidence-unmeasured detects a table verdict, not prose",
|
||||
len(f) == 1, f"{len(f)} finding(s), expected exactly 1")
|
||||
|
||||
# tier/chaos: missing tier trips; tier without chaos trips.
|
||||
with open(os.path.join(tmp, "research", "A.md"), "w") as fh:
|
||||
fh.write("# survey\nno tier here\n")
|
||||
with open(os.path.join(tmp, "research", "B.md"), "w") as fh:
|
||||
fh.write("tier: L (structural L)\n")
|
||||
f = check_survey_tier_and_chaos(tmp)
|
||||
rules = sorted(x.rule for x in f)
|
||||
check("tier/chaos detects both omissions",
|
||||
rules == ["chaos-recorded", "tier-declared"], f"{rules}")
|
||||
|
||||
# self-test: a tool without the flag trips.
|
||||
with open(os.path.join(tmp, "tools", "silent.py"), "w") as fh:
|
||||
fh.write("print(42)\n")
|
||||
f = check_reporting_tools_self_test(tmp)
|
||||
check("self-test detects a tool lacking --self-test",
|
||||
len(f) == 1 and "silent.py" in f[0].path, f"{len(f)} finding(s)")
|
||||
|
||||
# review trail: approved tier-L survey with no history trips.
|
||||
with open(os.path.join(tmp, "research", "C.md"), "w") as fh:
|
||||
fh.write("tier: L (structural L, chaos 3)\nstatus: approved\n")
|
||||
f = check_review_trail(tmp)
|
||||
check("review-trail detects a missing challenge/response",
|
||||
len(f) == 2, f"{len(f)} finding(s), expected 2")
|
||||
finally:
|
||||
shutil.rmtree(tmp, ignore_errors=True)
|
||||
|
||||
print("loop-lint self-test (positive control)")
|
||||
ok = True
|
||||
for name, passed, detail in results:
|
||||
print(f" [{'ok ' if passed else 'FAIL'}] {name}"
|
||||
+ (f" — {detail}" if detail else ""))
|
||||
ok &= passed
|
||||
return 0 if ok else 1
|
||||
|
||||
|
||||
def main():
|
||||
if "--self-test" in sys.argv:
|
||||
return self_test()
|
||||
|
||||
findings = []
|
||||
for c in CHECKS:
|
||||
findings.extend(c())
|
||||
|
||||
print("loop-lint — executable InnerLoop rules")
|
||||
if not findings:
|
||||
print(" no findings")
|
||||
return 0
|
||||
by_rule = {}
|
||||
for f in findings:
|
||||
by_rule.setdefault(f.rule, []).append(f)
|
||||
for rule, fs in sorted(by_rule.items()):
|
||||
print(f"\n{rule} ({len(fs)}):")
|
||||
for f in fs:
|
||||
print(str(f))
|
||||
print(f"\n{len(findings)} finding(s)")
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
|
|
@ -5,190 +5,30 @@ Compares the rule IDs declared in specs/GroundRules.md against the
|
|||
`covers:` lists in scenarios/ground/*.yaml. Exits non-zero when a
|
||||
scenario claims a rule the spec does not define, so coverage can never
|
||||
be inflated by a typo'd or invented rule ID.
|
||||
|
||||
Stated limit (InnerLoop implementation rule 4): this gate counts tags. It
|
||||
proves no rule is unclaimed and no claimed rule is invented. It does NOT
|
||||
prove a scenario exercises the rule it names.
|
||||
|
||||
Positive control (InnerLoop v1.1 §Step 5): the run asserts it actually
|
||||
found rules and scenarios. Before this was added, a broken spec regex
|
||||
yielded rules=[] and missing=[] and the tool exited 0 reporting "0/0" —
|
||||
the harness-does-nothing class, in the tool that reports our headline
|
||||
coverage number.
|
||||
|
||||
Usage:
|
||||
python3 tools/rule-coverage.py
|
||||
python3 tools/rule-coverage.py --self-test
|
||||
"""
|
||||
import glob
|
||||
import re
|
||||
import sys
|
||||
|
||||
RULE_RE = r"\*\*(GR-[A-Z]+\d+)"
|
||||
COVERS_RE = r"covers: \[(.*?)\]"
|
||||
AGGREGATE = "games/ground/src/lib.rs"
|
||||
# CB-WP-0003 T08: a provisional default with no expiry can shape the kernel
|
||||
# indefinitely while looking handled. CI warns; it does not break the build,
|
||||
# because the ruling is a ground-game decision we cannot make for them.
|
||||
PROVISIONAL_WARN_DAYS = 30
|
||||
ID_RE = r"GR-[A-Z]+\d+"
|
||||
spec = open("specs/GroundRules.md").read()
|
||||
rules = sorted(set(re.findall(r"\*\*(GR-[A-Z]+\d+)", spec)))
|
||||
|
||||
covered = set()
|
||||
for path in sorted(glob.glob("scenarios/ground/*.yaml")):
|
||||
match = re.search(r"covers: \[(.*?)\]", open(path).read(), re.S)
|
||||
if match:
|
||||
covered |= {c.strip() for c in match.group(1).split(",") if c.strip()}
|
||||
|
||||
def parse_rules(spec_text):
|
||||
return sorted(set(re.findall(RULE_RE, spec_text)))
|
||||
known = set(rules)
|
||||
hit = sorted(known & covered)
|
||||
missing = [r for r in rules if r not in covered]
|
||||
invented = sorted(covered - known)
|
||||
|
||||
|
||||
def parse_code_ids(text):
|
||||
"""Rule IDs named anywhere in the aggregate source (T09)."""
|
||||
return set(re.findall(ID_RE, text))
|
||||
|
||||
|
||||
def provisional_items(paths):
|
||||
"""(path, owner, raised) for every scenario encoding a U-item default."""
|
||||
out = []
|
||||
for path in paths:
|
||||
text = open(path).read()
|
||||
if not re.search(r"^provisional:\s*true", text, re.M):
|
||||
continue
|
||||
owner = re.search(r"^provisional_owner:\s*(\S+)", text, re.M)
|
||||
raised = re.search(r"^provisional_raised:\s*(\S+)", text, re.M)
|
||||
out.append((path,
|
||||
owner.group(1) if owner else None,
|
||||
raised.group(1) if raised else None))
|
||||
return out
|
||||
|
||||
|
||||
def parse_covers(text):
|
||||
match = re.search(COVERS_RE, text, re.S)
|
||||
if not match:
|
||||
return set()
|
||||
return {c.strip() for c in match.group(1).split(",") if c.strip()}
|
||||
|
||||
|
||||
def self_test():
|
||||
"""Each assertion pins a failure this tool must detect."""
|
||||
results = []
|
||||
|
||||
def check(name, ok, detail=""):
|
||||
results.append((name, ok, detail))
|
||||
|
||||
# The defect that motivated this control: a spec that parses to zero
|
||||
# rules must not be reportable as coverage.
|
||||
check("zero rules detected as a failure", parse_rules("no rules here") == [],
|
||||
"empty spec yields no rules; main() now aborts on this")
|
||||
# The matcher must actually match the real format.
|
||||
check("rule matcher works on real spec format",
|
||||
parse_rules("**GR-R06** something\n**GR-A12** other")
|
||||
== ["GR-A12", "GR-R06"])
|
||||
# covers: parsing, including the empty case.
|
||||
check("covers matcher works", parse_covers("covers: [GR-R06, GR-A12]")
|
||||
== {"GR-R06", "GR-A12"})
|
||||
check("missing covers yields empty set", parse_covers("no covers key") == set())
|
||||
# T09: the spec -> code link must be detectable.
|
||||
check("code-id matcher finds ids in source",
|
||||
parse_code_ids("// GR-R06: lead first\nfn f(){} // GR-A12")
|
||||
== {"GR-R06", "GR-A12"})
|
||||
check("code-id matcher finds none in unmarked source",
|
||||
parse_code_ids("fn f() { let x = 1; }") == set())
|
||||
# T08: every provisional scenario must carry an owner and a date.
|
||||
import glob as _g
|
||||
prov = provisional_items(sorted(_g.glob("scenarios/ground/*.yaml")))
|
||||
check("every provisional item has an owner and a raised date",
|
||||
bool(prov) and all(o and r for _, o, r in prov),
|
||||
f"{len(prov)} provisional item(s)")
|
||||
|
||||
print("rule-coverage self-test (positive control)")
|
||||
ok = True
|
||||
for name, passed, detail in results:
|
||||
print(f" [{'ok ' if passed else 'FAIL'}] {name}"
|
||||
+ (f" — {detail}" if detail else ""))
|
||||
ok &= passed
|
||||
return 0 if ok else 1
|
||||
|
||||
|
||||
def main():
|
||||
if "--self-test" in sys.argv:
|
||||
return self_test()
|
||||
|
||||
rules = parse_rules(open("specs/GroundRules.md").read())
|
||||
paths = sorted(glob.glob("scenarios/ground/*.yaml"))
|
||||
|
||||
# Positive control: refuse to report a percentage over nothing.
|
||||
if not rules:
|
||||
print("ERROR — no GR-rules parsed from specs/GroundRules.md; "
|
||||
"refusing to report coverage", file=sys.stderr)
|
||||
return 1
|
||||
if not paths:
|
||||
print("ERROR — no scenarios found in scenarios/ground/; "
|
||||
"refusing to report coverage", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
covered = set()
|
||||
for path in paths:
|
||||
covered |= parse_covers(open(path).read())
|
||||
|
||||
known = set(rules)
|
||||
hit = sorted(known & covered)
|
||||
missing = [r for r in rules if r not in covered]
|
||||
invented = sorted(covered - known)
|
||||
|
||||
# T09: the spec -> code -> scenario chain, made mechanical. A rule a
|
||||
# scenario claims should also be named in the aggregate, or the claim
|
||||
# rests on nothing but a tag.
|
||||
code_ids = parse_code_ids(open(AGGREGATE).read())
|
||||
unlinked = sorted((known & covered) - code_ids)
|
||||
phantom = sorted(code_ids - known)
|
||||
|
||||
pct = 100 * len(hit) // len(rules)
|
||||
linked = len((known & covered) & code_ids)
|
||||
print(f"AM-1 rule coverage: {len(hit)}/{len(rules)} ({pct}%) "
|
||||
f"over {len(paths)} scenarios")
|
||||
print(f"AM-1b spec->code link: {linked}/{len(hit)} claimed rules also "
|
||||
f"named in {AGGREGATE}")
|
||||
print(" NOTE: counts tags; does not prove a scenario exercises what it names")
|
||||
if unlinked:
|
||||
print(" unlinked (claimed by a scenario, absent from the aggregate):")
|
||||
print(" ", " ".join(unlinked))
|
||||
if phantom:
|
||||
print(" ERROR — rule id in code that the spec does not define:",
|
||||
" ".join(phantom), file=sys.stderr)
|
||||
return 1
|
||||
# T08: provisional items are reported with an owner and an age.
|
||||
prov = provisional_items(paths)
|
||||
if prov:
|
||||
import datetime
|
||||
today = datetime.date.today()
|
||||
print(f"\nprovisional U-item defaults: {len(prov)}")
|
||||
unowned, stale = [], []
|
||||
for path, owner, raised in prov:
|
||||
age = "?"
|
||||
if raised:
|
||||
try:
|
||||
age = (today - datetime.date.fromisoformat(raised)).days
|
||||
except ValueError:
|
||||
age = "?"
|
||||
name = path.split("/")[-1]
|
||||
print(f" {name:<34} owner={owner or 'NONE':<12} age={age}d")
|
||||
if not owner:
|
||||
unowned.append(name)
|
||||
if isinstance(age, int) and age > PROVISIONAL_WARN_DAYS:
|
||||
stale.append(f"{name} ({age}d)")
|
||||
if unowned:
|
||||
print(" WARN — provisional with no owner:", " ".join(unowned))
|
||||
if stale:
|
||||
print(f" WARN — provisional for over {PROVISIONAL_WARN_DAYS} days:",
|
||||
" ".join(stale))
|
||||
print(" NOTE: evidence files must list these; a ruling flips the "
|
||||
"scenario, not the kernel")
|
||||
|
||||
if missing:
|
||||
print(" uncovered:", " ".join(missing))
|
||||
if invented:
|
||||
print(" ERROR — claimed but not defined in the spec:", " ".join(invented),
|
||||
file=sys.stderr)
|
||||
return 1
|
||||
return 0 if not missing else 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
pct = 100 * len(hit) // len(rules) if rules else 0
|
||||
print(f"AM-1 rule coverage: {len(hit)}/{len(rules)} ({pct}%)")
|
||||
if missing:
|
||||
print(" uncovered:", " ".join(missing))
|
||||
if invented:
|
||||
print(" ERROR — claimed but not defined in the spec:", " ".join(invented))
|
||||
sys.exit(1)
|
||||
sys.exit(0 if not missing else 2)
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
---
|
||||
id: CB-WP-0003
|
||||
title: "Harden the inner loop: executable rules, session economics, dead policy"
|
||||
status: done
|
||||
status: proposed
|
||||
state_hub_workstream_id: "39d61dc0-870d-45c1-a595-bcf91f289dce"
|
||||
---
|
||||
|
||||
|
|
@ -55,7 +55,7 @@ accordingly.
|
|||
|
||||
```task
|
||||
id: CB-WP-0003-T01
|
||||
status: done
|
||||
status: todo
|
||||
priority: high
|
||||
state_hub_task_id: "3d5d45fb-f931-407a-be71-d2d727279d5e"
|
||||
```
|
||||
|
|
@ -89,7 +89,7 @@ rules and the three v1.1 rules are currently prose.
|
|||
|
||||
```task
|
||||
id: CB-WP-0003-T02
|
||||
status: done
|
||||
status: todo
|
||||
priority: high
|
||||
state_hub_task_id: "f861e67f-ed09-489d-963d-06697da9c08e"
|
||||
```
|
||||
|
|
@ -100,12 +100,7 @@ one with five assertions (AC-5..AC-9); `make cost` depends on
|
|||
`make cost-test`; `cost-test` runs in CI and in `make all`. `cargo bench
|
||||
-- --test` and cb-sim's empty-run check were already in place.
|
||||
|
||||
**Closed 2026-07-31 alongside T01** — both items below are done:
|
||||
tools/rule-coverage.py and tools/dep-weight.py gained `--self-test`
|
||||
(the former exposing a latent silent-pass on zero parsed rules), and
|
||||
`make loop-lint` fails CI when any `tools/*.py` lacks the flag.
|
||||
|
||||
**Scope as written:**
|
||||
**Remaining scope:**
|
||||
|
||||
- `tools/rule-coverage.py` and `tools/dep-weight.py` have positive-control
|
||||
logic but no `--self-test` entry point, so nothing verifies the control
|
||||
|
|
@ -124,7 +119,7 @@ dedup.
|
|||
|
||||
```task
|
||||
id: CB-WP-0003-T03
|
||||
status: done
|
||||
status: todo
|
||||
priority: medium
|
||||
state_hub_task_id: "1365b35d-4539-489a-beff-c072221e4702"
|
||||
```
|
||||
|
|
@ -158,7 +153,7 @@ rather than reporting a clean verify.
|
|||
|
||||
```task
|
||||
id: CB-WP-0003-T04
|
||||
status: done
|
||||
status: todo
|
||||
priority: high
|
||||
state_hub_task_id: "382724e6-efd5-4ef8-a438-9d31a65dacdb"
|
||||
```
|
||||
|
|
@ -203,7 +198,7 @@ attribution.
|
|||
|
||||
```task
|
||||
id: CB-WP-0003-T05
|
||||
status: done
|
||||
status: todo
|
||||
priority: medium
|
||||
state_hub_task_id: "26c920ee-2c09-4b45-a908-4d343532db09"
|
||||
```
|
||||
|
|
@ -234,7 +229,7 @@ Unblocked: depends on CB-WP-0002, now complete.
|
|||
|
||||
```task
|
||||
id: CB-WP-0003-T06
|
||||
status: done
|
||||
status: todo
|
||||
priority: low
|
||||
state_hub_task_id: "8f3a6147-54e8-48f9-8543-c13b0e0b0278"
|
||||
```
|
||||
|
|
@ -253,7 +248,7 @@ to reject.
|
|||
|
||||
```task
|
||||
id: CB-WP-0003-T07
|
||||
status: done
|
||||
status: todo
|
||||
priority: medium
|
||||
state_hub_task_id: "2f702821-7e64-4981-9885-82fa07f638aa"
|
||||
```
|
||||
|
|
@ -286,7 +281,7 @@ self-report of which one it was.
|
|||
|
||||
```task
|
||||
id: CB-WP-0003-T08
|
||||
status: done
|
||||
status: todo
|
||||
priority: low
|
||||
state_hub_task_id: "c2ee91ee-e551-48a7-b1a9-477c34c0690c"
|
||||
```
|
||||
|
|
@ -304,7 +299,7 @@ file must list them, not that the build breaks.
|
|||
|
||||
```task
|
||||
id: CB-WP-0003-T09
|
||||
status: done
|
||||
status: todo
|
||||
priority: low
|
||||
state_hub_task_id: "69ccc9ec-dc35-481c-8065-cef040f07f50"
|
||||
```
|
||||
|
|
@ -325,9 +320,9 @@ now price the evaluation itself.
|
|||
|
||||
```task
|
||||
id: CB-WP-0003-T11
|
||||
status: done
|
||||
status: todo
|
||||
priority: medium
|
||||
state_hub_task_id: "809215d8-8a5c-4b10-9445-9bc6db9bda42"
|
||||
state_hub_task_id: ""
|
||||
```
|
||||
|
||||
**New, from CB-WP-0002.** `benchmarks/baselines/model-prices.toml`
|
||||
|
|
@ -350,7 +345,7 @@ State Hub: a schema that cannot hold the fact it needs.
|
|||
|
||||
```task
|
||||
id: CB-WP-0003-T10
|
||||
status: done
|
||||
status: todo
|
||||
priority: low
|
||||
state_hub_task_id: "f43de208-92b1-4f0e-9236-a9d27f3ec451"
|
||||
```
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue