diff --git a/Makefile b/Makefile index 8e5a746..7bbbfe7 100644 --- a/Makefile +++ b/Makefile @@ -24,7 +24,7 @@ TOOLS := $(REPO)/tools # Every cargo recipe runs at the repo root; the shell does not persist cd. IN_REPO := cd $(REPO) && -.PHONY: check test sim bench bench-test coverage dep-weight cost cost-test cost-pin cost-budget cost-mix loop-lint self-tests env-test task-done status facts-check facts-gen mutation-check size-metrics runtime-metrics build-time loc all +.PHONY: check test sim bench bench-test coverage dep-weight cost cost-test cost-pin cost-budget cost-mix loop-lint self-tests env-test task-done status facts-check facts-gen mutation-check size-metrics runtime-metrics build-time am6 loc all ## fmt + clippy (deny warnings) + HashMap deny-lint check: @@ -42,6 +42,13 @@ dep-weight: coverage: $(PY) $(TOOLS)/rule-coverage.py +# AM-6 gate. Runs in RELEASE, where headroom is ~24x; the same assertion +# in debug has ~3.4x and flaked under load. The target is unchanged — only +# where it is measured. +am6: + $(IN_REPO) $(CARGO) test --release -p games-ground --all-features \ + am6_throughput -- --ignored --nocapture --test-threads=1 + # AM-9 peak RSS (fast, gated). AM-5 needs a clean build — see build-time. runtime-metrics: $(PY) $(TOOLS)/runtime-metrics.py --fast @@ -157,4 +164,4 @@ loc: printf '%-28s %s\n' $$d "$$(find $$d/src -name '*.rs' | xargs cat | grep -vcE '^\s*(//|$$)')"; \ done -all: check test sim coverage size-metrics runtime-metrics dep-weight self-tests env-test facts-check loop-lint bench-test +all: check test sim coverage size-metrics runtime-metrics am6 dep-weight self-tests env-test facts-check loop-lint bench-test diff --git a/facts.toml b/facts.toml index 09789a2..441d124 100644 --- a/facts.toml +++ b/facts.toml @@ -6,7 +6,7 @@ # `make facts-check` fails if this file disagrees with the # instruments, or if a tagged artifact disagrees with this file. -generated = "2026-07-31" +generated = "2026-08-01" pin = "fc76445" [am4a_loc] diff --git a/games/ground/src/lib.rs b/games/ground/src/lib.rs index a154df5..d966bbf 100644 --- a/games/ground/src/lib.rs +++ b/games/ground/src/lib.rs @@ -2244,7 +2244,18 @@ mod replay_probe { /// spec target. A test, not a bench — Criterion reports throughput and /// asserts nothing, which is why this row measured nothing for six /// passes. + /// + /// **`#[ignore]` on purpose, and this is the T04 correction.** The + /// assertion first ran inside `make all` and failed at 38,753 ev/s + /// against 341,280 measured in isolation — a 9x drop, because + /// `cargo test` runs test binaries and threads **concurrently**. A + /// throughput assertion inside a parallel harness measures contention, + /// not throughput. The fix is not a lower target (T01 forbade that, + /// and it would reproduce the defect being fixed) but a measurement + /// that only runs where it is valid: `make am6`, release, + /// `--test-threads=1`. #[test] + #[ignore = "throughput measurement — invalid under a parallel harness; run `make am6`"] fn am6_throughput_clears_the_spec_target() { let mut best = 0.0f64; let mut sampled = 0usize; @@ -2279,11 +2290,12 @@ mod replay_probe { ); assert!( best >= AM6_EVENTS_PER_SEC, - "AM-6 UNMET: {best:.0} events/s < {AM6_EVENTS_PER_SEC:.0} target \ - ({headroom:.2}x). debug_assertions={}. Reference: ~182k debug, \ - ~2.4M release on bnt-lap001. Do NOT lower the target to pass — \ - GameKernel §5 AM-6 is a spec value and lowering it needs an ADR.", - cfg!(debug_assertions) + "AM-6 UNMET: {best:.0} events/s < {AM6_EVENTS_PER_SEC:.0} \ + ({headroom:.2}x). Reference: ~1.7M via `make am6` on \ + bnt-lap001. Do NOT lower the target to pass — GameKernel §5 \ + AM-6 is a spec value and lowering it needs an ADR. If this \ + fired under a parallel harness, the measurement is invalid \ + rather than the target: run `make am6`.", ); } diff --git a/specs/GameKernel.md b/specs/GameKernel.md index 3a5f50a..f8d9a8f 100644 --- a/specs/GameKernel.md +++ b/specs/GameKernel.md @@ -165,7 +165,7 @@ evidence lands in `evidence/CB-EV-0001-game-kernel.md` with no | AM-3 | Synthetic-workload definition size: LOC to express the CB-RES-0001 synthetic game on our kernel | ~36 LOC (boardgame.io, measured) | ≤ 50 LOC | measured | | AM-4a | M-D2-DEP: third-party LOC, **shipped runtime** (`--no-default-features`) | boardgame.io: 120 npm packages / 3.9M LOC | **≤ 250,000 lines** | measured (`make dep-weight`) | | AM-4b | M-D2-DEP: third-party LOC, **dev toolchain** (default features) | as above | **≤ 350,000 lines** | measured (`make dep-weight`) | -| AM-4c | M-D2-DEP: own source per third-party 100k lines | — | reported, not targeted | measured (`make dep-weight`) | +| ~~AM-4c~~ | M-D2-DEP: own source per third-party 100k lines | — | **WITHDRAWN from the acceptance table 2026-08-01 (CB-WP-0006 T04)** — retained as a reported diagnostic in `make dep-weight`; see §5a | diagnostic | | AM-5 | M-D2-BLD: clean release build of headless workspace | n/a (npm install ~seconds; not comparable) | ≤ 60 s on bnt-lap001, recorded not gated | measured | | AM-6 | M-D3-THR: applied events/s, synthetic workload, same machine | boardgame.io ~1,100–1,900 moves/s (best config, degrading) | **≥ 100,000/s** (stipulated target, ADR-0002) | measured | | AM-7 | M-D3 scaling: throughput @100k events vs @5k; and snapshot+replay of 100k events | boardgame.io 0.45–0.66× @20–40k, DNF @100k | **≥ 0.9×** (flat), replay of 100k events ≤ 5 s, hash-identical | measured | @@ -175,6 +175,43 @@ evidence lands in `evidence/CB-EV-0001-game-kernel.md` with no | AM-11 | M-D4-SWAP **(unmet 2026-07-31 — the pair exists, the suite does not)**: null + reference impls passing one conformance suite | no candidate has the pattern | RNG and log storage each have ≥2 impls (real + test/null) under one suite | measured (bool) | | AM-12 | M-D2-TOK / M-D2-CST: tokens and USD per completed task | n/a — first pass sets our own baseline | recorded per task in the evidence cost log (price sheet 2026-07-31) | recorded, not gated | +### 5a. Why AM-4c was withdrawn from the acceptance table + +*(CB-WP-0006 T04, 2026-08-01. Tier S — amends one row, creates no +capability; chaos d4=2, no override. Follows the precedent ADR-0005 §4 set +for AM-10.)* + +AM-4c was `reported, not targeted`, so nothing could fail and it counted +against M-D1-MUT. The task was to give it a threshold or drop it. It is +dropped, for a reason that a threshold cannot fix: + +**The ratio has no monotone better direction.** INTENT's rule is *own the +semantics; assimilate the implementation*. A **rising** ratio can mean we +are properly owning semantics, or that we are reimplementing things we +should have assimilated. A **falling** ratio can mean good leverage, or +dependency bloat and implementation leaking into our semantics. Both +directions are ambiguous, and a target requires knowing which way is +better. + +**It is also redundant.** AM-4a and AM-4b already bound the denominator +(third-party LOC ceilings, ratified in ADR-0004) and AM-2 bounds own-source +density per rule. AM-4c is the ratio of two quantities that are each +already targeted; any threshold on it would be implied by those two or +would contradict them. + +Measured at withdrawal: **1,426** own lines per 100k third-party (shipped +runtime), **1,107** (dev toolchain). + +**Decided before Phase B, deliberately.** ADR-0005 predicts own-source +growth from the kernel work, which will move this ratio. Setting a +threshold after seeing that movement would be the retarget InnerLoop +§Step 4 forbids — so the decision was taken while the number was still +unaffected by the work that will change it. + +**M-D1-MUT keeps AM-4c in its denominator.** Withdrawing a row would +otherwise improve the metric from 7/14 to 7/13 without enforcing anything — +a score improved by deleting the question. + Comparisons against the event-sourcing 10⁵–10⁶/s estimate stay **parity** until a local Rust comparator is measured (open follow-up from the adversarial review). diff --git a/tools/__pycache__/cb-cost.cpython-312.pyc b/tools/__pycache__/cb-cost.cpython-312.pyc index eb81dcd..1e1506d 100644 Binary files a/tools/__pycache__/cb-cost.cpython-312.pyc and b/tools/__pycache__/cb-cost.cpython-312.pyc differ diff --git a/tools/__pycache__/dep-weight.cpython-312.pyc b/tools/__pycache__/dep-weight.cpython-312.pyc index c2effba..38e9bdd 100644 Binary files a/tools/__pycache__/dep-weight.cpython-312.pyc and b/tools/__pycache__/dep-weight.cpython-312.pyc differ diff --git a/tools/__pycache__/mutation-check.cpython-312.pyc b/tools/__pycache__/mutation-check.cpython-312.pyc index 3f12d37..dcaf2e7 100644 Binary files a/tools/__pycache__/mutation-check.cpython-312.pyc and b/tools/__pycache__/mutation-check.cpython-312.pyc differ diff --git a/tools/dep-weight.py b/tools/dep-weight.py index 3a62388..46a5739 100755 --- a/tools/dep-weight.py +++ b/tools/dep-weight.py @@ -186,6 +186,14 @@ def main(): f"{r['third_party_loc']:>9,} lines third-party " f"[{mark} target {limit:,}]" ) + # AM-4c: own source per 100k third-party lines. A DIAGNOSTIC, not a + # target — see specs/GameKernel.md §5 and CB-WP-0006 T04. The ratio + # has no monotone better direction, so it cannot carry a threshold. + for label in CONFIGS: + tp = report[label]["third_party_loc"] + if tp: + print(f" AM-4c {label:<18}{own / (tp / 100_000):>9,.0f} own " + f"lines per 100k third-party (diagnostic, not targeted)") delta = ( report["dev-toolchain"]["third_party_loc"] - report["shipped-runtime"]["third_party_loc"] diff --git a/tools/mutation-check.py b/tools/mutation-check.py index 31d12a4..a23a119 100644 --- a/tools/mutation-check.py +++ b/tools/mutation-check.py @@ -125,12 +125,22 @@ def rows(): mutate=("tools/dep-weight.py", '"dev-toolchain": 350_000,', '"dev-toolchain": 1_000,')), + # Deliberately RETAINED in this denominator after its withdrawal + # from the acceptance table. Dropping it would move M-D1-MUT from + # 7/14 to 7/13 without enforcing anything — a score improved by + # deleting the question. Row("AM-4c", "own source per third-party 100k lines", - unmutatable="declared `reported, not targeted` in the spec. " - "There is no threshold, so there is no property to " - "invert. Counts against the metric by ADR-0005 §1, " - "which is the honest treatment: an untargeted number " - "cannot fail."), + unmutatable="WITHDRAWN from the acceptance table 2026-08-01 " + "(CB-WP-0006 T04, GameKernel §5a) and retained here " + "as a diagnostic. The ratio has no monotone better " + "direction — rising can mean owning semantics or " + "reimplementing what should be assimilated; falling " + "can mean leverage or dependency bloat — so it " + "cannot carry a threshold. It is also redundant: " + "AM-4a/AM-4b bound the denominator and AM-2 bounds " + "own-source density. Kept in this denominator on " + "purpose, so the metric cannot be improved by " + "deleting rows."), Row("AM-5", "clean release build <= 60 s", unmutatable="now RECORDED (`make build-time`) and **breaching**: " @@ -144,8 +154,12 @@ def rows(): "itself."), Row("AM-6", ">= 100,000 applied events/s", - verify=CARGO + ["test", "-p", "games-ground", "--all-features", - "am6_throughput"], + # Release: the gate runs where headroom is ~24x. In debug the + # test only asserts a sanity floor, so a debug mutation run + # would need a far larger slowdown to register. + verify=CARGO + ["test", "--release", "-p", "games-ground", + "--all-features", "am6_throughput", "--", + "--ignored", "--test-threads=1"], # A PROPERTY mutation, not a threshold tweak: slow the fold hot # path and require the gate to notice. Raising the target # instead would only prove the comparison runs. @@ -153,7 +167,7 @@ def rows(): " fn fold(&mut self, event: &Self::Event) {\n" " match event {", " fn fold(&mut self, event: &Self::Event) {\n" - " for _ in 0..4000 { std::hint::black_box(0u8); }\n" + " for _ in 0..100_000 { std::hint::black_box(0u8); }\n" " match event {"), expect="AM-6 UNMET"), @@ -339,7 +353,12 @@ def report(only=None): red = tally.get("red", 0) total = len(rs) + withdrawn = sum(1 for r in rs if "WITHDRAWN" in (r.unmutatable or "")) print(f"\n M-D1-MUT: {red}/{total} rows enforced") + if withdrawn: + print(f" ({withdrawn} withdrawn row(s) retained in the denominator " + f"on purpose — a score\n improved by deleting the question " + f"is not an improvement)") for k in ("PARTIAL", "SURVIVED", "WRONG-REASON", "unmutatable", "inconclusive"): if tally.get(k): diff --git a/workplans/CB-WP-0006-instrument-the-table.md b/workplans/CB-WP-0006-instrument-the-table.md index b84bb02..083892e 100644 --- a/workplans/CB-WP-0006-instrument-the-table.md +++ b/workplans/CB-WP-0006-instrument-the-table.md @@ -246,6 +246,51 @@ kernel work in Phase B, so AM-4c will move against us. Setting its threshold *after* seeing that movement would be the retarget InnerLoop §Step 4 forbids — so it is set here, before Phase B, or not at all. +**Delivered: withdrawn from the acceptance table, retained as a +diagnostic.** `specs/GameKernel.md` §5a carries the argument. + +**The ratio has no monotone better direction.** INTENT's rule is *own the +semantics; assimilate the implementation*. Rising can mean owning +semantics properly, or reimplementing what should have been assimilated. +Falling can mean good leverage, or dependency bloat. A target requires +knowing which way is better, and this metric does not. + +**It is also redundant:** AM-4a/AM-4b bound the denominator and AM-2 bounds +own-source density per rule. AM-4c is a ratio of two already-targeted +quantities. + +Measured at withdrawal: **1,426** own lines per 100k third-party (shipped), +**1,107** (dev). `make dep-weight` now prints both, labelled +`diagnostic, not targeted` — it was never actually reported before. + +**M-D1-MUT keeps AM-4c in its denominator, deliberately**, and says so in +the output. Dropping it would move the score 7/14 → 7/13 without enforcing +anything: a score improved by deleting the question. + +### A T01 correction found here + +**The AM-6 gate failed inside `make all` at 38,753 ev/s against 341,280 +measured in isolation — a 9× drop.** `cargo test` runs test binaries and +threads concurrently, so **a throughput assertion inside a parallel +harness measures contention, not throughput.** T01's measurement was valid +and its gate placement was not. + +Fixed by running the measurement only where it is valid — `#[ignore]`, +plus `make am6` in release with `--test-threads=1`, now **2.0M ev/s, +20.2× headroom** — and *not* by lowering the target, which T01 explicitly +forbade and which would have reproduced the defect being fixed. My first +attempt did drift that way (a debug "sanity floor" of 50,000) and was +backed out: a second threshold is still a second chance to tune. + +**And the mutation went `SURVIVED` on the first run after the move** — +4,000 `black_box` iterations were calibrated against debug's 3.4× +headroom and are invisible against release's 20×. Raised to 100,000; back +to `red`. A weak mutation is not a fixed property of a row: **it can +become weak when the row's measurement conditions change.** + +**M-D1-MUT: 7 of 14** (unchanged — AM-4c was always going to stay +uncounted; what changed is that the reason is now correct and recorded). + ## Phase B — the kernel work inherited from CB-WP-0005 > These three were written in [CB-WP-0005](CB-WP-0005-assertion-coverage.md)