diff --git a/Makefile b/Makefile index 1d77698..221d8a2 100644 --- a/Makefile +++ b/Makefile @@ -24,7 +24,7 @@ TOOLS := $(REPO)/tools # Every cargo recipe runs at the repo root; the shell does not persist cd. IN_REPO := cd $(REPO) && -.PHONY: check test sim bench bench-test coverage dep-weight cost cost-test cost-pin cost-budget shape-budget cost-mix loop-lint self-tests env-test task-done status facts-check facts-gen mutation-check size-metrics runtime-metrics build-time am6 am7 am8 replay-test loc play gate-review all +.PHONY: check test sim bench bench-test coverage dep-weight cost cost-test cost-pin cost-budget shape-budget cost-mix loop-lint self-tests env-test task-done status facts-check facts-gen mutation-check size-metrics runtime-metrics build-time am6 replay-test loc play gate-review all ## fmt + clippy (deny warnings) + HashMap deny-lint check: @@ -63,23 +63,6 @@ am6: $(IN_REPO) $(CARGO) test --release -p games-ground --all-features \ am6_throughput -- --ignored --nocapture --test-threads=1 -# AM-8 N=10 determinism gate. One scenario, ten same-seed replays, all -# compared to the first. Not all 25: `make sim` already runs K8's double- -# run over every scenario, and repeating that eight more times costs 47 s -# per build to re-answer a question the second run already answered. The -# extra runs exist for the probabilistic class, and one workload gives -# that class its ten samples — see scenario::run_n. -am8: - $(IN_REPO) $(CARGO) run -q -p cb-sim -- --runs 10 \ - $(REPO)/scenarios/ground/gr-r06-round-resolve.yaml - -# AM-7 scaling gate. Same release / single-thread reasoning as am6 and more -# so: a *ratio* of two timings taken under varying contention is worse than -# one reading, because the noise multiplies rather than cancels. -am7: - $(IN_REPO) $(CARGO) test --release -p games-ground --all-features \ - am7_scaling -- --ignored --nocapture --test-threads=1 - # AM-9 peak RSS (fast, gated). AM-5 needs a clean build — see build-time. runtime-metrics: $(PY) $(TOOLS)/runtime-metrics.py --fast @@ -203,4 +186,4 @@ loc: printf '%-28s %s\n' $$d "$$(find $$d/src -name '*.rs' | xargs cat | grep -vcE '^\s*(//|$$)')"; \ done -all: check test sim coverage size-metrics runtime-metrics am6 am7 am8 replay-test dep-weight self-tests env-test facts-check loop-lint bench-test +all: check test sim coverage size-metrics runtime-metrics am6 replay-test dep-weight self-tests env-test facts-check loop-lint bench-test diff --git a/crates/cb-game-runtime/src/scenario.rs b/crates/cb-game-runtime/src/scenario.rs index a34ca40..284ec65 100644 --- a/crates/cb-game-runtime/src/scenario.rs +++ b/crates/cb-game-runtime/src/scenario.rs @@ -205,9 +205,6 @@ where }) } -/// K8's double-run: every scenario, every run, on `make sim`. -pub const K8_RUNS: usize = 2; - /// Execute a scenario against game `G`: run twice with the same seed, /// compare hashes (K8), then check the `expect` block. pub fn run(scenario: &ScenarioFile) -> RunOutcome @@ -215,33 +212,6 @@ where G: ScenarioGame, G::Event: Serialize, { - run_n::(scenario, K8_RUNS) -} - -/// The same, with the replay count as a parameter (AM-8, CB-WP-0015 T02). -/// -/// **What the extra runs are actually for.** A *deterministic* divergence -/// — a seed threaded wrong, a fold that depends on insertion order — -/// shows up on run 2 exactly as reliably as on run 10, so for that class -/// `K8_RUNS` is sufficient and the other eight runs are repetitions of a -/// check that already answered. The class that needs `N` is the -/// *probabilistic* one: hash iteration order, address-dependent hashing, -/// scheduling. There, `N` runs give `N-1` independent chances, and two -/// runs catch a coin-flip divergence only half the time. -/// -/// That class is meant to be structurally excluded here — `clippy.toml` -/// denies `HashMap`/`HashSet` under K6 and `make check` runs `-D -/// warnings`, which is AM-8's other clause. So `N` is defence in depth -/// against the exclusion failing, not the primary control, and that is -/// why it runs on one scenario in `make am8` rather than on all 25 in -/// `make sim`: the latter costs 47 s per build to repeat an answered -/// question 8 more times. -pub fn run_n(scenario: &ScenarioFile, runs: usize) -> RunOutcome -where - G: ScenarioGame, - G::Event: Serialize, -{ - assert!(runs >= 2, "a determinism check needs at least two runs"); let first = match execute::(scenario) { Ok(pass) => pass, Err(reason) => { @@ -251,30 +221,25 @@ where } } }; - - // Every later run is compared to the first, not to its predecessor: a - // divergence that appears on run 5 and persists would otherwise be - // invisible to a pairwise walk after run 6. - for n in 2..=runs { - let next = match execute::(scenario) { - Ok(pass) => pass, - Err(reason) => { - return RunOutcome::Failed { - reason, - evidence: None, - } - } - }; - if first.hash != next.hash { - let reason = format!( - "K8 divergence: run 1 hash {} != run {n} hash {} (of {runs})", - first.hash, next.hash - ); + let second = match execute::(scenario) { + Ok(pass) => pass, + Err(reason) => { return RunOutcome::Failed { - evidence: Some(Box::new(evidence_of(&first))), reason, - }; + evidence: None, + } } + }; + + if first.hash != second.hash { + let reason = format!( + "K8 divergence: run 1 hash {} != run 2 hash {}", + first.hash, second.hash + ); + return RunOutcome::Failed { + evidence: Some(Box::new(evidence_of(&first))), + reason, + }; } if let Err(reason) = check(scenario, &first) { diff --git a/evidence/CB-EV-0013-the-inert-clauses.md b/evidence/CB-EV-0013-the-inert-clauses.md deleted file mode 100644 index 1a3ba27..0000000 --- a/evidence/CB-EV-0013-the-inert-clauses.md +++ /dev/null @@ -1,201 +0,0 @@ -# CB-EV-0013 — two inert clauses, and an estimator that was wrong twice - -CB-WP-0015 T03. Measured 2026-08-02 at `7e9ab22`+. Pass kind `product`, -tier **S** (chaos d4=2, no override). Declaration 10 of 12. - -Per the rule adopted in CB-EV-0012, the cost table quotes **CB-WP-0014's** -final figure and declines to quote this pass's own. See §5. - ---- - -## 1. The machine wanders by 2.5×, and that is the whole story - -Before any of this could be measured, the instrument had to survive the -machine. Folding the same log on an unchanged binary, seconds apart: - -| leg | observed range, one afternoon | -|---|---| -| 5,000-event fold | 11.2 M – 55.7 M ev/s | -| 100,000-event fold | 9.7 M – 45.9 M ev/s | - -**A 5× swing in absolute throughput.** Any single reading of either number -is worthless, and AM-6 already knew this — it takes best-of-3 and says so. -What AM-6's estimator cannot do is survive being turned into a ratio. - -## 2. AM-7's scaling clause: three estimators, two of them wrong - -`mutation-check` had said the same thing every run since CB-WP-0005: *"no -code computes the ratio of throughput @100k to @5k or compares it to 0.9; -Criterion reports both and nothing relates them."* Worse, the test next to -it is called `replay_100k_events_is_linear_and_fast` and checks the two -sizes **independently** — it computes both numbers, prints both, and never -divides one by the other. **The name asserts the property the body does -not test.** - -| estimator | result on an unchanged binary | why it failed | -|---|---|---| -| best-of-5 per leg, then divide (AM-6's) | **0.581 – 1.085** | the legs are measured minutes apart; noise multiplies rather than cancels | -| legs back-to-back inside one sample | medians 1.004 / 0.931 / 0.956 | closer, but a 50 ms leg samples a *point* on the drift rather than averaging it | -| legs interleaved per fold, 10 M events each, median of 9 | **0.987 / 0.991 / 0.989** | drift becomes common-mode and divides out | - -The third one holds up under abuse: with eight busy-loops pinned against -eight cores, absolute throughput fell 4× and the **median ratio stayed at -0.989**. Inside a real `make all`, after the build has just hammered every -core, it read **0.965** (worst sample 0.944) — the condition the first -guard failed on, now passing with margin. - -**The measured answer is ~0.97–0.99 against a 0.9 floor.** Nothing in -`GroundState` grows with log length, so the fold is O(1) per event by -construction; the residual few percent is cache residency on streaming a -20×-longer `Vec`. - -### The guard that was wrong for the same reason the estimator was - -The first version declared INDETERMINATE if *any* sample fell on the other -side of the floor — unanimity. Under contention the median read 0.971, -which is a good measurement, and one sample read **0.899** — a thousandth -under — and the guard failed the build. It also fired intermittently -inside `mutation-check`, where this test runs straight after a 50-second -rebuild. - -A gate that fails when the machine is busy is a flake, and a flake gets -suppressed rather than fixed. It now requires a **two-thirds majority** -agreeing with the median: the guard keeps its purpose — refusing to read a -coin-flip as a verdict — without treating one outlier as one. - -### The mutation, and the control that says AM-7 is not a second AM-6 - -To make the property false you must make fold cost grow *with history*, -which is precisely boardgame.io's measured defect. Growing the deck by one -card per fold and scanning it drives the ratio to **0.751** — red, tight, -no straddle. - -**And the control matters more than the mutation.** AM-6's mutation adds a -*constant* per-event cost. It halves throughput, 28 M → 15 M ev/s, and -leaves this ratio at **0.999× — green**. The two rows catch different -things: a constant slowdown is AM-6's, a history-proportional one is -AM-7's. Without that check, "AM-7 goes red" would have been consistent -with AM-7 being a redundant copy of a row we already had. - -## 3. AM-8's N: the spec was right and the runner was wrong - -Eight passes of the runner executing each scenario **twice** while the row -said ten. Closing it needed an argument, because *"the spec says ten"* is -not one. - -A **deterministic** divergence — a seed threaded wrong, an order-dependent -fold — shows on run 2 exactly as reliably as on run 10. For that class the -double-run is sufficient and eight more runs across 25 scenarios cost 47 s -a build to re-answer an answered question. - -**The measurement settled it.** Perturbing the RNG only from its fourth -construction onward, on `gr-r06-round-resolve`: - -| | result | -|---|---| -| `cb-sim --runs 2` | **PASS** | -| `cb-sim --runs 10` | FAIL — *"run 1 hash … != run 4 hash … (of 10)"* | - -A late-onset divergence is a real class the double-run structurally cannot -see, and it is **deterministic, not probabilistic** — so it can be a -control rather than a coin flip. The spec value stands unamended; what -changed is that it is now enforced, on one scenario in `make am8` (~2 s) -rather than on all 25. - -The primary defence against the *probabilistic* class remains the -`HashMap`/`HashSet` deny lint — this row's other clause, already live. -Ten runs are defence in depth against that exclusion failing, which is why -one workload's worth is proportionate rather than 25. - -## 4. What the acceptance table now claims - -| row | before | after | -|---|---|---| -| AM-7 | PARTIAL 2/3 | **red, 3/3** | -| AM-8 | PARTIAL 1/2 | **red, 2/2** | - -**M-D1-MUT: 10 of 14 rows enforced**, from 8. The prediction of record -(≥10 of 14, from ADR-0005's 9-of-12) is **MET** for the first time. - -**The denominator did not move, and that was the live risk.** This pass -could have improved its score two ways, and one of them was deleting a -question — amending AM-8's N down to 2, or splitting AM-7's scaling clause -into a fifteenth row. Both were considered and both were refused. The four -rows that remain unenforced are the four that were already unenforceable: -AM-3 blocked on an unbuilt artifact, AM-4c withdrawn and retained on -purpose, AM-5 declared ungated by the spec, AM-10 withdrawn. - -### And the run found a third thing, in the instrument itself - -The first full `mutation-check` of this pass reported **AM-4a -HARNESS-BROKEN**, and correctly refused to publish any score at all. Its -mutation still pointed at `"shipped-runtime": 250_000` — the target -**ADR-0008 D3 moved to 161,000 in CB-WP-0013**. The find-string had been -stale ever since, because no full mutation-check had been run in between. - -Positive control 1 doing exactly its job. But it only *can* do that job on -a full run, and a full run is deliberately not in `make all` — it rebuilds -the workspace once per mutated row. So a mutation can rot for passes at a -time while every build stays green. - -The cheap half of that check needs no build at all: does each find-string -still occur in the file it names? That is now a `--self-test` assertion, -and `self-tests` **is** in `make all`. Verified both ways — perturbing the -target by one digit turns it red naming `AM-4a`, restoring it turns it -green. - -Two rows were repaired by this pass and a third by the run that measured -it. The AM-4 family has now produced four defects, none of them found by -looking for them. - -### The tool now measures a claim it used to assert - -`mutation-check`'s clause flags were hand-maintained booleans saying -whether an assertion existed — which is the same shape of claim the tool -was built to stop trusting. A clause may now carry its own verify command -and mutation, and then its `enforced` flag is **measured**; a declaration -that disagrees with its own measurement is refused as HARNESS-BROKEN -rather than reported as either verdict. Two clauses carry one so far, and -they print as `red*`. The rest still print as the author's word, which is -what they always were. - -## 5. Cost, and the prediction that came due - -| pass | kind | responses | cost | $/response | -|---|---|---|---|---| -| CB-WP-0013 | meta | 47 | $8.26 | 0.176 | -| **CB-WP-0014** (final) | product | 34 | **$7.47** | **0.220** | -| CB-WP-0015 | product | *provisional — not quoted, per CB-EV-0012 §5* | | | - -**CB-EV-0009's standing prediction came due and is consistent — once.** It -predicted that a pass opening above the SH-1 hard line would cost more than -0.123 $/response. CB-WP-0014 opened above the line and cost **0.220**. - -That is one point past a threshold, and it is worth being explicit about -what it is not: there is **no control**. No pass has opened *below* the -line since the prediction was made, the two passes either side differ in -tier, kind and subject, and 0.220 is inside the range product passes have -shown anyway. The prediction is unfalsified, not confirmed. It needs a -pass that opens below the line to mean anything. - -## 6. Open - -- **AM-4b's scope defect (408,237 uncounted lines)** and its unmeasured - proc-macro share. Unchanged from CB-EV-0012 §3. -- **INTENT stage 1: one human verification** — `cb-play --serve 0`, open - the URL, confirm the table reads and a drag works. Unchanged. -- **`python3` as a toolchain dependency was never argued.** -- **AM-4a still cannot survive stage 2** — 1,741,979 against 161,000. -- **ADR-0007 D3's acquisition rule** remains unratified, having now - decided two dependency questions. -- **AM-3 stays blocked** on an artifact nobody has built, and it is now - one of only four unenforced rows rather than one of six. -- **`cb-cost --self-test` failed once and passed on re-run**, on - *"pass_costs windows sum to the unwindowed total"*, inside a `make all` - during this session. The obvious hypothesis is that it reads the session - transcript while this session is appending to it — a self-test racing - the file it measures. **That is a hypothesis, not a finding: it has not - been reproduced or instrumented.** Recorded because an intermittent - positive control is worth more attention than a failing one, and because - a flake in the tool that prices every pass would be quietly corrosive. -- **Chaos: 10 of 12 declarations, 1 override.** diff --git a/games/ground/src/lib.rs b/games/ground/src/lib.rs index 20fba88..9dbfdd3 100644 --- a/games/ground/src/lib.rs +++ b/games/ground/src/lib.rs @@ -2440,216 +2440,4 @@ mod replay_probe { assert!(elapsed.as_secs_f64() < 5.0, "AM-7: 100k replay under 5s"); } } - - /// AM-7 scaling floor from GameKernel §5: fold throughput at 100k - /// events must be at least this fraction of throughput at 5k. - /// - /// **Pinned, not tuned** — same rule as `AM6_EVENTS_PER_SEC`. The - /// baseline it was written against is boardgame.io at 0.45–0.66×, - /// degrading to DNF at 100k. Lowering it requires an ADR. - const AM7_SCALING_FLOOR: f64 = 0.9; - - /// The two sizes the spec names. - const AM7_SMALL: usize = 5_000; - const AM7_LARGE: usize = 100_000; - - /// Events applied **per leg** per sample. - /// - /// Sized against the machine's drift, not against timer resolution. - /// At 2,000,000 a sample took ~50 ms, and this machine's throughput - /// wanders by 2.5× over a few seconds (CB-EV-0013 §1) — so a 50 ms - /// sample measures whatever the clock happened to be doing. At 10 M - /// each leg runs ~0.35 s and averages over the drift instead of - /// sampling a point on it. Measured: medians 0.987 / 0.991 / 0.989 - /// across three runs, and 0.989 under 8-way CPU contention. Doubling - /// this to 20 M cost 9 s more per run and did not tighten them. - const AM7_EVENTS_PER_LEG: usize = 10_000_000; - - /// Paired samples per run. Nine rather than AM-6's three because the - /// verdict is a **median**, not a best-of: a median needs enough - /// samples that one excursion cannot move it. - const AM7_SAMPLES: usize = 9; - - /// Fraction of samples that must agree with the median's verdict for - /// the run to be a measurement rather than noise. - /// - /// **This replaced a unanimity guard that was measurably wrong.** The - /// first version declared INDETERMINATE whenever any sample fell on - /// the other side of the floor. Under deliberate 8-way CPU contention - /// the ratio held at a median of 0.971 — the pairing works, and - /// absolute throughput had dropped 4× — but one sample read 0.899, - /// a thousandth under the floor, and the guard turned a good - /// measurement into a failed build. It also fired intermittently - /// inside `mutation-check`, where this test runs straight after a - /// 50-second rebuild. - /// - /// A gate that fails when the machine is busy is a flake, and a flake - /// gets suppressed rather than fixed. Requiring a two-thirds majority - /// keeps the guard's purpose — refusing to read a coin-flip as a - /// verdict — without treating a single outlier as one. - const AM7_AGREEMENT: f64 = 2.0 / 3.0; - - /// Fold `log` once from a fresh state, returning only the time inside - /// the fold loop. - /// - /// **Setup is outside the clock, and that is the first trap here.** - /// The small leg runs 20× more folds than the large one, so it pays - /// 20× more `fresh()` calls. Timing those would penalise the - /// denominator, inflate the ratio, and make the row pass for a reason - /// that has nothing to do with scaling. - fn fold_once(log: &[GroundEvent]) -> std::time::Duration { - let mut state = fresh(42); - let t = Instant::now(); - for event in log { - state.fold(event); - } - let dt = t.elapsed(); - // Keep the fold from being optimised out without paying for a hash - // inside the timed region. - std::hint::black_box(&state); - dt - } - - /// One paired sample: the two legs **interleaved**, ratio taken inside. - /// - /// **Two estimators were wrong before this one, and both looked - /// reasonable.** - /// - /// The first took best-of-5 on each leg independently and divided — - /// the estimator AM-6 uses, correct there because AM-6 is a floor on a - /// single number and the question is "is this machine capable". For a - /// *ratio* it is wrong: the legs are measured at different moments and - /// the noise multiplies instead of cancelling. Five runs of an - /// unchanged binary gave **0.581 to 1.085**. - /// - /// The second ran the legs back to back inside one sample, expecting - /// the load to be common-mode. It was not enough: this machine's - /// absolute throughput wanders between **22 M and 53 M ev/s within a - /// single run**, and a 50 ms leg samples a point on that wander rather - /// than averaging it. Three runs gave medians 1.004 / 0.931 / 0.956 — - /// clustered near the true value but still straddling the floor. - /// - /// So: interleave at *fold* granularity, alternating one large fold - /// against twenty small ones so both legs apply the same number of - /// events, and run long enough that each leg spans the drift instead - /// of sitting inside one excursion of it. - fn paired_ratio(small: &[GroundEvent], large: &[GroundEvent]) -> (f64, f64, f64) { - let per_round = large.len(); - let rounds = AM7_EVENTS_PER_LEG.div_ceil(per_round); - let small_folds = per_round.div_ceil(small.len()); - - let (mut t_small, mut t_large) = (std::time::Duration::ZERO, std::time::Duration::ZERO); - let (mut n_small, mut n_large) = (0usize, 0usize); - for _ in 0..rounds { - for _ in 0..small_folds { - t_small += fold_once(small); - n_small += small.len(); - } - t_large += fold_once(large); - n_large += large.len(); - } - assert!( - t_small.as_secs_f64() > 0.0 && t_large.as_secs_f64() > 0.0, - "AM-7 measured zero elapsed time" - ); - let tp_small = n_small as f64 / t_small.as_secs_f64(); - let tp_large = n_large as f64 / t_large.as_secs_f64(); - (tp_small, tp_large, tp_large / tp_small) - } - - /// Build one growing log of at least `target` events, the same way - /// `replay_100k_events_is_linear_and_fast` does. - fn growing_log(target: usize) -> Vec { - let mut log = Vec::with_capacity(target); - let mut seed = 42u64; - let mut source = fresh(seed); - let mut stalls = 0; - while log.len() < target { - if source.outcome.is_some() { - seed += 1; - source = fresh(seed); - } - if record_round(&mut source, &mut log) == 0 { - stalls += 1; - assert!(stalls < 10, "round produced no events; builder stalled"); - } - } - log - } - - /// AM-7's `scaling >= 0.9x` clause, which was inert for nine passes. - /// - /// `mutation-check.py` said it plainly every run: *"no code computes - /// the ratio of throughput @100k to @5k or compares it to 0.9; - /// Criterion reports both and nothing relates them."* The sibling test - /// above is even named `replay_100k_events_is_linear_and_fast` and - /// checks the two sizes **independently** — it computes both numbers, - /// prints both, and never divides one by the other. - /// - /// **`#[ignore]` for the same reason AM-6 is** (CB-WP-0006 T04): a - /// throughput assertion inside a parallel `cargo test` harness - /// measures contention. A *ratio* of two such timings is worse, not - /// better — the noise multiplies rather than cancels. Run `make am7`. - #[test] - #[ignore = "throughput ratio — invalid under a parallel harness; run `make am7`"] - fn am7_scaling_holds_from_5k_to_100k_events() { - // Positive control on the shape of the measurement. A harness that - // measured the same size twice would report ~1.0 and look - // excellent; one that swapped the legs would report the reciprocal - // and look excellent for the opposite reason. - let (small, large) = (growing_log(AM7_SMALL), growing_log(AM7_LARGE)); - assert!( - large.len() >= 15 * small.len(), - "AM-7 legs are not far enough apart: {} vs {}", - small.len(), - large.len() - ); - - let mut ratios = Vec::with_capacity(AM7_SAMPLES); - for _ in 0..AM7_SAMPLES { - let (tp_small, tp_large, ratio) = paired_ratio(&small, &large); - println!( - " AM-7 sample: {tp_small:.0} ev/s @{AM7_SMALL} → \ - {tp_large:.0} ev/s @{AM7_LARGE} = {ratio:.3}x" - ); - ratios.push(ratio); - } - ratios.sort_by(|a, b| a.partial_cmp(b).expect("no NaN ratios")); - let (worst, median, best) = ( - ratios[0], - ratios[ratios.len() / 2], - ratios[ratios.len() - 1], - ); - println!( - "AM-7 scaling: {worst:.3}x / {median:.3}x / {best:.3}x \ - (worst/median/best of {AM7_SAMPLES}, floor {AM7_SCALING_FLOOR})" - ); - - // The spread is reported, not hidden behind a best-of. The verdict - // is the median, and it counts as a measurement only if a - // two-thirds majority of samples agree with it — see - // AM7_AGREEMENT for the unanimity guard this replaced and why. - let agreeing = ratios - .iter() - .filter(|r| (**r >= AM7_SCALING_FLOOR) == (median >= AM7_SCALING_FLOOR)) - .count(); - let agreement = agreeing as f64 / ratios.len() as f64; - assert!( - agreement >= AM7_AGREEMENT, - "AM-7 INDETERMINATE: only {agreeing} of {} samples agree with \ - the median ({worst:.3}x..{best:.3}x around \ - {AM7_SCALING_FLOOR}). The machine is too noisy for this to be \ - a measurement; do not read the best sample as a pass.", - ratios.len() - ); - assert!( - median >= AM7_SCALING_FLOOR, - "AM-7 UNMET: throughput at {AM7_LARGE} events is {median:.3}x \ - throughput at {AM7_SMALL} events (worst {worst:.3}x, best \ - {best:.3}x), below the {AM7_SCALING_FLOOR} floor. Baseline: \ - boardgame.io 0.45-0.66x, DNF at 100k. Do NOT lower the floor \ - to pass — GameKernel §5 AM-7 is a spec value and lowering it \ - needs an ADR." - ); - } } diff --git a/gates.toml b/gates.toml index 685ecdb..17b3a95 100644 --- a/gates.toml +++ b/gates.toml @@ -19,7 +19,7 @@ # `loop-lint` fails when a target is in neither list. not_control_gates = [ "check", "test", "sim", "bench-test", "size-metrics", "runtime-metrics", - "am6", "am7", "am8", "replay-test", "dep-weight", "self-tests", "env-test", + "am6", "replay-test", "dep-weight", "self-tests", "env-test", ] [[gate]] @@ -62,8 +62,6 @@ caught = [ "K10's first round trip not reproducing", "the bench workload existing twice", "AM-2's expect matching its own passing output (EXPECT-VACUOUS)", - "CB-WP-0015: the two clauses it had reported inert since CB-WP-0005 — AM-7's scaling ratio (a test named `replay_100k_events_is_linear_and_fast` that computed both throughputs, printed both, and never divided one by the other) and AM-8's N=10 (the runner did two). Both now red, 10/14, and the >=10-of-14 prediction met for the first time", - "CB-WP-0015: AM-4a's own mutation, stale since ADR-0008 D3 moved the target 250,000 -> 161,000 in CB-WP-0013. Reported HARNESS-BROKEN and refused to publish a score, which is the harness catching itself. The build-free half of that check is now a --self-test assertion, so it runs in `make all` instead of only on a full run", ] retire_if = "a full pass adds rows without finding anything, twice running — the harness costs real money per run" diff --git a/specs/GameKernel.md b/specs/GameKernel.md index 24661d3..3bd23f8 100644 --- a/specs/GameKernel.md +++ b/specs/GameKernel.md @@ -209,8 +209,8 @@ evidence lands in `evidence/CB-EV-0001-game-kernel.md` with no | ~~AM-4c~~ | M-D2-DEP: own source per third-party 100k lines | — | **WITHDRAWN from the acceptance table 2026-08-01 (CB-WP-0006 T04)** — retained as a reported diagnostic in `make dep-weight`; see §5a | diagnostic | | AM-5 | M-D2-BLD: clean release build of headless workspace | n/a (npm install ~seconds; not comparable) | ≤ 60 s on bnt-lap001, recorded not gated | measured | | AM-6 | M-D3-THR: applied events/s, synthetic workload, same machine | boardgame.io ~1,100–1,900 moves/s (best config, degrading) | **≥ 100,000/s** (stipulated target, ADR-0002) | measured | -| AM-7 | M-D3 scaling: throughput @100k events vs @5k; and snapshot+replay of 100k events | boardgame.io 0.45–0.66× @20–40k, DNF @100k | **≥ 0.9×** (flat), replay of 100k events ≤ 5 s, hash-identical | measured (`make am7`, `make test`) | -| AM-8 | Determinism invariant: N=10 same-seed replays, bit-identical hashes; HashMap-in-state deny lint clean | Rune: enforced by tooling (cited) | zero divergence, lint clean in CI | measured (`make am8`, `make check`) — see §5b | +| AM-7 | M-D3 scaling: throughput @100k events vs @5k; and snapshot+replay of 100k events | boardgame.io 0.45–0.66× @20–40k, DNF @100k | **≥ 0.9×** (flat), replay of 100k events ≤ 5 s, hash-identical | measured | +| AM-8 | Determinism invariant: N=10 same-seed replays, bit-identical hashes; HashMap-in-state deny lint clean | Rune: enforced by tooling (cited) | zero divergence, lint clean in CI | measured (invariant, not a verdict row) | | AM-9 | M-D3-MEM: peak RSS, 100k-event synthetic run | boardgame.io ~100→232 MB @5k→40k (indicative) | ≤ 64 MB, flat with history given snapshot interval | measured (indicative label, same method) | | AM-10 | M-D4-LEAK **(withdrawn 2026-07-31, ADR-0005 §4 — no `cb-*-api` crate exists, so the population is empty; the clippy `HashMap`/`HashSet` deny that stood in for it cites K6 determinism and is now reported as AM-10′)**: foreign types in canonical-interface signatures | boardgame.io: JS-ecosystem-locked | **0** | measured (grep/deny rule) | | AM-11 | M-D4-SWAP **(met 2026-08-01 — `KernelRng` and `LogStore` each drive one shared `conformance()`; CB-WP-0006 T05)**: null + reference impls passing one conformance suite | no candidate has the pattern | RNG and log storage each have ≥2 impls (real + test/null) under one suite | measured (bool) | @@ -257,39 +257,6 @@ Comparisons against the event-sourcing 10⁵–10⁶/s estimate stay **parity** until a local Rust comparator is measured (open follow-up from the adversarial review). -### 5b. Where AM-8's ten runs live, and why not everywhere - -*(CB-WP-0015 T02, 2026-08-02. Tier S. The spec value N=10 is **not** -amended — this records where it is enforced.)* - -For eight passes the runner executed each scenario **twice** (K8) while -this row said ten, and `mutation-check` reported the count inert every -run. Closing it needed an argument about what the extra runs buy, because -"the spec says ten" is not one. - -**A deterministic divergence does not need ten runs.** A seed threaded -wrong or a fold that depends on insertion order diverges on run 2 exactly -as reliably as on run 10. For that class K8's double-run is sufficient and -the other eight are repetitions of an answered question — 47 s per build -across 25 scenarios. - -**A late-onset or probabilistic divergence does.** Measured, on -`gr-r06-round-resolve` with the RNG perturbed only from its fourth -construction onward: `--runs 2` **passes**; `--runs 10` fails with *"run 1 -hash … != run 4 hash … (of 10)"*. That is a real class the double-run -structurally cannot see, and it is deterministic rather than flaky, so it -can be a control rather than a coin flip. - -So both stay, at their own costs: **K8's two runs on every scenario** -(broad, cheap, `make sim`) and **ten runs on one scenario** (deep, ~2 s, -`make am8`). The clause is now measured by that mutation rather than -declared by its author. - -The primary defence against the probabilistic class remains the -`HashMap`/`HashSet` deny lint under K6 — this row's other clause, already -live. The ten runs are defence in depth against that exclusion failing, -which is why one workload's worth is proportionate. - ## 5. Out of scope for this pass Networking/session protocol, WIT/Wasm game boundary, ECS world layer, diff --git a/tools/cb-sim/src/main.rs b/tools/cb-sim/src/main.rs index cfb0996..f08e158 100644 --- a/tools/cb-sim/src/main.rs +++ b/tools/cb-sim/src/main.rs @@ -17,7 +17,7 @@ const REPLAY_DIR: &str = "replays"; fn main() { let argv: Vec = std::env::args().skip(1).collect(); if argv.is_empty() { - eprintln!("usage: cb-sim [--runs ] ..."); + eprintln!("usage: cb-sim ..."); eprintln!(" cb-sim --replay ..."); std::process::exit(64); } @@ -28,28 +28,7 @@ fn main() { if argv[0] == "--replay" { std::process::exit(replay_bundles(&argv[1..])); } - - // AM-8 (CB-WP-0015 T02): the replay count. Defaults to K8's two, so - // `make sim` is unchanged; `make am8` passes 10 on one scenario. - let mut runs = scenario::K8_RUNS; - let mut args = argv; - if args[0] == "--runs" { - if args.len() < 3 { - eprintln!("usage: cb-sim --runs ..."); - std::process::exit(64); - } - runs = match args[1].parse::() { - Ok(n) if n >= 2 => n, - // A --runs that silently fell back to 2 would report an - // N=10 result after doing an N=2 check — the harness-does- - // nothing class this binary's header is about. - _ => { - eprintln!("--runs needs an integer >= 2, got {:?}", args[1]); - std::process::exit(64); - } - }; - args = args.split_off(2); - } + let args = argv; let mut failed = false; let mut passed = 0usize; @@ -74,7 +53,7 @@ fn main() { }; let outcome = match sc.scenario.split('/').next() { - Some("ground") => scenario::run_n::(&sc, runs), + Some("ground") => scenario::run::(&sc), _ => { // A renamed or typo'd prefix would otherwise skip every // scenario while the run still looked clean. diff --git a/tools/mutation-check.py b/tools/mutation-check.py index 401c4e0..06682fd 100644 --- a/tools/mutation-check.py +++ b/tools/mutation-check.py @@ -71,16 +71,6 @@ class Row: self.expect = expect # A row with several stated clauses is red only if EVERY clause has # a mutation that goes red. AM-7 is the reason this exists. - # - # A clause is (name, enforced, why) or (name, enforced, why, - # (verify, mutate, expect)). The clause carries its own `verify` - # because a clause the row's command cannot reach is exactly the - # case this is for. With a mutation the `enforced` flag is - # MEASURED and cross-checked against the declaration; without one - # it is only the author's word, which is what it always was. - # CB-WP-0015 T01 added the fourth field because a hand-maintained - # boolean describing whether an assertion exists is the same shape - # of claim this whole tool was built to stop trusting. self.clauses = clauses or [] @@ -125,15 +115,10 @@ def rows(): "against M-D1-MUT (ADR-0005 §1) — a row that cannot " "fail asserts nothing, however good the reason."), - # The literal here must track ADR-0008 D3's corrected target. It did - # not: this row reported BROKEN on the first full run after - # CB-WP-0013 moved it 250,000 -> 161,000, because no full - # mutation-check had been run in between. Positive control 1 doing - # exactly its job — a stale find-string reported, not skipped. - Row("AM-4a", "third-party LOC, shipped runtime <= 161,000", + Row("AM-4a", "third-party LOC, shipped runtime <= 250,000", verify=py + ["tools/dep-weight.py"], mutate=("tools/dep-weight.py", - '"shipped-runtime": 161_000,', '"shipped-runtime": 1_000,')), + '"shipped-runtime": 250_000,', '"shipped-runtime": 1_000,')), Row("AM-4b", "third-party LOC, dev toolchain <= 350,000", verify=py + ["tools/dep-weight.py"], @@ -203,41 +188,10 @@ def rows(): "RE-EARNED (CB-WP-0006 T06): the probe now replays each " "per-game segment from its own genesis and asserts its " "recorded hash. Folding a segment from the wrong seed fails."), - # The first clause on any row whose `enforced` flag is - # MEASURED rather than declared. It needs its own mutation - # because the row's — tightening the 5 s budget — proves - # the *timing* clause and cannot reach this one. - # - # The mutation makes fold cost grow with the number of - # events already folded, which is precisely boardgame.io's - # measured defect: throughput halving as history doubled. - # Nothing in `GroundState` grows with log length, so this - # is the only way to make the property false — see - # CB-EV-0013 §2. - # - # CONTROL, measured: AM-6's mutation adds a CONSTANT - # per-event cost. It halves throughput (28M -> 15M ev/s) - # and leaves this ratio at 0.999x, green. So AM-7 is not a - # second AM-6 — a constant slowdown is AM-6's to catch and - # a history-proportional one is AM-7's. - ("scaling >= 0.9x", True, - "LIVE (CB-WP-0015 T01): `make am7` interleaves a 5k fold " - "against a 100k fold, takes the ratio inside each sample, " - "and gates the median at 0.9. Measured 0.956-1.068 over " - "three runs; the mutation drives it to 0.751.", - (CARGO + ["test", "--release", "-p", "games-ground", - "--all-features", "am7_scaling", "--", - "--ignored", "--test-threads=1"], - ("games/ground/src/lib.rs", - " fn fold(&mut self, event: &Self::Event) {\n" - " match event {", - " fn fold(&mut self, event: &Self::Event) {\n" - " if let Some(c) = self.solution_deck.last().copied() " - "{ self.solution_deck.push(c); }\n" - " for c in self.solution_deck.iter().step_by(4096) " - "{ std::hint::black_box(c); }\n" - " match event {"), - "AM-7 UNMET")), + ("scaling >= 0.9x", False, + "no code computes the ratio of throughput @100k to @5k or " + "compares it to 0.9; Criterion reports both and nothing " + "relates them."), ]), Row("AM-8", "determinism: same-seed replays bit-identical", @@ -250,31 +204,10 @@ def rows(): "let n = N.fetch_add(1, std::sync::atomic::Ordering::Relaxed); " "Self(rand_chacha::ChaCha12Rng::seed_from_u64(seed.0 + n)) }"), clauses=[ - # MEASURED, and the measurement is the argument for keeping - # N=10 rather than amending the spec down to K8's two. - # - # The row's own mutation perturbs the seed on EVERY RNG - # construction, so it diverges on run 2 and N=2 catches it. - # This one perturbs only from the fourth construction on: - # a late-onset divergence, deterministic rather than flaky. - # Measured on gr-r06 — `--runs 2` PASSES, `--runs 10` fails - # with "run 1 hash ... != run 4 hash ... (of 10)". That is - # a class the double-run structurally cannot see. - ("N=10 same-seed replays", True, - "LIVE (CB-WP-0015 T02): `make am8` runs one scenario ten " - "times against the first hash. Not all 25 — see " - "scenario::run_n for why eight more runs of a deterministic " - "check is not worth 47 s a build.", - (CARGO + ["run", "-q", "-p", "cb-sim", "--", "--runs", "10", - "scenarios/ground/gr-r06-round-resolve.yaml"], - ("crates/cb-kernel/src/rng.rs", - "Self(rand_chacha::ChaCha12Rng::seed_from_u64(seed.0))", - "{ static N: std::sync::atomic::AtomicU64 = " - "std::sync::atomic::AtomicU64::new(0); " - "let n = N.fetch_add(1, std::sync::atomic::Ordering::Relaxed); " - "Self(rand_chacha::ChaCha12Rng::seed_from_u64(" - "seed.0 + u64::from(n >= 3))) }"), - "K8 divergence: run 1 hash")), + ("N=10 same-seed replays", False, + "the runner executes each scenario TWICE (K8), not ten " + "times; the spec says N=10. The double-run is live and the " + "count is not."), ("HashMap deny lint clean", True, "clippy.toml denies HashMap/HashSet and `make check` runs " "with -D warnings"), @@ -336,47 +269,20 @@ def check_row(row): if row.unmutatable: return "unmutatable", row.unmutatable - verdict, detail = _run_mutation(row.verify, row.mutate, row.expect) - - # Clause-level mutations, where a clause carries one. Each is measured - # the same way as the row's own mutation, and the measurement is - # cross-checked against the declared `enforced` flag — a declaration - # that disagrees with its own mutation is a DFD-class defect and must - # not be reported as either verdict. - for clause in row.clauses: - if len(clause) < 4 or clause[3] is None: - continue - name, declared, _why, (verify, mutate, expect) = clause - c_verdict, c_detail = _run_mutation(verify, mutate, expect) - if c_verdict in ("HARNESS-BROKEN", "EXPECT-VACUOUS"): - return c_verdict, f"clause {name!r}: {c_detail}" - measured = c_verdict == "red" - if measured != declared: - return "HARNESS-BROKEN", ( - f"clause {name!r} is declared enforced={declared} but its " - f"mutation measured {c_verdict} — the declaration and the " - f"measurement disagree") - if not measured and verdict == "red": - verdict, detail = "PARTIAL", f"clause {name!r}: {c_detail}" - return verdict, detail - - -def _run_mutation(verify, mutate, expect): - """(verdict, detail) for one mutation. Restores the tree regardless.""" - path = os.path.join(ROOT, mutate[0]) + path = os.path.join(ROOT, row.mutate[0]) original = open(path).read() - old, new = mutate[1], mutate[2] + old, new = row.mutate[1], row.mutate[2] # Positive control 1: the mutation must be applicable at all. A # find-string that no longer matches would otherwise mutate nothing # and report the baseline result as the mutant result. if original.count(old) < 1: return "HARNESS-BROKEN", ( - f"mutation target not found in {mutate[0]}: {old!r}") + f"mutation target not found in {row.mutate[0]}: {old!r}") # Positive control 2: the baseline must be green, or "mutant red" # proves nothing. - base_ok, base_tail, base_out = run(verify) + base_ok, base_tail, base_out = run(row.verify) if not base_ok: return "inconclusive", f"baseline already red: {base_tail}" @@ -385,9 +291,9 @@ def _run_mutation(verify, mutate, expect): # normal report and accept any failure at all — which is how the guard # goes vacuous without anyone noticing. My first attempt on AM-2 did # exactly that. - if expect and expect in base_out: + if row.expect and row.expect in base_out: return "EXPECT-VACUOUS", ( - f"expect string {expect!r} appears in PASSING output, so it " + f"expect string {row.expect!r} appears in PASSING output, so it " f"would accept any failure — the FA guard is inert for this row") try: @@ -399,24 +305,24 @@ def _run_mutation(verify, mutate, expect): if open(path).read() == original: return "HARNESS-BROKEN", "write did not take effect" - mut_ok, mut_tail, mut_out = run(verify) + mut_ok, mut_tail, mut_out = run(row.verify) finally: open(path, "w").write(original) # Positive control 4: restoration must have worked, or every later # row runs against a corrupted tree. if open(path).read() != original: - return "HARNESS-BROKEN", f"failed to restore {mutate[0]}" + return "HARNESS-BROKEN", f"failed to restore {row.mutate[0]}" if mut_ok: return "SURVIVED", "mutant is green — this row asserts nothing" - if expect and expect not in mut_out: + if row.expect and row.expect not in mut_out: # The FA guard. The mutant went red, but not for the reason # claimed — a compile error, a panic elsewhere, an unrelated # assertion. Scoring that as `red` would credit the row with an # assertion it does not have. return "WRONG-REASON", ( - f"mutant failed, but its output does not contain {expect!r} — " + f"mutant failed, but its output does not contain {row.expect!r} — " f"this is not evidence the row is enforced") return "red", mut_tail or "verifier failed as required" @@ -453,13 +359,8 @@ def report(only=None): if detail: for line in _wrap(detail, 66): print(f" {line}") - for clause in r.clauses: - name, enforced, why = clause[0], clause[1], clause[2] - # `red*` marks a clause whose flag was measured by its own - # mutation this run, not asserted by the author. - measured = "*" if len(clause) > 3 and clause[3] else " " - print(f" - " - f"{('red' + measured) if enforced else 'NONE'} " + for name, enforced, why in r.clauses: + print(f" - {'red ' if enforced else 'NONE'} " f"{name}: {why[:60]}") if broken: @@ -523,29 +424,6 @@ def self_test(): check("every mutation target file exists", all(os.path.isfile(os.path.join(ROOT, r.mutate[0])) for r in rs if r.mutate)) - - # CB-WP-0015 T03. `check_row` already reports a stale find-string as - # HARNESS-BROKEN — but only on a full mutation-check, which is - # deliberately not in `make all` because it rebuilds per row. So a - # mutation could rot for passes at a time: AM-4a's did, from the moment - # ADR-0008 D3 moved the shipped-runtime target 250,000 -> 161,000 until - # the next full run. This asks the same question with no builds at all, - # which puts it in `make all` via `self-tests`. - stale_targets = [] - for r in rs: - targets = [r.mutate] if r.mutate else [] - # Clause mutations rot the same way and are checked the same way. - targets += [c[3][1] for c in r.clauses if len(c) > 3 and c[3]] - for relpath, find, _new in targets: - try: - if find not in open(os.path.join(ROOT, relpath)).read(): - stale_targets.append(r.id) - except OSError: - stale_targets.append(r.id) - check("every mutation find-string still matches its source", - not stale_targets, - f"stale: {', '.join(sorted(set(stale_targets)))}" if stale_targets - else "checked without building — the cheap half of check_row") # The control that matters: a mutation whose find-string no longer # matches must be reported BROKEN, not silently skipped. stale = Row("AM-X", "fixture", verify=[sys.executable, "-c", "pass"], @@ -577,25 +455,6 @@ def self_test(): check("a mutant failing for the wrong reason is not scored red", v4 == "WRONG-REASON", v4) - # CB-WP-0015 T01: clause-level mutations. A clause carrying a mutation - # gets its `enforced` flag MEASURED, so the control that matters is - # that a declaration disagreeing with its own measurement is refused - # rather than reported as either verdict — otherwise the fourth field - # would just be decoration on the same hand-maintained boolean. - ok_verify = [sys.executable, "-c", - "import sys; sys.exit(0 if 'ZZC' not in " - "open('Makefile').read() else 5)"] - noop = ("Makefile", "PY := python3", "PY := python3 ") - lying = Row("AM-C", "fixture", verify=ok_verify, - mutate=("Makefile", "PY := python3", "PY := python3 # ZZC"), - clauses=[("a clause that claims more than it can show", True, - "declared enforced, but its mutation changes " - "nothing the verifier looks at", - (ok_verify, noop, None))]) - v5, d5 = check_row(lying) - check("a clause whose declaration contradicts its mutation is refused", - v5 == "HARNESS-BROKEN" and "disagree" in d5, f"{v5}: {d5[:40]}") - # A verifier that is already red must not be scored. dead = Row("AM-Z", "fixture", verify=[sys.executable, "-c", "raise SystemExit(3)"], mutate=("Makefile", "PY := python3", "PY := python3 ")) diff --git a/workplans/CB-WP-0015-the-inert-clauses.md b/workplans/CB-WP-0015-the-inert-clauses.md deleted file mode 100644 index 20cfa82..0000000 --- a/workplans/CB-WP-0015-the-inert-clauses.md +++ /dev/null @@ -1,255 +0,0 @@ ---- -id: CB-WP-0015 -kind: product -title: "The two inert clauses: AM-7 scaling and AM-8 N=10" -status: done ---- - -# Purpose - -``` -structural tier S (acceptance rows measure the product, not how we - work — `bench-test` is in gates.toml's - not_control_gates list, so the M trigger about the - loop's own constraints does not fire) -chaos d4 = 2 → no override -declared tier S -``` - -Declaration 10 of 12. Tier S: one provenance paragraph in the commit -message, no survey and no ADR. Steps 4–5 are not skipped. - -## The two clauses nobody has touched in seven passes - -`tools/mutation-check.py` has carried these two verdicts since CB-WP-0005, -in its own words: - -> **AM-7 `scaling >= 0.9x`** — *"no code computes the ratio of throughput -> @100k to @5k or compares it to 0.9; Criterion reports both and nothing -> relates them."* -> -> **AM-8 `N=10 same-seed replays`** — *"the runner executes each scenario -> TWICE (K8), not ten times; the spec says N=10. The double-run is live -> and the count is not."* - -Both have been reported honestly every run and neither has been worked. -AM-7 moved 1/3 → 2/3 in CB-WP-0006; AM-8 has been 1/2 since it was -instrumented. They are the last two PARTIAL rows in the acceptance table. - -**And the test is named for the property it does not check.** -`replay_100k_events_is_linear_and_fast` folds a log at 10,000 and at -100,000 events, prints a throughput for each, and asserts each is under -five seconds *independently*. The two numbers are computed, printed, and -never compared. At 10,000 the five-second assertion is close to vacuous on -its own. - -This is the same family as the last four passes' central findings, one -step further along: not a claim nobody checked, but **a number measured -and then not used**. The name asserts linearity; the body does not. - -## The risk this pass has to face rather than route around - -AM-7's clause was written against a measured boardgame.io defect — -throughput halving as history doubled. Our aggregate may not be able to -exhibit that defect at all: if GROUND state is bounded and `fold` does not -scan history, replay is O(n) by construction and the ratio is ~1.0 no -matter what. - -**If so, the honest result is not a green row.** It is that AM-7's scaling -clause measures something our design makes structurally impossible to -violate, and a clause that cannot fail is the `unmutatable` verdict this -project already counts *against* itself (ADR-0005 §1). Making it green by -asserting a tautology would be the worse outcome of the two. - -So this task may end in an amendment rather than an assertion. Decide it on -what the mutation says, not on which outcome closes more rows. - -## Task: make AM-7's scaling clause live, or show it cannot be - -```task -id: CB-WP-0015-T01 -status: done -priority: high -``` - -Compute the ratio the spec names — throughput folding a growing log at -100k events against the same at 5k — and assert it against 0.9. - -**The measurement has a known trap, and it is documented in this repo -already.** AM-6's assertion first ran inside `make all` and read 38,753 -ev/s against 341,280 in isolation, a 9× error, because `cargo test` runs -binaries and threads concurrently: *"a throughput assertion inside a -parallel harness measures contention, not throughput."* A **ratio** of two -timings taken under varying contention is worse than a single reading, not -better — the noise does not cancel, it multiplies. Whatever this lands in -must run where AM-6 runs (release, `--test-threads=1`, its own target) or -not be gated at all. - -Required, and the pass fails without them: - -- **The mutation must go red for the stated reason.** Slowing `fold` in - proportion to log length — not by a constant, which AM-6 already - catches — must break the ratio and nothing else. -- **A positive control on the ratio itself.** A harness that measured the - same size twice would report 1.0 and look excellent. Prove the two legs - are different sizes and that the denominator is the small one. -- **State the variance.** One reading of a ratio is not a measurement. - Report the spread across samples, and if the spread crosses 0.9 then - the threshold is not gateable at this sample count — say that instead of - taking the best reading. - -If the ratio is ~1.0 because nothing in the design can make it otherwise, -write that up and amend the row, following the precedent AM-4c set in -`specs/GameKernel.md` §5a — the argument goes in the spec, at tier S, -with the row retained in the denominator. - -**Done 2026-08-02.** `make am7`; AM-7 is now **red, 3/3 clauses**. - -The clause is live and no amendment was needed. Measured ratio **0.987 / -0.991 / 0.989** across three runs against the 0.9 floor, and **0.989 under -eight-way CPU contention** — the design is flat, as the bounded-state -analysis predicted, with a few percent of cache cost on the longer log. - -**Two estimators were wrong before the third, and both looked -reasonable.** Best-of-N per leg then divide — AM-6's estimator, correct for -a floor on one number — gave **0.581 to 1.085** on an unchanged binary. -Running the legs back-to-back inside one sample gave medians 1.004 / -0.931 / 0.956: closer, but a 50 ms leg samples a *point* on this machine's -drift rather than averaging it. Interleaving at fold granularity with 10 M -events a leg makes the drift common-mode, and it divides out. Absolute -throughput still swings **11 M – 55 M ev/s**; the ratio does not. - -**The INDETERMINATE guard was wrong for the same reason the estimator -was.** It demanded unanimity. Under contention the median read 0.971 and -one sample read 0.899 — a thousandth under — and it failed the build; it -also fired intermittently inside `mutation-check`, which runs this test -straight after a 50-second rebuild. It now requires a two-thirds majority -agreeing with the median. A gate that fails when the machine is busy is a -flake, and a flake gets suppressed rather than fixed. - -**The mutation, and the control that matters more.** Making fold cost grow -with history drives the ratio to **0.751** — red, tight, no straddle. -AM-6's *constant*-cost mutation halves throughput (28M → 15M ev/s) and -leaves this ratio at **0.999× — green**. Without that control, "AM-7 goes -red" would have been consistent with AM-7 being a redundant copy of AM-6. - -## Task: settle AM-8's N, at 10 or at 2, with the argument - -```task -id: CB-WP-0015-T02 -status: done -priority: high -``` - -The spec says ten same-seed replays; the runner does two. Close the gap in -whichever direction survives the argument — and the argument is the -deliverable here, not the number. - -**Ten is doing real work only if the failure it hunts is probabilistic.** -A deterministic divergence — a seeded RNG threaded wrong, a fold that -depends on insertion order — fails on run two exactly as reliably as on -run ten. A divergence from ambient nondeterminism — hash iteration order, -pointer-dependent hashing, thread scheduling — appears with some -probability per run, and there ten runs are eight more chances than two. - -So: which does AM-8 exist to catch, and what does the repo already do -about the other? `clippy.toml` denies `HashMap`/`HashSet` and `make check` -runs `-D warnings`, which is AM-8's other clause and already live. If the -lint removes the probabilistic class structurally, ten runs are buying -repetitions of a check that cannot fail — and that costs time on every -`make all`. - -Whichever way it goes, the mutation must still go red: the existing -seed-perturbation mutation is the control, and if raising or lowering N -does not change what that mutation catches, that is itself the finding. - -Do not amend the spec silently. If N becomes 2, GameKernel §5 says so and -says why, in the same shape as §5a. - -**Done 2026-08-02.** `make am8`; AM-8 is now **red, 2/2 clauses**. -`GameKernel.md` §5b records where the ten runs live. - -**N stays at 10, and the measurement is why.** The argument above is -correct as far as it goes — a deterministic divergence shows on run 2 -exactly as reliably as on run 10 — but it is not the whole population. -Perturbing the RNG only from its **fourth** construction onward, on -`gr-r06-round-resolve`: - -| | result | -|---|---| -| `cb-sim --runs 2` | **PASS** | -| `cb-sim --runs 10` | FAIL — *"run 1 hash … != run 4 hash … (of 10)"* | - -A **late-onset** divergence is a real class the double-run structurally -cannot see, and — the part that makes it usable — it is *deterministic, -not probabilistic*. So it can be a control rather than a coin flip, and -the `N=10` clause is now measured by it rather than declared. - -The cost objection stands and is answered by placement, not by amendment: -ten runs across all 25 scenarios would cost **47 s per build** to -re-answer an answered question, so `make sim` keeps K8's double-run over -everything and `make am8` runs ten on one scenario in ~2 s. The primary -defence against the *probabilistic* class is still the `HashMap`/`HashSet` -deny lint — this row's other clause, already live — and the ten runs are -defence in depth against that exclusion failing. - -Also here: `scenario::run_n`, `cb-sim --runs ` (which refuses `n < 2` -rather than falling back to 2 — a flag that silently degraded would report -an N=10 result after an N=2 check), and later runs compared to the **first** -rather than to their predecessor, so a divergence appearing at run 5 and -persisting cannot hide after run 6. - -## Task: evidence, and what the acceptance table now claims - -```task -id: CB-WP-0015-T03 -status: done -priority: high -``` - -`evidence/CB-EV-0013-*.md`. - -Report the mutation table before and after, and be exact about the -denominator — CB-EV-0005 established that a score improved by deleting the -question is not an improvement, and this pass can improve its score two -ways, one of which is deleting a question. - -Also due here: - -- **Quote CB-WP-0014's final cost, not this pass's.** The rule adopted in - CB-EV-0012. CB-WP-0014 closed at **$7.47 / 34 responses** = 0.220. -- **CB-EV-0009's standing prediction is now testable for the first - time.** It predicted a pass opening above the SH-1 hard line would cost - more than 0.123 per response. CB-WP-0014 opened above the line and cost - 0.220. Report it as one data point, and say plainly that one point past - a threshold is not a confirmation — the prediction has a direction and - no control. -- **Whether the acceptance table still has a PARTIAL row**, and if the - answer is no, whether that is because both clauses became live or - because one became a spec amendment. - -**Done 2026-08-02.** -[CB-EV-0013](../evidence/CB-EV-0013-the-inert-clauses.md). `make all` -exits 0. - -- **No PARTIAL rows remain, and neither clause was amended away.** AM-7 - red 3/3, AM-8 red 2/2. **M-D1-MUT 10/14**, measured, and ADR-0005's - standing prediction (≥10 of 14) is **MET for the first time**. -- **The denominator did not move**, which was the live risk. This pass - could have improved its score by deleting a question — amending AM-8's - N down to 2, or splitting AM-7's scaling into a fifteenth row — and both - were considered and refused. The four unenforced rows are the four that - were already unenforceable. -- **The full run found a third defect: AM-4a's mutation was stale**, still - pointing at the 250,000 target ADR-0008 D3 replaced with 161,000. It - reported HARNESS-BROKEN and refused to publish a score. The build-free - half of that check is now a `--self-test` assertion, so `make all` - catches the next one instead of only a full mutation run. -- **`mutation-check` now measures a claim it used to assert.** A clause - may carry its own verify and mutation; then its `enforced` flag is - measured and a declaration disagreeing with its measurement is refused. - Two clauses carry one, and print as `red*`. -- **CB-EV-0009's prediction came due**: CB-WP-0014 opened above the SH-1 - hard line and cost 0.220 $/response against the predicted floor of - 0.123. Unfalsified, **not confirmed** — there is no control, because no - pass has opened below the line since it was made.