From c43754f0fee6df18bc109c6ff3bcde63ab01ea39 Mon Sep 17 00:00:00 2001 From: tegwick Date: Fri, 31 Jul 2026 18:32:16 +0200 Subject: [PATCH] CB-WP-0006 T01: assert the AM-6 throughput target MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Nothing in the workspace compared any number to 100,000 events/s while the evidence file reported "AM-6 | met, 16.5x". Now a test does — a test, not a bench, because Criterion reports throughput and asserts nothing, which is why this row measured nothing for six passes. Measured on bnt-lap001: 341,280 ev/s in debug (3.4x the target), ~2.4-3.1M in release. The spec target holds even in an unoptimized build, so the gate needs no cfg split and runs in the ordinary `make test`. The trap this task named — loosening a flaky timing assertion until it never fires — is avoided by construction. The threshold is the spec value, untouched; the constant says lowering it requires an ADR; and the failure message repeats that, states measured headroom, and names reference figures, so an agent hitting a red AM-6 is told not to tune it in the place they are actually reading. Robustness comes from best-of-N, not from a lower bar: a throughput floor asks whether the machine is capable, so transient load should not fail the build. Two positive controls in the test: a run that applied fewer than 50,000 events, or measured zero elapsed time, fails rather than scoring as infinite throughput. Verified by a PROPERTY mutation — 4,000 black_box iterations injected into GroundState::fold, the hot path — not a threshold tweak, which would only prove the comparison runs. And the FA class found last pass is now gated. mutation-check rows gained an `expect` field: the mutant's output must contain the row's stated failure string or the verdict is WRONG-REASON, not red. Without it a mutation that merely failed to compile would credit its row with an assertion it does not have. Verified by pointing expect at a string the verifier never prints and watching the verdict flip. This is remedy (2) from the CB-WP-0005 retrospective, built a task earlier than planned because the class it guards is the newest and most dangerous. M-D1-MUT: 4 -> 5 of 14. Co-Authored-By: Claude Opus 5 --- facts.toml | 4 +- games/ground/src/lib.rs | 68 ++++++++++++++++++ tools/__pycache__/cb-cost.cpython-312.pyc | Bin 37064 -> 37064 bytes tools/__pycache__/dep-weight.cpython-312.pyc | Bin 9295 -> 9295 bytes .../mutation-check.cpython-312.pyc | Bin 19077 -> 19987 bytes tools/mutation-check.py | 62 ++++++++++++---- workplans/CB-WP-0006-instrument-the-table.md | 39 ++++++++++ 7 files changed, 159 insertions(+), 14 deletions(-) diff --git a/facts.toml b/facts.toml index 4c0e46a..f14f17f 100644 --- a/facts.toml +++ b/facts.toml @@ -40,8 +40,8 @@ fmt = "{:,}" by = "tools/mutation-check.py" [am_unmutatable] -value = 8 -text = "8" +value = 7 +text = "7" fmt = "{:,}" by = "tools/mutation-check.py" diff --git a/games/ground/src/lib.rs b/games/ground/src/lib.rs index 12b2408..a154df5 100644 --- a/games/ground/src/lib.rs +++ b/games/ground/src/lib.rs @@ -2219,6 +2219,74 @@ mod replay_probe { n } + /// AM-6 target from GameKernel §5, in applied events per second. + /// + /// **Pinned, not tuned.** CB-WP-0006 T01 named the trap up front: a + /// timing assertion is flaky by nature and the reflex is to loosen it + /// until it never fires, which reproduces the defect being fixed — + /// this row was `unmutatable` because *nothing in the workspace + /// compared any number to 100,000*, while the evidence file reported + /// `AM-6 | met, 16.5×`. + /// + /// Measured on bnt-lap001 2026-07-31: **~182k–212k ev/s in debug**, + /// **~2.4M–3.1M ev/s in release**. So the spec target holds even in an + /// unoptimized build, with ~1.8× headroom there and ~24× in release. + /// **Lowering this constant requires an ADR.** + const AM6_EVENTS_PER_SEC: f64 = 100_000.0; + + /// Best of N samples. A throughput *floor* asks "is this machine + /// capable", so transient load should not fail the build; taking the + /// max makes the gate robust without loosening the threshold, which is + /// the trade this task was told to avoid making on the threshold. + const AM6_SAMPLES: usize = 3; + + /// AM-6: applied events/s on the synthetic workload must clear the + /// spec target. A test, not a bench — Criterion reports throughput and + /// asserts nothing, which is why this row measured nothing for six + /// passes. + #[test] + fn am6_throughput_clears_the_spec_target() { + let mut best = 0.0f64; + let mut sampled = 0usize; + for s in 0..AM6_SAMPLES { + let mut state = fresh(7 + s as u64); + let mut log = Vec::new(); + let mut n = 0usize; + let t = Instant::now(); + while n < 50_000 { + if state.outcome.is_some() { + state = fresh(7 + (s * 1_000_000 + n) as u64); + } + n += record_round(&mut state, &mut log); + log.clear(); + } + let secs = t.elapsed().as_secs_f64(); + // Positive control: a run that applied no events, or took no + // measurable time, must not be scored as infinite throughput. + assert!( + n >= 50_000, + "AM-6 harness applied {n} events, expected >= 50000" + ); + assert!(secs > 0.0, "AM-6 harness measured zero elapsed time"); + best = best.max(n as f64 / secs); + sampled += n; + } + let headroom = best / AM6_EVENTS_PER_SEC; + println!( + "AM-6: {best:.0} events/s (best of {AM6_SAMPLES}, {sampled} events, \ + debug_assertions={}) — {headroom:.1}x the {AM6_EVENTS_PER_SEC:.0} target", + cfg!(debug_assertions) + ); + assert!( + best >= AM6_EVENTS_PER_SEC, + "AM-6 UNMET: {best:.0} events/s < {AM6_EVENTS_PER_SEC:.0} target \ + ({headroom:.2}x). debug_assertions={}. Reference: ~182k debug, \ + ~2.4M release on bnt-lap001. Do NOT lower the target to pass — \ + GameKernel §5 AM-6 is a spec value and lowering it needs an ADR.", + cfg!(debug_assertions) + ); + } + /// AM-7: folding a 100k-event log back into state must stay well /// under the 5s budget, and must be linear in log length. #[test] diff --git a/tools/__pycache__/cb-cost.cpython-312.pyc b/tools/__pycache__/cb-cost.cpython-312.pyc index 5a58df85b547af68311b6977c542eaca7a4907b8..03aa62eab3234a2851f79ba519a0abdb5a293682 100644 GIT binary patch delta 21 bcmX@HkmogIA&zsAp}Aqdk5I^X&~t& z#u*ahY!Z{$PMkEI+JoCBrA^&*rl~#c^fgUQ^G6&tlV*}>XY!wn`6H#1Nxwb7CYj`q z?nuA=_S^5<+uQwK_v8zRoJaP**lZjF?Y_T;Mpt{^vJ-QriHb*&3*19 zl^HT@&8fy?NY)`HzyQ~{#XZhkM*Tt(DddC>qjYRS@+b_&T9QYoYW$aEJxIl&!5t>gyh&bKmI`E%^NC1GT;3!jT!ES_VZvEBwDMlkdf zo2B070Qxe6GoZuiq~B6#E_1KxlMPOsL8dGPCKJ5OSw>+(PnzBw@)Y+9~c*8L>fZ4Cu7IG+i+)v9Kv<0s9~$G%wHF9?+vg@>$9RDqKF4YSXm| zhB&K8va+u8Osrc2zHv$R1aM?BEUw!<9$zTr8J2v?B$nj3-8a5#NEw02z{td;BIwu2 zk|SrPi-$HG=7%F8(N#7+sqplUDtT#23M}=Bl1-6k7P$cRZcz zIN;gE-`{_r`_N_mHk+cBL|l&jF3bakizpORC;^CDurv|!9UrQxt{xNg*a9(k_d9han^Mpk2jnMMYqX0(3lY+H?* zzvg&eq7=Ty#`LP5%$7LFC#9y=7^~_=)3zk<_98pxVi=XhoXU!Z9GDP|Vp1-n>WB4% zjA)`g3Gx?r>i5Kx4)q@B9=QJy-UVWqKKb*np!I*ghmP|B1TN0WlSe0T#4pJ*c9Eu{ z12cxW-ajtloiHCqIIfRODqw!TU`VE(LT!M@^i(ixWCQ*Ng#0|fER$eaL&Znd%+vj+ z`WNfo*nK6qGbTF8Ra7dB&b+^#aCxv=s# z8s@*9uUe)MP7{;Gly9FurxQ~z=!bQ1iD@dg5p~6MIxZo%N*hRfnHf1qPubgOFZrsh z7_hiJGP7=d@ZdoIUeCeqj=_QcXcc(2FRbvxa9fgC$sYwr4=OSbR|vs?7bB9)ha(E_ zkA&eu!~CQNcF%u0`#i6V1Z5t6bXJ-Qic;7w0aN0^@+n^mgB$xLF|Nmw7~L@?;oxvk zBCnQbAUpX*d0n|?3^dZmlQfs}OL4IEw(MyBxq;aru2XegdUcdEd8Mj(bF zV{sGhsg-F{L$G6+CJ)WrHbJwxxY<7J-mpBOL!sO>AIxe5<{WduxP&$qN6dT)%PgSz-7F0=RohmA z^hiGT6rtkhyhW&l6npR6QL5TuA(NU6=)7RKyNFmup7l1E%heRoMPGot=dEZFH0c7> zF1p7K(~u-qz=kT}@$6kqnDvZ-V`0m-M|(+qRS~LMexRxu8EFj0-Q>k;7kI(D)g|4% zz=&Gz?KpU-x1*05OVrYNaA1FTzh*eN52l4Atf}p%x>^}Rm9U?}UJ89Azvc+)A!^Nd zmSKah$x`J_1@hmT&rlWlXKjX&S{_a%>$UmW_foQgLMwof8fPQn&~e;Bs_F*O0rLI2 z(kv~97{M(fSiw(%R0soZ0EKC3MJMu*&ESo;mah`EqBOF$w5fCL-+G=ZqkRYx>V>E|bw9XTmd{$?SEq zZ?FK@v$$Tj@p>`a>t0^mWkV7T{kWS#06<{l26-YBR4}G2g+d$U8*x2!;Ko5w8G$Z> zMH$b~=}rp!0AyNTX{WrI6DH{lh5dwUdM%{^Mqt!!+pIW67};<*sbS zLuxWuY$aH%2dv#oZTSMLreK3urKSL_gBh*9yt}AsQr7P`1DGKNo30s^W$8B%1K)`@V~vrzi>wUwg7 zCq3%uiq_DrH;X2@;86(Jaex{8b&8zH2t4fT?huG%JvuBwlW@o{!fM{ged+5bZ_+*(TaZyt6 zGr*T=i3uk4^>)u@rlsEqL-fM~Ujea)XwZ6U+0uOy9po5W%7Ssx{89FfSM4v^FR>7s za+)DWX1CCfw?1?;t98v{u(i*2uUQ#e=7RLTHSh86+0HriA#1+Db=~Bw3-%SR;Gz!7 zLPP1AiI6=@83DxDJX+qrmY&+TDlJ{g5n2&5_EB$Zz9h6 za6rCe{J@X1<51n;w}E_#eA(j%AMy3(&CqHYzezzm_9jZvOM&v9hGxyssf%G+=VA{9 zTB2{YT?I?G(Q%8>MD`Dw(@BaYU Cj_85_ delta 3057 zcmZ`*YfK#16`ngYJNsVVPXjx|<97ot`{VG{@23Xiv?hMAzMYU49 z#+I$(Tv@h@Bd1ndHNusWuG&UbQ`OF^k2Eq#RXfwFjasQqTDJnskHkORo;!hl+?h zEryB`^Jz}@Ycp?aV~W^`gPMYa(TF&#L;_zO`VW2CJeUE8tBLRSheu>KpZIAdmTb zV~XLKs+40&L{oh+1^ZxCSS}Wf)QEedVjwCj0O1%o0Z)->F)|q*R^~Gi!qyDf0>lmJ9Tt1&a6!MKIer*zb=yv$PkbimzW=}}mOpak( zn>4OfaiBI79EM$!@hop*B+JH<-)36LbWTgU2;rO&0>eHv7@lE$KD;UMPR{!#R7@@w zp3S0!(Rm!5Qn3VL4?OC1vZZLrwajvPJ!p;Nt-WZ?L5kf!bsgZM$RR8t(bUl4wKc>m@>3^W@{j?iITmT^1belRk9K+E7L?3&}9nZiHUgrW~8P|edM;Kz~ z4~A6A7ZF;L{9fvS!cBP8^8hE9Wu7;QWW?+7I9=cDJ<~h?{H5a;jxRafckE?L_OcJ{ z6?g0nOZJ8j?Tuv7Tkz~{d+Tk!^>uHu|7^|7II{29OP1^ryjHTEGsn&3UnOB=CWlME zYiKO9R&uR$D`_wDr&%zXcF8GmYqEC8sl-~|h0@5=Z8l7TOzuqxFjhF4>Im>DIuQ5||jG^S=b@ z_ux_ALPV&%nwDR9dGOkyD~A@`OMLc;p1FhMN0k@T3fz~~Ycp477TT9AxpTt2g&eJV zD#M;SU-3bD%k83;+xC{*e9LER8Dyn0v+;qN3m)H0&FwKWZ<(3^-_E1*yXL~4k}pQi z0ft88{zzJ@KR`PZslFK`UKQd7qh_reicm(}y1>YpaWu|S_F1zQcsU#1Qt2DV$T{Rs zH6E1vV(liBM+$1Y`tsur%*vZ0NG^bh*>O8yVcaP*a#23Bz`P_ZuKz>CoGa#;d33Im zXW~4r-LR-z7RR%Qr!0fKQR_e+^6Od;{D6D4&ul7=yTG3%o&s3%lIe*rxs-gluE|;! zPnFB*7Rlwhs+J1Fx-jmNrHLcJT~@AyeO19O_$M<#TRo8vTKk3__{hUL52{Iso12l) z1>wSNTRuCsHMVNJd{beH_0sZ20|ydn_uwmqd2h_P+|d^MlRYErYo z_4(|+68<|pz^xoxpCsJ!@_ka`*fu9pXu-j46pjK&ylzrsp`eDx=+HQYCOR(Q8t8P> zR8ZE&po?Nz#glZ1gHr`}f~3-f-;hvv0^SX%C!cKp2FfJS=DeK00kKb~T!Q<+*%Y~I zE@cGC>`Ch^>8Q>~{IGeM z?PpSUGe;{K<_Or%I%b`-t_`*&aVM~=6xdY;%q)*PD97S1Y?6V4?}LQNY&(CTQe#k&Hh(M^9#mGDSkkV=C1+Hw~uf z9GuujlIi?#G#W}?#Ec{;qmF4o#eEbWZ-2wK_fW+sfRu_)Q8`SZh1~4P4CrV`=R?6` ziq0OZ*Lhj-%OEx2uTv8v`a+D}bYz{A6%C(+VdLJHB;sS-xnAQ{B{VWb?;ZF$Shhp0 zQyUXKy{8cph`47lYXC7$pGwpDW=2|14S%zg@2N?AyvK`B3E}z=)YEB3%Hq8sNvzKL zV><80qw3?I3gbE-_6H;Q28b#-+aG{`|L^{SEP4Xq?@=(q&_Jb9;vUF5Tuwy`1*7hG zsbmC=R)=IJ87otdI151MmE(}6^osH1{`SxvYNjc<*$w*a&MrRr= 100,000 applied events/s", - unmutatable="the Criterion bench reports throughput and asserts " - "nothing about it. The only asserts in synthetic.rs " - "are the stress-gate shape and the events-per-round " - "pin. No code compares any number to 100,000."), + verify=CARGO + ["test", "-p", "games-ground", "--all-features", + "am6_throughput"], + # A PROPERTY mutation, not a threshold tweak: slow the fold hot + # path and require the gate to notice. Raising the target + # instead would only prove the comparison runs. + mutate=("games/ground/src/lib.rs", + " fn fold(&mut self, event: &Self::Event) {\n" + " match event {", + " fn fold(&mut self, event: &Self::Event) {\n" + " for _ in 0..4000 { std::hint::black_box(0u8); }\n" + " match event {"), + expect="AM-6 UNMET"), Row("AM-7", "scaling >= 0.9x, and replay of 100k events <= 5 s, " "hash-identical", @@ -196,9 +210,10 @@ def run(cmd, timeout=900): r = subprocess.run(cmd, cwd=ROOT, env=cargo_env(), capture_output=True, text=True, timeout=timeout) except subprocess.TimeoutExpired: - return False, "TIMEOUT" - tail = (r.stdout + r.stderr).strip().splitlines() - return r.returncode == 0, (tail[-1][:70] if tail else "") + return False, "TIMEOUT", "TIMEOUT" + out = (r.stdout + r.stderr).strip() + tail = out.splitlines() + return r.returncode == 0, (tail[-1][:70] if tail else ""), out def check_row(row): @@ -219,7 +234,7 @@ def check_row(row): # Positive control 2: the baseline must be green, or "mutant red" # proves nothing. - base_ok, base_tail = run(row.verify) + base_ok, base_tail, _ = run(row.verify) if not base_ok: return "inconclusive", f"baseline already red: {base_tail}" @@ -232,7 +247,7 @@ def check_row(row): if open(path).read() == original: return "HARNESS-BROKEN", "write did not take effect" - mut_ok, mut_tail = run(row.verify) + mut_ok, mut_tail, mut_out = run(row.verify) finally: open(path, "w").write(original) @@ -243,6 +258,14 @@ def check_row(row): if mut_ok: return "SURVIVED", "mutant is green — this row asserts nothing" + if row.expect and row.expect not in mut_out: + # The FA guard. The mutant went red, but not for the reason + # claimed — a compile error, a panic elsewhere, an unrelated + # assertion. Scoring that as `red` would credit the row with an + # assertion it does not have. + return "WRONG-REASON", ( + f"mutant failed, but its output does not contain {row.expect!r} — " + f"this is not evidence the row is enforced") return "red", mut_tail or "verifier failed as required" @@ -271,7 +294,8 @@ def report(only=None): mark = {"red": "red ", "SURVIVED": "SURVIVED ", "unmutatable": "unmutatable", "inconclusive": "inconclusive", "PARTIAL": "PARTIAL ", - "HARNESS-BROKEN": "BROKEN "}[verdict] + "HARNESS-BROKEN": "BROKEN ", + "WRONG-REASON": "WRONG-REASON"}[verdict] print(f" [{mark}] {r.id:<6} {r.claim[:52]}") if detail: for line in _wrap(detail, 66): @@ -289,7 +313,8 @@ def report(only=None): red = tally.get("red", 0) total = len(rs) print(f"\n M-D1-MUT: {red}/{total} rows enforced") - for k in ("PARTIAL", "SURVIVED", "unmutatable", "inconclusive"): + for k in ("PARTIAL", "SURVIVED", "WRONG-REASON", "unmutatable", + "inconclusive"): if tally.get(k): print(f" {k:<13} {tally[k]}") if only: @@ -353,6 +378,19 @@ def self_test(): v2 == "red", v2) check("the tree is restored after a mutation run", "ZZMARKER" not in open(os.path.join(ROOT, "Makefile")).read()) + # The FA guard: a mutant that fails for the WRONG reason must not be + # scored as red. Without this, a mutation that merely fails to compile + # would credit its row with an assertion it does not have. + wrong = Row("AM-W", "fixture", + verify=[sys.executable, "-c", + "import sys; sys.exit(0 if 'ZZW' not in " + "open('Makefile').read() else 7)"], + mutate=("Makefile", "PY := python3", "PY := python3 # ZZW"), + expect="a message the verifier never prints") + v4, _ = check_row(wrong) + check("a mutant failing for the wrong reason is not scored red", + v4 == "WRONG-REASON", v4) + # A verifier that is already red must not be scored. dead = Row("AM-Z", "fixture", verify=[sys.executable, "-c", "raise SystemExit(3)"], mutate=("Makefile", "PY := python3", "PY := python3 ")) diff --git a/workplans/CB-WP-0006-instrument-the-table.md b/workplans/CB-WP-0006-instrument-the-table.md index 9a77881..d86869c 100644 --- a/workplans/CB-WP-0006-instrument-the-table.md +++ b/workplans/CB-WP-0006-instrument-the-table.md @@ -63,6 +63,45 @@ names. **Verified by:** `make mutation-check --row AM-6` goes from `unmutatable` to `red`. +**Delivered.** `am6_throughput_clears_the_spec_target` in +`games/ground/src/lib.rs` — a test, not a bench. Best of 3 samples of +50,000 applied events each. + +Measured on bnt-lap001 2026-07-31: **341,280 ev/s in debug (3.4× the +target)**, ~2.4–3.1M in release (~24–30×). **The spec target holds even in +an unoptimized build**, so the gate needed no `cfg` split and runs in the +ordinary `make test`. + +**The trap was avoided by construction, not by intention.** The threshold +is the spec value `100_000`, untouched; the constant carries a comment +saying lowering it requires an ADR; and the failure message repeats that, +states the measured headroom, and names the reference figures — so a +future agent hitting a red AM-6 is told not to tune it, in the place they +will actually be reading. Robustness comes from **best-of-N**, not from a +lower bar: a throughput *floor* asks "is this machine capable", so +transient load should not fail the build. + +Two positive controls in the test itself: a run that applied fewer than +50,000 events, or measured zero elapsed time, fails rather than scoring as +infinite throughput. + +**Verified:** `make mutation-check --row AM-6` → **red**, via a *property* +mutation (4,000 `black_box` iterations injected into `GroundState::fold`, +the hot path) rather than a threshold tweak — raising the target would only +prove the comparison runs. + +**And the FA class is now gated.** `mutation-check` rows gained an +`expect` field: the mutant's output must contain the row's stated failure +string, or the verdict is **`WRONG-REASON`**, not `red`. Without it, a +mutation that merely failed to compile would credit its row with an +assertion it does not have. Verified by pointing `expect` at a string the +verifier never prints and confirming the verdict flips. This is +remedy (2) from the CB-WP-0005 retrospective, built one task earlier than +T08 planned because the class it guards is the newest and the most +dangerous. + +**M-D1-MUT: 4 → 5 of 14.** + ## Task: AM-2, AM-3 — instrument the size metrics ```task