diff --git a/games/ground/Cargo.toml b/games/ground/Cargo.toml index 9d0768b..1f8e627 100644 --- a/games/ground/Cargo.toml +++ b/games/ground/Cargo.toml @@ -27,5 +27,10 @@ serde_json.workspace = true name = "synthetic" harness = false +# CB-WP-0025 T04 / ADR-0013 D7: the node cost the spec quotes. +[[bench]] +name = "search" +harness = false + [lints] workspace = true diff --git a/games/ground/benches/search.rs b/games/ground/benches/search.rs new file mode 100644 index 0000000..75c7fd5 --- /dev/null +++ b/games/ground/benches/search.rs @@ -0,0 +1,128 @@ +//! CB-WP-0025 T04 — what one search node actually costs. +//! +//! **ADR-0013 D7 exists because two measurements disagreed by 5×.** The +//! survey published 112–161 µs/node from a timer that bracketed two +//! `setup`s and a whole greedy game (C1). The author's re-measurement said +//! 3.0–4.1 µs with `Instant::now()` around each call; the adversarial +//! reviewer's isolation said 15.6–20.4 µs. Both agreed the published +//! figure was wrong by 1–2 orders and neither established which +//! replacement was right. +//! +//! So the spec quotes **this** and nothing else. `criterion` handles the +//! things hand-rolled timing gets wrong here: per-call clock overhead +//! against a ~microsecond subject, warm-up, and run-to-run variance — +//! which is what let the survey's figure move 161 → 112 between two runs +//! of the same unmodified binary. +//! +//! Two subjects, because a search node is not one call: +//! +//! * `legal_commands` — enumerating a seat's options; +//! * `validate + fold` — taking one branch, which any search does per +//! child and which the survey never separated out. + +use cb_game_runtime::{ScenarioGame, Setup}; +use cb_kernel::{Actor, Aggregate, PlayerId}; +use criterion::{criterion_group, criterion_main, BatchSize, Criterion}; +use games_ground::bot::{legal_commands, play, GreedyPolicy, Policy}; +use games_ground::GroundState; +use std::collections::BTreeMap; + +fn setup(players: u8, seed: u64) -> GroundState { + GroundState::setup( + &Setup { + players, + preset: format!("standard-{players}p"), + patch: BTreeMap::new(), + }, + seed, + ) + .expect("preset") +} + +/// A **mid-game state at a real decision point** for `seat`. +/// +/// Not a fresh deal: at deal time most branches do not exist yet, and a +/// node cost taken there would flatter any search proposal. +/// +/// **And not a fixed step count either.** The first version stopped at +/// step 20 for every seat count, which put 2p and 4p in a state where +/// seat 0 had *no* legal commands at all — so the benchmark reported +/// ~120 ns (the cost of returning an empty `Vec`) and silently skipped +/// `validate_fold` because there was nothing to validate. A fixture that +/// measures the empty case and calls it a node cost is the same defect +/// class this whole pass exists to correct, one layer down. +/// +/// So: advance until the seat genuinely has a choice, and assert it. +fn midgame(players: u8, seed: u64, seat: PlayerId) -> GroundState { + let mut ps: Vec> = (0..players) + .map(|_| Box::new(GreedyPolicy) as Box) + .collect(); + let game = play(setup(players, seed), &mut ps).expect("a complete game"); + let mut state = setup(players, seed); + let mut best: Option = None; + for (i, (actor, cmd)) in game.steps.iter().enumerate() { + // Past the opening, take the first state where the seat has a real + // branch. `> 1` rather than `> 0`: a forced move is not a node. + if i >= 8 && legal_commands(&state, seat).len() > 1 { + best = Some(state.clone()); + break; + } + if let Ok(events) = state.validate(*actor, cmd) { + for e in &events { + state.fold(e); + } + } + } + let state = best.expect("a mid-game state where the seat has a choice"); + assert!( + legal_commands(&state, seat).len() > 1, + "benchmark fixture has no branch to measure — it would time the empty case" + ); + state +} + +fn bench(c: &mut Criterion) { + for players in [2u8, 3, 4] { + let seat = PlayerId(0); + let state = midgame(players, 7, seat); + let width = legal_commands(&state, seat).len(); + println!(" fixture {players}p: {width} legal commands at the measured node"); + + c.bench_function(&format!("legal_commands/{players}p"), |b| { + b.iter(|| std::hint::black_box(legal_commands(&state, seat))) + }); + + // A search must COPY the state per branch (or undo, which we do + // not have). `iter_batched` excludes setup from the timing, so + // without this the budget would rest on an unmeasured span — + // which is the exact mistake C1 caught in the survey. + c.bench_function(&format!("clone/{players}p"), |b| { + b.iter(|| std::hint::black_box(state.clone())) + }); + + // One branch taken: what a search pays per CHILD, on top of + // enumeration. The survey folded this into "us/node" without + // separating it, and a search's real cost is enumeration once plus + // this per child. + let legal = legal_commands(&state, seat); + if let Some(cmd) = legal.first() { + c.bench_function(&format!("validate_fold/{players}p"), |b| { + b.iter_batched( + || state.clone(), + |mut s| { + if let Ok(events) = s.validate(Actor::Player(seat), cmd) { + for e in &events { + s.fold(e); + } + } + std::hint::black_box(s) + }, + BatchSize::SmallInput, + ) + }); + } + } +} + +criterion_group!(benches, bench); +criterion_main!(benches); diff --git a/specs/RetrospectiveAnalysis.md b/specs/RetrospectiveAnalysis.md new file mode 100644 index 0000000..3fdee9b --- /dev/null +++ b/specs/RetrospectiveAnalysis.md @@ -0,0 +1,196 @@ +# RetrospectiveAnalysis — was this deal winnable, and how hard is the game + +v1.0 — CB-WP-0025 T04, 2026-08-05. Normative. Implements +[ADR-0013](../decisions/ADR-0013-could-we-have-won.md). Admissibility of +anything this produces is governed by +[GameDesign.md](GameDesign.md) §1. + +**Two capabilities, one machine.** *Was this deal winnable?* is a search +over a finished game. *How hard is the game?* is that search run over many +deals and counted — **not** a bot's win rate (§4.1). + +--- + +## 1. The question, and its name + +> **"Given the deal as it actually was, was there a line of play that +> reached the threshold?"** + +**Never labelled "how you should have played."** The distinction is the +honest content of the feature: the tool answers a question about the +*deal*, and a label promising advice about the *player* turns a true +answer into a false lesson. + +**Strategy fusion does not apply and must not be invoked as an objection.** +Fusion (Frank, Basin & Matsubara 1998) is a defect of aggregating over +determinizations to choose a move. After the game there is **one world** — +the deal is known — so a line found in it is executable in the only world +there is. This is why the affordable option is also the honest one. + +## 2. The witness + +A witness is a sequence of joint selections that, replayed from the +recorded initial state, ends with `group_success == true`. + +### 2.1 It must replay — hard gate, not a metric + +> **100% of emitted witnesses replay through the existing scenario runner +> and end in `group_success`.** + +Not a target: a **gate**. A witness that does not replay asserts the +opposite of the truth to a player who just lost, which is worse than +emitting nothing. + +### 2.2 Every move carries its information dependence + +ADR-0013 D2. Each move in a witness is marked: + +| mark | meaning | +|---|---| +| `visible` | everything the move depended on was in the acting seat's projection at that point — the Problem face-up, the Solution in that seat's own hand | +| `hidden` | it was not | + +Computed from `project()`, which already exists and whose hiding rules are +already asserted by `games_ground::view`. + +**This replaces the structural boundary the survey wanted.** Searching a +`GroundView` is not implementable — a view cannot `fold` events — so the +guarantee moved from *the search cannot see it* to *the answer says which +moves needed it*. A witness reads: + +> *"This deal was winnable. Two of these six moves needed a card you had +> no way to know was coming."* + +**Falsifier:** a witness whose moves are all `visible` but which no seat +could have chosen means the marking is wrong. A test constructs that case. + +### 2.3 Wording when nothing is found + +> **"No winning line found in the last K rounds."** + +**Never "unwinnable".** A bounded search cannot establish unwinnability, +and this sentence is what the player reads. + +## 3. The bound + +**Exhaustive over the last `K` rounds**, table treated as one co-operative +agent choosing joint selections. `K = 2` by default. + +Bounded in **rounds**, not nodes: *"winnable from round 4"* means something +to a player; *"winnable within 100,000 nodes"* does not. A node budget is a +secondary cut that aborts with a stated reason so a wide table cannot hang +the page. + +### 3.1 Measured cost, and what it permits + +`cargo bench -p games-ground --bench search` — the single source for these +numbers (ADR-0013 D7). Mid-game states at real decision points: + +| seats | branch width | `legal_commands` | `clone` | `validate+fold` | +|---|---:|---:|---:|---:| +| 2 | 5 | 4.06 µs | 378 ns | 696 ns | +| 3 | 8 | 4.13 µs | 432 ns | 508 ns | +| 4 | 11 | 4.76 µs | 639 ns | **3.76 µs** | + +**Per-child cost is not uniform** — `validate+fold` ranges 0.5–3.8 µs +depending on which command is taken, because some resolve cascades and +some do not. **Budgets use the upper end**, so ~5 µs per child +(clone + validate + fold). + +Joint branching over the last two rounds, from the measured per-seat +widths: + +| seats | joint / 2 rounds | at ~5 µs/child | +|---|---:|---:| +| 2 | ~5×10² | negligible | +| 3 | ~1.6×10⁵ | **~0.8 s** | +| 4 | ~5.7×10⁵ | **~2.9 s** | + +**So `K = 2` is affordable at two, three and four seats, and is not at +five or six** — joint branching there exceeds 10⁶ per round. Five and six +seats require a smaller `K`, and the tool must reduce it and **say that it +did** rather than silently searching less. + +**The published 112–161 µs/node figure is withdrawn** (CB-RES-0008 §1.2, +challenge C1) and must not be quoted from anywhere. + +## 4. Difficulty + +### 4.1 A bot's win rate is not a difficulty + +**Normative prohibition**, because this project already made the error and +nearly exported it: + +> A win rate from a single policy **may not be reported as a difficulty**. + +Measured, on identical deals: `GreedyPolicy` wins **100%** at five and six +seats where a `FirstLegal` policy — take `legal[0]`, no heuristic — wins +**0%**; at two seats `FirstLegal` (77.5%) *beats* greedy (66.0%). Two +unsophisticated agents span the entire range. + +And a measure that improves when the *measurer* improves is not measuring +the subject: a better bot would make the game "easier" with no rule +changing. + +### 4.2 What is reported instead + +> **Winnable fraction** — over N deals at a seat count, the proportion in +> which the search finds a winning line within its bound. + +A property of the **deal distribution and the threshold**, which is what +`ground-game` tunes. Ships as a table, never one number: + +| column | what it is | +|---|---| +| winnable fraction | can the deal be won at all — bounded, `K` stated | +| reference-policy win rate | what a **named** policy achieves | +| skill gap | the difference — how much play has to supply | + +**It is a lower bound and must be labelled one.** A `K`-round search cannot +see a line that required round 1, so the figure is +**"winnable-from-round-(6−K)"**, never "winnable". + +**Every rate carries its policy, N, seed range and K in the number's +name**, not in a footnote — GameDesign §1.2, and the reason the withdrawn +finding was inadmissible. + +### 4.3 Resolution — the number that makes it usable + +> The smallest threshold change the measurement can distinguish, with its +> N. + +*"We can tell a threshold of 5 from 7 but not 7 from 8"* is more useful to +`ground-game` than any rate with no error bar, and it is what makes the +figure a tuning instrument rather than a statistic. + +## 5. The instruments must be able to fail + +ADR-0013 D5, and GameDesign §1.3. Before **any** figure from these tools is +quoted anywhere: + +- **positive controls** — a deal constructed to be unwinnable returns + none; a deal constructed to be winnable returns a witness that replays; +- **`--self-test`**, wired into `make self-tests` like every other + reporting tool; +- **one command regenerates the figure** (`make difficulty`); +- **the policy panel is plural** — at least `greedy`, `random` and + `first-legal`. The spread between them is the finding §4.1 rests on, and + reporting one policy would restore the error. + +**`games/ground/examples/difficulty-baseline.rs` currently satisfies none +of the first three** and is inadmissible until it does. It has no +assertions, no self-test, and no `make` target — nothing can turn it red, +which under CB-WP-0022 T05's `role` distinction makes it a `default` +artifact wearing a `counterexample` label. + +## 6. Falsifiers for this spec + +- **§2.2 fails** if a witness is emitted whose moves are all `visible` but + which no seat could have chosen. Then the marking must be derived from + the search rather than checked after it. +- **§4.2 fails** if the winnable fraction turns out to be ~100% or ~0% at + every seat count and threshold — it would then have no resolution (§4.3) + and be as useless as the bot rate it replaced. +- **§3 fails** if `K = 2` proves unaffordable in practice at four seats; + the measured 2.9 s is a projection from branch widths, not a timing of + the real search. diff --git a/workplans/CB-WP-0025-could-we-have-won.md b/workplans/CB-WP-0025-could-we-have-won.md index cec037a..1e9741c 100644 --- a/workplans/CB-WP-0025-could-we-have-won.md +++ b/workplans/CB-WP-0025-could-we-have-won.md @@ -127,34 +127,34 @@ per dimension — a number or a reproducible comparison, not an impression. [CB-RES-0008](../research/CB-RES-0008-could-we-have-won.md), with a runnable baseline (`games/ground/examples/difficulty-baseline.rs`). -**The baseline produced a finding before any solver exists, and it is the -biggest thing in this pass.** A greedy bot wins **200 of 200** games at -five and six seats — with a median margin of +3 and 11.8–12.0 points -available against a threshold of 9. The curve is 66% / 82.5% / 95% / 100% -/ 100% across 2/3/4/5/6 seats. +> **Everything this record originally claimed was withdrawn by T02 the +> same day.** Kept as a pointer rather than rewritten, because a claim +> retracted silently is how three earlier wrong premises survived +> (ADR-0012 D5). -The row-level table shows why: available points go **6 / 9 / 12** against -thresholds **5 / 7 / 9**, so the ratio *rises* with seat count (1.20 → -1.29 → 1.33) while the table also gains actions per round. **Three -multipliers pointing the same way.** +**Claimed:** a greedy bot wins 200/200 at five and six seats, so the game +is too easy there; the points-to-threshold ratio rises with seat count and +explains the curve; `legal_commands` costs 112–161 µs, so exhaustive +search is out; and all of this explains the maintainer's report. -**It also explains the maintainer's report without a solver.** *"I felt it -was too easy but then we lost"* — 66% at two seats is a real game; 100% at -six is not. Both halves are true of different seat counts. +**Withdrawn:** the win rate is `GreedyPolicy`'s, not the game's (C4 — +`FirstLegal` scores 0% on the same deals); the ratio is identical at 3p +and 4p, which differ by 12.5 points (C2); the cost figure was wrong by +30–50× and exhaustive search is affordable (C1, C6); and the maintainer's +losses were 3-player games on the pre-ruling deal (C5). -**Cost measured, and it rules out the obvious approach.** Branching is -small (mean 4.7–9.1) but `legal_commands` costs **112–161 µs** per call, -because it filters candidates through full `validate`. Exhaustive search -is out at every seat count; ~10⁴–10⁵ nodes is 1.4–14 s, which is the -budget the ADR must design inside. +**What survives:** the harness exists and runs, the 6/9/12 arithmetic is +right against `Problems.csv`, and the branching widths (4.7–9.1) hold. See +[CB-RES-0008](../research/CB-RES-0008-could-we-have-won.md) for the +corrected text and +[the response](../history/260805-could-we-have-won-response.md) for the +full accounting. -**Prior art names the trap.** Determinized search (PIMC) suffers *strategy -fusion* — Frank, Basin & Matsubara 1998 — where the search picks different -actions in states a real player cannot tell apart. A witness built that -way may require knowing what was on top of the deck. **And it would still -replay green**, so the checkability benchmark does not catch it. Honesty -and checkability are different properties, stated so T03 cannot conflate -them. +**Prior art named the trap** — determinized search suffers *strategy +fusion* (Frank, Basin & Matsubara 1998) — **and T03 then established it +does not apply here.** After the game there is one world, so a line found +in it is executable in it. Fusion is an obstacle to a *playing* engine, +which this is not. ## Task: adversarial review @@ -189,47 +189,27 @@ Tier L requires it. Exactly one round: challenge, then response, trail in [response](../history/260805-could-we-have-won-response.md). Separate agent, as in CB-WP-0022. -**Six of seven conceded, and the survey's headline finding is withdrawn — -not softened.** +**Six of seven conceded. The survey's headline finding is WITHDRAWN.** -**C4 is the one that kills it, and the reviewer ranked it fourth.** -Measured: a `FirstLegal` policy — take `legal[0]`, no heuristic — scores -**0% at five and six seats** where greedy scores 100%, and **77.5% at two -seats** where greedy scores 66%. **Two unsophisticated agents span the -entire range at the same seat count**, so *"the game is too easy at 5–6 -seats"* is a statement about `GreedyPolicy`, not about GROUND. Their -offered rescue — greedy hits the 12-point ceiling every time, so it is a -rules claim — dies on the same data. +**C4 kills it and the reviewer ranked it fourth.** A `FirstLegal` policy — +`legal[0]`, no heuristic — scores **0% at five and six seats** where greedy +scores 100%, and **77.5% at two** where greedy scores 66%. Two +unsophisticated agents span the whole range, so *"too easy at 5–6 seats"* +was about `GreedyPolicy`, not GROUND. -**C1: the per-node cost was wrong by 30–50×.** The timer bracketed two -`setup`s, a whole greedy game and a validate+fold replay, then divided by -player decisions. **The tell was in my own published output**: the figure -*fell* as branching *rose*, which no per-enumeration cost can do. -Re-measured at **3.0–4.1 µs**; the reviewer got 15.6–20.4 by a different -isolation and **that discrepancy is unsettled** — T04 benchmarks it. - -**C6: exhaustive search is not out**, which changes T03's premise. With -C1's correction the budget is ~10⁵–10⁶ nodes and bounded endgame search -fits — so the ADR cannot open with *"exhaustive is impossible, therefore -determinized sampling"*, especially since sampling carries strategy fusion -that exhaustive search does not. - -**C3: the finding failed the admissibility rule this project wrote nine -hours earlier** — sums where GROUND-WP-0004 T02 requires per-priority -rows, and a harness with no assertions, no `--self-test` and no `make` -target, so nothing can turn it red. A `default` artifact wearing a -`counterexample` label. - -**C2: the ratio explains nothing** — 3p and 4p share deal, threshold and -ratio, and differ by 12.5 points of win rate. **C5: the "explains the -maintainer's report" claim is contradicted by `lib.rs:2487`**, which -records his losses as 3-player on the pre-ruling deal, unwinnable at 6 -against 7. +**C1**: the node cost was wrong by 30–50× — the timer bracketed whole +games — and **the tell was in my own output**, falling as branching rose. +**C6**: exhaustive search is *not* out, which changes T03's premise. +**C3**: the finding failed the admissibility rule this project wrote nine +hours earlier. **C2**: 3p and 4p share deal, threshold and ratio and differ +by 12.5 points. **C5**: `lib.rs:2487` contradicts the "explains the +maintainer" claim — his losses were 3-player on the pre-ruling deal, +unwinnable at 6 against 7. **T06 was pointed at GROUND-WP-0005, which is blocked on this number.** -Sending it would have invited threshold changes on one bot's behaviour — -the fifth wrong premise to reach ground-game. **Both tier-L reviews in -this project have now caught a false headline that every gate passed.** +Sending it would have been the fifth wrong premise to reach ground-game. +**Both tier-L reviews here have now caught a false headline that every +gate passed.** ## Task: decide @@ -277,24 +257,21 @@ moved the ground under both. property of the deal distribution and the threshold, and is what GROUND-WP-0005 actually needs. **The bot rate never was.** -**D2** searches `GroundState` (the survey's view-only boundary is not -implementable — a view cannot fold events) and moves the guarantee to a -checkable per-move `visible`/`hidden` marking computed from `project()`. -A witness reads *"you could have won, but two of these six moves needed a -card you had no way to know was coming."* **D3** is bounded exhaustive -over the last K rounds — measured affordable at 2–4 seats once C6 -corrected the premise — with normative wording: *"no winning line found in -the last K rounds"*, never *"unwinnable"*. **D5** makes the harness an -instrument before any figure is quoted (C3). **D6**: no new crate, no -port — **the L declaration was an over-declaration and is recorded as -one**. **D7**: the node cost is disputed 5× and T04 must benchmark it; -neither figure may be cited, including by the ADR. +**D2** searches `GroundState` — the survey's view-only boundary is not +implementable, since a view cannot fold events — and moves the guarantee to +a checkable per-move `visible`/`hidden` marking from `project()`. **D3** +bounded exhaustive over the last K rounds, with *"no winning line found in +the last K rounds"* normative, never *"unwinnable"*. **D5** makes the +harness an instrument before any figure is quoted (C3). **D6** no new +crate and no port — **the L declaration was an over-declaration, recorded +as one**. **D7** the node cost is disputed 5×; T04 benchmarks it and +neither existing figure may be cited. ## Task: specify ```task id: CB-WP-0025-T04 -status: todo +status: done priority: high state_hub_task_id: "7c3ca25d-0001-44ef-a07d-0ab2d51b09a5" ``` @@ -320,6 +297,37 @@ this produces ships a runnable reproduction and a row-level table** — never a summed figure. A difficulty number is arithmetic, and it is exactly the kind that has already gone wrong twice. +**Done 2026-08-05.** +[specs/RetrospectiveAnalysis.md](../specs/RetrospectiveAnalysis.md) v1.0, +and `games/ground/benches/search.rs` for D7's disputed number. + +**The benchmark's own first fixture was defective — the same defect class, +one layer down.** A fixed stop at step 20 put 2p and 4p where seat 0 had +**no legal commands**, so it timed an empty `Vec` (~120 ns) and silently +skipped `validate_fold`. It now advances to a real branch **and asserts +it**. + +Measured at real decision points (table in +[RetrospectiveAnalysis §3.1](../specs/RetrospectiveAnalysis.md)): +`legal_commands` 4.06–4.76 µs, `clone` 378–639 ns, `validate+fold` +**0.5–3.8 µs**. + +**`clone` is in there because a search must copy state per branch**, and +`iter_batched` excludes setup from timing — leaving the budget resting on +an unmeasured span, which is precisely C1's mistake. **Per-child cost is +not uniform**: `validate+fold` ranges 0.5–3.8 µs by command, so budgets +use the upper end (~5 µs/child). + +**That settles D3's affordability with real numbers**: joint branching +over the last two rounds is ~5×10² / 1.6×10⁵ / 5.7×10⁵ at 2/3/4 seats → +negligible / **0.8 s** / **2.9 s**. `K = 2` holds at two to four seats and +**does not at five or six**, where the tool must reduce `K` and *say so* +rather than silently search less. + +**§4.1 is a normative prohibition**, not a preference: a single-policy win +rate may not be reported as a difficulty. The spec carries the measured +reason — greedy 100% vs first-legal 0% on identical deals. + ## Task: build the witness ```task