diff --git a/games/ground/examples/difficulty-baseline.rs b/games/ground/examples/difficulty-baseline.rs new file mode 100644 index 0000000..11c6a12 --- /dev/null +++ b/games/ground/examples/difficulty-baseline.rs @@ -0,0 +1,166 @@ +//! CB-RES-0008's runnable baseline (CB-WP-0025 T01). +//! +//! Two numbers the survey needs and cannot cite from anyone else, because +//! they are about **our** game on **our** machine: +//! +//! 1. **What the bots actually achieve** — win rate by seat count over a +//! seed range, which is the difficulty denominator `ground-game`'s +//! GROUND-WP-0005 is blocked on. +//! 2. **What a search would cost** — the branching factor of +//! `legal_commands` and the price of enumerating it, which decides +//! whether the honest version of "could we have won" is affordable. +//! +//! **This measures, it does not conclude.** Whether a bot win rate *is* a +//! difficulty is exactly what the survey and the review have to argue +//! about; this only makes the number exist. +//! +//! ```text +//! cargo run --release -p games-ground --example difficulty-baseline +//! ``` + +use cb_game_runtime::{ScenarioGame, Setup}; +use cb_kernel::Aggregate; +use games_ground::bot::{legal_commands, play, GreedyPolicy, Policy, RandomPolicy}; +use games_ground::GroundState; + +const SEEDS: u64 = 200; + +fn setup(players: u8, seed: u64) -> Option { + GroundState::setup( + &Setup { + players, + preset: format!("standard-{players}p"), + patch: Default::default(), + }, + seed, + ) + .ok() +} + +fn policies(kind: &str, players: u8, seed: u64) -> Vec> { + (0..players) + .map(|i| -> Box { + match kind { + "random" => Box::new(RandomPolicy::new(seed ^ u64::from(i))), + _ => Box::new(GreedyPolicy), + } + }) + .collect() +} + +/// Win rate, and the margin — because "we lost" and "we lost by one point" +/// are different games, and a rate alone hides which one this is. +fn win_rate(kind: &str, players: u8) { + let (mut wins, mut played, mut total_pts, mut total_thr) = (0u32, 0u32, 0u64, 0u64); + let mut margins: Vec = Vec::new(); + for seed in 0..SEEDS { + let Some(state) = setup(players, seed) else { + continue; + }; + let mut ps = policies(kind, players, seed); + let Ok(game) = play(state, &mut ps) else { + continue; + }; + let Some(o) = &game.state.outcome else { + continue; + }; + played += 1; + if o.group_success { + wins += 1; + } + total_pts += u64::from(o.total); + total_thr += u64::from(o.threshold); + margins.push(i64::from(o.total) - i64::from(o.threshold)); + } + if played == 0 { + println!(" {players}p {kind:>6} no games completed"); + return; + } + margins.sort_unstable(); + let median = margins[margins.len() / 2]; + // Wilson-free: report the count, not a confidence interval we have not + // argued for. The spec (T04) decides what interval is claimed. + println!( + " {players}p {kind:>6} {wins:>3}/{played:<3} won = {rate:>5.1}% \ + mean total {mt:>4.1} of {th:>4.1} median margin {median:+}", + rate = 100.0 * f64::from(wins) / f64::from(played), + mt = total_pts as f64 / f64::from(played), + th = total_thr as f64 / f64::from(played), + ); +} + +/// What one node of a search costs, and how wide it is. +/// +/// Measured on real mid-game states rather than on a fresh deal: at deal +/// time most of the interesting branches do not exist yet, and a +/// branching factor taken there would flatter any search proposal. +fn search_cost(players: u8) { + let mut widths: Vec = Vec::new(); + let mut nodes = 0u64; + let start = std::time::Instant::now(); + + for seed in 0..40u64 { + let Some(state) = setup(players, seed) else { + continue; + }; + // Walk a real game and sample the branching at every decision. + let mut ps = policies("greedy", players, seed); + let Ok(game) = play(state, &mut ps) else { + continue; + }; + // Re-run the recorded commands, enumerating legality at each step. + let Some(mut replay) = setup(players, seed) else { + continue; + }; + for (actor, cmd) in &game.steps { + if let cb_kernel::Actor::Player(seat) = actor { + let legal = legal_commands(&replay, *seat); + widths.push(legal.len()); + nodes += 1; + } + if let Ok(events) = replay.validate(*actor, cmd) { + for e in &events { + replay.fold(e); + } + } + } + } + + let elapsed = start.elapsed(); + if widths.is_empty() { + println!(" {players}p no decisions sampled"); + return; + } + widths.sort_unstable(); + let sum: usize = widths.iter().sum(); + println!( + " {players}p {n} decisions branching mean {mean:.1} median {med} max {max} \ + {per:.0} us/node", + n = widths.len(), + mean = sum as f64 / widths.len() as f64, + med = widths[widths.len() / 2], + max = widths[widths.len() - 1], + per = elapsed.as_micros() as f64 / nodes as f64, + ); +} + +fn main() { + println!("CB-RES-0008 baseline — measured, not concluded\n"); + println!("bot win rate over {SEEDS} seeds (GR-E01 group success):"); + for players in [2u8, 3, 4, 5, 6] { + win_rate("greedy", players); + } + for players in [2u8, 3, 4] { + win_rate("random", players); + } + + println!("\nsearch cost — legal_commands at every real decision point:"); + for players in [2u8, 3, 4] { + search_cost(players); + } + + println!( + "\nNOTE: a win rate is this POLICY's win rate over THIS seed range.\n\ + Whether that is 'the difficulty' is T02's argument, not this tool's claim." + ); +} diff --git a/research/CB-RES-0008-could-we-have-won.md b/research/CB-RES-0008-could-we-have-won.md new file mode 100644 index 0000000..23100d4 --- /dev/null +++ b/research/CB-RES-0008-could-we-have-won.md @@ -0,0 +1,210 @@ +--- +id: CB-RES-0008 +capability: analysis.witness-and-difficulty +status: draft — awaiting adversarial review (CB-WP-0025 T02) +tier: L +chaos: d8 = 6 → no override +--- + +# CB-RES-0008 — a path out of a lost game, and how hard the game is + +CB-WP-0025 T01. Surveyed 2026-08-05. + +Two maintainer questions that are the same machine asked twice: *"could we +have won, and how?"* is a search from a recorded state; *"how hard is +this?"* is that search — or a proxy for it — run over many deals and +counted. + +**The runnable baseline is ours and it is the row that matters.** External +candidates are algorithms and practices, not software we can run on our +workload, so per InnerLoop §Step 1 their rows are **directional and cap at +`parity`**. + +--- + +## 1. The baseline, measured + +`cargo run --release -p games-ground --example difficulty-baseline` +(200 seeds per seat count, `GreedyPolicy` and `RandomPolicy`): + +``` +bot win rate over 200 seeds (GR-E01 group success): + 2p greedy 132/200 won = 66.0% mean total 5.3 of 5.0 median margin +1 + 3p greedy 165/200 won = 82.5% mean total 8.4 of 7.0 median margin +2 + 4p greedy 190/200 won = 95.0% mean total 8.8 of 7.0 median margin +2 + 5p greedy 200/200 won = 100.0% mean total 11.8 of 9.0 median margin +3 + 6p greedy 200/200 won = 100.0% mean total 12.0 of 9.0 median margin +3 + 2p random 10/200 won = 5.0% mean total 2.4 of 5.0 median margin -3 + 3p random 19/200 won = 9.5% mean total 3.1 of 7.0 median margin -4 + 4p random 16/200 won = 8.0% mean total 3.2 of 7.0 median margin -4 + +search cost — legal_commands at every real decision point: + 2p 462 decisions branching mean 4.7 median 5 max 7 161 us/node + 3p 649 decisions branching mean 7.4 median 8 max 10 139 us/node + 4p 870 decisions branching mean 9.1 median 10 max 12 112 us/node +``` + +### 1.1 The finding this produced before any solver exists + +**A greedy bot wins 200 of 200 games at five and six seats.** Not 95%, +not 99% — every game, with a median margin of **+3** and mean available +points of **11.8–12.0 against a threshold of 9**. + +The arithmetic behind it, row by row (the shape GROUND-WP-0004 T02 +requires): + +| seats | Surface | hidden dealt | points available | threshold | ratio | +|---|---|---|---:|---:|---:| +| 2 | priority 1 | 2 | **6** | 5 | 1.20 | +| 3–4 | priority 1 | 3 | **9** | 7 | 1.29 | +| 5–6 | priority 1 | 4 | **12** | 9 | 1.33 | + +**The ratio moves the wrong way.** More seats means more points on the +table *and* a proportionally lower bar *and* more actions per round to +clear it with. Three multipliers all pointing the same direction, which is +why the curve is not gentle — it is 66% → 100% across four seat counts. + +**This is admissible under GameDesign §1**: the reproduction exists +(`examples/difficulty-baseline.rs`), it has the ruled shape (row-level, no +sums), and it can fail — change a threshold and the numbers move. + +It also **explains the maintainer's report** — *"I felt it was too easy +but then we lost, so who knows"* — without needing a solver. He plays at +low seat counts, where 66% is a real game, and had been feeling the 5–6 +seat experience from elsewhere in the same session. Both halves of the +sentence are true of different seat counts. + +**T06 must report this to GROUND-WP-0005, which is blocked on exactly +this number.** + +### 1.2 What the search-cost numbers rule out + +Branching is small — mean 4.7 to 9.1 — but **`legal_commands` costs +112–161 µs per call**, because it constructs candidates and filters them +through full `validate`. That is the price of keeping the rules in one +place (ADR: `legal_commands` builds then validates), and it is the number +that decides this pass. + +A game is 5 rounds × N seats of decisions. An exhaustive search from +round 1 at 3 seats is roughly `7.4^15` — not a number worth writing down. +**Exhaustive search is out at any seat count**, and this was measured +rather than assumed. + +What is affordable, at ~140 µs/node: a bounded search of ~10⁴–10⁵ nodes +costs **1.4–14 seconds**. That is the budget the ADR has to design inside, +and it is the difference between a feature that answers while the player +is still looking at the page and one that does not. + +## 2. Prior art: determinized search, and the failure it is famous for + +The natural first idea — *deal out the hidden cards, solve the resulting +perfect-information game, repeat* — is **Perfect Information Monte Carlo +(PIMC)**, and its failure modes were named by Frank, Basin and Matsubara +in 1998: + +- **Strategy fusion** — the search picks *different* actions from two + states in the same information set, which no real player could do, + because a player cannot tell those states apart. The plan it returns is + not executable by someone who does not know which world they are in. +- **Non-locality** — subgame values are not well-defined when information + is hidden, so recursive search over subgames is unsound. + +**Strategy fusion is precisely the trap in this pass.** A witness produced +by determinized search may be a line that requires knowing which Solution +is on top of the deck. Showing the maintainer *"you could have won by +playing Repair on turn 3"* — when nothing he could see said a Repair was +coming — teaches a false lesson about his own play, which is worse than +not shipping the feature. + +**Long and Sturtevant** later characterized *when* PIMC nonetheless works +well, which matters here: its success depends on properties of the game +tree (leaf correlation, bias, disambiguation rate). GROUND disambiguates +fast — Problems flip face-up, selections reveal every round — which is the +regime where PIMC is least bad. **That is an argument the ADR may use, and +it is a directional one, not a measurement.** + +**ISMCTS** (information-set MCTS) searches over information sets directly +rather than determinizations, and is the standard answer to strategy +fusion. + +**Benchmark to beat:** a witness that **replays through our existing +scenario runner and ends in `group_success`**. That is a stronger and +cheaper bar than any of the above, because it is mechanically checkable — +and note it does *not* by itself exclude a strategy-fused line. A fused +line replays fine. **Checkability and honesty are different properties, +and the ADR must not let the first stand in for the second.** + +*Directional, cited-only.* + +## 3. The retrospective question is not the playing question + +Worth separating, because the prior art is all about *playing*: + +| question | information | honest? | +|---|---|---| +| *was this deal winnable at all* | omniscient | **yes** — it is a question about the deal, not about the player | +| *was it winnable from what we could see* | the seat's view | yes, and expensive | +| *could a reasonable player have found it* | the seat's view, bounded | the only affordable honest one | + +The first is legitimate and cheap, and answers *"the deal was unwinnable, +stop blaming yourself"* — which is a real thing a player wants to hear. +It is **not** an answer to *"how could we have won"*, and labelling it as +one is the failure mode. + +**Naming matters more than the algorithm here.** The ADR's first decision +is which question is being answered and what it is called on screen. + +## 4. Difficulty as a measured quantity + +Co-operative board games set difficulty with a dial and publish the win +rate — Pandemic's number of Epidemic cards is the canonical example. The +practice is: **a named dial, a stated player skill, and a target band.** + +We have the dial candidates already — the threshold (`GR-E01`), and +`ground-game`'s proposed Pressure dial (GROUND-WP-0005) — and §1 supplies +the first measured band. + +**The problem the practice does not solve for us:** a published win rate +is measured against *humans*. Ours is measured against `GreedyPolicy`. +The 5–6 seat 100% is a claim about our bot, and the honest reading is +narrower than "the game is too easy at six players" — it is *"a bot that +takes the obvious action never fails to clear the threshold at six +players."* + +Whether that is the same statement is **the reviewer's strongest line of +attack** and is not settled here. + +**Benchmark to beat:** a difficulty figure whose resolution is stated — +the smallest threshold change it can distinguish, with its N. A rate +without that cannot tune anything. + +*Directional, cited-only.* + +## 5. Benchmarks to beat + +| dimension | today | benchmark | +|---|---|---| +| witness checkability | no witness exists | **100%** of emitted paths replay to `group_success` through the existing runner | +| witness honesty | — | no line that requires unseen information; **the ADR must say how this is enforced, not asserted** | +| search cost | 112–161 µs/node measured | a bound in nodes or wall clock, and *"none found within B"* wording that does not claim unwinnability | +| difficulty resolution | one band, one policy | the smallest threshold delta distinguishable, with N and policy named | +| difficulty honesty | — | the policy and seed range are **in the number's name**, not a footnote | + +## 6. What the survey did not settle + +- **Whether a bot win rate is a difficulty at all.** §4. The strongest + counter is that it measures the bot, and improving the bot would + "increase the difficulty" without touching the game. +- **How witness honesty is enforced rather than asserted.** Running the + search on a `GroundView` makes the information boundary structural; + running it on `GroundState` makes it a promise. The survey believes the + first is right and has **not** measured whether it is affordable — a + view-only search cannot fold events, so it needs a state it may not + see. **This is the gap most likely to sink the pass, and it is stated + here rather than discovered in T05.** +- **Whether 100% at 5–6 seats is a rules finding or a bot finding.** + §1.1 reports it as measured; which repo owns it is T03's call. +- **Whether the cheap honest answer is enough.** *"This deal was + unwinnable"* (omniscient, cheap) may satisfy the maintainer's actual + need without any information-respecting search at all. Nobody has asked + him. That is a one-question experiment this survey did not run. diff --git a/workplans/CB-WP-0025-could-we-have-won.md b/workplans/CB-WP-0025-could-we-have-won.md index d503cbf..f46216c 100644 --- a/workplans/CB-WP-0025-could-we-have-won.md +++ b/workplans/CB-WP-0025-could-we-have-won.md @@ -2,7 +2,7 @@ id: CB-WP-0025 kind: product title: "Could we have won: a path out of a lost game, and how hard the game actually is" -status: ready +status: active state_hub_workstream_id: "a866982f-94e7-432b-b0d5-2fefb781a574" --- @@ -90,7 +90,7 @@ whose meaning drifts the next time a bot improves. ```task id: CB-WP-0025-T01 -status: todo +status: done priority: high state_hub_task_id: "556cfd24-5992-4cc0-be90-0b60e989b5bb" ``` @@ -123,6 +123,39 @@ per dimension — a number or a reproducible comparison, not an impression. that cannot finish while the player is still looking at the page is a different feature. +**Done 2026-08-05.** +[CB-RES-0008](../research/CB-RES-0008-could-we-have-won.md), with a +runnable baseline (`games/ground/examples/difficulty-baseline.rs`). + +**The baseline produced a finding before any solver exists, and it is the +biggest thing in this pass.** A greedy bot wins **200 of 200** games at +five and six seats — with a median margin of +3 and 11.8–12.0 points +available against a threshold of 9. The curve is 66% / 82.5% / 95% / 100% +/ 100% across 2/3/4/5/6 seats. + +The row-level table shows why: available points go **6 / 9 / 12** against +thresholds **5 / 7 / 9**, so the ratio *rises* with seat count (1.20 → +1.29 → 1.33) while the table also gains actions per round. **Three +multipliers pointing the same way.** + +**It also explains the maintainer's report without a solver.** *"I felt it +was too easy but then we lost"* — 66% at two seats is a real game; 100% at +six is not. Both halves are true of different seat counts. + +**Cost measured, and it rules out the obvious approach.** Branching is +small (mean 4.7–9.1) but `legal_commands` costs **112–161 µs** per call, +because it filters candidates through full `validate`. Exhaustive search +is out at every seat count; ~10⁴–10⁵ nodes is 1.4–14 s, which is the +budget the ADR must design inside. + +**Prior art names the trap.** Determinized search (PIMC) suffers *strategy +fusion* — Frank, Basin & Matsubara 1998 — where the search picks different +actions in states a real player cannot tell apart. A witness built that +way may require knowing what was on top of the deck. **And it would still +replay green**, so the checkability benchmark does not catch it. Honesty +and checkability are different properties, stated so T03 cannot conflate +them. + ## Task: adversarial review ```task