From 469d00d6790d500620ea91ba70c79286da775311 Mon Sep 17 00:00:00 2001 From: tegwick Date: Wed, 5 Aug 2026 17:52:46 +0200 Subject: [PATCH] CB-WP-0025 T01: survey -- the baseline found the game before the solver did MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CB-RES-0008 with a runnable baseline (games/ground/examples/difficulty-baseline.rs), and the measurement produced a finding before any solver exists. A GREEDY BOT WINS 200 OF 200 GAMES AT FIVE AND SIX SEATS. Median margin +3, 11.8-12.0 points available against a threshold of 9. The curve is 66% / 82.5% / 95% / 100% / 100% across 2/3/4/5/6 seats. Row-level, as GROUND-WP-0004 T02 requires: available points 6 / 9 / 12 against thresholds 5 / 7 / 9, so the ratio RISES with seat count (1.20, 1.29, 1.33) while the table also gains actions per round to clear it with. Three multipliers pointing the same direction. It also explains the maintainer's report without needing a solver at all: "I felt it was too easy but then we lost" is two true statements about different seat counts. Cost measured and it rules out the obvious approach. Branching is small (mean 4.7-9.1) but legal_commands costs 112-161 us per call because it filters candidates through full validate. Exhaustive search is out at every seat count; 10^4-10^5 nodes is 1.4-14 seconds, which is the budget the ADR must design inside. Prior art names the trap: determinized search (PIMC) suffers strategy fusion (Frank, Basin & Matsubara 1998) -- the search picks different actions in states a real player cannot distinguish, so the witness may require knowing what was on top of the deck. Such a line still replays green, so the checkability benchmark does not catch it. Honesty and checkability are different properties; stated explicitly so T03 cannot conflate them. The survey states its own most likely killer up front (§6): a view-only search cannot fold events, so making the information boundary structural rather than a promise may not be affordable. Better found here than in T05. Co-Authored-By: Claude Opus 5 --- games/ground/examples/difficulty-baseline.rs | 166 +++++++++++++++ research/CB-RES-0008-could-we-have-won.md | 210 +++++++++++++++++++ workplans/CB-WP-0025-could-we-have-won.md | 37 +++- 3 files changed, 411 insertions(+), 2 deletions(-) create mode 100644 games/ground/examples/difficulty-baseline.rs create mode 100644 research/CB-RES-0008-could-we-have-won.md diff --git a/games/ground/examples/difficulty-baseline.rs b/games/ground/examples/difficulty-baseline.rs new file mode 100644 index 0000000..11c6a12 --- /dev/null +++ b/games/ground/examples/difficulty-baseline.rs @@ -0,0 +1,166 @@ +//! CB-RES-0008's runnable baseline (CB-WP-0025 T01). +//! +//! Two numbers the survey needs and cannot cite from anyone else, because +//! they are about **our** game on **our** machine: +//! +//! 1. **What the bots actually achieve** — win rate by seat count over a +//! seed range, which is the difficulty denominator `ground-game`'s +//! GROUND-WP-0005 is blocked on. +//! 2. **What a search would cost** — the branching factor of +//! `legal_commands` and the price of enumerating it, which decides +//! whether the honest version of "could we have won" is affordable. +//! +//! **This measures, it does not conclude.** Whether a bot win rate *is* a +//! difficulty is exactly what the survey and the review have to argue +//! about; this only makes the number exist. +//! +//! ```text +//! cargo run --release -p games-ground --example difficulty-baseline +//! ``` + +use cb_game_runtime::{ScenarioGame, Setup}; +use cb_kernel::Aggregate; +use games_ground::bot::{legal_commands, play, GreedyPolicy, Policy, RandomPolicy}; +use games_ground::GroundState; + +const SEEDS: u64 = 200; + +fn setup(players: u8, seed: u64) -> Option { + GroundState::setup( + &Setup { + players, + preset: format!("standard-{players}p"), + patch: Default::default(), + }, + seed, + ) + .ok() +} + +fn policies(kind: &str, players: u8, seed: u64) -> Vec> { + (0..players) + .map(|i| -> Box { + match kind { + "random" => Box::new(RandomPolicy::new(seed ^ u64::from(i))), + _ => Box::new(GreedyPolicy), + } + }) + .collect() +} + +/// Win rate, and the margin — because "we lost" and "we lost by one point" +/// are different games, and a rate alone hides which one this is. +fn win_rate(kind: &str, players: u8) { + let (mut wins, mut played, mut total_pts, mut total_thr) = (0u32, 0u32, 0u64, 0u64); + let mut margins: Vec = Vec::new(); + for seed in 0..SEEDS { + let Some(state) = setup(players, seed) else { + continue; + }; + let mut ps = policies(kind, players, seed); + let Ok(game) = play(state, &mut ps) else { + continue; + }; + let Some(o) = &game.state.outcome else { + continue; + }; + played += 1; + if o.group_success { + wins += 1; + } + total_pts += u64::from(o.total); + total_thr += u64::from(o.threshold); + margins.push(i64::from(o.total) - i64::from(o.threshold)); + } + if played == 0 { + println!(" {players}p {kind:>6} no games completed"); + return; + } + margins.sort_unstable(); + let median = margins[margins.len() / 2]; + // Wilson-free: report the count, not a confidence interval we have not + // argued for. The spec (T04) decides what interval is claimed. + println!( + " {players}p {kind:>6} {wins:>3}/{played:<3} won = {rate:>5.1}% \ + mean total {mt:>4.1} of {th:>4.1} median margin {median:+}", + rate = 100.0 * f64::from(wins) / f64::from(played), + mt = total_pts as f64 / f64::from(played), + th = total_thr as f64 / f64::from(played), + ); +} + +/// What one node of a search costs, and how wide it is. +/// +/// Measured on real mid-game states rather than on a fresh deal: at deal +/// time most of the interesting branches do not exist yet, and a +/// branching factor taken there would flatter any search proposal. +fn search_cost(players: u8) { + let mut widths: Vec = Vec::new(); + let mut nodes = 0u64; + let start = std::time::Instant::now(); + + for seed in 0..40u64 { + let Some(state) = setup(players, seed) else { + continue; + }; + // Walk a real game and sample the branching at every decision. + let mut ps = policies("greedy", players, seed); + let Ok(game) = play(state, &mut ps) else { + continue; + }; + // Re-run the recorded commands, enumerating legality at each step. + let Some(mut replay) = setup(players, seed) else { + continue; + }; + for (actor, cmd) in &game.steps { + if let cb_kernel::Actor::Player(seat) = actor { + let legal = legal_commands(&replay, *seat); + widths.push(legal.len()); + nodes += 1; + } + if let Ok(events) = replay.validate(*actor, cmd) { + for e in &events { + replay.fold(e); + } + } + } + } + + let elapsed = start.elapsed(); + if widths.is_empty() { + println!(" {players}p no decisions sampled"); + return; + } + widths.sort_unstable(); + let sum: usize = widths.iter().sum(); + println!( + " {players}p {n} decisions branching mean {mean:.1} median {med} max {max} \ + {per:.0} us/node", + n = widths.len(), + mean = sum as f64 / widths.len() as f64, + med = widths[widths.len() / 2], + max = widths[widths.len() - 1], + per = elapsed.as_micros() as f64 / nodes as f64, + ); +} + +fn main() { + println!("CB-RES-0008 baseline — measured, not concluded\n"); + println!("bot win rate over {SEEDS} seeds (GR-E01 group success):"); + for players in [2u8, 3, 4, 5, 6] { + win_rate("greedy", players); + } + for players in [2u8, 3, 4] { + win_rate("random", players); + } + + println!("\nsearch cost — legal_commands at every real decision point:"); + for players in [2u8, 3, 4] { + search_cost(players); + } + + println!( + "\nNOTE: a win rate is this POLICY's win rate over THIS seed range.\n\ + Whether that is 'the difficulty' is T02's argument, not this tool's claim." + ); +} diff --git a/research/CB-RES-0008-could-we-have-won.md b/research/CB-RES-0008-could-we-have-won.md new file mode 100644 index 0000000..23100d4 --- /dev/null +++ b/research/CB-RES-0008-could-we-have-won.md @@ -0,0 +1,210 @@ +--- +id: CB-RES-0008 +capability: analysis.witness-and-difficulty +status: draft — awaiting adversarial review (CB-WP-0025 T02) +tier: L +chaos: d8 = 6 → no override +--- + +# CB-RES-0008 — a path out of a lost game, and how hard the game is + +CB-WP-0025 T01. Surveyed 2026-08-05. + +Two maintainer questions that are the same machine asked twice: *"could we +have won, and how?"* is a search from a recorded state; *"how hard is +this?"* is that search — or a proxy for it — run over many deals and +counted. + +**The runnable baseline is ours and it is the row that matters.** External +candidates are algorithms and practices, not software we can run on our +workload, so per InnerLoop §Step 1 their rows are **directional and cap at +`parity`**. + +--- + +## 1. The baseline, measured + +`cargo run --release -p games-ground --example difficulty-baseline` +(200 seeds per seat count, `GreedyPolicy` and `RandomPolicy`): + +``` +bot win rate over 200 seeds (GR-E01 group success): + 2p greedy 132/200 won = 66.0% mean total 5.3 of 5.0 median margin +1 + 3p greedy 165/200 won = 82.5% mean total 8.4 of 7.0 median margin +2 + 4p greedy 190/200 won = 95.0% mean total 8.8 of 7.0 median margin +2 + 5p greedy 200/200 won = 100.0% mean total 11.8 of 9.0 median margin +3 + 6p greedy 200/200 won = 100.0% mean total 12.0 of 9.0 median margin +3 + 2p random 10/200 won = 5.0% mean total 2.4 of 5.0 median margin -3 + 3p random 19/200 won = 9.5% mean total 3.1 of 7.0 median margin -4 + 4p random 16/200 won = 8.0% mean total 3.2 of 7.0 median margin -4 + +search cost — legal_commands at every real decision point: + 2p 462 decisions branching mean 4.7 median 5 max 7 161 us/node + 3p 649 decisions branching mean 7.4 median 8 max 10 139 us/node + 4p 870 decisions branching mean 9.1 median 10 max 12 112 us/node +``` + +### 1.1 The finding this produced before any solver exists + +**A greedy bot wins 200 of 200 games at five and six seats.** Not 95%, +not 99% — every game, with a median margin of **+3** and mean available +points of **11.8–12.0 against a threshold of 9**. + +The arithmetic behind it, row by row (the shape GROUND-WP-0004 T02 +requires): + +| seats | Surface | hidden dealt | points available | threshold | ratio | +|---|---|---|---:|---:|---:| +| 2 | priority 1 | 2 | **6** | 5 | 1.20 | +| 3–4 | priority 1 | 3 | **9** | 7 | 1.29 | +| 5–6 | priority 1 | 4 | **12** | 9 | 1.33 | + +**The ratio moves the wrong way.** More seats means more points on the +table *and* a proportionally lower bar *and* more actions per round to +clear it with. Three multipliers all pointing the same direction, which is +why the curve is not gentle — it is 66% → 100% across four seat counts. + +**This is admissible under GameDesign §1**: the reproduction exists +(`examples/difficulty-baseline.rs`), it has the ruled shape (row-level, no +sums), and it can fail — change a threshold and the numbers move. + +It also **explains the maintainer's report** — *"I felt it was too easy +but then we lost, so who knows"* — without needing a solver. He plays at +low seat counts, where 66% is a real game, and had been feeling the 5–6 +seat experience from elsewhere in the same session. Both halves of the +sentence are true of different seat counts. + +**T06 must report this to GROUND-WP-0005, which is blocked on exactly +this number.** + +### 1.2 What the search-cost numbers rule out + +Branching is small — mean 4.7 to 9.1 — but **`legal_commands` costs +112–161 µs per call**, because it constructs candidates and filters them +through full `validate`. That is the price of keeping the rules in one +place (ADR: `legal_commands` builds then validates), and it is the number +that decides this pass. + +A game is 5 rounds × N seats of decisions. An exhaustive search from +round 1 at 3 seats is roughly `7.4^15` — not a number worth writing down. +**Exhaustive search is out at any seat count**, and this was measured +rather than assumed. + +What is affordable, at ~140 µs/node: a bounded search of ~10⁴–10⁵ nodes +costs **1.4–14 seconds**. That is the budget the ADR has to design inside, +and it is the difference between a feature that answers while the player +is still looking at the page and one that does not. + +## 2. Prior art: determinized search, and the failure it is famous for + +The natural first idea — *deal out the hidden cards, solve the resulting +perfect-information game, repeat* — is **Perfect Information Monte Carlo +(PIMC)**, and its failure modes were named by Frank, Basin and Matsubara +in 1998: + +- **Strategy fusion** — the search picks *different* actions from two + states in the same information set, which no real player could do, + because a player cannot tell those states apart. The plan it returns is + not executable by someone who does not know which world they are in. +- **Non-locality** — subgame values are not well-defined when information + is hidden, so recursive search over subgames is unsound. + +**Strategy fusion is precisely the trap in this pass.** A witness produced +by determinized search may be a line that requires knowing which Solution +is on top of the deck. Showing the maintainer *"you could have won by +playing Repair on turn 3"* — when nothing he could see said a Repair was +coming — teaches a false lesson about his own play, which is worse than +not shipping the feature. + +**Long and Sturtevant** later characterized *when* PIMC nonetheless works +well, which matters here: its success depends on properties of the game +tree (leaf correlation, bias, disambiguation rate). GROUND disambiguates +fast — Problems flip face-up, selections reveal every round — which is the +regime where PIMC is least bad. **That is an argument the ADR may use, and +it is a directional one, not a measurement.** + +**ISMCTS** (information-set MCTS) searches over information sets directly +rather than determinizations, and is the standard answer to strategy +fusion. + +**Benchmark to beat:** a witness that **replays through our existing +scenario runner and ends in `group_success`**. That is a stronger and +cheaper bar than any of the above, because it is mechanically checkable — +and note it does *not* by itself exclude a strategy-fused line. A fused +line replays fine. **Checkability and honesty are different properties, +and the ADR must not let the first stand in for the second.** + +*Directional, cited-only.* + +## 3. The retrospective question is not the playing question + +Worth separating, because the prior art is all about *playing*: + +| question | information | honest? | +|---|---|---| +| *was this deal winnable at all* | omniscient | **yes** — it is a question about the deal, not about the player | +| *was it winnable from what we could see* | the seat's view | yes, and expensive | +| *could a reasonable player have found it* | the seat's view, bounded | the only affordable honest one | + +The first is legitimate and cheap, and answers *"the deal was unwinnable, +stop blaming yourself"* — which is a real thing a player wants to hear. +It is **not** an answer to *"how could we have won"*, and labelling it as +one is the failure mode. + +**Naming matters more than the algorithm here.** The ADR's first decision +is which question is being answered and what it is called on screen. + +## 4. Difficulty as a measured quantity + +Co-operative board games set difficulty with a dial and publish the win +rate — Pandemic's number of Epidemic cards is the canonical example. The +practice is: **a named dial, a stated player skill, and a target band.** + +We have the dial candidates already — the threshold (`GR-E01`), and +`ground-game`'s proposed Pressure dial (GROUND-WP-0005) — and §1 supplies +the first measured band. + +**The problem the practice does not solve for us:** a published win rate +is measured against *humans*. Ours is measured against `GreedyPolicy`. +The 5–6 seat 100% is a claim about our bot, and the honest reading is +narrower than "the game is too easy at six players" — it is *"a bot that +takes the obvious action never fails to clear the threshold at six +players."* + +Whether that is the same statement is **the reviewer's strongest line of +attack** and is not settled here. + +**Benchmark to beat:** a difficulty figure whose resolution is stated — +the smallest threshold change it can distinguish, with its N. A rate +without that cannot tune anything. + +*Directional, cited-only.* + +## 5. Benchmarks to beat + +| dimension | today | benchmark | +|---|---|---| +| witness checkability | no witness exists | **100%** of emitted paths replay to `group_success` through the existing runner | +| witness honesty | — | no line that requires unseen information; **the ADR must say how this is enforced, not asserted** | +| search cost | 112–161 µs/node measured | a bound in nodes or wall clock, and *"none found within B"* wording that does not claim unwinnability | +| difficulty resolution | one band, one policy | the smallest threshold delta distinguishable, with N and policy named | +| difficulty honesty | — | the policy and seed range are **in the number's name**, not a footnote | + +## 6. What the survey did not settle + +- **Whether a bot win rate is a difficulty at all.** §4. The strongest + counter is that it measures the bot, and improving the bot would + "increase the difficulty" without touching the game. +- **How witness honesty is enforced rather than asserted.** Running the + search on a `GroundView` makes the information boundary structural; + running it on `GroundState` makes it a promise. The survey believes the + first is right and has **not** measured whether it is affordable — a + view-only search cannot fold events, so it needs a state it may not + see. **This is the gap most likely to sink the pass, and it is stated + here rather than discovered in T05.** +- **Whether 100% at 5–6 seats is a rules finding or a bot finding.** + §1.1 reports it as measured; which repo owns it is T03's call. +- **Whether the cheap honest answer is enough.** *"This deal was + unwinnable"* (omniscient, cheap) may satisfy the maintainer's actual + need without any information-respecting search at all. Nobody has asked + him. That is a one-question experiment this survey did not run. diff --git a/workplans/CB-WP-0025-could-we-have-won.md b/workplans/CB-WP-0025-could-we-have-won.md index d503cbf..f46216c 100644 --- a/workplans/CB-WP-0025-could-we-have-won.md +++ b/workplans/CB-WP-0025-could-we-have-won.md @@ -2,7 +2,7 @@ id: CB-WP-0025 kind: product title: "Could we have won: a path out of a lost game, and how hard the game actually is" -status: ready +status: active state_hub_workstream_id: "a866982f-94e7-432b-b0d5-2fefb781a574" --- @@ -90,7 +90,7 @@ whose meaning drifts the next time a bot improves. ```task id: CB-WP-0025-T01 -status: todo +status: done priority: high state_hub_task_id: "556cfd24-5992-4cc0-be90-0b60e989b5bb" ``` @@ -123,6 +123,39 @@ per dimension — a number or a reproducible comparison, not an impression. that cannot finish while the player is still looking at the page is a different feature. +**Done 2026-08-05.** +[CB-RES-0008](../research/CB-RES-0008-could-we-have-won.md), with a +runnable baseline (`games/ground/examples/difficulty-baseline.rs`). + +**The baseline produced a finding before any solver exists, and it is the +biggest thing in this pass.** A greedy bot wins **200 of 200** games at +five and six seats — with a median margin of +3 and 11.8–12.0 points +available against a threshold of 9. The curve is 66% / 82.5% / 95% / 100% +/ 100% across 2/3/4/5/6 seats. + +The row-level table shows why: available points go **6 / 9 / 12** against +thresholds **5 / 7 / 9**, so the ratio *rises* with seat count (1.20 → +1.29 → 1.33) while the table also gains actions per round. **Three +multipliers pointing the same way.** + +**It also explains the maintainer's report without a solver.** *"I felt it +was too easy but then we lost"* — 66% at two seats is a real game; 100% at +six is not. Both halves are true of different seat counts. + +**Cost measured, and it rules out the obvious approach.** Branching is +small (mean 4.7–9.1) but `legal_commands` costs **112–161 µs** per call, +because it filters candidates through full `validate`. Exhaustive search +is out at every seat count; ~10⁴–10⁵ nodes is 1.4–14 s, which is the +budget the ADR must design inside. + +**Prior art names the trap.** Determinized search (PIMC) suffers *strategy +fusion* — Frank, Basin & Matsubara 1998 — where the search picks different +actions in states a real player cannot tell apart. A witness built that +way may require knowing what was on top of the deck. **And it would still +replay green**, so the checkability benchmark does not catch it. Honesty +and checkability are different properties, stated so T03 cannot conflate +them. + ## Task: adversarial review ```task