//! CB-WP-0025 T06 — the difficulty table, and the instrument that can fail. //! //! Implements `specs/RetrospectiveAnalysis.md` §4. Replaces //! `difficulty-baseline.rs`, which CB-WP-0022's admissibility rule and //! this pass's own review both found inadmissible: it had no assertions, //! no `--self-test` and no `make` target, so nothing could turn it red. //! //! ## What it will not print //! //! **A single policy's win rate as a difficulty** (§4.1). Measured on //! identical deals, `GreedyPolicy` wins 100% at five and six seats where //! `FirstLegal` wins 0%. The panel is plural for that reason, and the //! spread is reported rather than hidden. //! //! ```text //! cargo run --release -p games-ground --example difficulty [--self-test] //! ``` use cb_game_runtime::{ScenarioGame, Setup}; use cb_kernel::{Aggregate, PlayerId}; use games_ground::bot::{play, Choice, GreedyPolicy, Policy, RandomPolicy}; use games_ground::search::{winnable_within, Verdict}; use games_ground::{GroundCommand, GroundState}; /// Seeds per cell. Small by default: the winnable fraction needs an /// exhaustive search per deal, which is the expensive half (§3.1). const SEEDS: u64 = 60; /// §3's measured limit: `K=1` is exhaustible, `K=2` is not. const K: u8 = 1; const BUDGET: usize = 2_000_000; /// A policy with no heuristic at all. **It is in the panel because it is /// what falsified the survey's headline** — it beats greedy at two seats /// and scores zero at six. struct FirstLegal; impl Policy for FirstLegal { fn name(&self) -> &'static str { "first-legal" } fn choose( &mut self, _s: &GroundState, _seat: PlayerId, _legal: &[GroundCommand], _may_pass: bool, ) -> Choice { Choice::Command(0) } } fn setup(players: u8, seed: u64) -> Option { GroundState::setup( &Setup { players, preset: format!("standard-{players}p"), patch: Default::default(), }, seed, ) .ok() } fn policies(kind: &str, players: u8, seed: u64) -> Vec> { (0..players) .map(|i| -> Box { match kind { "random" => Box::new(RandomPolicy::new(seed ^ u64::from(i))), "first-legal" => Box::new(FirstLegal), _ => Box::new(GreedyPolicy), } }) .collect() } /// Games a named policy actually wins. fn policy_wins(kind: &str, players: u8) -> (u32, u32) { let (mut wins, mut played) = (0, 0); for seed in 0..SEEDS { let Some(state) = setup(players, seed) else { continue; }; let mut ps = policies(kind, players, seed); let Ok(game) = play(state, &mut ps) else { continue; }; if let Some(o) = &game.state.outcome { played += 1; if o.group_success { wins += 1; } } } (wins, played) } /// Rewind a played game to the start of its last `k` rounds. fn last_rounds(players: u8, seed: u64, k: usize) -> Option { let mut ps = policies("greedy", players, seed); let game = play(setup(players, seed)?, &mut ps).ok()?; let total = game .steps .iter() .filter(|(_, c)| matches!(c, GroundCommand::EndRound)) .count(); let mut st = setup(players, seed)?; let mut ends = 0usize; for (a, c) in &game.steps { if let Ok(ev) = st.validate(*a, c) { for e in &ev { st.fold(e); } } if matches!(c, GroundCommand::EndRound) { ends += 1; if ends >= total.saturating_sub(k) { break; } } } Some(st) } /// In what fraction of deals does a winning line exist in the last `K` /// rounds? /// /// **Less policy-dependent than a win rate, but NOT policy-free, and /// saying otherwise would repeat this pass's own error in a subtler /// form.** The position searched is the one `GreedyPolicy` left at the /// start of the last round, so the figure is *"winnable from where greedy /// got to"*. A genuinely policy-free measure would search from round 1, /// which §3.1 measured as unaffordable. /// /// What it does buy: the last round's outcome no longer depends on which /// agent plays it, so the measure is insensitive to exactly the variation /// that made the bot rate meaningless (§4.1). /// /// Deals where the search was cut by its budget are **not counted either /// way** — they are reported separately, because folding "we stopped /// looking" into "not winnable" is exactly the collapse §2.3 forbids. fn winnable_fraction(players: u8) -> (u32, u32, u32) { let (mut yes, mut decided, mut undecided) = (0, 0, 0); for seed in 0..SEEDS { let Some(state) = last_rounds(players, seed, K as usize) else { continue; }; match winnable_within(&state, K, BUDGET) { Verdict::Winnable { .. } => { yes += 1; decided += 1; } Verdict::NoneFound { exhausted: true, .. } => decided += 1, Verdict::NoneFound { exhausted: false, .. } => undecided += 1, } } (yes, decided, undecided) } fn report() { println!("difficulty — specs/RetrospectiveAnalysis.md §4\n"); println!( " winnable-from-round-{} over {SEEDS} seeds, budget {BUDGET} nodes", 6 - K ); println!(" policy win rates over the same {SEEDS} seeds\n"); println!(" seats winnable greedy random first-legal spread undecided"); for players in [2u8, 3, 4, 5, 6] { let (yes, decided, undecided) = winnable_fraction(players); let pct = |(w, n): (u32, u32)| { if n == 0 { -1.0 } else { 100.0 * f64::from(w) / f64::from(n) } }; let g = pct(policy_wins("greedy", players)); let r = pct(policy_wins("random", players)); let f = pct(policy_wins("first-legal", players)); let spread = [g, r, f].iter().cloned().fold(f64::MIN, f64::max) - [g, r, f].iter().cloned().fold(f64::MAX, f64::min); let wf = if decided == 0 { "n/a".to_string() } else { format!("{:.0}%", 100.0 * f64::from(yes) / f64::from(decided)) }; println!( " {players}p {wf:>6} {g:>5.1}% {r:>5.1}% {f:>5.1}% \ {spread:>5.1} {undecided:>3}" ); } println!( "\n WINNABLE is conditioned on GREEDY's play up to the last round —\n \ it is 'winnable from where greedy got to', not a property of the\n \ deal alone. Searching from round 1 is unaffordable (spec §3.1).\n\n \ It is also a LOWER BOUND: a K={K} search cannot see a line that\n \ needed an earlier round. `undecided` are deals whose search hit the\n \ node budget — they are excluded from the fraction, not counted as\n \ unwinnable.\n\n \ SPREAD is the range across three policies. Where it is large, no\n \ single policy's rate says anything about the game (§4.1)." ); } fn self_test() -> i32 { let mut ok = true; let mut check = |name: &str, cond: bool, detail: String| { ok &= cond; println!( " [{}] {name}{}", if cond { "ok " } else { "FAIL" }, if detail.is_empty() { String::new() } else { format!(" — {detail}") } ); }; // The control the old harness never had: a search that finds a line // must produce one that REPLAYS. §2.1 is a gate, not a metric. let state = last_rounds(3, 7, K as usize).expect("a 3p game"); match winnable_within(&state, K, BUDGET) { Verdict::Winnable { line, .. } => { let mut replay = state.clone(); let mut good = true; for m in &line { match replay.validate(m.actor, &m.command) { Ok(ev) => { for e in &ev { replay.fold(e); } } Err(_) => good = false, } } let won = replay.outcome.as_ref().is_some_and(|o| o.group_success); check( "a witness replays to a win", good && won, format!("{} moves", line.len()), ); } Verdict::NoneFound { .. } => check( "a witness replays to a win", false, "3p seed 7 found no line — the fixture moved".into(), ), } // The negative: a search must be able to return nothing, exhaustively. let lost = last_rounds(2, 7, 1).expect("a 2p game"); match winnable_within(&lost, 1, BUDGET) { Verdict::NoneFound { exhausted, nodes } => check( "an unwinnable position is reported as searched-out", exhausted && nodes > 100, format!("{nodes} nodes"), ), Verdict::Winnable { .. } => check( "an unwinnable position is reported as searched-out", false, "found a win in a game 2p seed 7 lost".into(), ), } // A budget of one must NOT claim exhaustion — the distinction §2.3 // rests on. match winnable_within(&state, K, 1) { Verdict::NoneFound { exhausted, .. } => check( "a budget cut is not reported as exhaustion", !exhausted, String::new(), ), Verdict::Winnable { .. } => check( "a budget cut is not reported as exhaustion", false, String::new(), ), } // §4.1's reason, asserted rather than asserted-about: the panel must // actually disagree, or reporting three policies is ceremony. let g = policy_wins("greedy", 6); let f = policy_wins("first-legal", 6); check( "the policy panel disagrees, so no single rate is a difficulty", g.0 != f.0, format!("greedy {}/{}, first-legal {}/{}", g.0, g.1, f.0, f.1), ); println!("difficulty self-test (positive control)"); i32::from(!ok) } fn main() { if std::env::args().any(|a| a == "--self-test") { std::process::exit(self_test()); } report(); }