clay-borg/games/ground/examples/difficulty-baseline.rs

//! CB-RES-0008's runnable baseline (CB-WP-0025 T01).
//!
//! Two numbers the survey needs and cannot cite from anyone else, because
//! they are about **our** game on **our** machine:
//!
//! 1. **What the bots actually achieve** — win rate by seat count over a
//!    seed range, which is the difficulty denominator `ground-game`'s
//!    GROUND-WP-0005 is blocked on.
//! 2. **What a search would cost** — the branching factor of
//!    `legal_commands` and the price of enumerating it, which decides
//!    whether the honest version of "could we have won" is affordable.
//!
//! **This measures, it does not conclude.** Whether a bot win rate *is* a
//! difficulty is exactly what the survey and the review have to argue
//! about; this only makes the number exist.
//!
//! ```text
//! cargo run --release -p games-ground --example difficulty-baseline
//! ```

use cb_game_runtime::{ScenarioGame, Setup};
use cb_kernel::Aggregate;
use games_ground::bot::{legal_commands, play, GreedyPolicy, Policy, RandomPolicy};
use games_ground::GroundState;

const SEEDS: u64 = 200;

fn setup(players: u8, seed: u64) -> Option<GroundState> {
    GroundState::setup(
        &Setup {
            players,
            preset: format!("standard-{players}p"),
            patch: Default::default(),
        },
        seed,
    )
    .ok()
}

fn policies(kind: &str, players: u8, seed: u64) -> Vec<Box<dyn Policy>> {
    (0..players)
        .map(|i| -> Box<dyn Policy> {
            match kind {
                "random" => Box::new(RandomPolicy::new(seed ^ u64::from(i))),
                _ => Box::new(GreedyPolicy),
            }
        })
        .collect()
}

/// Win rate, and the margin — because "we lost" and "we lost by one point"
/// are different games, and a rate alone hides which one this is.
fn win_rate(kind: &str, players: u8) {
    let (mut wins, mut played, mut total_pts, mut total_thr) = (0u32, 0u32, 0u64, 0u64);
    let mut margins: Vec<i64> = Vec::new();
    for seed in 0..SEEDS {
        let Some(state) = setup(players, seed) else {
            continue;
        };
        let mut ps = policies(kind, players, seed);
        let Ok(game) = play(state, &mut ps) else {
            continue;
        };
        let Some(o) = &game.state.outcome else {
            continue;
        };
        played += 1;
        if o.group_success {
            wins += 1;
        }
        total_pts += u64::from(o.total);
        total_thr += u64::from(o.threshold);
        margins.push(i64::from(o.total) - i64::from(o.threshold));
    }
    if played == 0 {
        println!("  {players}p {kind:>6}  no games completed");
        return;
    }
    margins.sort_unstable();
    let median = margins[margins.len() / 2];
    // Wilson-free: report the count, not a confidence interval we have not
    // argued for. The spec (T04) decides what interval is claimed.
    println!(
        "  {players}p {kind:>6}  {wins:>3}/{played:<3} won = {rate:>5.1}%   \
         mean total {mt:>4.1} of {th:>4.1}   median margin {median:+}",
        rate = 100.0 * f64::from(wins) / f64::from(played),
        mt = total_pts as f64 / f64::from(played),
        th = total_thr as f64 / f64::from(played),
    );
}

/// What one node of a search costs, and how wide it is.
///
/// Measured on real mid-game states rather than on a fresh deal: at deal
/// time most of the interesting branches do not exist yet, and a
/// branching factor taken there would flatter any search proposal.
fn search_cost(players: u8) {
    let mut widths: Vec<usize> = Vec::new();
    let mut nodes = 0u64;
    // **The clock brackets `legal_commands` AND NOTHING ELSE.**
    //
    // The first version started it before the seed loop, so it timed two
    // `setup`s, a whole greedy game and a full validate+fold replay, then
    // divided by the number of player decisions — reporting 112–161 µs
    // for a call that costs ~16–20. The adversarial review caught it (C1),
    // and the tell was in the published output: the figure FELL as seat
    // count rose while branching rose, which is backwards for a
    // per-enumeration cost. Accumulate only the call.
    let mut spent = std::time::Duration::ZERO;

    for seed in 0..40u64 {
        let Some(state) = setup(players, seed) else {
            continue;
        };
        // Walk a real game and sample the branching at every decision.
        let mut ps = policies("greedy", players, seed);
        let Ok(game) = play(state, &mut ps) else {
            continue;
        };
        // Re-run the recorded commands, enumerating legality at each step.
        let Some(mut replay) = setup(players, seed) else {
            continue;
        };
        for (actor, cmd) in &game.steps {
            if let cb_kernel::Actor::Player(seat) = actor {
                let t0 = std::time::Instant::now();
                let legal = legal_commands(&replay, *seat);
                spent += t0.elapsed();
                widths.push(legal.len());
                nodes += 1;
            }
            if let Ok(events) = replay.validate(*actor, cmd) {
                for e in &events {
                    replay.fold(e);
                }
            }
        }
    }

    if widths.is_empty() {
        println!("  {players}p  no decisions sampled");
        return;
    }
    widths.sort_unstable();
    let sum: usize = widths.iter().sum();
    println!(
        "  {players}p  {n} decisions   branching mean {mean:.1}  median {med}  max {max}   \
         {per:.1} us/call",
        n = widths.len(),
        mean = sum as f64 / widths.len() as f64,
        med = widths[widths.len() / 2],
        max = widths[widths.len() - 1],
        per = spent.as_nanos() as f64 / nodes as f64 / 1000.0,
    );
}

fn main() {
    println!("CB-RES-0008 baseline — measured, not concluded\n");
    println!("bot win rate over {SEEDS} seeds (GR-E01 group success):");
    for players in [2u8, 3, 4, 5, 6] {
        win_rate("greedy", players);
    }
    for players in [2u8, 3, 4] {
        win_rate("random", players);
    }

    println!("\nsearch cost — legal_commands at every real decision point:");
    for players in [2u8, 3, 4] {
        search_cost(players);
    }

    println!(
        "\nNOTE: a win rate is this POLICY's win rate over THIS seed range.\n\
         Whether that is 'the difficulty' is T02's argument, not this tool's claim."
    );
}