clay-borg/games/ground/examples/difficulty.rs

//! CB-WP-0025 T06 — the difficulty table, and the instrument that can fail.
//!
//! Implements `specs/RetrospectiveAnalysis.md` §4. Replaces
//! `difficulty-baseline.rs`, which CB-WP-0022's admissibility rule and
//! this pass's own review both found inadmissible: it had no assertions,
//! no `--self-test` and no `make` target, so nothing could turn it red.
//!
//! ## What it will not print
//!
//! **A single policy's win rate as a difficulty** (§4.1). Measured on
//! identical deals, `GreedyPolicy` wins 100% at five and six seats where
//! `FirstLegal` wins 0%. The panel is plural for that reason, and the
//! spread is reported rather than hidden.
//!
//! ```text
//! cargo run --release -p games-ground --example difficulty [--self-test]
//! ```

use cb_game_runtime::{ScenarioGame, Setup};
use cb_kernel::{Aggregate, PlayerId};
use games_ground::bot::{play, Choice, GreedyPolicy, Policy, RandomPolicy};
use games_ground::search::{winnable_within, Verdict};
use games_ground::{GroundCommand, GroundState};

/// Seeds per cell. Small by default: the winnable fraction needs an
/// exhaustive search per deal, which is the expensive half (§3.1).
const SEEDS: u64 = 60;
/// §3's measured limit: `K=1` is exhaustible, `K=2` is not.
const K: u8 = 1;
const BUDGET: usize = 2_000_000;

/// A policy with no heuristic at all. **It is in the panel because it is
/// what falsified the survey's headline** — it beats greedy at two seats
/// and scores zero at six.
struct FirstLegal;
impl Policy for FirstLegal {
    fn name(&self) -> &'static str {
        "first-legal"
    }
    fn choose(
        &mut self,
        _s: &GroundState,
        _seat: PlayerId,
        _legal: &[GroundCommand],
        _may_pass: bool,
    ) -> Choice {
        Choice::Command(0)
    }
}

fn setup(players: u8, seed: u64) -> Option<GroundState> {
    GroundState::setup(
        &Setup {
            players,
            preset: format!("standard-{players}p"),
            patch: Default::default(),
        },
        seed,
    )
    .ok()
}

fn policies(kind: &str, players: u8, seed: u64) -> Vec<Box<dyn Policy>> {
    (0..players)
        .map(|i| -> Box<dyn Policy> {
            match kind {
                "random" => Box::new(RandomPolicy::new(seed ^ u64::from(i))),
                "first-legal" => Box::new(FirstLegal),
                _ => Box::new(GreedyPolicy),
            }
        })
        .collect()
}

/// Games a named policy actually wins.
fn policy_wins(kind: &str, players: u8) -> (u32, u32) {
    let (mut wins, mut played) = (0, 0);
    for seed in 0..SEEDS {
        let Some(state) = setup(players, seed) else {
            continue;
        };
        let mut ps = policies(kind, players, seed);
        let Ok(game) = play(state, &mut ps) else {
            continue;
        };
        if let Some(o) = &game.state.outcome {
            played += 1;
            if o.group_success {
                wins += 1;
            }
        }
    }
    (wins, played)
}

/// Rewind a played game to the start of its last `k` rounds.
fn last_rounds(players: u8, seed: u64, k: usize) -> Option<GroundState> {
    let mut ps = policies("greedy", players, seed);
    let game = play(setup(players, seed)?, &mut ps).ok()?;
    let total = game
        .steps
        .iter()
        .filter(|(_, c)| matches!(c, GroundCommand::EndRound))
        .count();
    let mut st = setup(players, seed)?;
    let mut ends = 0usize;
    for (a, c) in &game.steps {
        if let Ok(ev) = st.validate(*a, c) {
            for e in &ev {
                st.fold(e);
            }
        }
        if matches!(c, GroundCommand::EndRound) {
            ends += 1;
            if ends >= total.saturating_sub(k) {
                break;
            }
        }
    }
    Some(st)
}

/// In what fraction of deals does a winning line exist in the last `K`
/// rounds?
///
/// **Less policy-dependent than a win rate, but NOT policy-free, and
/// saying otherwise would repeat this pass's own error in a subtler
/// form.** The position searched is the one `GreedyPolicy` left at the
/// start of the last round, so the figure is *"winnable from where greedy
/// got to"*. A genuinely policy-free measure would search from round 1,
/// which §3.1 measured as unaffordable.
///
/// What it does buy: the last round's outcome no longer depends on which
/// agent plays it, so the measure is insensitive to exactly the variation
/// that made the bot rate meaningless (§4.1).
///
/// Deals where the search was cut by its budget are **not counted either
/// way** — they are reported separately, because folding "we stopped
/// looking" into "not winnable" is exactly the collapse §2.3 forbids.
fn winnable_fraction(players: u8) -> (u32, u32, u32) {
    let (mut yes, mut decided, mut undecided) = (0, 0, 0);
    for seed in 0..SEEDS {
        let Some(state) = last_rounds(players, seed, K as usize) else {
            continue;
        };
        match winnable_within(&state, K, BUDGET) {
            Verdict::Winnable { .. } => {
                yes += 1;
                decided += 1;
            }
            Verdict::NoneFound {
                exhausted: true, ..
            } => decided += 1,
            Verdict::NoneFound {
                exhausted: false, ..
            } => undecided += 1,
        }
    }
    (yes, decided, undecided)
}

fn report() {
    println!("difficulty — specs/RetrospectiveAnalysis.md §4\n");
    println!(
        "  winnable-from-round-{} over {SEEDS} seeds, budget {BUDGET} nodes",
        6 - K
    );
    println!("  policy win rates over the same {SEEDS} seeds\n");
    println!("  seats  winnable    greedy      random   first-legal   spread  undecided");

    for players in [2u8, 3, 4, 5, 6] {
        let (yes, decided, undecided) = winnable_fraction(players);
        let pct = |(w, n): (u32, u32)| {
            if n == 0 {
                -1.0
            } else {
                100.0 * f64::from(w) / f64::from(n)
            }
        };
        let g = pct(policy_wins("greedy", players));
        let r = pct(policy_wins("random", players));
        let f = pct(policy_wins("first-legal", players));
        let spread = [g, r, f].iter().cloned().fold(f64::MIN, f64::max)
            - [g, r, f].iter().cloned().fold(f64::MAX, f64::min);
        let wf = if decided == 0 {
            "n/a".to_string()
        } else {
            format!("{:.0}%", 100.0 * f64::from(yes) / f64::from(decided))
        };
        println!(
            "   {players}p    {wf:>6}     {g:>5.1}%      {r:>5.1}%      {f:>5.1}%     \
             {spread:>5.1}      {undecided:>3}"
        );
    }

    println!(
        "\n  WINNABLE is conditioned on GREEDY's play up to the last round —\n  \
         it is 'winnable from where greedy got to', not a property of the\n  \
         deal alone. Searching from round 1 is unaffordable (spec §3.1).\n\n  \
         It is also a LOWER BOUND: a K={K} search cannot see a line that\n  \
         needed an earlier round. `undecided` are deals whose search hit the\n  \
         node budget — they are excluded from the fraction, not counted as\n  \
         unwinnable.\n\n  \
         SPREAD is the range across three policies. Where it is large, no\n  \
         single policy's rate says anything about the game (§4.1)."
    );
}

fn self_test() -> i32 {
    let mut ok = true;
    let mut check = |name: &str, cond: bool, detail: String| {
        ok &= cond;
        println!(
            "  [{}] {name}{}",
            if cond { "ok " } else { "FAIL" },
            if detail.is_empty() {
                String::new()
            } else {
                format!("  — {detail}")
            }
        );
    };

    // The control the old harness never had: a search that finds a line
    // must produce one that REPLAYS. §2.1 is a gate, not a metric.
    let state = last_rounds(3, 7, K as usize).expect("a 3p game");
    match winnable_within(&state, K, BUDGET) {
        Verdict::Winnable { line, .. } => {
            let mut replay = state.clone();
            let mut good = true;
            for m in &line {
                match replay.validate(m.actor, &m.command) {
                    Ok(ev) => {
                        for e in &ev {
                            replay.fold(e);
                        }
                    }
                    Err(_) => good = false,
                }
            }
            let won = replay.outcome.as_ref().is_some_and(|o| o.group_success);
            check(
                "a witness replays to a win",
                good && won,
                format!("{} moves", line.len()),
            );
        }
        Verdict::NoneFound { .. } => check(
            "a witness replays to a win",
            false,
            "3p seed 7 found no line — the fixture moved".into(),
        ),
    }

    // The negative: a search must be able to return nothing, exhaustively.
    let lost = last_rounds(2, 7, 1).expect("a 2p game");
    match winnable_within(&lost, 1, BUDGET) {
        Verdict::NoneFound { exhausted, nodes } => check(
            "an unwinnable position is reported as searched-out",
            exhausted && nodes > 100,
            format!("{nodes} nodes"),
        ),
        Verdict::Winnable { .. } => check(
            "an unwinnable position is reported as searched-out",
            false,
            "found a win in a game 2p seed 7 lost".into(),
        ),
    }

    // A budget of one must NOT claim exhaustion — the distinction §2.3
    // rests on.
    match winnable_within(&state, K, 1) {
        Verdict::NoneFound { exhausted, .. } => check(
            "a budget cut is not reported as exhaustion",
            !exhausted,
            String::new(),
        ),
        Verdict::Winnable { .. } => check(
            "a budget cut is not reported as exhaustion",
            false,
            String::new(),
        ),
    }

    // §4.1's reason, asserted rather than asserted-about: the panel must
    // actually disagree, or reporting three policies is ceremony.
    let g = policy_wins("greedy", 6);
    let f = policy_wins("first-legal", 6);
    check(
        "the policy panel disagrees, so no single rate is a difficulty",
        g.0 != f.0,
        format!("greedy {}/{}, first-legal {}/{}", g.0, g.1, f.0, f.1),
    );

    println!("difficulty self-test (positive control)");
    i32::from(!ok)
}

fn main() {
    if std::env::args().any(|a| a == "--self-test") {
        std::process::exit(self_test());
    }
    report();
}