Compare commits
2 commits
1edda54217
...
1f0f652920
| Author | SHA1 | Date | |
|---|---|---|---|
| 1f0f652920 | |||
| 469d00d679 |
5 changed files with 1185 additions and 3 deletions
176
games/ground/examples/difficulty-baseline.rs
Normal file
176
games/ground/examples/difficulty-baseline.rs
Normal file
|
|
@ -0,0 +1,176 @@
|
|||
//! CB-RES-0008's runnable baseline (CB-WP-0025 T01).
|
||||
//!
|
||||
//! Two numbers the survey needs and cannot cite from anyone else, because
|
||||
//! they are about **our** game on **our** machine:
|
||||
//!
|
||||
//! 1. **What the bots actually achieve** — win rate by seat count over a
|
||||
//! seed range, which is the difficulty denominator `ground-game`'s
|
||||
//! GROUND-WP-0005 is blocked on.
|
||||
//! 2. **What a search would cost** — the branching factor of
|
||||
//! `legal_commands` and the price of enumerating it, which decides
|
||||
//! whether the honest version of "could we have won" is affordable.
|
||||
//!
|
||||
//! **This measures, it does not conclude.** Whether a bot win rate *is* a
|
||||
//! difficulty is exactly what the survey and the review have to argue
|
||||
//! about; this only makes the number exist.
|
||||
//!
|
||||
//! ```text
|
||||
//! cargo run --release -p games-ground --example difficulty-baseline
|
||||
//! ```
|
||||
|
||||
use cb_game_runtime::{ScenarioGame, Setup};
|
||||
use cb_kernel::Aggregate;
|
||||
use games_ground::bot::{legal_commands, play, GreedyPolicy, Policy, RandomPolicy};
|
||||
use games_ground::GroundState;
|
||||
|
||||
const SEEDS: u64 = 200;
|
||||
|
||||
fn setup(players: u8, seed: u64) -> Option<GroundState> {
|
||||
GroundState::setup(
|
||||
&Setup {
|
||||
players,
|
||||
preset: format!("standard-{players}p"),
|
||||
patch: Default::default(),
|
||||
},
|
||||
seed,
|
||||
)
|
||||
.ok()
|
||||
}
|
||||
|
||||
fn policies(kind: &str, players: u8, seed: u64) -> Vec<Box<dyn Policy>> {
|
||||
(0..players)
|
||||
.map(|i| -> Box<dyn Policy> {
|
||||
match kind {
|
||||
"random" => Box::new(RandomPolicy::new(seed ^ u64::from(i))),
|
||||
_ => Box::new(GreedyPolicy),
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Win rate, and the margin — because "we lost" and "we lost by one point"
|
||||
/// are different games, and a rate alone hides which one this is.
|
||||
fn win_rate(kind: &str, players: u8) {
|
||||
let (mut wins, mut played, mut total_pts, mut total_thr) = (0u32, 0u32, 0u64, 0u64);
|
||||
let mut margins: Vec<i64> = Vec::new();
|
||||
for seed in 0..SEEDS {
|
||||
let Some(state) = setup(players, seed) else {
|
||||
continue;
|
||||
};
|
||||
let mut ps = policies(kind, players, seed);
|
||||
let Ok(game) = play(state, &mut ps) else {
|
||||
continue;
|
||||
};
|
||||
let Some(o) = &game.state.outcome else {
|
||||
continue;
|
||||
};
|
||||
played += 1;
|
||||
if o.group_success {
|
||||
wins += 1;
|
||||
}
|
||||
total_pts += u64::from(o.total);
|
||||
total_thr += u64::from(o.threshold);
|
||||
margins.push(i64::from(o.total) - i64::from(o.threshold));
|
||||
}
|
||||
if played == 0 {
|
||||
println!(" {players}p {kind:>6} no games completed");
|
||||
return;
|
||||
}
|
||||
margins.sort_unstable();
|
||||
let median = margins[margins.len() / 2];
|
||||
// Wilson-free: report the count, not a confidence interval we have not
|
||||
// argued for. The spec (T04) decides what interval is claimed.
|
||||
println!(
|
||||
" {players}p {kind:>6} {wins:>3}/{played:<3} won = {rate:>5.1}% \
|
||||
mean total {mt:>4.1} of {th:>4.1} median margin {median:+}",
|
||||
rate = 100.0 * f64::from(wins) / f64::from(played),
|
||||
mt = total_pts as f64 / f64::from(played),
|
||||
th = total_thr as f64 / f64::from(played),
|
||||
);
|
||||
}
|
||||
|
||||
/// What one node of a search costs, and how wide it is.
|
||||
///
|
||||
/// Measured on real mid-game states rather than on a fresh deal: at deal
|
||||
/// time most of the interesting branches do not exist yet, and a
|
||||
/// branching factor taken there would flatter any search proposal.
|
||||
fn search_cost(players: u8) {
|
||||
let mut widths: Vec<usize> = Vec::new();
|
||||
let mut nodes = 0u64;
|
||||
// **The clock brackets `legal_commands` AND NOTHING ELSE.**
|
||||
//
|
||||
// The first version started it before the seed loop, so it timed two
|
||||
// `setup`s, a whole greedy game and a full validate+fold replay, then
|
||||
// divided by the number of player decisions — reporting 112–161 µs
|
||||
// for a call that costs ~16–20. The adversarial review caught it (C1),
|
||||
// and the tell was in the published output: the figure FELL as seat
|
||||
// count rose while branching rose, which is backwards for a
|
||||
// per-enumeration cost. Accumulate only the call.
|
||||
let mut spent = std::time::Duration::ZERO;
|
||||
|
||||
for seed in 0..40u64 {
|
||||
let Some(state) = setup(players, seed) else {
|
||||
continue;
|
||||
};
|
||||
// Walk a real game and sample the branching at every decision.
|
||||
let mut ps = policies("greedy", players, seed);
|
||||
let Ok(game) = play(state, &mut ps) else {
|
||||
continue;
|
||||
};
|
||||
// Re-run the recorded commands, enumerating legality at each step.
|
||||
let Some(mut replay) = setup(players, seed) else {
|
||||
continue;
|
||||
};
|
||||
for (actor, cmd) in &game.steps {
|
||||
if let cb_kernel::Actor::Player(seat) = actor {
|
||||
let t0 = std::time::Instant::now();
|
||||
let legal = legal_commands(&replay, *seat);
|
||||
spent += t0.elapsed();
|
||||
widths.push(legal.len());
|
||||
nodes += 1;
|
||||
}
|
||||
if let Ok(events) = replay.validate(*actor, cmd) {
|
||||
for e in &events {
|
||||
replay.fold(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if widths.is_empty() {
|
||||
println!(" {players}p no decisions sampled");
|
||||
return;
|
||||
}
|
||||
widths.sort_unstable();
|
||||
let sum: usize = widths.iter().sum();
|
||||
println!(
|
||||
" {players}p {n} decisions branching mean {mean:.1} median {med} max {max} \
|
||||
{per:.1} us/call",
|
||||
n = widths.len(),
|
||||
mean = sum as f64 / widths.len() as f64,
|
||||
med = widths[widths.len() / 2],
|
||||
max = widths[widths.len() - 1],
|
||||
per = spent.as_nanos() as f64 / nodes as f64 / 1000.0,
|
||||
);
|
||||
}
|
||||
|
||||
fn main() {
|
||||
println!("CB-RES-0008 baseline — measured, not concluded\n");
|
||||
println!("bot win rate over {SEEDS} seeds (GR-E01 group success):");
|
||||
for players in [2u8, 3, 4, 5, 6] {
|
||||
win_rate("greedy", players);
|
||||
}
|
||||
for players in [2u8, 3, 4] {
|
||||
win_rate("random", players);
|
||||
}
|
||||
|
||||
println!("\nsearch cost — legal_commands at every real decision point:");
|
||||
for players in [2u8, 3, 4] {
|
||||
search_cost(players);
|
||||
}
|
||||
|
||||
println!(
|
||||
"\nNOTE: a win rate is this POLICY's win rate over THIS seed range.\n\
|
||||
Whether that is 'the difficulty' is T02's argument, not this tool's claim."
|
||||
);
|
||||
}
|
||||
461
history/260805-could-we-have-won-challenge.md
Normal file
461
history/260805-could-we-have-won-challenge.md
Normal file
|
|
@ -0,0 +1,461 @@
|
|||
# 260805 — challenge to CB-RES-0008
|
||||
|
||||
Adversarial review, one round, per InnerLoop §Step 2. Target: the survey
|
||||
`research/CB-RES-0008-could-we-have-won.md`, the harness
|
||||
`games/ground/examples/difficulty-baseline.rs`, and the T01 §judgment in
|
||||
`CB-WP-0025:126-157`.
|
||||
|
||||
**Fidelity note, first.** Run in a separate agent session with only the
|
||||
files. There is one harness and one repo, so per §Step 2 this review
|
||||
**inherits the author's sampling** and does not report a clean verify on
|
||||
that basis. What it substitutes is **mutation**: every quoted number was
|
||||
traced to the expression that produces it, and the expression was changed.
|
||||
A mutant copy of the harness (`examples/zz-review-mutant.rs`) was written,
|
||||
run, and deleted; `git status` is clean and no file in the repo was
|
||||
modified. Every number below is reproducible by re-creating that mutant
|
||||
from the diffs quoted inline.
|
||||
|
||||
Three of the four headline claims move under mutation. One of them moves
|
||||
by a factor of seven.
|
||||
|
||||
Ranked. **C1 and C2 land hardest.** C7 is marked weak.
|
||||
|
||||
---
|
||||
|
||||
## C1 — `legal_commands` does not cost 112–161 µs. It costs 15.6–20.4 µs. The timer is around the wrong thing.
|
||||
|
||||
The survey, §1.2:
|
||||
|
||||
> *"**`legal_commands` costs 112–161 µs per call**, because it constructs
|
||||
> candidates and filters them through full `validate`. … it is the number
|
||||
> that decides this pass."*
|
||||
|
||||
Read the timer's scope. `difficulty-baseline.rs:100` starts the clock:
|
||||
|
||||
```rust
|
||||
100: let start = std::time::Instant::now();
|
||||
102: for seed in 0..40u64 {
|
||||
103: let Some(state) = setup(players, seed) else { continue }; // deal #1
|
||||
108: let Ok(game) = play(state, &mut ps) else { continue }; // A WHOLE GREEDY GAME
|
||||
112: let Some(mut replay) = setup(players, seed) else { continue };// deal #2
|
||||
115: for (actor, cmd) in &game.steps {
|
||||
118: let legal = legal_commands(&replay, *seat); // the thing being claimed
|
||||
121: if let Ok(events) = replay.validate(*actor, cmd) { // + full replay
|
||||
122: for e in &events { replay.fold(e); }
|
||||
129: let elapsed = start.elapsed();
|
||||
143: per = elapsed.as_micros() as f64 / nodes as f64,
|
||||
```
|
||||
|
||||
`elapsed` is **two `setup`s, a complete five-round greedy game (which
|
||||
itself calls `legal_commands` on every step and runs the whole ranking
|
||||
loop), a second deal, and a full validate+fold replay of every command,
|
||||
player and system alike** — divided by the count of *player* decision
|
||||
points only. It is not a per-node search cost. It is a per-game cost with
|
||||
a per-node denominator.
|
||||
|
||||
**Mutation.** Bracket `legal_commands` alone (`Instant::now()` immediately
|
||||
before line 118, accumulate on the next line), and separately time the
|
||||
`GroundState::clone` a real search must also pay per node:
|
||||
|
||||
```
|
||||
2p 462 nodes mean width 4.7 ORIGINAL 112 us legal_commands ALONE 15.6 us clone 1.3 us
|
||||
3p 649 nodes mean width 7.4 ORIGINAL 115 us legal_commands ALONE 18.0 us clone 1.4 us
|
||||
4p 870 nodes mean width 9.1 ORIGINAL 116 us legal_commands ALONE 20.4 us clone 1.4 us
|
||||
```
|
||||
|
||||
**The claimed number is 6–8× the measured one.** And the mutant supplies
|
||||
the diagnostic that should have caught it in the survey: the survey's
|
||||
figures **fall** with seat count (161 → 139 → 112) while branching
|
||||
**rises** (4.7 → 7.4 → 9.1). If the number were the cost of constructing
|
||||
and validating candidates, it would rise with the number of candidates.
|
||||
`legal_commands` alone does exactly that — 15.6 → 18.0 → 20.4. The
|
||||
survey's number falls because the fixed per-game overhead is being
|
||||
amortised over more nodes at higher seat counts. **The reported quantity
|
||||
varies inversely with the mechanism the prose gives for it**, in the
|
||||
survey's own printed table, and the survey did not notice.
|
||||
|
||||
**Secondary, and it is enough on its own.** The number is not stable.
|
||||
Three runs of the *unmodified* example on this machine:
|
||||
|
||||
```
|
||||
run 1 (survey) 2p 161 3p 139 4p 112
|
||||
run 2 2p 132 3p 128 4p 139
|
||||
run 3 (mutant) 2p 112 3p 115 4p 116
|
||||
```
|
||||
|
||||
The 2p figure moved 161 → 112 between runs, and the seat-count ordering
|
||||
inverted. The survey quotes `112–161 µs/node` in three places
|
||||
(`:44`, `:83`, `:189`) as a **measured range across seat counts**. It is a
|
||||
range across *runs*, of a quantity that is mostly loop overhead. Anyone
|
||||
can settle this: run the example twice.
|
||||
|
||||
**What this changes in the design, which is why it is C1.** §1.2 concludes
|
||||
*"at ~140 µs/node: a bounded search of ~10⁴–10⁵ nodes costs 1.4–14
|
||||
seconds. **That is the budget the ADR has to design inside.**"* At the
|
||||
measured ~17 µs for `legal_commands` plus ~1.4 µs for the clone, ~19
|
||||
µs/node, the same 1.4–14 s buys **~10⁵–10⁶ nodes**. T03 is about to pick a
|
||||
node bound one order of magnitude too small, and T04 is about to write it
|
||||
into an acceptance table.
|
||||
|
||||
**Required:** re-scope the timer to the call the prose names, re-quote,
|
||||
and state the run-to-run spread rather than a single range. If the intent
|
||||
was "the cost of one node of a search that replays from a scenario", say
|
||||
so and price the clone-and-fold node separately — but then it is not
|
||||
`legal_commands`'s cost and §1.2's causal sentence must go.
|
||||
|
||||
## C2 — The ratio does not explain the curve, and the survey's own table proves it: two rows with an identical ratio are 12.5 points apart
|
||||
|
||||
§1.1 is the pass's headline finding:
|
||||
|
||||
> *"**The ratio moves the wrong way.** … Three multipliers all pointing the
|
||||
> same direction, which is why the curve is not gentle — it is 66% → 100%
|
||||
> across four seat counts."*
|
||||
|
||||
| seats | ratio (survey) | greedy win rate |
|
||||
|---|---|---|
|
||||
| 2 | 1.20 | 66.0% |
|
||||
| 3 | 1.29 | 82.5% |
|
||||
| 4 | **1.29** | **95.0%** |
|
||||
| 5 | 1.33 | 100% |
|
||||
| 6 | **1.33** | **100%** |
|
||||
|
||||
**3p and 4p have the same deal, the same threshold, and the same ratio,
|
||||
and differ by 12.5 points of win rate** — a jump as large as either of the
|
||||
two between-ratio jumps. The ratio takes three distinct values across five
|
||||
rows; seat count takes five. The explanatory variable and the confound are
|
||||
not separated anywhere in the survey, and the one comparison that
|
||||
separates them (3p vs 4p) points at seat count, not at the ratio.
|
||||
|
||||
It gets worse for the ratio when the arithmetic is done at row level,
|
||||
which §1.1 claims to have done and has not (see C3). Claimable point
|
||||
values are `{2, 2, 2}` at 2p, `{2, 2, 2, 3}` at 3–4p, `{2, 2, 2, 3, 3}` at
|
||||
5–6p (`editions/ground-darvo-r0/Problems.csv`, priorities 0–4 of every
|
||||
scenario; all four scenarios carry the same value vector, checked). So the
|
||||
**achievable** totals are not continuous, and the useful quantity is *how
|
||||
much of the board must be claimed*:
|
||||
|
||||
| seats | achievable totals | threshold | Problems that must be claimed | effective slack |
|
||||
|---|---|---:|---|---:|
|
||||
| 2 | 0, 2, 4, 6 | 5 | **3 of 3** — a full clear | **1.00** |
|
||||
| 3–4 | 0, 2, 3, 4, 5, 6, 7, 9 | 7 | 3 of 4, incl. the 3-pointer | 1.29 |
|
||||
| 5–6 | 0 … 12 | 9 | 4 of 5 | 1.20 |
|
||||
|
||||
A threshold of 5 at 2p is **identical to a threshold of 6** — nothing sums
|
||||
to 5. The survey's `1.20` is not slack; the real slack at 2p is 1.00, and
|
||||
`games/ground/src/lib.rs:2504` already says so in as many words (*"2+2+2
|
||||
against a threshold of 5 means a full clear"*). And on this measure the
|
||||
sequence is **1.00 / 1.29 / 1.20 — not monotone**, so "the ratio moves the
|
||||
wrong way" reverses at the seat count the finding is loudest about.
|
||||
|
||||
**Confirmed directly.** Instrumenting the harness to record whether the
|
||||
group claimed *every* Problem on the board:
|
||||
|
||||
```
|
||||
2p greedy 132/200 won cleared-the-board 132/200 AVAIL 6
|
||||
3p greedy 165/200 won cleared-the-board 161/200 AVAIL 9
|
||||
4p greedy 190/200 won cleared-the-board 190/200 AVAIL 9
|
||||
5p greedy 200/200 won cleared-the-board 186/200 AVAIL 12
|
||||
6p greedy 200/200 won cleared-the-board 200/200 AVAIL 12
|
||||
```
|
||||
|
||||
At 2p, wins == board-clears **exactly** (132 = 132): the 2-player game is
|
||||
not "a real game at 66%", it is pass/fail on a full clear. The mechanism
|
||||
driving the curve is the third item the survey lists last and never
|
||||
quantifies — more seats means more hands, which means more matching
|
||||
Solutions, which means a higher fraction of a *fixed* pool gets claimed.
|
||||
More seats add **no points**; they add claimants for the same 5 Problems.
|
||||
|
||||
**Required:** withdraw "three multipliers all pointing the same direction"
|
||||
or measure it. The separating experiment is cheap and was not run: hold
|
||||
seats fixed and move the threshold, or hold the threshold fixed and move
|
||||
`hidden_depth` (`games/ground/src/edition.rs:123-130`). Until one of those
|
||||
runs, the finding handed to GROUND-WP-0005 tells them to tune the wrong
|
||||
dial — and T06 ships it into another repo.
|
||||
|
||||
## C3 — The finding is inadmissible under GameDesign §1 on two of the three clauses, and the survey asserts all three
|
||||
|
||||
§1.1:
|
||||
|
||||
> *"**This is admissible under GameDesign §1**: the reproduction exists
|
||||
> (`examples/difficulty-baseline.rs`), it has the ruled shape (row-level,
|
||||
> no sums), and it can fail — change a threshold and the numbers move."*
|
||||
|
||||
**Clause 3, "can fail" — fails.** `difficulty-baseline.rs` contains **zero
|
||||
assertions**:
|
||||
|
||||
```
|
||||
$ grep -c "assert" games/ground/examples/difficulty-baseline.rs
|
||||
0
|
||||
$ grep -rn "difficulty" Makefile gates.toml facts.toml
|
||||
(no output)
|
||||
```
|
||||
|
||||
It prints and exits 0. Delete the win-rate line, change a threshold,
|
||||
break the deal — it still exits 0. *"The numbers move"* is **sensitivity**,
|
||||
not failability; GameDesign §1.3 requires *"the artifact must be capable
|
||||
of going red, and the register records its current colour."* An artifact
|
||||
with no assertion has no colour. It is not a `counterexample` (nothing
|
||||
alarms) and it is not a `default` (it encodes no choice) — it is a third
|
||||
thing, an **observation**, and CB-WP-0022 T05's role table has no row for
|
||||
it. That is the distinction this repo paid for four days ago.
|
||||
|
||||
This also violates InnerLoop §Measurement validity twice over: *"every
|
||||
tool that reports a number exposes `--self-test`, and that self-test runs
|
||||
before the number is produced"* — there is none, and the example is not
|
||||
wired into `make` at all — and *"state the divisor used to convert raw
|
||||
timings into the metric's unit, pinned by a test"* — `nodes` at
|
||||
`:143` is unpinned. **Mutation:** run the win-rate loop over 40 seeds
|
||||
instead of 200 (`const SEEDS: u64 = 40`):
|
||||
|
||||
```
|
||||
6p greedy 40/40 = 100.0% mean total 12.0 of 9.0 median margin +3
|
||||
```
|
||||
|
||||
Nothing in the harness objects to a denominator that shrank by 5×. Three
|
||||
`continue`s (`:57`, `:62`, `:65`) can silently drop games out of `played`
|
||||
and there is no `assert_eq!(played, SEEDS)`. *(To the author's credit,
|
||||
`played` is printed, and on the current run it is genuinely 200/200 — see
|
||||
§What survives. The control is missing, not the work.)*
|
||||
|
||||
**Clause 2, "the ruled shape" — fails.** GameDesign §1.2, quoting
|
||||
GROUND-WP-0004 T02: *"an arithmetic finding ships a row-level table —
|
||||
Surface and each hidden priority listed **separately** — never 'sum of
|
||||
file'"*, and *"the artifact **prints the rows** it came from. A finding
|
||||
stating a total without its rows is inadmissible even if the total is
|
||||
right."*
|
||||
|
||||
The survey's table is:
|
||||
|
||||
| seats | Surface | hidden dealt | points available | threshold | ratio |
|
||||
|---|---|---|---:|---:|---|
|
||||
| 2 | priority 1 | 2 | **6** | 5 | 1.20 |
|
||||
|
||||
`6`, `9`, `12` are **sums**. The hidden priorities are collapsed into a
|
||||
count (`hidden dealt: 2`), which is the exact shape the ruling forbids.
|
||||
The `Surface` column says *"priority 1"* in all three rows, and Surface is
|
||||
`hidden_priority` **0** in `Problems.csv` — the cell is either wrong or
|
||||
meaningless, and it is the cell the ruled shape is about.
|
||||
|
||||
And the named artifact prints **none of this**. `difficulty-baseline.rs`
|
||||
never prints available points, thresholds by row, or priorities; it prints
|
||||
win rates and timings. The reproduction cited for the 6/9/12 finding does
|
||||
not compute 6/9/12. The thing that does is
|
||||
`games_ground::gd0001_group_success_is_reachable_at_every_seat_count`
|
||||
(`lib.rs:2508`) — which the survey does not name, and which *also* sums
|
||||
(`lib.rs:2512`: `.map(|p| p.value).sum()`).
|
||||
|
||||
**Required:** either name `gd0001` and give it the row-level print the
|
||||
ruling requires, or drop the admissibility claim. This is the third time a
|
||||
correct total has shipped with the wrong shape (`"12 in the file"`,
|
||||
`"4/6/9"`, and now this), and it is the failure GameDesign §1.2 was
|
||||
written this week to stop.
|
||||
|
||||
## C4 — 66% at two seats is a `GreedyPolicy` defect, not a difficulty. A policy with no heuristic at all scores 77.5%.
|
||||
|
||||
The task the survey sets itself in §4 and defers in §6 — *"whether a bot
|
||||
win rate is a difficulty at all"* — is settled against it by one mutation.
|
||||
Add two zero-knowledge policies: `FirstLegal` (always `Choice::Command(0)`)
|
||||
and `LastLegal` (always the last offered command), 200 seeds each:
|
||||
|
||||
```
|
||||
2p greedy 132/200 = 66.0% 2p first 155/200 = 77.5% 2p last 0/200 2p random 10/200 = 5.0%
|
||||
3p greedy 165/200 = 82.5% 3p first 50/200 = 25.0% 3p last 0/200 3p random 19/200 = 9.5%
|
||||
4p greedy 190/200 = 95.0% 4p first 64/200 = 32.0% 4p last 0/200 4p random 16/200 = 8.0%
|
||||
5p greedy 200/200 = 100.0% 5p first 0/200 = 0.0% 5p last 0/200 5p random 6/200 = 3.0%
|
||||
6p greedy 200/200 = 100.0% 6p first 0/200 = 0.0% 6p last 0/200 6p random 7/200 = 3.5%
|
||||
```
|
||||
|
||||
**At two seats, taking the first command in canonical order beats the
|
||||
stated heuristic by 11.5 points.** So:
|
||||
|
||||
1. **The 66% is not the game's 2-player difficulty.** It is the point at
|
||||
which `GreedyPolicy`'s ranking (`bot.rs:326-380`) becomes *worse than
|
||||
no ranking*. Somebody improving the bot next week moves this row to
|
||||
~78% and "the 2-player game got easier" without a rule changing — which
|
||||
is the objection §6 says is the strongest and leaves open. It is not
|
||||
open; it is demonstrated, on the row the survey leans on hardest.
|
||||
2. **The greedy/random pair brackets nothing.** The task asked whether the
|
||||
two figures bracket anything meaningful. They do not: a third trivial
|
||||
policy escapes the bracket from above at 2p (77.5% > 66%) and falls
|
||||
below `random` at 5–6p (0% < 3%). The interval `[random, greedy]` is
|
||||
not a range of achievable play; it is two arbitrary points.
|
||||
3. **`GreedyPolicy` never passes.** `choose` ignores `may_pass`
|
||||
(`bot.rs:388-405`) and always returns a `Command`. At Reveal the driver
|
||||
loops while anything is legal (`bot.rs:501-510`), so greedy takes
|
||||
*every* available Reveal action — Bonds, GROUND modes, DARVO targets —
|
||||
until the offer set empties. That is not "the obvious action"; it is
|
||||
maximal action. The survey's honest reading at §4 (*"a bot that takes
|
||||
the obvious action"*) understates what is being measured.
|
||||
|
||||
**Where this challenge stops, and it stops in the author's favour.** The
|
||||
6-seat row survives, for a reason the survey never gives. At 6p available
|
||||
points are 12 and greedy's mean total is **12.0 with 200/200 board
|
||||
clears** — greedy attains the **theoretical maximum in every single deal**.
|
||||
No policy can beat it, so *"a greedy bot wins 200/200 at six seats"* is
|
||||
not a bot claim at 6p: it is the claim that the threshold (9) sits below a
|
||||
ceiling (12) that ordinary play reaches every time. **That argument is
|
||||
available in the harness's own output and the survey does not make it** —
|
||||
it concedes the ground at §4 and §6 instead. Make it, and the 6-seat
|
||||
finding is defensible on rules grounds. The 2p, 3p and 4p rows are not,
|
||||
and 5p (186/200 clears) is intermediate.
|
||||
|
||||
**Required:** restate §1.1 as a claim about seat counts 5–6 only,
|
||||
supported by the ceiling argument, and withdraw the 66%/82.5%/95% figures
|
||||
as difficulty statements — or ship the bracket (≥3 policies) and name the
|
||||
number `greedy-200seed-win-rate`, which §5's own benchmark row already
|
||||
demands and §1.1 does not do.
|
||||
|
||||
## C5 — "It explains the maintainer's report" — the repo already contains a better explanation, and it is in a doc comment the survey did not read
|
||||
|
||||
§1.1:
|
||||
|
||||
> *"It also **explains the maintainer's report** … He plays at low seat
|
||||
> counts, where 66% is a real game, and had been feeling the 5–6 seat
|
||||
> experience from elsewhere in the same session."*
|
||||
|
||||
`games/ground/src/lib.rs:2487-2498`:
|
||||
|
||||
> *"This test used to assert the opposite, and it was right to: **the
|
||||
> maintainer played several 3-player games on 2026-08-03 and could not win
|
||||
> any of them**, because GR-S01 dealt 2/3/4 Problems worth 3/6/10 against
|
||||
> thresholds of 5/7/9."*
|
||||
|
||||
At 3 players on the pre-ruling deal there were **6 points on the table
|
||||
against a threshold of 7**. The games he lost were not 66%-likely; they
|
||||
were **arithmetically unwinnable**, at 3 seats, and the deal was ruled the
|
||||
next day (`2da19a4`, 2026-08-04). *"I felt it was too easy but then we
|
||||
lost, so who knows"* is fully explained by: the pre-ruling deal made
|
||||
losing certain, and the ruling that fixed it landed after he played.
|
||||
|
||||
The survey's explanation requires (a) that he plays at 2 seats, (b) that
|
||||
66% is the relevant rate, and (c) an unevidenced claim that he *"had been
|
||||
feeling the 5–6 seat experience from elsewhere in the same session"*.
|
||||
None of the three is sourced anywhere in the repo. The competing
|
||||
explanation is sourced, dated, and sitting in the test that was inverted
|
||||
because of it.
|
||||
|
||||
This matters beyond tidiness: §1.1's claim to *explain the report* is what
|
||||
elevates the finding from "a bot measurement" to "the answer to the
|
||||
maintainer's question", and it is the sentence CB-WP-0025:141-143 repeats.
|
||||
If the report is already explained by a bug that is already fixed, then
|
||||
the 5–6 seat finding is a **new, separate** finding and should be reported
|
||||
as one — which is a better outcome for the pass, not a worse one.
|
||||
|
||||
**Required:** delete the explanation, or ask him. §6 already concedes the
|
||||
one-question experiment was not run (*"Nobody has asked him"*) — that
|
||||
concession applies to this sentence too, and §1.1 states as settled what
|
||||
§6 lists as unsettled.
|
||||
|
||||
## C6 — "Exhaustive search is out at any seat count" is false at two seats, and it is answering a question T05 does not ask
|
||||
|
||||
§1.2: *"An exhaustive search from round 1 at 3 seats is roughly `7.4^15` —
|
||||
not a number worth writing down. **Exhaustive search is out at any seat
|
||||
count**, and this was measured rather than assumed."*
|
||||
|
||||
The exponent survives (see §What survives). Three things about the
|
||||
conclusion do not.
|
||||
|
||||
**(a) Two seats.** Measured decisions per game at 2p: 462/40 = 11.6, mean
|
||||
width 4.7. `4.7^11.6 ≈ 5.8 × 10⁷` nodes. At C1's corrected ~17 µs that is
|
||||
**~16 minutes**, single-threaded, no pruning, no transposition. Slow, and
|
||||
plainly not "out". The survey computed the 3-seat figure and generalised
|
||||
to "any seat count" without computing the 2-seat one — at the seat count
|
||||
§1.1 says the maintainer plays.
|
||||
|
||||
**(b) The wrong root.** T05's feature runs on **a recorded lost game**.
|
||||
Nobody asks "could we have won from round 1"; they ask it after the loss,
|
||||
and the useful witness starts at the divergence, typically the last one or
|
||||
two rounds. Depth 2 rounds × 3 seats = 6: `7.4^6 ≈ 1.6 × 10⁵` nodes ×
|
||||
~19 µs = **~3 seconds**. Exhaustive search over the final two rounds is
|
||||
affordable *today*, at 3 seats, with no algorithm at all. That is a
|
||||
materially different ADR than "bounded search, PIMC or ISMCTS, 10⁴ nodes".
|
||||
|
||||
**(c) `branching^depth` is the tree, not the state space.** GROUND is
|
||||
co-operative — all seats share one objective, so this is single-agent
|
||||
planning, not adversarial search, and single-agent planning transposes.
|
||||
The state that determines the answer is roughly (round, claimed-set,
|
||||
hands, stress): at 5–6 seats the claimed-set is a subset of **five**
|
||||
Problems. `2^5 × 5 rounds = 160` scoring-relevant classes. A search that
|
||||
memoises collapses `7.4^15` to something that does not need a bound at
|
||||
all. The survey does not mention memoisation, transposition, or the
|
||||
co-operative structure once, and it rules out the class of search that
|
||||
would benefit most from all three.
|
||||
|
||||
**Required:** compute the exponent for the question T05 asks (search from
|
||||
a recorded state, not from round 1), state the 2-seat figure, and either
|
||||
argue that transposition does not help here or stop concluding
|
||||
"exhaustive is out **at any seat count**". §5's `search cost` benchmark
|
||||
row inherits the wrong bound as written.
|
||||
|
||||
## C7 — (weak) The replay loop discards rejections silently, but currently has none
|
||||
|
||||
`difficulty-baseline.rs:121`: `if let Ok(events) = replay.validate(...)`.
|
||||
A rejection is dropped on the floor: the replay state would stop
|
||||
advancing, `legal_commands` would then be sampled on a **stale state**,
|
||||
and the branching figures would be measured against a game that had
|
||||
diverged from the one `play` produced. Nothing reports it.
|
||||
|
||||
**Mutation, and it clears the author.** Counting the two arms:
|
||||
|
||||
```
|
||||
2p replay validate: 1062 ok / 0 REJECTED
|
||||
3p replay validate: 1249 ok / 0 REJECTED
|
||||
4p replay validate: 1470 ok / 0 REJECTED
|
||||
```
|
||||
|
||||
The replay is faithful today, so no quoted number is contaminated. Marked
|
||||
**weak**: this is a missing control, not a wrong number, and it costs one
|
||||
`else { panic! }` to close. Do not spend the response round on it beyond
|
||||
adding the arm.
|
||||
|
||||
---
|
||||
|
||||
## Verdict
|
||||
|
||||
**Not approvable as written.** C1, C2, C3 and C4 each require a change to
|
||||
the survey, not a clarification. C5 requires a deletion. C6 requires a
|
||||
recomputation.
|
||||
|
||||
| # | verdict |
|
||||
|---|---|
|
||||
| **C1** | **lands hardest.** `legal_commands` is 15.6–20.4 µs, not 112–161. The timer brackets two deals, a whole greedy game and a full replay, over a player-decision denominator. The claimed number varies *inversely* with the mechanism given for it, and moves 161→112 between runs of the unmodified example. The ADR's node budget is off by ~10×. |
|
||||
| **C2** | **lands, equally hard on the finding.** 3p and 4p share a ratio and differ by 12.5 points of win rate — the survey's own table falsifies "the ratio explains the curve". Row-level slack is 1.00/1.29/1.20, not monotone. The separating experiment was never run, and T06 ships this to another repo. |
|
||||
| **C3** | **lands.** Zero assertions, no `--self-test`, not in `make`; "can fail" is asserted about an artifact that cannot go red. And 6/9/12 are sums, printed by nothing — the exact shape GameDesign §1.2 forbids, for the third time. |
|
||||
| **C4** | **lands.** A policy with no heuristic scores 77.5% at 2p against greedy's 66%. §6's "strongest counter" is not open, it is demonstrated. Partly self-repairing: the 6-seat row is rescuable by a ceiling argument the survey has the data for and does not make. |
|
||||
| **C5** | **lands, narrow.** The maintainer's lost games are already explained by the pre-ruling deal (`lib.rs:2489`, 3p, 6 points against a threshold of 7). §1.1 states as settled what §6 lists as unasked. |
|
||||
| **C6** | **lands, moderate.** "Out at any seat count" is ~16 min at 2p and ~3 s over the last two rounds at 3p — and the co-operative, small-state-space structure that makes memoisation work is never mentioned. |
|
||||
| **C7** | **weak.** Missing control, currently clean (0 rejections at all three seat counts). |
|
||||
|
||||
**What survives.** Four things were attacked and held:
|
||||
|
||||
- **`nodes` and `widths.len()` are the same denominator.** Both are
|
||||
incremented inside the same `if let Actor::Player` arm
|
||||
(`:118-120`), and the printed `n` equals the divisor on every run
|
||||
(462/649/870). What would have falsified it: `nodes` counting system
|
||||
steps too, which would have deflated the per-node figure by a further
|
||||
~2.3×. It does not.
|
||||
- **No games were dropped.** Instrumenting all three `continue` arms
|
||||
gives `setup 0, play 0, outcome 0` at every seat count — the quoted
|
||||
`/200` denominators are genuinely 200. What would have falsified it: any
|
||||
non-zero drop, which would have made "200 of 200" a survivor-biased
|
||||
rate. There is none. *(The control is still absent — C3.)*
|
||||
- **`GreedyPolicy` is a real heuristic, not first-legal.** C4's `FirstLegal`
|
||||
mutant diverges from it at every seat count (0% vs 100% at 6p). What
|
||||
would have falsified it: the two policies producing identical rates,
|
||||
which would have meant the ranking at `bot.rs:326` was inert.
|
||||
- **The 6/9/12 arithmetic itself, and the 5×N exponent.** Surface 2 + hidden
|
||||
{2,2}/{2,2,3}/{2,2,3,3} = 6/9/12 against 5/7/9 checks out against
|
||||
`Problems.csv` (all four scenarios carry the identical value vector) and
|
||||
`edition.rs:119-130`. Measured decisions per game — 11.6/16.2/21.8 vs
|
||||
the survey's 5×N of 10/15/20 — are within 10%, so C6 attacks the
|
||||
conclusion, not the exponent. What would have falsified either: a
|
||||
scenario in the edition with different values, or a decision count far
|
||||
from 5N. Neither exists.
|
||||
|
||||
**The single challenge that forces a change to the design: C1.** Every
|
||||
other challenge changes what the survey *says*. C1 changes what T03 will
|
||||
*decide*: the affordable budget is ~10⁵–10⁶ nodes, not ~10⁴–10⁵, and at
|
||||
that budget C6's exhaustive-over-the-last-two-rounds search comes into
|
||||
range — which is a different ADR, with a different honesty story, than a
|
||||
bounded PIMC/ISMCTS design chosen because exhaustive was ruled out. The
|
||||
number that "decides this pass" was measured around the wrong brackets.
|
||||
197
history/260805-could-we-have-won-response.md
Normal file
197
history/260805-could-we-have-won-response.md
Normal file
|
|
@ -0,0 +1,197 @@
|
|||
# 260805 — response to the challenge on CB-RES-0008
|
||||
|
||||
One round, per InnerLoop §Step 2. Separate agent, as with CB-RES-0007.
|
||||
|
||||
**Six challenges conceded, one noted. The survey's headline finding is
|
||||
withdrawn.** And one challenge lands harder than the reviewer scored it —
|
||||
C4, which they ranked fourth, is the one that kills the claim.
|
||||
|
||||
---
|
||||
|
||||
## C1 — the timer measured almost everything except the thing it named — **conceded**
|
||||
|
||||
`difficulty-baseline.rs` started the clock **before the seed loop**, so
|
||||
"µs/node" included two `setup`s, an entire greedy game, and a full
|
||||
validate+fold replay — then divided by the number of *player decisions*.
|
||||
|
||||
**The tell was in my own published output and I did not look at it.** The
|
||||
figure **fell** as seat count rose — 161 → 139 → 112 — while branching
|
||||
**rose** 4.7 → 7.4 → 9.1. A per-enumeration cost cannot do that. The
|
||||
denominator was growing faster than the numerator because the numerator
|
||||
was mostly fixed per-game work.
|
||||
|
||||
Re-measured with the clock bracketing `legal_commands` and nothing else:
|
||||
|
||||
```
|
||||
2p 462 decisions branching mean 4.7 median 5 max 7 3.0 us/call
|
||||
3p 649 decisions branching mean 7.4 median 8 max 10 3.5 us/call
|
||||
4p 870 decisions branching mean 9.1 median 10 max 12 4.1 us/call
|
||||
```
|
||||
|
||||
Now it rises with branching, which is the direction the mechanism
|
||||
predicts.
|
||||
|
||||
**A discrepancy I am not going to paper over.** The reviewer measured
|
||||
**15.6–20.4 µs**; I measure **3.0–4.1 µs** bracketing each call with
|
||||
`Instant::now()`. We disagree by ~5×, which is small against the 30–50×
|
||||
correction we agree on but is not nothing. Two different isolation methods
|
||||
gave two different answers and **neither of us has established which is
|
||||
right** — so T04 must settle the per-node cost with a proper benchmark
|
||||
(this repo has `criterion` already), not with either of these numbers.
|
||||
|
||||
**This is the wrong-denominator family again** — CB-EV-0019 §1 named it
|
||||
with three instances, and this is a fourth, in the *numerator* this time:
|
||||
a number computed correctly over the wrong span.
|
||||
|
||||
## C4 — **conceded, and the reviewer under-ranked it. This is the one that kills the finding.**
|
||||
|
||||
They ranked it fourth. Measured, it is first.
|
||||
|
||||
A `FirstLegal` policy — take `legal[0]`, no heuristic at all — against
|
||||
`GreedyPolicy`, 200 seeds each:
|
||||
|
||||
| seats | greedy | firstlegal |
|
||||
|---|---:|---:|
|
||||
| 2 | 66.0% | **77.5%** |
|
||||
| 3 | 82.5% | 25.0% |
|
||||
| 4 | 95.0% | 32.0% |
|
||||
| 5 | **100.0%** | **0.0%** |
|
||||
| 6 | **100.0%** | **0.0%** |
|
||||
|
||||
**At five and six seats, one unsophisticated policy wins every game and
|
||||
another wins none.** The spread between two dumb agents at the same seat
|
||||
count is the entire range.
|
||||
|
||||
The survey said *"a greedy bot wins 200 of 200 at five and six seats"* and
|
||||
concluded the game is too easy there. **That conclusion does not follow
|
||||
from a measurement whose value ranges over 0–100% depending on which
|
||||
trivial agent you pick.** It is a statement about `GreedyPolicy`, not
|
||||
about GROUND.
|
||||
|
||||
**The reviewer offered me a rescue and it does not survive either.** They
|
||||
suggested that greedy hitting the 12-point ceiling in 200/200 deals makes
|
||||
the 6p row a rules claim — the ceiling is reachable, therefore the game is
|
||||
clearable. FirstLegal's 0% at 6p refutes it: reachable-by-greedy is not
|
||||
reachable-by-an-agent, and "the ceiling can be hit" was never in doubt
|
||||
anyway. What was claimed is that it is *easy* to hit, and nothing here
|
||||
supports that.
|
||||
|
||||
**And at two seats greedy is worse than no heuristic at all** (66% vs
|
||||
77.5%), so the 66% figure — which I used as "a real game" — is partly a
|
||||
GreedyPolicy defect, not a property of the deal.
|
||||
|
||||
**The headline finding is withdrawn.** Not softened: withdrawn.
|
||||
|
||||
## C2 — the ratio does not explain the curve — **conceded**
|
||||
|
||||
I wrote *"three multipliers all pointing the same direction"* over a table
|
||||
of ratios 1.20 / 1.29 / 1.33.
|
||||
|
||||
**3-player and 4-player share the same deal, the same threshold and
|
||||
therefore the same ratio (9 points against 7, 1.29) — and differ by 12.5
|
||||
points of win rate** (82.5% vs 95%). My own harness prints `of 7.0` for
|
||||
both. A quantity identical across two rows cannot explain a difference
|
||||
between them.
|
||||
|
||||
The mechanism the reviewer identifies — more hands claiming a **fixed
|
||||
five-Problem pool** — is consistent with the data and mine was not. I
|
||||
constructed a story from a table I had built and did not test it against
|
||||
the rows sitting next to each other in my own output.
|
||||
|
||||
## C3 — the finding is inadmissible under GameDesign §1 — **conceded**
|
||||
|
||||
The survey claimed its finding met the admissibility rule this project
|
||||
wrote nine hours earlier. It fails two of three clauses:
|
||||
|
||||
- **Clause 2 (ruled shape).** 6/9/12 are **sums**. GROUND-WP-0004 T02
|
||||
requires Surface and each hidden priority listed *separately*, and
|
||||
explicitly forbids "deal depth N". My table has a `hidden dealt` column
|
||||
containing a count. That is the exact shape the ruling exists to
|
||||
prohibit, and I reproduced it while citing the ruling.
|
||||
- **Clause 3 (can fail).** The harness has **no assertions, no
|
||||
`--self-test`, and is not in any `make` target**. Nothing can turn it
|
||||
red. Under CB-WP-0022 T05's own distinction it is a `default`-role
|
||||
artifact — it prints what the code does — wearing a `counterexample`
|
||||
label.
|
||||
|
||||
**A reproduction that cannot fail is a printout.** The rule was one day
|
||||
old and I broke it in the first pass that used it.
|
||||
|
||||
## C5 — the "explains the maintainer's report" claim is contradicted by this repo — **conceded**
|
||||
|
||||
I wrote that 66%-at-2p and 100%-at-6p together explain *"I felt it was too
|
||||
easy but then we lost."*
|
||||
|
||||
`games/ground/src/lib.rs:2487-2493`, in a test comment I did not read:
|
||||
|
||||
> *"the maintainer played several 3-player games on 2026-08-03 and could
|
||||
> not win any of them, because GR-S01 dealt 2/3/4 Problems worth 3/6/10
|
||||
> against thresholds of 5/7/9."*
|
||||
|
||||
**Three-player games, on the pre-ruling deal, arithmetically unwinnable at
|
||||
6 against 7.** Not 2-player, not a 66% coin-flip, and nothing to do with
|
||||
the curve I fitted to it.
|
||||
|
||||
**This is the same defect as CB-WP-0022's C1**, four days later: a claim
|
||||
about the maintainer's own experience, contradicted by a record in this
|
||||
repo, written by a pass that had the record available. The remedy there
|
||||
was to check `git log`; the remedy here was to read the test comment
|
||||
attached to the very test that inverted.
|
||||
|
||||
## C6 — "exhaustive search is out at any seat count" is false — **conceded, and it changes the ADR**
|
||||
|
||||
I asserted `7.4^15` and stopped thinking. The reviewer measured ~16 min for
|
||||
a full 2p game and ~3 s over the last two rounds at 3p.
|
||||
|
||||
Combined with C1's correction, **the affordable budget is ~10⁵–10⁶ nodes,
|
||||
not ~10⁴–10⁵**, and bounded exhaustive search over the endgame is inside
|
||||
it. That is a different starting point for T03: the ADR cannot open with
|
||||
*"exhaustive is impossible, therefore determinized sampling"*, because the
|
||||
premise is false and the alternative carries strategy fusion that
|
||||
exhaustive search does not.
|
||||
|
||||
Neither I nor the reviewer considered transposition or the co-operative
|
||||
single-agent framing, which cut the exponent further.
|
||||
|
||||
## C7 — the replay loop discards rejections silently — **noted**
|
||||
|
||||
Correct, and the reviewer measured **zero rejections**, so no published
|
||||
number is contaminated. It is still a silent `if let Ok(...)`, and it will
|
||||
lie the moment the replay diverges. Fixed alongside the rest.
|
||||
|
||||
## What survives, and what would have falsified it
|
||||
|
||||
- **`nodes == widths.len()`** — the denominator is the one the label says.
|
||||
- **No games were dropped**: all 200 seeds completed at every seat count,
|
||||
so the win-rate denominators are real.
|
||||
- **`GreedyPolicy` is a genuine heuristic**, not first-legal in disguise —
|
||||
which is what makes C4's comparison meaningful rather than trivial.
|
||||
- **The 6/9/12 arithmetic itself** is right against `Problems.csv` and
|
||||
GR-S01. It is the *shape* that fails C3 and the *interpretation* that
|
||||
fails C2 and C4, not the arithmetic.
|
||||
- **The 5×N exponent** was measured at 11.6/16.2/21.8 against my claimed
|
||||
10/15/20 — close enough that the exponent was not where C6's error was.
|
||||
|
||||
---
|
||||
|
||||
## What the review changed
|
||||
|
||||
| | before | after |
|
||||
|---|---|---|
|
||||
| headline finding | *the game is too easy at 5–6 seats* | **withdrawn** — one dumb policy wins 100%, another 0% |
|
||||
| per-node cost | 112–161 µs | 3.0–4.1 µs (mine) vs 15.6–20.4 (theirs); **unsettled, T04 must benchmark** |
|
||||
| exhaustive search | ruled out | **available** for bounded endgame; ADR's premise changes |
|
||||
| the ratio story | three multipliers | 3p and 4p share the ratio and differ by 12.5 points |
|
||||
| admissibility | claimed | **fails clauses 2 and 3** of a rule one day old |
|
||||
| explains the maintainer | asserted | contradicted by `lib.rs:2487` |
|
||||
|
||||
**The number was going to ground-game.** T06 exists to report to
|
||||
GROUND-WP-0005, which is *blocked waiting on a difficulty baseline*. Had
|
||||
this pass proceeded, they would have been told the game is too easy at
|
||||
five and six seats and invited to move thresholds — on the strength of one
|
||||
bot's behaviour, when a second bot of the same sophistication scores zero.
|
||||
|
||||
**That would have been the fifth wrong premise this project has sent
|
||||
them, and the second stopped by an adversarial review rather than by a
|
||||
control.** Two for two: both tier-L reviews in this project have caught a
|
||||
false headline that every gate passed.
|
||||
268
research/CB-RES-0008-could-we-have-won.md
Normal file
268
research/CB-RES-0008-could-we-have-won.md
Normal file
|
|
@ -0,0 +1,268 @@
|
|||
---
|
||||
id: CB-RES-0008
|
||||
capability: analysis.witness-and-difficulty
|
||||
status: reviewed 2026-08-05 — headline finding WITHDRAWN (C4); numbers corrected (C1)
|
||||
tier: L
|
||||
chaos: d8 = 6 → no override
|
||||
---
|
||||
|
||||
# CB-RES-0008 — a path out of a lost game, and how hard the game is
|
||||
|
||||
CB-WP-0025 T01. Surveyed 2026-08-05.
|
||||
|
||||
Two maintainer questions that are the same machine asked twice: *"could we
|
||||
have won, and how?"* is a search from a recorded state; *"how hard is
|
||||
this?"* is that search — or a proxy for it — run over many deals and
|
||||
counted.
|
||||
|
||||
**The runnable baseline is ours and it is the row that matters.** External
|
||||
candidates are algorithms and practices, not software we can run on our
|
||||
workload, so per InnerLoop §Step 1 their rows are **directional and cap at
|
||||
`parity`**.
|
||||
|
||||
---
|
||||
|
||||
## 1. The baseline, measured
|
||||
|
||||
`cargo run --release -p games-ground --example difficulty-baseline`
|
||||
(200 seeds per seat count, `GreedyPolicy` and `RandomPolicy`):
|
||||
|
||||
```
|
||||
bot win rate over 200 seeds (GR-E01 group success):
|
||||
2p greedy 132/200 won = 66.0% mean total 5.3 of 5.0 median margin +1
|
||||
3p greedy 165/200 won = 82.5% mean total 8.4 of 7.0 median margin +2
|
||||
4p greedy 190/200 won = 95.0% mean total 8.8 of 7.0 median margin +2
|
||||
5p greedy 200/200 won = 100.0% mean total 11.8 of 9.0 median margin +3
|
||||
6p greedy 200/200 won = 100.0% mean total 12.0 of 9.0 median margin +3
|
||||
2p random 10/200 won = 5.0% mean total 2.4 of 5.0 median margin -3
|
||||
3p random 19/200 won = 9.5% mean total 3.1 of 7.0 median margin -4
|
||||
4p random 16/200 won = 8.0% mean total 3.2 of 7.0 median margin -4
|
||||
|
||||
search cost — legal_commands at every real decision point:
|
||||
2p 462 decisions branching mean 4.7 median 5 max 7 3.0 us/call
|
||||
3p 649 decisions branching mean 7.4 median 8 max 10 3.5 us/call
|
||||
4p 870 decisions branching mean 9.1 median 10 max 12 4.1 us/call
|
||||
```
|
||||
|
||||
*(Search-cost row re-measured after C1. The first published figures —
|
||||
161/139/112 "us/node" — timed two `setup`s, a whole greedy game and a
|
||||
validate+fold replay. See §1.2.)*
|
||||
|
||||
### 1.1 WITHDRAWN — the finding this claimed, and why it is not one
|
||||
|
||||
> **Withdrawn 2026-08-05 by the adversarial review (C4), before it left
|
||||
> the repo.** The section is kept, struck through, because a claim
|
||||
> retracted silently is how three earlier wrong premises survived
|
||||
> (ADR-0012 D5).
|
||||
>
|
||||
> A `FirstLegal` policy — take `legal[0]`, no heuristic — scores
|
||||
> **0% at five and six seats** where `GreedyPolicy` scores 100%, and
|
||||
> **77.5% at two seats** where greedy scores 66%. Two unsophisticated
|
||||
> agents span the entire range at the same seat count. **A measurement
|
||||
> that does that is about the policy, not about the game.**
|
||||
>
|
||||
> | seats | greedy | firstlegal |
|
||||
> |---|---:|---:|
|
||||
> | 2 | 66.0% | **77.5%** |
|
||||
> | 5 | **100.0%** | **0.0%** |
|
||||
> | 6 | **100.0%** | **0.0%** |
|
||||
>
|
||||
> The arithmetic below is right; the **interpretation** is not, and the
|
||||
> table also fails GameDesign §1.2 by reporting **sums** where
|
||||
> GROUND-WP-0004 T02 requires per-priority rows.
|
||||
|
||||
~~**A greedy bot wins 200 of 200 games at five and six seats.**~~ Not 95%,
|
||||
not 99% — every game, with a median margin of **+3** and mean available
|
||||
points of **11.8–12.0 against a threshold of 9**.
|
||||
|
||||
The arithmetic behind it, row by row (the shape GROUND-WP-0004 T02
|
||||
requires):
|
||||
|
||||
| seats | Surface | hidden dealt | points available | threshold | ratio |
|
||||
|---|---|---|---:|---:|---:|
|
||||
| 2 | priority 1 | 2 | **6** | 5 | 1.20 |
|
||||
| 3–4 | priority 1 | 3 | **9** | 7 | 1.29 |
|
||||
| 5–6 | priority 1 | 4 | **12** | 9 | 1.33 |
|
||||
|
||||
**The ratio moves the wrong way.** More seats means more points on the
|
||||
table *and* a proportionally lower bar *and* more actions per round to
|
||||
clear it with. Three multipliers all pointing the same direction, which is
|
||||
why the curve is not gentle — it is 66% → 100% across four seat counts.
|
||||
|
||||
~~**This is admissible under GameDesign §1.**~~ **It is not (C3), and the
|
||||
rule it fails was one day old.**
|
||||
|
||||
- **Clause 2, ruled shape.** 6/9/12 are **sums**, and the `hidden dealt`
|
||||
column is a *count*. GROUND-WP-0004 T02 requires Surface and each hidden
|
||||
priority listed separately and explicitly forbids "deal depth N". The
|
||||
table reproduces the prohibited shape while citing the ruling.
|
||||
- **Clause 3, can fail.** The harness has **no assertions, no
|
||||
`--self-test`, and is in no `make` target**. Nothing can turn it red.
|
||||
Under CB-WP-0022 T05's own `role` distinction it is a `default` artifact
|
||||
— it prints what the code does — wearing a `counterexample` label.
|
||||
|
||||
**A reproduction that cannot fail is a printout.** T05 must fix the
|
||||
harness before any figure from it is quoted again.
|
||||
|
||||
~~It also **explains the maintainer's report** — *"I felt it was too easy
|
||||
but then we lost, so who knows"* — without needing a solver. He plays at
|
||||
low seat counts, where 66% is a real game, and had been feeling the 5–6
|
||||
seat experience from elsewhere in the same session. Both halves of the
|
||||
sentence are true of different seat counts.~~
|
||||
|
||||
**Also withdrawn (C5).** `games/ground/src/lib.rs:2487-2493` records what
|
||||
actually happened: *"the maintainer played several 3-player games on
|
||||
2026-08-03 and could not win any of them, because GR-S01 dealt 2/3/4
|
||||
Problems worth 3/6/10 against thresholds of 5/7/9."* Three seats, on the
|
||||
pre-ruling deal, **arithmetically unwinnable at 6 against 7** — nothing to
|
||||
do with the curve fitted to it here.
|
||||
|
||||
~~**T06 must report this to GROUND-WP-0005.**~~ **It must not.**
|
||||
GROUND-WP-0005 is blocked waiting on a difficulty baseline, and this is
|
||||
not one. Sending it would have invited ground-game to move thresholds on
|
||||
the strength of one bot's behaviour — the **fifth** wrong premise this
|
||||
project has sent them, and the second stopped by a review rather than by
|
||||
a control.
|
||||
|
||||
### 1.2 What the search-cost numbers rule out
|
||||
|
||||
Branching is small — mean 4.7 to 9.1.
|
||||
|
||||
> **CORRECTED (C1). The published figure of 112–161 µs/node was wrong by
|
||||
> 30–50×**: the timer bracketed two `setup`s, a whole greedy game and a
|
||||
> validate+fold replay, divided by player-decision count. The tell was in
|
||||
> the output — it **fell** as branching **rose**, which no per-enumeration
|
||||
> cost can do.
|
||||
>
|
||||
> Re-measured, clock around `legal_commands` only: **3.0 / 3.5 / 4.1 µs**
|
||||
> at 2/3/4 seats, now rising with branching. **The reviewer measured
|
||||
> 15.6–20.4 µs by a different isolation and we have not settled which is
|
||||
> right** — T04 must benchmark it properly (`criterion` is already a
|
||||
> dev-dependency) rather than adopt either.
|
||||
|
||||
~~**Exhaustive search is out at any seat count.**~~ **False (C6).** The
|
||||
reviewer measured ~16 minutes for a full 2p game and **~3 seconds over the
|
||||
last two rounds at 3p**. With C1's correction the affordable budget is
|
||||
**~10⁵–10⁶ nodes**, and bounded exhaustive search over the endgame is
|
||||
inside it.
|
||||
|
||||
**This changes T03's starting point.** The ADR cannot open with
|
||||
*"exhaustive is impossible, therefore determinized sampling"* — the
|
||||
premise is false, and the alternative carries strategy fusion that
|
||||
exhaustive search does not. Neither the survey nor the review considered
|
||||
transposition or the co-operative single-agent framing, which cut the
|
||||
exponent further.
|
||||
|
||||
## 2. Prior art: determinized search, and the failure it is famous for
|
||||
|
||||
The natural first idea — *deal out the hidden cards, solve the resulting
|
||||
perfect-information game, repeat* — is **Perfect Information Monte Carlo
|
||||
(PIMC)**, and its failure modes were named by Frank, Basin and Matsubara
|
||||
in 1998:
|
||||
|
||||
- **Strategy fusion** — the search picks *different* actions from two
|
||||
states in the same information set, which no real player could do,
|
||||
because a player cannot tell those states apart. The plan it returns is
|
||||
not executable by someone who does not know which world they are in.
|
||||
- **Non-locality** — subgame values are not well-defined when information
|
||||
is hidden, so recursive search over subgames is unsound.
|
||||
|
||||
**Strategy fusion is precisely the trap in this pass.** A witness produced
|
||||
by determinized search may be a line that requires knowing which Solution
|
||||
is on top of the deck. Showing the maintainer *"you could have won by
|
||||
playing Repair on turn 3"* — when nothing he could see said a Repair was
|
||||
coming — teaches a false lesson about his own play, which is worse than
|
||||
not shipping the feature.
|
||||
|
||||
**Long and Sturtevant** later characterized *when* PIMC nonetheless works
|
||||
well, which matters here: its success depends on properties of the game
|
||||
tree (leaf correlation, bias, disambiguation rate). GROUND disambiguates
|
||||
fast — Problems flip face-up, selections reveal every round — which is the
|
||||
regime where PIMC is least bad. **That is an argument the ADR may use, and
|
||||
it is a directional one, not a measurement.**
|
||||
|
||||
**ISMCTS** (information-set MCTS) searches over information sets directly
|
||||
rather than determinizations, and is the standard answer to strategy
|
||||
fusion.
|
||||
|
||||
**Benchmark to beat:** a witness that **replays through our existing
|
||||
scenario runner and ends in `group_success`**. That is a stronger and
|
||||
cheaper bar than any of the above, because it is mechanically checkable —
|
||||
and note it does *not* by itself exclude a strategy-fused line. A fused
|
||||
line replays fine. **Checkability and honesty are different properties,
|
||||
and the ADR must not let the first stand in for the second.**
|
||||
|
||||
*Directional, cited-only.*
|
||||
|
||||
## 3. The retrospective question is not the playing question
|
||||
|
||||
Worth separating, because the prior art is all about *playing*:
|
||||
|
||||
| question | information | honest? |
|
||||
|---|---|---|
|
||||
| *was this deal winnable at all* | omniscient | **yes** — it is a question about the deal, not about the player |
|
||||
| *was it winnable from what we could see* | the seat's view | yes, and expensive |
|
||||
| *could a reasonable player have found it* | the seat's view, bounded | the only affordable honest one |
|
||||
|
||||
The first is legitimate and cheap, and answers *"the deal was unwinnable,
|
||||
stop blaming yourself"* — which is a real thing a player wants to hear.
|
||||
It is **not** an answer to *"how could we have won"*, and labelling it as
|
||||
one is the failure mode.
|
||||
|
||||
**Naming matters more than the algorithm here.** The ADR's first decision
|
||||
is which question is being answered and what it is called on screen.
|
||||
|
||||
## 4. Difficulty as a measured quantity
|
||||
|
||||
Co-operative board games set difficulty with a dial and publish the win
|
||||
rate — Pandemic's number of Epidemic cards is the canonical example. The
|
||||
practice is: **a named dial, a stated player skill, and a target band.**
|
||||
|
||||
We have the dial candidates already — the threshold (`GR-E01`), and
|
||||
`ground-game`'s proposed Pressure dial (GROUND-WP-0005) — and §1 supplies
|
||||
the first measured band.
|
||||
|
||||
**The problem the practice does not solve for us:** a published win rate
|
||||
is measured against *humans*. Ours is measured against `GreedyPolicy`.
|
||||
The 5–6 seat 100% is a claim about our bot, and the honest reading is
|
||||
narrower than "the game is too easy at six players" — it is *"a bot that
|
||||
takes the obvious action never fails to clear the threshold at six
|
||||
players."*
|
||||
|
||||
Whether that is the same statement is **the reviewer's strongest line of
|
||||
attack** and is not settled here.
|
||||
|
||||
**Benchmark to beat:** a difficulty figure whose resolution is stated —
|
||||
the smallest threshold change it can distinguish, with its N. A rate
|
||||
without that cannot tune anything.
|
||||
|
||||
*Directional, cited-only.*
|
||||
|
||||
## 5. Benchmarks to beat
|
||||
|
||||
| dimension | today | benchmark |
|
||||
|---|---|---|
|
||||
| witness checkability | no witness exists | **100%** of emitted paths replay to `group_success` through the existing runner |
|
||||
| witness honesty | — | no line that requires unseen information; **the ADR must say how this is enforced, not asserted** |
|
||||
| search cost | 112–161 µs/node measured | a bound in nodes or wall clock, and *"none found within B"* wording that does not claim unwinnability |
|
||||
| difficulty resolution | one band, one policy | the smallest threshold delta distinguishable, with N and policy named |
|
||||
| difficulty honesty | — | the policy and seed range are **in the number's name**, not a footnote |
|
||||
|
||||
## 6. What the survey did not settle
|
||||
|
||||
- **Whether a bot win rate is a difficulty at all.** §4. The strongest
|
||||
counter is that it measures the bot, and improving the bot would
|
||||
"increase the difficulty" without touching the game.
|
||||
- **How witness honesty is enforced rather than asserted.** Running the
|
||||
search on a `GroundView` makes the information boundary structural;
|
||||
running it on `GroundState` makes it a promise. The survey believes the
|
||||
first is right and has **not** measured whether it is affordable — a
|
||||
view-only search cannot fold events, so it needs a state it may not
|
||||
see. **This is the gap most likely to sink the pass, and it is stated
|
||||
here rather than discovered in T05.**
|
||||
- **Whether 100% at 5–6 seats is a rules finding or a bot finding.**
|
||||
§1.1 reports it as measured; which repo owns it is T03's call.
|
||||
- **Whether the cheap honest answer is enough.** *"This deal was
|
||||
unwinnable"* (omniscient, cheap) may satisfy the maintainer's actual
|
||||
need without any information-respecting search at all. Nobody has asked
|
||||
him. That is a one-question experiment this survey did not run.
|
||||
|
|
@ -2,7 +2,7 @@
|
|||
id: CB-WP-0025
|
||||
kind: product
|
||||
title: "Could we have won: a path out of a lost game, and how hard the game actually is"
|
||||
status: ready
|
||||
status: active
|
||||
state_hub_workstream_id: "a866982f-94e7-432b-b0d5-2fefb781a574"
|
||||
---
|
||||
|
||||
|
|
@ -90,7 +90,7 @@ whose meaning drifts the next time a bot improves.
|
|||
|
||||
```task
|
||||
id: CB-WP-0025-T01
|
||||
status: todo
|
||||
status: done
|
||||
priority: high
|
||||
state_hub_task_id: "556cfd24-5992-4cc0-be90-0b60e989b5bb"
|
||||
```
|
||||
|
|
@ -123,11 +123,44 @@ per dimension — a number or a reproducible comparison, not an impression.
|
|||
that cannot finish while the player is still looking at the page is a
|
||||
different feature.
|
||||
|
||||
**Done 2026-08-05.**
|
||||
[CB-RES-0008](../research/CB-RES-0008-could-we-have-won.md), with a
|
||||
runnable baseline (`games/ground/examples/difficulty-baseline.rs`).
|
||||
|
||||
**The baseline produced a finding before any solver exists, and it is the
|
||||
biggest thing in this pass.** A greedy bot wins **200 of 200** games at
|
||||
five and six seats — with a median margin of +3 and 11.8–12.0 points
|
||||
available against a threshold of 9. The curve is 66% / 82.5% / 95% / 100%
|
||||
/ 100% across 2/3/4/5/6 seats.
|
||||
|
||||
The row-level table shows why: available points go **6 / 9 / 12** against
|
||||
thresholds **5 / 7 / 9**, so the ratio *rises* with seat count (1.20 →
|
||||
1.29 → 1.33) while the table also gains actions per round. **Three
|
||||
multipliers pointing the same way.**
|
||||
|
||||
**It also explains the maintainer's report without a solver.** *"I felt it
|
||||
was too easy but then we lost"* — 66% at two seats is a real game; 100% at
|
||||
six is not. Both halves are true of different seat counts.
|
||||
|
||||
**Cost measured, and it rules out the obvious approach.** Branching is
|
||||
small (mean 4.7–9.1) but `legal_commands` costs **112–161 µs** per call,
|
||||
because it filters candidates through full `validate`. Exhaustive search
|
||||
is out at every seat count; ~10⁴–10⁵ nodes is 1.4–14 s, which is the
|
||||
budget the ADR must design inside.
|
||||
|
||||
**Prior art names the trap.** Determinized search (PIMC) suffers *strategy
|
||||
fusion* — Frank, Basin & Matsubara 1998 — where the search picks different
|
||||
actions in states a real player cannot tell apart. A witness built that
|
||||
way may require knowing what was on top of the deck. **And it would still
|
||||
replay green**, so the checkability benchmark does not catch it. Honesty
|
||||
and checkability are different properties, stated so T03 cannot conflate
|
||||
them.
|
||||
|
||||
## Task: adversarial review
|
||||
|
||||
```task
|
||||
id: CB-WP-0025-T02
|
||||
status: todo
|
||||
status: done
|
||||
priority: high
|
||||
state_hub_task_id: "bdd9fc97-4706-4d8e-b286-91ff53339680"
|
||||
```
|
||||
|
|
@ -151,6 +184,53 @@ Tier L requires it. Exactly one round: challenge, then response, trail in
|
|||
*produces* them, but the reviewer should press whether that is a
|
||||
distinction worth a separate capability.
|
||||
|
||||
**Done 2026-08-05.** Trail:
|
||||
[challenge](../history/260805-could-we-have-won-challenge.md),
|
||||
[response](../history/260805-could-we-have-won-response.md).
|
||||
Separate agent, as in CB-WP-0022.
|
||||
|
||||
**Six of seven conceded, and the survey's headline finding is withdrawn —
|
||||
not softened.**
|
||||
|
||||
**C4 is the one that kills it, and the reviewer ranked it fourth.**
|
||||
Measured: a `FirstLegal` policy — take `legal[0]`, no heuristic — scores
|
||||
**0% at five and six seats** where greedy scores 100%, and **77.5% at two
|
||||
seats** where greedy scores 66%. **Two unsophisticated agents span the
|
||||
entire range at the same seat count**, so *"the game is too easy at 5–6
|
||||
seats"* is a statement about `GreedyPolicy`, not about GROUND. Their
|
||||
offered rescue — greedy hits the 12-point ceiling every time, so it is a
|
||||
rules claim — dies on the same data.
|
||||
|
||||
**C1: the per-node cost was wrong by 30–50×.** The timer bracketed two
|
||||
`setup`s, a whole greedy game and a validate+fold replay, then divided by
|
||||
player decisions. **The tell was in my own published output**: the figure
|
||||
*fell* as branching *rose*, which no per-enumeration cost can do.
|
||||
Re-measured at **3.0–4.1 µs**; the reviewer got 15.6–20.4 by a different
|
||||
isolation and **that discrepancy is unsettled** — T04 benchmarks it.
|
||||
|
||||
**C6: exhaustive search is not out**, which changes T03's premise. With
|
||||
C1's correction the budget is ~10⁵–10⁶ nodes and bounded endgame search
|
||||
fits — so the ADR cannot open with *"exhaustive is impossible, therefore
|
||||
determinized sampling"*, especially since sampling carries strategy fusion
|
||||
that exhaustive search does not.
|
||||
|
||||
**C3: the finding failed the admissibility rule this project wrote nine
|
||||
hours earlier** — sums where GROUND-WP-0004 T02 requires per-priority
|
||||
rows, and a harness with no assertions, no `--self-test` and no `make`
|
||||
target, so nothing can turn it red. A `default` artifact wearing a
|
||||
`counterexample` label.
|
||||
|
||||
**C2: the ratio explains nothing** — 3p and 4p share deal, threshold and
|
||||
ratio, and differ by 12.5 points of win rate. **C5: the "explains the
|
||||
maintainer's report" claim is contradicted by `lib.rs:2487`**, which
|
||||
records his losses as 3-player on the pre-ruling deal, unwinnable at 6
|
||||
against 7.
|
||||
|
||||
**T06 was pointed at GROUND-WP-0005, which is blocked on this number.**
|
||||
Sending it would have invited threshold changes on one bot's behaviour —
|
||||
the fifth wrong premise to reach ground-game. **Both tier-L reviews in
|
||||
this project have now caught a false headline that every gate passed.**
|
||||
|
||||
## Task: decide
|
||||
|
||||
```task
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue