diff --git a/.gitignore b/.gitignore index ea8c4bf..035c4f4 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,4 @@ /target + +# K10 replay bundles are generated artifacts of a failing run. +replays/ diff --git a/Makefile b/Makefile index 7bbbfe7..a8221ba 100644 --- a/Makefile +++ b/Makefile @@ -24,7 +24,7 @@ TOOLS := $(REPO)/tools # Every cargo recipe runs at the repo root; the shell does not persist cd. IN_REPO := cd $(REPO) && -.PHONY: check test sim bench bench-test coverage dep-weight cost cost-test cost-pin cost-budget cost-mix loop-lint self-tests env-test task-done status facts-check facts-gen mutation-check size-metrics runtime-metrics build-time am6 loc all +.PHONY: check test sim bench bench-test coverage dep-weight cost cost-test cost-pin cost-budget cost-mix loop-lint self-tests env-test task-done status facts-check facts-gen mutation-check size-metrics runtime-metrics build-time am6 replay-test loc all ## fmt + clippy (deny warnings) + HashMap deny-lint check: @@ -42,6 +42,11 @@ dep-weight: coverage: $(PY) $(TOOLS)/rule-coverage.py +# K10: a bundle must re-execute, and must be able to fail. Four controls +# from ADR-0005 §6, including truncate-by-one-byte and mutated-seed. +replay-test: + $(PY) $(TOOLS)/replay-test.py + # AM-6 gate. Runs in RELEASE, where headroom is ~24x; the same assertion # in debug has ~3.4x and flaked under load. The target is unchanged — only # where it is measured. @@ -87,6 +92,7 @@ self-tests: $(PY) $(TOOLS)/mutation-check.py --self-test $(PY) $(TOOLS)/size-metrics.py --self-test $(PY) $(TOOLS)/runtime-metrics.py --self-test + $(PY) $(TOOLS)/replay-test.py --self-test # T01 positive control: prove the environment fix, do not assume it. Runs # every tool from a foreign working directory with a PATH that has no @@ -164,4 +170,4 @@ loc: printf '%-28s %s\n' $$d "$$(find $$d/src -name '*.rs' | xargs cat | grep -vcE '^\s*(//|$$)')"; \ done -all: check test sim coverage size-metrics runtime-metrics am6 dep-weight self-tests env-test facts-check loop-lint bench-test +all: check test sim coverage size-metrics runtime-metrics am6 replay-test dep-weight self-tests env-test facts-check loop-lint bench-test diff --git a/crates/cb-game-runtime/src/lib.rs b/crates/cb-game-runtime/src/lib.rs index de392b4..54dc504 100644 --- a/crates/cb-game-runtime/src/lib.rs +++ b/crates/cb-game-runtime/src/lib.rs @@ -4,6 +4,10 @@ #[cfg(feature = "scenarios")] pub mod scenario; +/// K10 replay bundles. Dev-only: a shipped game runtime writes no bundles. +#[cfg(feature = "scenarios")] +pub mod replay; + #[cfg(feature = "scenarios")] pub use scenario::{parse_actor, run, CommandStep, RunOutcome, ScenarioFile, ScenarioGame, Setup}; diff --git a/crates/cb-game-runtime/src/replay.rs b/crates/cb-game-runtime/src/replay.rs new file mode 100644 index 0000000..a42679c --- /dev/null +++ b/crates/cb-game-runtime/src/replay.rs @@ -0,0 +1,216 @@ +//! K10 replay bundles (GameKernel §2.4, MetricsAndScenarios §4). +//! +//! K10: *"A replay bundle (`.cbreplay`) contains manifest, command log, +//! initial snapshot, and failed expectations; the runner's `--replay` +//! re-executes it bit-identically."* Unimplemented until CB-WP-0006 T06 — +//! `games/` had no `replays/` directory, no `.cbreplay` reader or writer, +//! and `cb-sim` had no flag parsing at all, so `--replay` had nowhere to +//! go. INTENT names this design decision **8 of 10**. +//! +//! MetricsAndScenarios §4 states the point: *"A bug report without a +//! replay bundle is information; with one, it is work an agent can +//! start."* +//! +//! **Dev-only**, behind the `scenarios` feature, so it is charged to +//! AM-4b (9.4% headroom) and not to the shipped runtime. +//! +//! ## Why replay can fail, and must be able to +//! +//! ADR-0005 §6 required four controls, two of which the adversarial +//! reviewer supplied. The two that shape this module: +//! +//! * the comparison hash is **read out of the bundle**, written by the +//! process that produced it — never recomputed in the replaying process, +//! which would be `assert_eq!(h, h)` and is exactly the AM-7 defect; +//! * the recorded **seed must reproduce the recorded initial snapshot**, +//! so mutating the manifest seed makes replay fail. Without that check a +//! replay restored from the snapshot would ignore the seed entirely and +//! the control could not bite. + +use std::path::{Path, PathBuf}; + +use cb_events::store::{parse, LogStore}; +use cb_events::{state_hash_hex, FileLogStore}; +use serde::{Deserialize, Serialize}; + +use crate::scenario::{CommandStep, ScenarioGame}; +use crate::{ScenarioFile, Setup}; + +pub const MANIFEST: &str = "manifest.yaml"; +pub const COMMANDS: &str = "commands.log"; +pub const INITIAL: &str = "initial.snapshot"; +pub const EXPECTED: &str = "expected.yaml"; + +/// `manifest.yaml` — everything needed to re-execute, and the hash the +/// replay must reproduce. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Manifest { + pub scenario: String, + pub seed: u64, + pub setup: Setup, + pub schema_ver: u16, + /// Written by the producing process. The replay compares against + /// **this**, never against a value it computed itself. + pub end_state_hash: String, + /// Hash of the initial state, so a mutated seed is detectable. + pub initial_state_hash: String, + pub commit: String, +} + +/// `expected.yaml` — the assertions that failed, expected vs actual. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Expected { + pub reason: String, + pub end_state: serde_json::Value, +} + +#[derive(Debug)] +pub struct ReplayReport { + pub scenario: String, + pub commands: usize, + pub hash: String, +} + +fn err(context: &str, e: impl std::fmt::Display) -> String { + format!("{context}: {e}") +} + +/// Write a `.cbreplay` bundle. Called when a scenario fails. +pub fn write_bundle( + dir: &Path, + scenario: &ScenarioFile, + initial_state: &serde_json::Value, + initial_state_hash: &str, + end_state: &serde_json::Value, + end_state_hash: &str, + reason: &str, +) -> Result { + let slug = scenario.scenario.replace('/', "-"); + let bundle = dir.join(format!("{slug}.cbreplay")); + // A stale bundle from a previous run would replay the old failure. + let _ = std::fs::remove_dir_all(&bundle); + std::fs::create_dir_all(&bundle).map_err(|e| err("create bundle", e))?; + + std::fs::write( + bundle.join(INITIAL), + serde_json::to_vec_pretty(initial_state).map_err(|e| err("encode initial", e))?, + ) + .map_err(|e| err("write initial", e))?; + + // The command stream goes through the K11 framing, so a truncated + // bundle is detected rather than replayed short. + let mut log = FileLogStore::open(bundle.join(COMMANDS)).map_err(|e| err("open log", e))?; + for step in &scenario.commands { + let bytes = serde_json::to_vec(step).map_err(|e| err("encode command", e))?; + log.append(&bytes).map_err(|e| err("append command", e))?; + } + + let manifest = Manifest { + scenario: scenario.scenario.clone(), + seed: scenario.seed, + setup: scenario.setup.clone(), + schema_ver: 1, + end_state_hash: end_state_hash.to_string(), + initial_state_hash: initial_state_hash.to_string(), + commit: option_env!("CB_COMMIT").unwrap_or("unknown").to_string(), + }; + std::fs::write( + bundle.join(MANIFEST), + serde_yaml::to_string(&manifest).map_err(|e| err("encode manifest", e))?, + ) + .map_err(|e| err("write manifest", e))?; + + std::fs::write( + bundle.join(EXPECTED), + serde_yaml::to_string(&Expected { + reason: reason.to_string(), + end_state: end_state.clone(), + }) + .map_err(|e| err("encode expected", e))?, + ) + .map_err(|e| err("write expected", e))?; + + Ok(bundle) +} + +/// Re-execute a bundle and require it to reproduce the recorded hash. +/// +/// Returns `Err` when the bundle is corrupt, incomplete, internally +/// inconsistent, or does not reproduce — all of which are the point. A +/// round-trip that cannot fail proves nothing. +pub fn replay(bundle: &Path) -> Result +where + G: ScenarioGame, +{ + let manifest: Manifest = serde_yaml::from_str( + &std::fs::read_to_string(bundle.join(MANIFEST)).map_err(|e| err("read manifest", e))?, + ) + .map_err(|e| err("parse manifest", e))?; + + // Control: the recorded seed and setup must reproduce the recorded + // initial snapshot. Without this the seed would be inert — replay + // restores from the snapshot — and a mutated-seed bundle would pass. + // Hash the TYPED aggregate, matching how the bundle was written. + let rebuilt = G::setup(&manifest.setup, manifest.seed)?; + let rebuilt_hash = state_hash_hex(&rebuilt); + if rebuilt_hash != manifest.initial_state_hash { + return Err(format!( + "bundle is inconsistent: seed {} and setup do not reproduce the \ + recorded initial state ({} != {})", + manifest.seed, rebuilt_hash, manifest.initial_state_hash + )); + } + + let initial: serde_json::Value = serde_json::from_slice( + &std::fs::read(bundle.join(INITIAL)).map_err(|e| err("read initial", e))?, + ) + .map_err(|e| err("parse initial", e))?; + let mut state: G = serde_json::from_value(initial).map_err(|e| err("restore initial", e))?; + + // K11 framing: a truncated or corrupt command log is rejected here. + let raw = std::fs::read(bundle.join(COMMANDS)).map_err(|e| err("read commands", e))?; + let records = parse(&raw).map_err(|e| err("command log", e))?; + + let mut applied = 0usize; + for record in &records { + let step: CommandStep = + serde_json::from_slice(record).map_err(|e| err("decode step", e))?; + let (actor, command) = G::parse_command(&step)?; + if let Ok(produced) = state.validate(actor, &command) { + for event in produced { + state.fold(&event); + } + } + applied += 1; + } + + // Positive control: a bundle that replayed nothing must not be + // reported as a successful reproduction. + if applied == 0 { + return Err("bundle contains no commands; nothing was replayed".to_string()); + } + + let hash = state_hash_hex(&state); + if hash != manifest.end_state_hash { + return Err(format!( + "replay did not reproduce: {} != recorded {}", + hash, manifest.end_state_hash + )); + } + + Ok(ReplayReport { + scenario: manifest.scenario, + commands: applied, + hash, + }) +} + +/// Read a bundle's raw command log. Used by the negative controls. +pub fn raw_command_log(bundle: &Path) -> Result, String> { + std::fs::read(bundle.join(COMMANDS)).map_err(|e| err("read commands", e)) +} + +/// Overwrite a bundle's raw command log. Used by the negative controls. +pub fn set_raw_command_log(bundle: &Path, bytes: &[u8]) -> Result<(), String> { + std::fs::write(bundle.join(COMMANDS), bytes).map_err(|e| err("write commands", e)) +} diff --git a/crates/cb-game-runtime/src/scenario.rs b/crates/cb-game-runtime/src/scenario.rs index ed47391..284ec65 100644 --- a/crates/cb-game-runtime/src/scenario.rs +++ b/crates/cb-game-runtime/src/scenario.rs @@ -83,8 +83,24 @@ impl ScenarioFile { /// Result of executing one scenario (twice, per the K8 double-run rule). #[derive(Debug)] pub enum RunOutcome { - Passed { covers: Vec }, - Failed { reason: String }, + Passed { + covers: Vec, + }, + Failed { + reason: String, + /// The failing run's states, so a K10 bundle can be written. + /// `None` when execution failed before producing a pass. + evidence: Option>, + }, +} + +/// What a `.cbreplay` bundle needs from a failed run (K10). +#[derive(Debug)] +pub struct FailureEvidence { + pub initial: serde_json::Value, + pub initial_hash: String, + pub end_state: serde_json::Value, + pub end_state_hash: String, } /// A game aggregate the scenario runner can drive (GameKernel K17). The @@ -118,9 +134,18 @@ pub fn parse_actor(actor: &str) -> Result { /// One execution pass: returns the end state as JSON, its hash, the /// emitted events, and the indices of rejected commands. -struct Pass { - state: serde_json::Value, - hash: String, +pub(crate) struct Pass { + /// The state *before* any command ran — `Pass` previously carried only + /// the end state, so a K10 bundle had no `initial.snapshot` to write. + pub(crate) initial: serde_json::Value, + /// Hash of the initial state, taken over the **typed** aggregate. + /// Hashing `serde_json::Value` instead gives a different canonical + /// form — `Value`'s map is key-sorted, a struct serializes in + /// declaration order — so a bundle written one way and verified the + /// other never reproduces. That defect cost the first K10 round trip. + pub(crate) initial_hash: String, + pub(crate) state: serde_json::Value, + pub(crate) hash: String, events: Vec, rejected: BTreeSet, } @@ -136,6 +161,8 @@ where if !scenario.setup.patch.is_empty() { state = apply_patch(&state, &scenario.setup.patch)?; } + let initial = serde_json::to_value(&state).map_err(|e| format!("initial encode: {e}"))?; + let initial_hash = state_hash_hex(&state); let mut log: EventLog = EventLog::new(); let mut events = Vec::new(); let mut rejected = BTreeSet::new(); @@ -169,6 +196,8 @@ where } Ok(Pass { + initial, + initial_hash, state: serde_json::to_value(&state).map_err(|e| format!("state encode: {e}"))?, hash: state_hash_hex(&state), events, @@ -185,24 +214,39 @@ where { let first = match execute::(scenario) { Ok(pass) => pass, - Err(reason) => return RunOutcome::Failed { reason }, + Err(reason) => { + return RunOutcome::Failed { + reason, + evidence: None, + } + } }; let second = match execute::(scenario) { Ok(pass) => pass, - Err(reason) => return RunOutcome::Failed { reason }, + Err(reason) => { + return RunOutcome::Failed { + reason, + evidence: None, + } + } }; if first.hash != second.hash { + let reason = format!( + "K8 divergence: run 1 hash {} != run 2 hash {}", + first.hash, second.hash + ); return RunOutcome::Failed { - reason: format!( - "K8 divergence: run 1 hash {} != run 2 hash {}", - first.hash, second.hash - ), + evidence: Some(Box::new(evidence_of(&first))), + reason, }; } if let Err(reason) = check(scenario, &first) { - return RunOutcome::Failed { reason }; + return RunOutcome::Failed { + evidence: Some(Box::new(evidence_of(&first))), + reason, + }; } RunOutcome::Passed { @@ -210,6 +254,15 @@ where } } +fn evidence_of(pass: &Pass) -> FailureEvidence { + FailureEvidence { + initial: pass.initial.clone(), + initial_hash: pass.initial_hash.clone(), + end_state: pass.state.clone(), + end_state_hash: pass.hash.clone(), + } +} + fn check(scenario: &ScenarioFile, pass: &Pass) -> Result<(), String> { let expected_rejects: BTreeSet = scenario.expect.rejects.iter().copied().collect(); if pass.rejected != expected_rejects { diff --git a/evidence/CB-EV-0001-game-kernel.md b/evidence/CB-EV-0001-game-kernel.md index 1b8108b..217abf1 100644 --- a/evidence/CB-EV-0001-game-kernel.md +++ b/evidence/CB-EV-0001-game-kernel.md @@ -33,7 +33,7 @@ the property is false? A row can be *measured* and still enforce nothing. | AM-6 throughput | ≥100,000 events/s | 2,017,009 events/s (`make am6`) | **met, 20.2×** | **yes** — CB-WP-0006 T01 | | AM-7 scaling | ≥0.9× at 20× workload | 1.08× | **met** | **no** — no code computes the ratio | | AM-7 replay, timing | 100k events ≤5s | 2.18 ms (CI 2.14–2.23) | **met, 2,290×** | **yes** | -| AM-7 replay, hash-identical | bit-identical fold | — | **WITHDRAWN** | **no** — mutation-proven inert | +| AM-7 replay, hash-identical | bit-identical fold | per-segment replay reproduces each recorded hash | **re-earned 2026-08-01** | **yes** — CB-WP-0006 T06 | | AM-8 determinism | zero divergence, 10 replays | 1 distinct hash / 10 runs (one-off probe) | **met, narrow** | **partial** — the runner double-runs; nothing enforces N=10 | | AM-8 lint | fmt + clippy clean | clean, `-D warnings` | **met** | **yes** | | AM-10 foreign types | 0 in `cb-*-api` signatures | — | **WITHDRAWN** | **no** — no such crate; population empty | @@ -61,7 +61,7 @@ above. | row | as committed | corrected to | found by | |---|---|---|---| -| **AM-7 replay** | `met, 2,290×` | split: timing **met**, `hash-identical` **withdrawn** | mutation — folding the log from `fresh(999)` instead of `fresh(42)`, an unrelated genesis state, leaves the test green. The hash reaches only a `println!`. It could not be asserted as written anyway: the log spans games seeded 42, 43, 44… so it is not a replay of anything | +| **AM-7 replay** | `met, 2,290×` | split: timing **met**, `hash-identical` **withdrawn** — *re-earned 2026-08-01, CB-WP-0006 T06* | mutation — folding the log from `fresh(999)` instead of `fresh(42)`, an unrelated genesis state, leaves the test green. The hash reaches only a `println!`. It could not be asserted as written anyway: the log spans games seeded 42, 43, 44… so it is not a replay of anything | | **AM-10** | `met` | **withdrawn as written**; restated as AM-10′ | adversarial review — there is no `cb-*-api` crate, so `0 foreign types` was true over an empty set. What was actually measured is a `clippy.toml` deny of `HashMap`/`HashSet` whose stated reason cites **K6 (determinism)**, reported under a **D4 leak** row | | **AM-11** | `met, narrow` | **unmet** | adversarial review — M-D4-SWAP is a bool over "the same conformance suite"; `grep -rn conformance` over every `.rs` returns one doc comment describing future work, and the RNG pair is exercised by two separate, non-shared tests | | **AM-12** | `$248.46 session` | **$93.15** | CB-WP-0002 re-derivation — the original figure double-counted per-line and priced a three-model session at one model's rate. It had been stale in this file since, untagged and therefore invisible to `facts-check` | diff --git a/facts.toml b/facts.toml index f4a061b..ab9e8da 100644 --- a/facts.toml +++ b/facts.toml @@ -70,8 +70,8 @@ fmt = "{:,}" by = "tools/rule-coverage.py" [k_linked] -value = 15 -text = "15" +value = 16 +text = "16" fmt = "{:,}" by = "tools/rule-coverage.py" @@ -82,8 +82,8 @@ fmt = "{:,}" by = "tools/rule-coverage.py" [k_unlinked] -value = 'K10 K14 K18' -text = "K10 K14 K18" +value = 'K14 K18' +text = "K14 K18" fmt = "{}" by = "tools/rule-coverage.py" diff --git a/games/ground/src/lib.rs b/games/ground/src/lib.rs index ce519aa..b7f655d 100644 --- a/games/ground/src/lib.rs +++ b/games/ground/src/lib.rs @@ -2381,19 +2381,54 @@ mod replay_probe { fn replay_100k_events_is_linear_and_fast() { for target in [10_000usize, 100_000] { let mut log = Vec::with_capacity(target); - let mut source = fresh(42); - let mut games = 0u64; + // Per-game segments with the hash of the state that produced + // them. AM-7's `hash-identical` clause was withdrawn because + // the fold ran a multi-seed log into a single genesis state + // and then never compared the hash to anything. Segments make + // the comparison meaningful: each is a real replay. + let mut segments: Vec<(u64, usize, String)> = Vec::new(); + let mut seed = 42u64; + let mut source = fresh(seed); + let mut seg_start = 0usize; let mut stalls = 0; while log.len() < target { if source.outcome.is_some() { - games += 1; - source = fresh(42 + games); + segments.push((seed, seg_start, state_hash_hex(&source))); + seg_start = log.len(); + seed += 1; + source = fresh(seed); } if record_round(&mut source, &mut log) == 0 { stalls += 1; assert!(stalls < 10, "round produced no events; builder stalled"); } } + segments.push((seed, seg_start, state_hash_hex(&source))); + + // AM-7 hash-identical, re-earned (CB-WP-0006 T06). Replay each + // segment from its own genesis and require the recorded hash. + let mut ends: Vec = segments.iter().skip(1).map(|s| s.1).collect(); + ends.push(log.len()); + let mut verified = 0usize; + for ((seg_seed, start, want), end) in segments.iter().zip(ends) { + let mut st = fresh(*seg_seed); + for e in &log[*start..end] { + st.fold(e); + } + assert_eq!( + &state_hash_hex(&st), + want, + "AM-7 hash-identical UNMET: replaying segment seeded \ + {seg_seed} ({start}..{end}) did not reproduce its \ + recorded state hash" + ); + verified += 1; + } + // Positive control: verifying zero segments would pass vacuously. + assert!( + verified >= 2, + "AM-7 needs several segments to verify, got {verified}" + ); let start = Instant::now(); let mut state = fresh(42); for event in &log { diff --git a/history/260801-cb-wp-0006-log.md b/history/260801-cb-wp-0006-log.md index e91e3e5..b7e1cb5 100644 --- a/history/260801-cb-wp-0006-log.md +++ b/history/260801-cb-wp-0006-log.md @@ -268,3 +268,69 @@ never demonstrate. with AM-2, AM-5 and AM-9 added, AM-6 moved to enforced, and the headline total corrected from 4 to 8 — it had gone stale within the same workplan that produced it. + +## CB-WP-0006-T06 + +**Delivered: K10 replay bundles, `cb-sim --replay`, and AM-7's withdrawn +hash clause re-earned.** + +INTENT design decision 8 of 10, unimplemented for six passes. `cb-sim` had +no flag parsing at all, so `--replay` had nowhere to go. + +**The bundle** (`crates/cb-game-runtime/src/replay.rs`, dev-only behind +`scenarios`, charged to AM-4b): manifest, `commands.log`, +`initial.snapshot`, `expected.yaml`. The command stream goes through the +**K11 framing built in T05**, so a truncated bundle is detected rather +than replayed short — the two tasks compose rather than duplicating. + +**The D2 correction the reviewer forced was real.** This was not "a +directory of four files": `Pass` carried only the *end* state, `RunOutcome::Failed` +was a formatted `String`, and `scenario.rs` created an `EventLog`, +appended to it and never read it. All three had to change. + +### The bug that proves the round trip is load-bearing + +The first round trip **failed to reproduce**. Cause: `state_hash_hex` over +a `serde_json::Value` is a **different canonical form** than over the +typed aggregate — `Value`'s map is key-sorted, a struct serializes in +declaration order. The bundle was written with one basis and verified with +the other. + +Worth stating plainly: a round trip that had been written to recompute its +own comparison value would have **passed** this bug. It failed precisely +because the recorded hash came from the producing process — control 2, +which exists for exactly this. + +### The four controls, all executable + +`make replay-test`, **14/14**: + +| control | how it is proven | +|---|---| +| something actually fails | a committed fixture in `scenarios/fixtures/` (outside the corpus, `covers: []`, so it neither fails `make sim` nor inflates AM-1); exactly one bundle produced; all four files present **and non-empty** | +| hash read from the bundle | tamper only the recorded hash → replay fails. A recomputed comparison would still pass | +| truncation detected | log short by one byte → rejected; length prefix corrupted → rejected | +| replay can fail | mutate the manifest seed → rejected. This bites only because replay re-derives the initial state from seed+setup and checks it against the recorded snapshot; restoring from the snapshot alone would leave the seed inert | + +Plus a control on the controls: the bundle must still replay after every +mutation is reverted, so the suite cannot leave the tree broken. + +### AM-7 re-earned + +The probe now records a hash per **per-game segment** and replays each +from its own genesis. Folding a segment from the wrong seed fails: + +```text +AM-7 hash-identical UNMET: replaying segment seeded 42 (0..64) did not +reproduce its recorded state hash +``` + +That is the clause ADR-0005 §4 withdrew as mutation-proven inert. The +`scaling >= 0.9x` clause is still unenforced, so AM-7 stays **PARTIAL** — +reported, not rounded up. + +**Kernel coverage 15/18 → 16/18**; K10 is no longer unlinked. `facts-check` +immediately caught the spec's copy of that number going stale — the DFD +gate earning its place on a number that moved the same hour. + +**M-D1-MUT: 8 of 14** (unchanged; AM-7 remains partial). diff --git a/scenarios/fixtures/k10-deliberate-failure.yaml b/scenarios/fixtures/k10-deliberate-failure.yaml new file mode 100644 index 0000000..dd4cb93 --- /dev/null +++ b/scenarios/fixtures/k10-deliberate-failure.yaml @@ -0,0 +1,39 @@ +# K10 fixture (CB-WP-0006 T06). This scenario is MEANT TO FAIL. +# +# ADR-0005 §6 control 1: all 21 real scenarios pass, so a corpus sweep +# would round-trip zero bundles and print `ok`. Something must actually +# fail, on purpose, for the bundle writer to be exercised at all. +# +# It lives in scenarios/fixtures/ rather than scenarios/ground/ so that +# `make sim` and `make coverage` do not pick it up: it is a harness +# fixture, not a rule claim, and its `covers` list is deliberately empty +# so it cannot inflate AM-1. +scenario: ground/k10-deliberate-failure +description: > + Deliberately wrong expectation, so cb-sim fails and writes a .cbreplay + bundle. Exercised only by `make replay-test`. +covers: [] +seed: 42 +setup: + players: 3 + preset: standard-3p +commands: + - actor: P1 + cmd: select_action + args: { action: ATTACK, target: P2 } + - actor: P2 + cmd: select_action + args: { action: SUPPORT, target: P3 } + - actor: P3 + cmd: select_action + args: { action: INVESTIGATE, problem: 3 } + - actor: SYSTEM + cmd: reveal + - actor: SYSTEM + cmd: resolve + - actor: SYSTEM + cmd: end_round +expect: + # Deliberately wrong: the round is 2 after end_round, never 99. + state: + round: 99 diff --git a/specs/MetricsAndScenarios.md b/specs/MetricsAndScenarios.md index 372a8e7..f8c7de5 100644 --- a/specs/MetricsAndScenarios.md +++ b/specs/MetricsAndScenarios.md @@ -53,8 +53,9 @@ and add capability-specific rows only when these don't cover the claim. > `58/58 (100%)` read as "all rules". > > `make coverage` now reports a second, separate denominator: -> **15 of 18** K-rules are named across the source. -> Unlinked: **K10 K14 K18**. +> **16 of 18** K-rules are named across the source. +> Unlinked: **K14 K18**. +> (K10 was unlinked until CB-WP-0006 T06 implemented replay bundles.) > > **Kernel rules are link-only, by design.** They are kernel invariants, > not game rules: there is no kernel aggregate, setup preset or command diff --git a/tools/__pycache__/cb-cost.cpython-312.pyc b/tools/__pycache__/cb-cost.cpython-312.pyc index c395c99..5f3d95e 100644 Binary files a/tools/__pycache__/cb-cost.cpython-312.pyc and b/tools/__pycache__/cb-cost.cpython-312.pyc differ diff --git a/tools/__pycache__/dep-weight.cpython-312.pyc b/tools/__pycache__/dep-weight.cpython-312.pyc index 3e04619..151a5e2 100644 Binary files a/tools/__pycache__/dep-weight.cpython-312.pyc and b/tools/__pycache__/dep-weight.cpython-312.pyc differ diff --git a/tools/__pycache__/mutation-check.cpython-312.pyc b/tools/__pycache__/mutation-check.cpython-312.pyc index abfc54b..384ffd2 100644 Binary files a/tools/__pycache__/mutation-check.cpython-312.pyc and b/tools/__pycache__/mutation-check.cpython-312.pyc differ diff --git a/tools/cb-sim/src/main.rs b/tools/cb-sim/src/main.rs index eba47eb..f08e158 100644 --- a/tools/cb-sim/src/main.rs +++ b/tools/cb-sim/src/main.rs @@ -6,16 +6,30 @@ //! no "tolerable" non-zero exit: a silent skip is the failure mode this //! binary exists to catch. -use cb_game_runtime::{scenario, RunOutcome, ScenarioFile}; +use cb_game_runtime::{replay, scenario, RunOutcome, ScenarioFile}; use games_ground::GroundState; +use std::path::{Path, PathBuf}; + +/// Where a failing run drops its `.cbreplay` bundle +/// (MetricsAndScenarios §2). +const REPLAY_DIR: &str = "replays"; fn main() { - let args: Vec = std::env::args().skip(1).collect(); - if args.is_empty() { + let argv: Vec = std::env::args().skip(1).collect(); + if argv.is_empty() { eprintln!("usage: cb-sim ..."); + eprintln!(" cb-sim --replay ..."); std::process::exit(64); } + // K10: until CB-WP-0006 T06 this binary had no flag parsing at all, so + // `--replay` had nowhere to go — every argument was treated as a + // scenario path. + if argv[0] == "--replay" { + std::process::exit(replay_bundles(&argv[1..])); + } + let args = argv; + let mut failed = false; let mut passed = 0usize; let mut covered: Vec = Vec::new(); @@ -64,8 +78,28 @@ fn main() { covered.extend(covers); passed += 1; } - RunOutcome::Failed { reason } => { + RunOutcome::Failed { reason, evidence } => { println!("FAIL {} — {reason}", sc.scenario); + // K10: a failure becomes work an agent can start, not just + // information (MetricsAndScenarios §4). + if let Some(ev) = evidence { + match std::fs::create_dir_all(REPLAY_DIR) + .map_err(|e| e.to_string()) + .and_then(|()| { + replay::write_bundle( + Path::new(REPLAY_DIR), + &sc, + &ev.initial, + &ev.initial_hash, + &ev.end_state, + &ev.end_state_hash, + &reason, + ) + }) { + Ok(path) => println!(" bundle {}", path.display()), + Err(e) => eprintln!(" bundle NOT written: {e}"), + } + } failed = true; } } @@ -86,3 +120,38 @@ fn main() { std::process::exit(1); } } + +/// `cb-sim --replay ...` — re-execute recorded bundles. +fn replay_bundles(paths: &[String]) -> i32 { + if paths.is_empty() { + eprintln!("usage: cb-sim --replay ..."); + return 64; + } + let mut ok = 0usize; + let mut bad = false; + for p in paths { + let bundle = PathBuf::from(p); + match replay::replay::(&bundle) { + Ok(r) => { + println!( + "REPLAY {} — {} commands, hash {} reproduced", + r.scenario, + r.commands, + &r.hash[..12] + ); + ok += 1; + } + Err(e) => { + println!("REPLAY FAIL {} — {e}", bundle.display()); + bad = true; + } + } + } + // Same positive control as the scenario path: a run that replayed + // nothing must not report success. + if ok == 0 { + eprintln!("FAIL — no bundle replayed; refusing to report success"); + bad = true; + } + i32::from(bad) +} diff --git a/tools/mutation-check.py b/tools/mutation-check.py index 145b03b..c3a578e 100644 --- a/tools/mutation-check.py +++ b/tools/mutation-check.py @@ -184,11 +184,10 @@ def rows(): clauses=[ ("timing <= 5 s", True, "the elapsed assert is live; tightening it to 0.0 s goes red"), - ("hash-identical", False, - "MUTATION-PROVEN SURVIVED: folding from fresh(999) instead " - "of fresh(42) leaves the test green. The hash reaches only a " - "println!. It could not be asserted as written anyway — the " - "log spans games seeded 42, 43, 44..."), + ("hash-identical", True, + "RE-EARNED (CB-WP-0006 T06): the probe now replays each " + "per-game segment from its own genesis and asserts its " + "recorded hash. Folding a segment from the wrong seed fails."), ("scaling >= 0.9x", False, "no code computes the ratio of throughput @100k to @5k or " "compares it to 0.9; Criterion reports both and nothing " diff --git a/tools/replay-test.py b/tools/replay-test.py new file mode 100644 index 0000000..9c51288 --- /dev/null +++ b/tools/replay-test.py @@ -0,0 +1,195 @@ +#!/usr/bin/env python3 +"""K10 acceptance: a replay bundle must re-execute, and must be able to fail. + +CB-WP-0006 T06, implementing ADR-0005 §6's four controls verbatim. The +reviewer supplied two of them, and both are the kind a round-trip test +usually omits. + +Without controls, `make replay-test` reports `ok` under at least four +silent failures: + + * no scenario fails, so zero bundles are round-tripped; + * the writer emits nothing and identical error strings satisfy + "same failure both times"; + * the replayed hash is compared to one recomputed in the same process — + `assert_eq!(h, h)`, which is the AM-7 defect exactly; + * the truncation path is never exercised, so K11's operative clause is + dead code while the gate is green. + +So each control below **corrupts a real bundle and requires the replay to +reject it**. A round-trip that cannot fail proves nothing. + +Usage: + python3 tools/replay-test.py + python3 tools/replay-test.py --self-test +""" +import os +import shutil +import subprocess +import sys + +from repo import ROOT, cargo_bin, cargo_env, enter_root + +FIXTURE = "scenarios/fixtures/k10-deliberate-failure.yaml" +REPLAY_DIR = "replays" +BUNDLE = f"{REPLAY_DIR}/ground-k10-deliberate-failure.cbreplay" +# cb_events::store — magic (5) + version (1). +HEADER_LEN = 6 + + +def sim(*args): + """(exit code, combined output) for one cb-sim invocation.""" + cargo = cargo_bin() + r = subprocess.run( + [cargo, "run", "-q", "-p", "cb-sim", "--", *args], + cwd=ROOT, env=cargo_env(), capture_output=True, text=True) + return r.returncode, (r.stdout + r.stderr) + + +def produce_bundle(): + """Run the deliberately-failing fixture; return (bundles, output).""" + shutil.rmtree(os.path.join(ROOT, REPLAY_DIR), ignore_errors=True) + code, out = sim(FIXTURE) + base = os.path.join(ROOT, REPLAY_DIR) + bundles = sorted(d for d in os.listdir(base)) if os.path.isdir(base) else [] + return code, out, bundles + + +def read(name): + with open(os.path.join(ROOT, BUNDLE, name), "rb") as fh: + return fh.read() + + +def write(name, data): + with open(os.path.join(ROOT, BUNDLE, name), "wb") as fh: + fh.write(data) + + +def run(): + results = [] + + def check(name, ok, detail=""): + results.append((name, ok, detail)) + return ok + + print("K10 replay acceptance — a bundle must re-execute, and must be " + "able to fail\n") + + # --- Control 1: something must actually fail, or zero bundles exist --- + code, out, bundles = produce_bundle() + check("the fixture fails (all 21 real scenarios pass)", code != 0, + f"cb-sim exit {code}") + check("exactly one bundle was produced", len(bundles) == 1, + f"{len(bundles)} bundle(s): {bundles}") + for f in ("manifest.yaml", "commands.log", "initial.snapshot", + "expected.yaml"): + check(f"bundle contains {f}", + os.path.isfile(os.path.join(ROOT, BUNDLE, f))) + # A writer that emitted empty files would satisfy "contains" above. + check("bundle files are non-empty", + all(len(read(f)) > 0 for f in + ("manifest.yaml", "commands.log", "initial.snapshot", + "expected.yaml"))) + + good_log = read("commands.log") + good_manifest = read("manifest.yaml") + + # --- The happy path, once, so the negatives mean something --- + code, out = sim("--replay", BUNDLE) + check("a well-formed bundle replays and reproduces the recorded hash", + code == 0 and "reproduced" in out, out.strip().splitlines()[0][:70]) + + # --- Control 2: the hash is READ FROM the bundle, not recomputed --- + # Corrupt only the recorded hash. If the replay recomputed its own + # comparison value, this would still pass — `assert_eq!(h, h)`. + m = good_manifest.decode() + tampered = m.replace("end_state_hash: ", "end_state_hash: ff", 1) + write("manifest.yaml", tampered.encode()) + code, out = sim("--replay", BUNDLE) + check("a tampered recorded hash fails the replay", + code != 0 and "did not reproduce" in out, + "proves the comparison value comes from the bundle") + write("manifest.yaml", good_manifest) + + # --- Control 3a: truncate the command log by one byte --- + write("commands.log", good_log[:-1]) + code, out = sim("--replay", BUNDLE) + check("a command log short by one byte is rejected", + code != 0 and "truncated tail" in out, out.strip().splitlines()[0][:70]) + + # --- Control 3b: corrupt the length prefix --- + bad = bytearray(good_log) + bad[HEADER_LEN] = 0xFF + bad[HEADER_LEN + 1] = 0xFF + write("commands.log", bytes(bad)) + code, out = sim("--replay", BUNDLE) + check("a corrupted length prefix is rejected", + code != 0 and ("truncated tail" in out or "exceeds" in out), + out.strip().splitlines()[0][:70]) + write("commands.log", good_log) + + # --- Control 4: a mutated seed must fail to reproduce --- + m = good_manifest.decode() + seed_line = next(ln for ln in m.splitlines() if ln.startswith("seed:")) + mutated = m.replace(seed_line, "seed: 999", 1) + check("the seed mutation actually changed the manifest", mutated != m) + write("manifest.yaml", mutated.encode()) + code, out = sim("--replay", BUNDLE) + check("a mutated seed fails the replay", + code != 0 and "inconsistent" in out, + "without this the seed would be inert — replay restores from the " + "snapshot") + write("manifest.yaml", good_manifest) + + # --- Restored bundle must still replay, or the controls corrupted it --- + code, out = sim("--replay", BUNDLE) + check("the bundle still replays after every control restored it", + code == 0, "controls must not leave the tree broken") + + ok = True + for name, passed, detail in results: + print(f" [{'ok ' if passed else 'FAIL'}] {name}" + + (f"\n {detail}" if detail else "")) + ok &= passed + print(f"\n {sum(1 for _, p, _ in results if p)}/{len(results)} controls " + f"passed") + return 0 if ok else 1 + + +def self_test(): + """Positive controls on the harness itself.""" + results = [] + + def check(name, ok, detail=""): + results.append((name, ok, detail)) + + check("the deliberately-failing fixture is committed", + os.path.isfile(os.path.join(ROOT, FIXTURE)), FIXTURE) + # It must not be in the corpus `make sim` and `make coverage` sweep, or + # it would fail the build and pollute AM-1. + check("the fixture is outside scenarios/ground/", + "scenarios/ground/" not in FIXTURE, + "a deliberately-failing scenario in the corpus would fail `make sim`") + text = open(os.path.join(ROOT, FIXTURE)).read() + check("the fixture claims no rules", "covers: []" in text, + "a fixture that claimed rules would inflate AM-1") + check("cb-sim is locatable", bool(cargo_bin())) + + print("replay-test self-test (positive control)") + ok = True + for name, passed, detail in results: + print(f" [{'ok ' if passed else 'FAIL'}] {name}" + + (f" — {detail}" if detail else "")) + ok &= passed + return 0 if ok else 1 + + +def main(): + enter_root() + if "--self-test" in sys.argv: + return self_test() + return run() + + +if __name__ == "__main__": + sys.exit(main())