coulomb/clay-borg
4
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

CB-WP-0006 T06: K10 replay bundles, --replay, and AM-7 re-earned

Browse source 
INTENT design decision 8 of 10, unimplemented for six passes. cb-sim had
no flag parsing at all, so --replay had nowhere to go.

The bundle is manifest + commands.log + initial.snapshot + expected.yaml,
dev-only behind the scenarios feature and charged to AM-4b. The command
stream goes through the K11 framing built in T05, so a truncated bundle is
detected rather than replayed short — the two tasks compose rather than
duplicating.

The reviewer's D2 correction was real: this was not "a directory of four
files". Pass carried only the end state, RunOutcome::Failed was a
formatted String, and scenario.rs created an EventLog, appended to it and
never read it. All three had to change.

The first round trip failed to reproduce, and the cause is worth keeping:
state_hash_hex over a serde_json::Value is a different canonical form than
over the typed aggregate — Value's map is key-sorted, a struct serializes
in declaration order. The bundle was written with one basis and verified
with the other. A round trip written to recompute its own comparison value
would have PASSED this bug; it failed because the recorded hash came from
the producing process, which is control 2's entire purpose.

make replay-test implements ADR-0005 §6's four controls, 14/14: a
committed deliberately-failing fixture outside the corpus with covers: []
so it neither fails `make sim` nor inflates AM-1; a tampered recorded hash
must fail; a log short by one byte and a corrupted length prefix must be
rejected; and a mutated manifest seed must fail — which bites only because
replay re-derives the initial state from seed+setup and checks it against
the recorded snapshot, since restoring from the snapshot alone would leave
the seed inert. Plus a control on the controls: the bundle must still
replay after every mutation is reverted.

AM-7's hash-identical clause is re-earned. The probe records a hash per
per-game segment and replays each from its own genesis; folding from the
wrong seed now fails. That is the clause ADR-0005 §4 withdrew as
mutation-proven inert. The scaling >= 0.9x clause is still unenforced, so
AM-7 stays PARTIAL — reported, not rounded up.

Kernel coverage 15/18 -> 16/18. facts-check immediately caught the spec's
copy of that number going stale, on a number that moved the same hour.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
tegwick 2026-08-01 11:05:37 +02:00
parent 6037467478
commit 1edadac9a2
17 changed files with 721 additions and 35 deletions
Download patch file Download diff file Expand all files Collapse all files

BIN
tools/__pycache__/cb-cost.cpython-312.pyc
View file

Binary file not shown.

BIN
tools/__pycache__/dep-weight.cpython-312.pyc
View file

Binary file not shown.

BIN
tools/__pycache__/mutation-check.cpython-312.pyc
View file

Binary file not shown.

77
tools/cb-sim/src/main.rs
View file

@ -6,16 +6,30 @@
//! no "tolerable" non-zero exit: a silent skip is the failure mode this
//! binary exists to catch.
use cb_game_runtime::{scenario, RunOutcome, ScenarioFile};
use cb_game_runtime::{replay, scenario, RunOutcome, ScenarioFile};
use games_ground::GroundState;
use std::path::{Path, PathBuf};
/// Where a failing run drops its `.cbreplay` bundle
/// (MetricsAndScenarios §2).
const REPLAY_DIR: &str = "replays";
fn main() {
let args: Vec<String> = std::env::args().skip(1).collect();
if args.is_empty() {
let argv: Vec<String> = std::env::args().skip(1).collect();
if argv.is_empty() {
eprintln!("usage: cb-sim <scenario.yaml>...");
eprintln!(" cb-sim --replay <bundle.cbreplay>...");
std::process::exit(64);
}
// K10: until CB-WP-0006 T06 this binary had no flag parsing at all, so
// `--replay` had nowhere to go — every argument was treated as a
// scenario path.
if argv[0] == "--replay" {
std::process::exit(replay_bundles(&argv[1..]));
}
let args = argv;
let mut failed = false;
let mut passed = 0usize;
let mut covered: Vec<String> = Vec::new();
@ -64,8 +78,28 @@ fn main() {
covered.extend(covers);
passed += 1;
}
RunOutcome::Failed { reason } => {
RunOutcome::Failed { reason, evidence } => {
println!("FAIL {} — {reason}", sc.scenario);
// K10: a failure becomes work an agent can start, not just
// information (MetricsAndScenarios §4).
if let Some(ev) = evidence {
match std::fs::create_dir_all(REPLAY_DIR)
.map_err(|e| e.to_string())
.and_then(|()| {
replay::write_bundle(
Path::new(REPLAY_DIR),
&sc,
&ev.initial,
&ev.initial_hash,
&ev.end_state,
&ev.end_state_hash,
&reason,
)
}) {
Ok(path) => println!(" bundle {}", path.display()),
Err(e) => eprintln!(" bundle NOT written: {e}"),
}
}
failed = true;
}
}
@ -86,3 +120,38 @@ fn main() {
std::process::exit(1);
}
}
/// `cb-sim --replay <bundle>...` — re-execute recorded bundles.
fn replay_bundles(paths: &[String]) -> i32 {
if paths.is_empty() {
eprintln!("usage: cb-sim --replay <bundle.cbreplay>...");
return 64;
}
let mut ok = 0usize;
let mut bad = false;
for p in paths {
let bundle = PathBuf::from(p);
match replay::replay::<GroundState>(&bundle) {
Ok(r) => {
println!(
"REPLAY {} — {} commands, hash {} reproduced",
r.scenario,
r.commands,
&r.hash[..12]
);
ok += 1;
}
Err(e) => {
println!("REPLAY FAIL {} — {e}", bundle.display());
bad = true;
}
}
}
// Same positive control as the scenario path: a run that replayed
// nothing must not report success.
if ok == 0 {
eprintln!("FAIL — no bundle replayed; refusing to report success");
bad = true;
}
i32::from(bad)
}

9
tools/mutation-check.py
View file

@ -184,11 +184,10 @@ def rows():
clauses=[
("timing <= 5 s", True,
"the elapsed assert is live; tightening it to 0.0 s goes red"),
("hash-identical", False,
"MUTATION-PROVEN SURVIVED: folding from fresh(999) instead "
"of fresh(42) leaves the test green. The hash reaches only a "
"println!. It could not be asserted as written anyway — the "
"log spans games seeded 42, 43, 44..."),
("hash-identical", True,
"RE-EARNED (CB-WP-0006 T06): the probe now replays each "
"per-game segment from its own genesis and asserts its "
"recorded hash. Folding a segment from the wrong seed fails."),
("scaling >= 0.9x", False,
"no code computes the ratio of throughput @100k to @5k or "
"compares it to 0.9; Criterion reports both and nothing "

195
tools/replay-test.py Normal file
View file

@ -0,0 +1,195 @@
#!/usr/bin/env python3
"""K10 acceptance: a replay bundle must re-execute, and must be able to fail.
CB-WP-0006 T06, implementing ADR-0005 §6's four controls verbatim. The
reviewer supplied two of them, and both are the kind a round-trip test
usually omits.
Without controls, `make replay-test` reports `ok` under at least four
silent failures:
* no scenario fails, so zero bundles are round-tripped;
* the writer emits nothing and identical error strings satisfy
"same failure both times";
* the replayed hash is compared to one recomputed in the same process —
`assert_eq!(h, h)`, which is the AM-7 defect exactly;
* the truncation path is never exercised, so K11's operative clause is
dead code while the gate is green.
So each control below **corrupts a real bundle and requires the replay to
reject it**. A round-trip that cannot fail proves nothing.
Usage:
python3 tools/replay-test.py
python3 tools/replay-test.py --self-test
"""
import os
import shutil
import subprocess
import sys
from repo import ROOT, cargo_bin, cargo_env, enter_root
FIXTURE = "scenarios/fixtures/k10-deliberate-failure.yaml"
REPLAY_DIR = "replays"
BUNDLE = f"{REPLAY_DIR}/ground-k10-deliberate-failure.cbreplay"
# cb_events::store — magic (5) + version (1).
HEADER_LEN = 6
def sim(*args):
"""(exit code, combined output) for one cb-sim invocation."""
cargo = cargo_bin()
r = subprocess.run(
[cargo, "run", "-q", "-p", "cb-sim", "--", *args],
cwd=ROOT, env=cargo_env(), capture_output=True, text=True)
return r.returncode, (r.stdout + r.stderr)
def produce_bundle():
"""Run the deliberately-failing fixture; return (bundles, output)."""
shutil.rmtree(os.path.join(ROOT, REPLAY_DIR), ignore_errors=True)
code, out = sim(FIXTURE)
base = os.path.join(ROOT, REPLAY_DIR)
bundles = sorted(d for d in os.listdir(base)) if os.path.isdir(base) else []
return code, out, bundles
def read(name):
with open(os.path.join(ROOT, BUNDLE, name), "rb") as fh:
return fh.read()
def write(name, data):
with open(os.path.join(ROOT, BUNDLE, name), "wb") as fh:
fh.write(data)
def run():
results = []
def check(name, ok, detail=""):
results.append((name, ok, detail))
return ok
print("K10 replay acceptance — a bundle must re-execute, and must be "
"able to fail\n")
# --- Control 1: something must actually fail, or zero bundles exist ---
code, out, bundles = produce_bundle()
check("the fixture fails (all 21 real scenarios pass)", code != 0,
f"cb-sim exit {code}")
check("exactly one bundle was produced", len(bundles) == 1,
f"{len(bundles)} bundle(s): {bundles}")
for f in ("manifest.yaml", "commands.log", "initial.snapshot",
"expected.yaml"):
check(f"bundle contains {f}",
os.path.isfile(os.path.join(ROOT, BUNDLE, f)))
# A writer that emitted empty files would satisfy "contains" above.
check("bundle files are non-empty",
all(len(read(f)) > 0 for f in
("manifest.yaml", "commands.log", "initial.snapshot",
"expected.yaml")))
good_log = read("commands.log")
good_manifest = read("manifest.yaml")
# --- The happy path, once, so the negatives mean something ---
code, out = sim("--replay", BUNDLE)
check("a well-formed bundle replays and reproduces the recorded hash",
code == 0 and "reproduced" in out, out.strip().splitlines()[0][:70])
# --- Control 2: the hash is READ FROM the bundle, not recomputed ---
# Corrupt only the recorded hash. If the replay recomputed its own
# comparison value, this would still pass — `assert_eq!(h, h)`.
m = good_manifest.decode()
tampered = m.replace("end_state_hash: ", "end_state_hash: ff", 1)
write("manifest.yaml", tampered.encode())
code, out = sim("--replay", BUNDLE)
check("a tampered recorded hash fails the replay",
code != 0 and "did not reproduce" in out,
"proves the comparison value comes from the bundle")
write("manifest.yaml", good_manifest)
# --- Control 3a: truncate the command log by one byte ---
write("commands.log", good_log[:-1])
code, out = sim("--replay", BUNDLE)
check("a command log short by one byte is rejected",
code != 0 and "truncated tail" in out, out.strip().splitlines()[0][:70])
# --- Control 3b: corrupt the length prefix ---
bad = bytearray(good_log)
bad[HEADER_LEN] = 0xFF
bad[HEADER_LEN + 1] = 0xFF
write("commands.log", bytes(bad))
code, out = sim("--replay", BUNDLE)
check("a corrupted length prefix is rejected",
code != 0 and ("truncated tail" in out or "exceeds" in out),
out.strip().splitlines()[0][:70])
write("commands.log", good_log)
# --- Control 4: a mutated seed must fail to reproduce ---
m = good_manifest.decode()
seed_line = next(ln for ln in m.splitlines() if ln.startswith("seed:"))
mutated = m.replace(seed_line, "seed: 999", 1)
check("the seed mutation actually changed the manifest", mutated != m)
write("manifest.yaml", mutated.encode())
code, out = sim("--replay", BUNDLE)
check("a mutated seed fails the replay",
code != 0 and "inconsistent" in out,
"without this the seed would be inert — replay restores from the "
"snapshot")
write("manifest.yaml", good_manifest)
# --- Restored bundle must still replay, or the controls corrupted it ---
code, out = sim("--replay", BUNDLE)
check("the bundle still replays after every control restored it",
code == 0, "controls must not leave the tree broken")
ok = True
for name, passed, detail in results:
print(f" [{'ok ' if passed else 'FAIL'}] {name}"
+ (f"\n {detail}" if detail else ""))
ok &= passed
print(f"\n {sum(1 for _, p, _ in results if p)}/{len(results)} controls "
f"passed")
return 0 if ok else 1
def self_test():
"""Positive controls on the harness itself."""
results = []
def check(name, ok, detail=""):
results.append((name, ok, detail))
check("the deliberately-failing fixture is committed",
os.path.isfile(os.path.join(ROOT, FIXTURE)), FIXTURE)
# It must not be in the corpus `make sim` and `make coverage` sweep, or
# it would fail the build and pollute AM-1.
check("the fixture is outside scenarios/ground/",
"scenarios/ground/" not in FIXTURE,
"a deliberately-failing scenario in the corpus would fail `make sim`")
text = open(os.path.join(ROOT, FIXTURE)).read()
check("the fixture claims no rules", "covers: []" in text,
"a fixture that claimed rules would inflate AM-1")
check("cb-sim is locatable", bool(cargo_bin()))
print("replay-test self-test (positive control)")
ok = True
for name, passed, detail in results:
print(f" [{'ok ' if passed else 'FAIL'}] {name}"
+ (f" — {detail}" if detail else ""))
ok &= passed
return 0 if ok else 1
def main():
enter_root()
if "--self-test" in sys.argv:
return self_test()
return run()
if __name__ == "__main__":
sys.exit(main())