tegwick
edab11c0e4
Some checks failed
ci / check (push) Failing after 3s
make trials reported "positions unreachable: 4, target 0 — the recording exists but the position moved". All four were false. A recording holds one hash, the final state, and reachability asked whether the note's hash was in that file — so a mid-game note could never match, and a post-game note from any but the last game could not either. Instance 8 of the ADR-0018 family: vary only WHEN a note was written and the answer flips, with nothing having moved. The root cause was not the metric. play again reused state belonging to a game: it overwrote the previous game's recording (data loss), never cleared the journal (game 2's log opened with game 1's commands), and so a note's command index pointed into a recording without those commands. Fixing reachability alone would have gone green while a session still destroyed its own evidence. A note now binds by (game, after) — an index into the recording's own commands list, which a reader can replay to. The hash keeps a job as the integrity check at the end of a game, where it can actually fail. Game 1 keeps the path it was given, so GameDesign §5's documented invocation is unchanged; later games get -2, -3 and nothing is overwritten. Legacy 5-column logs stay readable and are reported as legacy, never as orphans — an unsubstantiated orphan claim is the defect being fixed. All three fixes mutation-proven, including at the call site via a real two-game session. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
355 lines
16 KiB
Python
355 lines
16 KiB
Python
#!/usr/bin/env python3
|
|
"""trials — what the players said, and where (CB-WP-0027 T04).
|
|
|
|
**Surfacing is the deliverable, not storage.** ADR-0014 D6: a commentary
|
|
feature that stores comments and shows them nowhere would be this
|
|
project's signature failure in a new medium — after the message that sat
|
|
unread for four days and the ten rulings that arrived and were never
|
|
collected.
|
|
|
|
So this reports every note from every trial log, with its position and its
|
|
age, and flags the ones whose position can no longer be reached.
|
|
|
|
Reuses `design.py`'s parsing shape because the trial log deliberately
|
|
reuses `FindingRegister.md`'s idiom: a table between HTML-comment markers
|
|
inside a readable Markdown file.
|
|
"""
|
|
|
|
import os, re, sys, glob, datetime
|
|
|
|
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
TRIALS = os.path.join(ROOT, "trials")
|
|
|
|
BEGIN = "<!-- trial-log:begin -->"
|
|
END = "<!-- trial-log:end -->"
|
|
|
|
# GameDesign §3.1's figure, shared rather than reinvented: a note that
|
|
# cannot expire ages into an apparent finding.
|
|
NOTE_EXPIRY_DAYS = 30
|
|
|
|
# The trial-log table's columns, named once. `write_trial_log` in
|
|
# `hotseat.rs` writes them; this reads them; a row that is not this shape
|
|
# is an error rather than a row to skip.
|
|
COLUMNS = ("n", "game", "after", "round", "step", "state_hash", "comment")
|
|
|
|
# ADR-0019 D4. Logs written before the binding was fixed. They are parsed
|
|
# and reported, and they are NOT called orphans: their positions were
|
|
# never checkable by anything this repo can run, and an unsubstantiated
|
|
# orphan claim is the exact defect ADR-0019 exists to delete.
|
|
LEGACY_COLUMNS = ("n", "round", "step", "state_hash", "comment")
|
|
|
|
|
|
def parse(text):
|
|
"""Rows between the markers. A log without the block is an error, not
|
|
an empty log — silently reporting zero notes for a file full of them
|
|
is precisely the failure this tool exists to prevent."""
|
|
if BEGIN not in text or END not in text:
|
|
raise ValueError("no trial-log block")
|
|
block = text.split(BEGIN)[1].split(END)[0]
|
|
rows = []
|
|
for line in block.splitlines():
|
|
line = line.strip()
|
|
if not line.startswith("|") or line.startswith("|---"):
|
|
continue
|
|
cells = [c.strip() for c in line.strip("|").split("|")]
|
|
if cells[0] == "n":
|
|
continue
|
|
# CB-WP-0032. A wrong column count RAISES; it used to `continue`.
|
|
#
|
|
# The docstring above already says silently reporting zero notes
|
|
# for a file full of them is the failure this tool exists to
|
|
# prevent -- and a `continue` here did exactly that for every row
|
|
# at once. Adding a sixth column to the trial log (which CB-WP-0032
|
|
# considered, and did not do) would have emptied every existing
|
|
# log without a word.
|
|
if len(cells) == len(COLUMNS):
|
|
rows.append(dict(zip(COLUMNS, cells)))
|
|
elif len(cells) == len(LEGACY_COLUMNS):
|
|
row = dict(zip(LEGACY_COLUMNS, cells))
|
|
row["legacy"] = True
|
|
rows.append(row)
|
|
else:
|
|
raise ValueError(
|
|
f"trial-log row has {len(cells)} columns, expected "
|
|
f"{len(COLUMNS)} or {len(LEGACY_COLUMNS)}: {line!r}"
|
|
)
|
|
return rows
|
|
|
|
|
|
def recording_for(md_path, row):
|
|
"""The recording holding this note's game (ADR-0019 D2).
|
|
|
|
Game 1 keeps the base path; later games carry `-2`, `-3`. A legacy row
|
|
names no game, so it gets the base path and is judged as legacy.
|
|
"""
|
|
stem = os.path.splitext(md_path)[0]
|
|
n = int(row.get("game", 1) or 1)
|
|
path = f"{stem}.yaml" if n <= 1 else f"{stem}-{n}.yaml"
|
|
return path if os.path.exists(path) else None
|
|
|
|
|
|
def command_count(recording):
|
|
"""How many commands the recording holds — the range a note's `after`
|
|
must fall inside. Counted from the `commands:` block, because that is
|
|
the list `after` indexes into."""
|
|
text = open(recording).read()
|
|
if "commands:" not in text:
|
|
return 0
|
|
block = text.split("commands:", 1)[1].split("\nexpect:", 1)[0]
|
|
return sum(1 for line in block.splitlines() if line.startswith("- actor:"))
|
|
|
|
|
|
def logs(root=TRIALS):
|
|
"""Every trial log, with its recording beside it if there is one."""
|
|
out = []
|
|
for path in sorted(glob.glob(os.path.join(root, "*.md"))):
|
|
try:
|
|
rows = parse(open(path).read())
|
|
except ValueError as e:
|
|
# The REASON, not a guess at it. This said "no trial-log block
|
|
# -- not a trial log?" for every failure, which is now wrong
|
|
# for two of them: a malformed row is not a missing block, and
|
|
# telling a maintainer to check for the wrong thing costs more
|
|
# than saying nothing.
|
|
print(f" WARN {os.path.basename(path)}: {e}")
|
|
continue
|
|
recording = os.path.splitext(path)[0] + ".yaml"
|
|
out.append((path, rows, recording if os.path.exists(recording) else None))
|
|
return out
|
|
|
|
|
|
def reachability(row, recording):
|
|
"""Can this note's position be reached? One of three answers.
|
|
|
|
**"No recording" and "the hash is not in the recording" are different
|
|
states and must not be one number.** A note written mid-game is
|
|
pending — the recording is only written when the game ends — while a
|
|
note whose hash is absent from a recording that *exists* means the
|
|
position moved. Collapsing them makes every live session report
|
|
orphans, and a metric that cries wolf is one nobody reads, which is
|
|
the failure this whole pass is about.
|
|
|
|
Orphans are reported, never deleted (ADR-0014 D3): a moved position is
|
|
worth knowing, by the same reasoning that rewrote `gr-e01` rather than
|
|
retiring it.
|
|
"""
|
|
if row.get("legacy"):
|
|
return "legacy"
|
|
if not recording:
|
|
return "pending"
|
|
after = int(row["after"])
|
|
n = command_count(recording)
|
|
if after > n:
|
|
# The note points past the end of its own game.
|
|
return "orphan"
|
|
if after == n:
|
|
# The one position the recording states outright. A mismatch here
|
|
# is real: the note claims the end of a game the recording does
|
|
# not agree with.
|
|
return "ok" if row["state_hash"] in open(recording).read() else "orphan"
|
|
# Reachable by replaying `after` commands of this recording.
|
|
return "ok"
|
|
|
|
|
|
def age_days(path, today):
|
|
"""Dated from the log's filename (`YYYY-MM-DD-slug.md`), falling back
|
|
to mtime — the name is the intent, the mtime is what happened."""
|
|
m = re.match(r"(\d{4}-\d{2}-\d{2})", os.path.basename(path))
|
|
if m:
|
|
try:
|
|
return (today - datetime.date.fromisoformat(m.group(1))).days
|
|
except ValueError:
|
|
pass
|
|
return (today - datetime.date.fromtimestamp(os.path.getmtime(path))).days
|
|
|
|
|
|
def report(root=TRIALS, today=None):
|
|
today = today or datetime.date.today()
|
|
if not os.path.isdir(root):
|
|
print("trials — no trials/ directory yet\n")
|
|
print(" Play one: cb-play --serve 0 --record trials/<date>-<slug>.yaml \\")
|
|
print(" --trial trials/<date>-<slug>.md")
|
|
return 0
|
|
|
|
found = logs(root)
|
|
print("trials — what the players said, and where\n")
|
|
total, orphans, pending, expired, legacy = 0, 0, 0, 0, 0
|
|
for path, rows, _ in found:
|
|
name = os.path.basename(path)
|
|
age = age_days(path, today)
|
|
print(f" {name} ({len(rows)} note(s), {age}d)")
|
|
for r in rows:
|
|
total += 1
|
|
# ADR-0019 D2: each note names its own game, and each game has
|
|
# its own recording. One recording per FILE was the assumption
|
|
# that made `play again` overwrite the evidence.
|
|
rec = recording_for(path, r)
|
|
state = reachability(r, rec)
|
|
orphans += state == "orphan"
|
|
pending += state == "pending"
|
|
legacy += state == "legacy"
|
|
expired += age > NOTE_EXPIRY_DAYS
|
|
mark = {"orphan": "orphan", "pending": " .. ",
|
|
"legacy": "legacy", "ok": " "}[state]
|
|
where = f"g{r.get('game', '?')}+{r.get('after', '?')}"
|
|
print(f" {mark} {where:<7} r{r['round']:<2} {r['step']:<14} "
|
|
f"{r['comment'][:60]}")
|
|
print()
|
|
|
|
print(f" trial logs {len(found)}")
|
|
print(f" notes {total}")
|
|
print(f" positions unreachable {orphans} target 0"
|
|
+ (" <-- a note points past the end of its own game" if orphans else ""))
|
|
if pending:
|
|
print(f" awaiting a recording {pending}"
|
|
" (normal during a live session; the recording lands at game end)")
|
|
if legacy:
|
|
print(f" legacy binding {legacy}"
|
|
" (written before ADR-0019; position not checkable, NOT orphaned)")
|
|
print(f" notes past {NOTE_EXPIRY_DAYS} days {expired} target 0")
|
|
if total and not orphans and not legacy:
|
|
print("\n Every note points at a position a reader can reach.")
|
|
print("\n Raw notes stay in this repo (ADR-0014 D4). A note reaches"
|
|
"\n ground-game only by being promoted to a register finding, by a"
|
|
"\n human, with the wording chosen then.")
|
|
return 0
|
|
|
|
|
|
def self_test():
|
|
"""Positive controls, including for the REPORTING path.
|
|
|
|
`design-baseline.py` had a green self-test and an unexercised
|
|
reporting path, and the reporting path is where it rotted (ADR-0012
|
|
D8). So this runs `report` against a fixture and checks what it says.
|
|
"""
|
|
import tempfile, io, contextlib
|
|
|
|
ok = True
|
|
|
|
def check(name, cond, detail=""):
|
|
nonlocal ok
|
|
ok = ok and bool(cond)
|
|
print(f" [{'ok ' if cond else 'FAIL'}] {name}" + (f" — {detail}" if detail else ""))
|
|
|
|
def log_of(*rows):
|
|
head = ("| n | game | after | round | step | state_hash | comment |\n"
|
|
"|---|---|---|---|---|---|---|\n")
|
|
return f"# Trial log\n\n{BEGIN}\n\n{head}" + "".join(rows) + f"\n{END}\n"
|
|
|
|
good = log_of("| 1 | 1 | 4 | 3 | Select | abc123def456 | why is SOLVE doing nothing |\n")
|
|
legacy = (f"# Trial log\n\n{BEGIN}\n\n"
|
|
"| n | round | step | state_hash | comment |\n|---|---|---|---|---|\n"
|
|
"| 1 | 3 | Select | abc123def456 | why is SOLVE doing nothing |\n"
|
|
f"\n{END}\n")
|
|
|
|
def recording(commands, final_hash):
|
|
"""A recording with `commands` commands and one final hash — the
|
|
shape ADR-0019 D3 reads: a list to index into, plus the one
|
|
position the file states outright."""
|
|
body = "commands:\n"
|
|
for _ in range(commands):
|
|
body += "- actor: P1\n cmd: select_action\n args: {}\n"
|
|
return body + f"expect:\n state_hash: {final_hash}\n"
|
|
check("a trial log parses", len(parse(good)) == 1)
|
|
check("a legacy 5-column log still parses", len(parse(legacy)) == 1,
|
|
"ADR-0019 D4 \u2014 old logs stay readable")
|
|
check("a legacy row is MARKED legacy", parse(legacy)[0].get("legacy") is True)
|
|
check("the header row is not a note", all(r["n"] != "n" for r in parse(good)))
|
|
|
|
# CB-WP-0032. A row of the wrong shape must be LOUD. This used to
|
|
# `continue`, so a format change would have reported zero notes for
|
|
# every log at once -- the failure the parser's own docstring names.
|
|
# Six columns is neither the current shape (7) nor the legacy one (5),
|
|
# so it must RAISE rather than be skipped.
|
|
six = log_of("| 1 | 1 | 4 | 3 | Select | abc123def456 |\n")
|
|
try:
|
|
parse(six)
|
|
check("a row with the wrong column count is an error", False)
|
|
except ValueError as e:
|
|
check("a row with the wrong column count is an error", "6 columns" in str(e),
|
|
"a silent skip would empty every log at once")
|
|
|
|
# And the surrounding walk must report the REASON it failed.
|
|
with tempfile.TemporaryDirectory() as d:
|
|
with open(os.path.join(d, "2026-08-07-bad.md"), "w") as fh:
|
|
fh.write(six)
|
|
buf = io.StringIO()
|
|
with contextlib.redirect_stdout(buf):
|
|
logs(d)
|
|
check("the walk names the real reason", "6 columns" in buf.getvalue(),
|
|
"it used to blame a missing block for every failure")
|
|
|
|
try:
|
|
parse("# Trial log\n\nno block here\n")
|
|
check("a file with no block is an error, not an empty log", False)
|
|
except ValueError:
|
|
check("a file with no block is an error, not an empty log", True,
|
|
"silently reporting zero is the failure this tool prevents")
|
|
|
|
with tempfile.TemporaryDirectory() as d:
|
|
open(os.path.join(d, "2026-08-06-x.md"), "w").write(good)
|
|
row = parse(good)[0] # game 1, after 4, hash abc123def456
|
|
check("a note with no recording is PENDING, not orphaned",
|
|
reachability(row, None) == "pending",
|
|
"a live session must not report orphans")
|
|
|
|
rec = os.path.join(d, "2026-08-06-x.yaml")
|
|
|
|
# ADR-0019 D3. A MID-GAME note is reachable by replaying `after`
|
|
# commands. Under the old check this case was IMPOSSIBLE to pass:
|
|
# the recording holds one hash, so every mid-game note was an
|
|
# orphan whatever had happened.
|
|
open(rec, "w").write(recording(10, "zzz"))
|
|
check("a mid-game note inside its game is reachable",
|
|
reachability(row, rec) == "ok",
|
|
"the old check called every one of these an orphan")
|
|
|
|
# And it can still say NO: past the end of its own game.
|
|
open(rec, "w").write(recording(2, "zzz"))
|
|
check("a note past the end of its game is an orphan",
|
|
reachability(row, rec) == "orphan",
|
|
"without this the metric cannot go red")
|
|
|
|
# At the end, the hash is the integrity check — and it can fail.
|
|
end_row = parse(log_of(
|
|
"| 1 | 1 | 4 | 3 | after the end | abc123def456 | done |\n"))[0]
|
|
open(rec, "w").write(recording(4, "abc123def456"))
|
|
check("an end-of-game note agreeing with the recording is reachable",
|
|
reachability(end_row, rec) == "ok")
|
|
open(rec, "w").write(recording(4, "something-else"))
|
|
check("an end-of-game note whose hash disagrees is an orphan",
|
|
reachability(end_row, rec) == "orphan",
|
|
"the position moved \u2014 the case the flag exists for")
|
|
|
|
# A legacy row is never called an orphan (ADR-0019 D4).
|
|
open(rec, "w").write(recording(2, "no-match"))
|
|
check("a legacy row is LEGACY, never orphan",
|
|
reachability(parse(legacy)[0], rec) == "legacy",
|
|
"an unsubstantiated orphan claim is the defect being fixed")
|
|
|
|
# ADR-0019 D2: game 2 reads its OWN recording.
|
|
open(os.path.join(d, "g.md"), "w").write("x")
|
|
open(os.path.join(d, "g.yaml"), "w").write(recording(1, "one"))
|
|
open(os.path.join(d, "g-2.yaml"), "w").write(recording(9, "two"))
|
|
g2 = parse(log_of("| 1 | 2 | 5 | 1 | Select | two | second game |\n"))[0]
|
|
check("a note from game 2 is judged against game 2's recording",
|
|
recording_for(os.path.join(d, "g.md"), g2).endswith("g-2.yaml")
|
|
and reachability(g2, recording_for(os.path.join(d, "g.md"), g2)) == "ok",
|
|
"one recording per FILE is what let play again overwrite it")
|
|
|
|
# THE control design-baseline.py lacked: exercise the reporting
|
|
# path and assert on what it printed.
|
|
buf = io.StringIO()
|
|
with contextlib.redirect_stdout(buf):
|
|
report(d, datetime.date(2026, 8, 6))
|
|
out = buf.getvalue()
|
|
check("the reporting path runs and names the note",
|
|
"why is SOLVE doing nothing" in out, "not just the parser")
|
|
check("the reporting path counts", "notes 1" in out)
|
|
|
|
print("trials self-test (positive control)")
|
|
return 0 if ok else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(self_test() if "--self-test" in sys.argv else report())
|