tegwick
de25c4cae0
Some checks failed
ci / check (push) Failing after 4s
CB-WP-0045 left "nothing explains what a scope does" open and gave a FALSE reason: that scoping is our Variant so no printed sentence exists. The H2 package ships Rules_Text.csv -- twenty-two passages of player-facing rules -- and nothing in clay-borg had ever read the file. A wrong reason for an open item is worse than an open item; it retires the question. Filed as F26 against ground-game: a package that adds a FILE is invisible where one that adds a column is not. The scope rule now renders under the table in the edition's own words, only when a non-global scope is in play, matched by heading rather than row number, and absent (never paraphrased) if the edition drops it. The trial log stamps its variant on the begin marker -- a session property, not an eighth column -- read off state.variant rather than the --variant flag, because a bare `state.variant = v` leaves H2 inert and a flag-stamped log would put false provenance on real player words. An unstamped log reports `unrecorded`, never `ground-darvo-r0`. Six mutations, six red. The sixth is the finding: every trials.py fixture built its marker out of BEGIN, so nine checks followed BEGIN away from what hotseat.rs writes and stayed green while real logs broke. A fixture built from the constant under test cannot test the constant -- the control is now a literal, asserted from both sides. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
460 lines
21 KiB
Python
460 lines
21 KiB
Python
#!/usr/bin/env python3
|
|
"""trials — what the players said, and where (CB-WP-0027 T04).
|
|
|
|
**Surfacing is the deliverable, not storage.** ADR-0014 D6: a commentary
|
|
feature that stores comments and shows them nowhere would be this
|
|
project's signature failure in a new medium — after the message that sat
|
|
unread for four days and the ten rulings that arrived and were never
|
|
collected.
|
|
|
|
So this reports every note from every trial log, with its position and its
|
|
age, and flags the ones whose position can no longer be reached.
|
|
|
|
Reuses `design.py`'s parsing shape because the trial log deliberately
|
|
reuses `FindingRegister.md`'s idiom: a table between HTML-comment markers
|
|
inside a readable Markdown file.
|
|
"""
|
|
|
|
import os
|
|
import re, sys, glob, datetime
|
|
|
|
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
TRIALS = os.path.join(ROOT, "trials")
|
|
|
|
# The begin marker carries attributes since CB-WP-0046, so it is matched
|
|
# by PREFIX. A bare `<!-- trial-log:begin -->` is still a valid marker —
|
|
# every log written before this one has it.
|
|
BEGIN = "<!-- trial-log:begin"
|
|
END = "<!-- trial-log:end -->"
|
|
|
|
# Which rules the session was played under (CB-WP-0046).
|
|
#
|
|
# **A session property, not a note property**, so it rides the marker
|
|
# rather than becoming an eighth column. The variant is fixed at startup
|
|
# and cannot change between games, so a per-row column would repeat one
|
|
# fact on every line and force a third accepted row width on a parser
|
|
# whose docstring is about exactly that hazard.
|
|
VARIANT_ATTR = re.compile(r"variant=([A-Za-z0-9._-]+)")
|
|
|
|
# GameDesign §3.1's figure, shared rather than reinvented: a note that
|
|
# cannot expire ages into an apparent finding.
|
|
NOTE_EXPIRY_DAYS = 30
|
|
|
|
# The trial-log table's columns, named once. `write_trial_log` in
|
|
# `hotseat.rs` writes them; this reads them; a row that is not this shape
|
|
# is an error rather than a row to skip.
|
|
COLUMNS = ("n", "game", "after", "round", "step", "state_hash", "comment")
|
|
|
|
# ADR-0019 D4. Logs written before the binding was fixed. They are parsed
|
|
# and reported, and they are NOT called orphans: their positions were
|
|
# never checkable by anything this repo can run, and an unsubstantiated
|
|
# orphan claim is the exact defect ADR-0019 exists to delete.
|
|
LEGACY_COLUMNS = ("n", "round", "step", "state_hash", "comment")
|
|
|
|
|
|
def parse(text):
|
|
"""Rows between the markers. A log without the block is an error, not
|
|
an empty log — silently reporting zero notes for a file full of them
|
|
is precisely the failure this tool exists to prevent."""
|
|
if BEGIN not in text or END not in text:
|
|
raise ValueError("no trial-log block")
|
|
# Split off the marker's own line so an attribute cannot be read as
|
|
# a table row, and so `variant=` is available to every row below it.
|
|
tail = text.split(BEGIN, 1)[1].split(END)[0]
|
|
marker, _, block = tail.partition("\n")
|
|
hit = VARIANT_ATTR.search(marker)
|
|
variant = hit.group(1) if hit else None
|
|
rows = []
|
|
for line in block.splitlines():
|
|
line = line.strip()
|
|
if not line.startswith("|") or line.startswith("|---"):
|
|
continue
|
|
cells = [c.strip() for c in line.strip("|").split("|")]
|
|
if cells[0] == "n":
|
|
continue
|
|
# CB-WP-0032. A wrong column count RAISES; it used to `continue`.
|
|
#
|
|
# The docstring above already says silently reporting zero notes
|
|
# for a file full of them is the failure this tool exists to
|
|
# prevent -- and a `continue` here did exactly that for every row
|
|
# at once. Adding a sixth column to the trial log (which CB-WP-0032
|
|
# considered, and did not do) would have emptied every existing
|
|
# log without a word.
|
|
if len(cells) == len(COLUMNS):
|
|
row = dict(zip(COLUMNS, cells))
|
|
elif len(cells) == len(LEGACY_COLUMNS):
|
|
row = dict(zip(LEGACY_COLUMNS, cells))
|
|
row["legacy"] = True
|
|
else:
|
|
raise ValueError(
|
|
f"trial-log row has {len(cells)} columns, expected "
|
|
f"{len(COLUMNS)} or {len(LEGACY_COLUMNS)}: {line!r}"
|
|
)
|
|
# **`None`, not a default of "baseline"** — a log written before
|
|
# the marker carried the attribute does not say which rules were
|
|
# played, and answering an unrecorded question with the common
|
|
# case is how a baseline note gets read as an H2 result.
|
|
row["variant"] = variant
|
|
rows.append(row)
|
|
return rows
|
|
|
|
|
|
def recording_for(md_path, row):
|
|
"""The recording holding this note's game (ADR-0019 D2).
|
|
|
|
Game 1 keeps the base path; later games carry `-2`, `-3`. A legacy row
|
|
names no game, so it gets the base path and is judged as legacy.
|
|
"""
|
|
stem = os.path.splitext(md_path)[0]
|
|
n = int(row.get("game", 1) or 1)
|
|
path = f"{stem}.yaml" if n <= 1 else f"{stem}-{n}.yaml"
|
|
return path if os.path.exists(path) else None
|
|
|
|
|
|
def command_count(recording):
|
|
"""How many commands the recording holds — the range a note's `after`
|
|
must fall inside. Counted from the `commands:` block, because that is
|
|
the list `after` indexes into."""
|
|
text = open(recording).read()
|
|
if "commands:" not in text:
|
|
return 0
|
|
block = text.split("commands:", 1)[1].split("\nexpect:", 1)[0]
|
|
return sum(1 for line in block.splitlines() if line.startswith("- actor:"))
|
|
|
|
|
|
def logs(root=TRIALS):
|
|
"""Every trial log, with its recording beside it if there is one."""
|
|
out = []
|
|
for path in sorted(glob.glob(os.path.join(root, "*.md"))):
|
|
try:
|
|
rows = parse(open(path).read())
|
|
except ValueError as e:
|
|
# The REASON, not a guess at it. This said "no trial-log block
|
|
# -- not a trial log?" for every failure, which is now wrong
|
|
# for two of them: a malformed row is not a missing block, and
|
|
# telling a maintainer to check for the wrong thing costs more
|
|
# than saying nothing.
|
|
print(f" WARN {os.path.basename(path)}: {e}")
|
|
continue
|
|
recording = os.path.splitext(path)[0] + ".yaml"
|
|
out.append((path, rows, recording if os.path.exists(recording) else None))
|
|
return out
|
|
|
|
|
|
def reachability(row, recording):
|
|
"""Can this note's position be reached? One of three answers.
|
|
|
|
**"No recording" and "the hash is not in the recording" are different
|
|
states and must not be one number.** A note written mid-game is
|
|
pending — the recording is only written when the game ends — while a
|
|
note whose hash is absent from a recording that *exists* means the
|
|
position moved. Collapsing them makes every live session report
|
|
orphans, and a metric that cries wolf is one nobody reads, which is
|
|
the failure this whole pass is about.
|
|
|
|
Orphans are reported, never deleted (ADR-0014 D3): a moved position is
|
|
worth knowing, by the same reasoning that rewrote `gr-e01` rather than
|
|
retiring it.
|
|
"""
|
|
if row.get("legacy"):
|
|
return "legacy"
|
|
if not recording:
|
|
return "pending"
|
|
after = int(row["after"])
|
|
n = command_count(recording)
|
|
if after > n:
|
|
# The note points past the end of its own game.
|
|
return "orphan"
|
|
if after == n:
|
|
# The one position the recording states outright. A mismatch here
|
|
# is real: the note claims the end of a game the recording does
|
|
# not agree with.
|
|
return "ok" if row["state_hash"] in open(recording).read() else "orphan"
|
|
# Reachable by replaying `after` commands of this recording.
|
|
return "ok"
|
|
|
|
|
|
def age_days(path, today):
|
|
"""Dated from the log's filename (`YYYY-MM-DD-slug.md`), falling back
|
|
to mtime — the name is the intent, the mtime is what happened."""
|
|
m = re.match(r"(\d{4}-\d{2}-\d{2})", os.path.basename(path))
|
|
if m:
|
|
try:
|
|
return (today - datetime.date.fromisoformat(m.group(1))).days
|
|
except ValueError:
|
|
pass
|
|
return (today - datetime.date.fromtimestamp(os.path.getmtime(path))).days
|
|
|
|
|
|
def report(root=TRIALS, today=None):
|
|
today = today or datetime.date.today()
|
|
if not os.path.isdir(root):
|
|
print("trials — no trials/ directory yet\n")
|
|
print(" Play one: cb-play --serve 0 --record trials/<date>-<slug>.yaml \\")
|
|
print(" --trial trials/<date>-<slug>.md")
|
|
return 0
|
|
|
|
found = logs(root)
|
|
print("trials — what the players said, and where\n")
|
|
total, orphans, pending, expired, legacy = 0, 0, 0, 0, 0
|
|
by_variant = {}
|
|
for path, rows, _ in found:
|
|
name = os.path.basename(path)
|
|
age = age_days(path, today)
|
|
# CB-WP-0046: which rules were played. `unrecorded` is said out
|
|
# loud rather than filled in with the common case — a note read
|
|
# under the wrong rules is worse than a note whose rules are
|
|
# openly unknown.
|
|
rules = (rows[0].get("variant") if rows else None) or "unrecorded"
|
|
print(f" {name} ({len(rows)} note(s), {age}d, rules {rules})")
|
|
for r in rows:
|
|
total += 1
|
|
key = r.get("variant") or "unrecorded"
|
|
by_variant[key] = by_variant.get(key, 0) + 1
|
|
# ADR-0019 D2: each note names its own game, and each game has
|
|
# its own recording. One recording per FILE was the assumption
|
|
# that made `play again` overwrite the evidence.
|
|
rec = recording_for(path, r)
|
|
state = reachability(r, rec)
|
|
orphans += state == "orphan"
|
|
pending += state == "pending"
|
|
legacy += state == "legacy"
|
|
expired += age > NOTE_EXPIRY_DAYS
|
|
mark = {"orphan": "orphan", "pending": " .. ",
|
|
"legacy": "legacy", "ok": " "}[state]
|
|
where = f"g{r.get('game', '?')}+{r.get('after', '?')}"
|
|
print(f" {mark} {where:<7} r{r['round']:<2} {r['step']:<14} "
|
|
f"{r['comment'][:60]}")
|
|
print()
|
|
|
|
print(f" trial logs {len(found)}")
|
|
print(f" notes {total}")
|
|
print(f" positions unreachable {orphans} target 0"
|
|
+ (" <-- a note points past the end of its own game" if orphans else ""))
|
|
if pending:
|
|
print(f" awaiting a recording {pending}"
|
|
" (normal during a live session; the recording lands at game end)")
|
|
if legacy:
|
|
print(f" legacy binding {legacy}"
|
|
" (written before ADR-0019; position not checkable, NOT orphaned)")
|
|
print(f" notes past {NOTE_EXPIRY_DAYS} days {expired} target 0")
|
|
# A tally by rules, because the question a reader now has is "was this
|
|
# said about H2 or about the baseline" and counting the logs by hand
|
|
# is how that gets answered wrong.
|
|
if by_variant:
|
|
print(" notes by rules "
|
|
+ ", ".join(f"{v} {n}" for v, n in sorted(by_variant.items())))
|
|
if "unrecorded" in by_variant:
|
|
print(" (unrecorded = written before"
|
|
" CB-WP-0046 stamped the marker)")
|
|
if total and not orphans and not legacy:
|
|
print("\n Every note points at a position a reader can reach.")
|
|
print("\n Raw notes stay in this repo (ADR-0014 D4). A note reaches"
|
|
"\n ground-game only by being promoted to a register finding, by a"
|
|
"\n human, with the wording chosen then.")
|
|
return 0
|
|
|
|
|
|
def self_test():
|
|
"""Positive controls, including for the REPORTING path.
|
|
|
|
`design-baseline.py` had a green self-test and an unexercised
|
|
reporting path, and the reporting path is where it rotted (ADR-0012
|
|
D8). So this runs `report` against a fixture and checks what it says.
|
|
"""
|
|
import tempfile, io, contextlib
|
|
|
|
ok = True
|
|
|
|
def check(name, cond, detail=""):
|
|
nonlocal ok
|
|
ok = ok and bool(cond)
|
|
print(f" [{'ok ' if cond else 'FAIL'}] {name}" + (f" — {detail}" if detail else ""))
|
|
|
|
# `BEGIN` is a PREFIX since CB-WP-0046, so a fixture has to close the
|
|
# marker itself — which is the point: the closing text is what the
|
|
# attribute sits inside.
|
|
OPEN = BEGIN + " -->"
|
|
|
|
def log_of(*rows, marker=None):
|
|
head = ("| n | game | after | round | step | state_hash | comment |\n"
|
|
"|---|---|---|---|---|---|---|\n")
|
|
return (f"# Trial log\n\n{marker or OPEN}\n\n{head}"
|
|
+ "".join(rows) + f"\n{END}\n")
|
|
|
|
good = log_of("| 1 | 1 | 4 | 3 | Select | abc123def456 | why is SOLVE doing nothing |\n")
|
|
legacy = (f"# Trial log\n\n{OPEN}\n\n"
|
|
"| n | round | step | state_hash | comment |\n|---|---|---|---|---|\n"
|
|
"| 1 | 3 | Select | abc123def456 | why is SOLVE doing nothing |\n"
|
|
f"\n{END}\n")
|
|
|
|
def recording(commands, final_hash):
|
|
"""A recording with `commands` commands and one final hash — the
|
|
shape ADR-0019 D3 reads: a list to index into, plus the one
|
|
position the file states outright."""
|
|
body = "commands:\n"
|
|
for _ in range(commands):
|
|
body += "- actor: P1\n cmd: select_action\n args: {}\n"
|
|
return body + f"expect:\n state_hash: {final_hash}\n"
|
|
check("a trial log parses", len(parse(good)) == 1)
|
|
check("a legacy 5-column log still parses", len(parse(legacy)) == 1,
|
|
"ADR-0019 D4 \u2014 old logs stay readable")
|
|
check("a legacy row is MARKED legacy", parse(legacy)[0].get("legacy") is True)
|
|
check("the header row is not a note", all(r["n"] != "n" for r in parse(good)))
|
|
|
|
# CB-WP-0046 — WHICH RULES THE SESSION PLAYED.
|
|
row = "| 1 | 1 | 4 | 3 | Select | abc123def456 | it felt harder |\n"
|
|
stamped = log_of(row, marker=BEGIN + " variant=h2-scoped-problem-stress -->")
|
|
check("a stamped log reports its variant",
|
|
parse(stamped)[0]["variant"] == "h2-scoped-problem-stress")
|
|
check("an unstamped log says NOTHING, not 'baseline'",
|
|
parse(log_of(row))[0]["variant"] is None,
|
|
"answering an unrecorded question with the common case is how a "
|
|
"baseline note gets read as an H2 result")
|
|
# The attribute must not be mistaken for a table row, and adding it
|
|
# must not change how many notes the log has.
|
|
check("the marker is not counted as a note", len(parse(stamped)) == 1)
|
|
check("stamping does not disturb the row", parse(stamped)[0]["after"] == "4")
|
|
# The marker is not a THIRD row width: the table shape is untouched,
|
|
# which is why the parser needed no new accepted width.
|
|
check("the stamped table has the same columns as the unstamped one",
|
|
set(parse(stamped)[0]) - {"variant"} == set(parse(good)[0]) - {"variant"})
|
|
# **A LITERAL marker, not one built from `BEGIN`.**
|
|
#
|
|
# Every fixture above composes its marker out of `BEGIN`, so all of
|
|
# them follow `BEGIN` wherever it goes — including away from what
|
|
# `hotseat.rs` actually writes. Reverting `BEGIN` to an exact match
|
|
# broke real stamped logs and every one of those checks stayed green.
|
|
# The format is shared across two languages and this is the only
|
|
# check that holds this side to the bytes the other side emits.
|
|
literal = ("# Trial log\n\n"
|
|
"<!-- trial-log:begin variant=h2-scoped-problem-stress -->\n\n"
|
|
"| n | game | after | round | step | state_hash | comment |\n"
|
|
"|---|---|---|---|---|---|---|\n"
|
|
"| 1 | 1 | 4 | 3 | Select | abc123def456 | it felt harder |\n"
|
|
"\n<!-- trial-log:end -->\n")
|
|
# Caught, because the failure mode is `parse` REFUSING the writer's
|
|
# output — and an uncaught traceback names a line number where a
|
|
# named check names the defect.
|
|
try:
|
|
literal_rows = parse(literal)
|
|
except ValueError:
|
|
literal_rows = []
|
|
check("the bytes hotseat.rs writes are the bytes this parses",
|
|
len(literal_rows) == 1
|
|
and literal_rows[0]["variant"] == "h2-scoped-problem-stress",
|
|
"hotseat.rs's the_trial_log_records_which_rules_were_played "
|
|
"asserts this same string from the writing side")
|
|
# And the marker as it was written before CB-WP-0046, also literal.
|
|
try:
|
|
older = parse(literal.replace(
|
|
"<!-- trial-log:begin variant=h2-scoped-problem-stress -->",
|
|
"<!-- trial-log:begin -->"))
|
|
except ValueError:
|
|
older = []
|
|
check("a pre-CB-WP-0046 marker still opens a log", len(older) == 1,
|
|
"ADR-0019 D4's discipline: old logs stay readable")
|
|
|
|
check("a legacy log can still be stamped",
|
|
parse(f"# Trial log\n\n{BEGIN} variant=ground-darvo-r0 -->\n\n"
|
|
"| n | round | step | state_hash | comment |\n|---|---|---|---|---|\n"
|
|
"| 1 | 3 | Select | abc123def456 | x |\n"
|
|
f"\n{END}\n")[0]["variant"] == "ground-darvo-r0")
|
|
|
|
# CB-WP-0032. A row of the wrong shape must be LOUD. This used to
|
|
# `continue`, so a format change would have reported zero notes for
|
|
# every log at once -- the failure the parser's own docstring names.
|
|
# Six columns is neither the current shape (7) nor the legacy one (5),
|
|
# so it must RAISE rather than be skipped.
|
|
six = log_of("| 1 | 1 | 4 | 3 | Select | abc123def456 |\n")
|
|
try:
|
|
parse(six)
|
|
check("a row with the wrong column count is an error", False)
|
|
except ValueError as e:
|
|
check("a row with the wrong column count is an error", "6 columns" in str(e),
|
|
"a silent skip would empty every log at once")
|
|
|
|
# And the surrounding walk must report the REASON it failed.
|
|
with tempfile.TemporaryDirectory() as d:
|
|
with open(os.path.join(d, "2026-08-07-bad.md"), "w") as fh:
|
|
fh.write(six)
|
|
buf = io.StringIO()
|
|
with contextlib.redirect_stdout(buf):
|
|
logs(d)
|
|
check("the walk names the real reason", "6 columns" in buf.getvalue(),
|
|
"it used to blame a missing block for every failure")
|
|
|
|
try:
|
|
parse("# Trial log\n\nno block here\n")
|
|
check("a file with no block is an error, not an empty log", False)
|
|
except ValueError:
|
|
check("a file with no block is an error, not an empty log", True,
|
|
"silently reporting zero is the failure this tool prevents")
|
|
|
|
with tempfile.TemporaryDirectory() as d:
|
|
open(os.path.join(d, "2026-08-06-x.md"), "w").write(good)
|
|
row = parse(good)[0] # game 1, after 4, hash abc123def456
|
|
check("a note with no recording is PENDING, not orphaned",
|
|
reachability(row, None) == "pending",
|
|
"a live session must not report orphans")
|
|
|
|
rec = os.path.join(d, "2026-08-06-x.yaml")
|
|
|
|
# ADR-0019 D3. A MID-GAME note is reachable by replaying `after`
|
|
# commands. Under the old check this case was IMPOSSIBLE to pass:
|
|
# the recording holds one hash, so every mid-game note was an
|
|
# orphan whatever had happened.
|
|
open(rec, "w").write(recording(10, "zzz"))
|
|
check("a mid-game note inside its game is reachable",
|
|
reachability(row, rec) == "ok",
|
|
"the old check called every one of these an orphan")
|
|
|
|
# And it can still say NO: past the end of its own game.
|
|
open(rec, "w").write(recording(2, "zzz"))
|
|
check("a note past the end of its game is an orphan",
|
|
reachability(row, rec) == "orphan",
|
|
"without this the metric cannot go red")
|
|
|
|
# At the end, the hash is the integrity check — and it can fail.
|
|
end_row = parse(log_of(
|
|
"| 1 | 1 | 4 | 3 | after the end | abc123def456 | done |\n"))[0]
|
|
open(rec, "w").write(recording(4, "abc123def456"))
|
|
check("an end-of-game note agreeing with the recording is reachable",
|
|
reachability(end_row, rec) == "ok")
|
|
open(rec, "w").write(recording(4, "something-else"))
|
|
check("an end-of-game note whose hash disagrees is an orphan",
|
|
reachability(end_row, rec) == "orphan",
|
|
"the position moved \u2014 the case the flag exists for")
|
|
|
|
# A legacy row is never called an orphan (ADR-0019 D4).
|
|
open(rec, "w").write(recording(2, "no-match"))
|
|
check("a legacy row is LEGACY, never orphan",
|
|
reachability(parse(legacy)[0], rec) == "legacy",
|
|
"an unsubstantiated orphan claim is the defect being fixed")
|
|
|
|
# ADR-0019 D2: game 2 reads its OWN recording.
|
|
open(os.path.join(d, "g.md"), "w").write("x")
|
|
open(os.path.join(d, "g.yaml"), "w").write(recording(1, "one"))
|
|
open(os.path.join(d, "g-2.yaml"), "w").write(recording(9, "two"))
|
|
g2 = parse(log_of("| 1 | 2 | 5 | 1 | Select | two | second game |\n"))[0]
|
|
check("a note from game 2 is judged against game 2's recording",
|
|
recording_for(os.path.join(d, "g.md"), g2).endswith("g-2.yaml")
|
|
and reachability(g2, recording_for(os.path.join(d, "g.md"), g2)) == "ok",
|
|
"one recording per FILE is what let play again overwrite it")
|
|
|
|
# THE control design-baseline.py lacked: exercise the reporting
|
|
# path and assert on what it printed.
|
|
buf = io.StringIO()
|
|
with contextlib.redirect_stdout(buf):
|
|
report(d, datetime.date(2026, 8, 6))
|
|
out = buf.getvalue()
|
|
check("the reporting path runs and names the note",
|
|
"why is SOLVE doing nothing" in out, "not just the parser")
|
|
check("the reporting path counts", "notes 1" in out)
|
|
|
|
print("trials self-test (positive control)")
|
|
return 0 if ok else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(self_test() if "--self-test" in sys.argv else report())
|