From 104aad6f3c8ed1cb85fc52f28fa5f29688a79864 Mon Sep 17 00:00:00 2001 From: tegwick Date: Mon, 12 Oct 2026 18:46:45 +0200 Subject: [PATCH] Checkpoint runner: stricter assessment prompt; debug output for rejected assessments Co-Authored-By: Claude Opus 5.5 Assistant: claude-code Assistant-Model: opus Assistant-Process: 1494246@bnt-lap001 Assistant-Session: caf6de6f-f9e4-47ce-a584-49baa271fa4b --- checkpoints/run_checkpoint.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/checkpoints/run_checkpoint.py b/checkpoints/run_checkpoint.py index 789f9a2..525dffc 100644 --- a/checkpoints/run_checkpoint.py +++ b/checkpoints/run_checkpoint.py @@ -89,10 +89,12 @@ def assessment(observations: list[dict]) -> tuple[str | None, str | None]: facts = "\n".join(f"- {o['indicator']} {o['jurisdiction']} {o.get('labels') or ''}: {o['label']} = " f"{o['value']} {o.get('unit') or ''} (period {o['period']})" for o in ok) prompt = ( - "You assess AI adoption observations for a research checkpoint. Use ONLY the facts below. " - "Do not add any number, date or statistic that is not literally in the facts. " - "Write 2-4 short bullet points: what the observations say about the pace of AI adoption per " - "jurisdiction (US, EU, DE), and which differences stand out. No forecasts.\n\nFacts:\n" + facts) + "You assess AI adoption observations for a research checkpoint. Use ONLY the facts below.\n" + "Rules: (a) every number you write must appear literally in the facts; (b) do not compute " + "differences, ratios, growth rates or percentage points; (c) use '-' bullets, never numbered " + "lists; (d) no forecasts.\n" + "Write two to four short bullets on what the observations say about AI adoption per " + "jurisdiction (US, EU, DE) and which contrasts stand out (describe them in words).\n\nFacts:\n" + facts) body = json.dumps({"prompt": prompt, "config": {"temperature": 0.1, "max_tokens": 500}}).encode() try: req = urllib.request.Request(url + "/execute", data=body, headers={"Content-Type": "application/json"}) @@ -103,6 +105,8 @@ def assessment(observations: list[dict]) -> tuple[str | None, str | None]: allowed = _numbers(facts) extra = sorted(n for n in _numbers(text) if n not in allowed and n.replace(",", ".") not in allowed) if extra: + if os.environ.get("CHECKPOINT_DEBUG"): + print("REJECTED ASSESSMENT:\n" + text, file=sys.stderr) return None, f"assessment rejected: numbers not in fetched data: {extra[:8]}" return text, None