Preregister matched pattern-language presentation pilot

Assistant: codex
Assistant-Model: gpt-6-astra
Assistant-Session: 01a0cffe-206f-7323-89d3-b20ca4cf4bf9

research/evaluations/language-pilot-01/run.py

@@ -0,0 +1,124 @@
+"""Bounded, immutable 12-call presentation pilot; inference via llm-connect only."""
+import argparse
+import hashlib
+import json
+from pathlib import Path
+import secrets
+import sys
+import time
+from types import SimpleNamespace
+
+ROOT = Path(__file__).resolve().parent
+REPO = ROOT.parents[2]
+sys.path.insert(0, str(REPO / 'scripts'))
+from research_bench import LLMConnectAdapter
+
+
+def sha(data):
+    return hashlib.sha256(data).hexdigest()
+
+
+def read(name):
+    return json.loads((ROOT / name).read_text())
+
+
+def write(path, data):
+    with path.open('x') as handle:
+        json.dump(data, handle, indent=2)
+        handle.write('\n')
+
+
+def validate():
+    manifest = read('freeze.json')
+    for name, digest in manifest['files'].items():
+        if sha((REPO / name).read_bytes()) != digest:
+            raise ValueError('Frozen input changed: ' + name)
+    scenarios = read('scenarios.json')
+    assert len(scenarios) == 6
+    assignments = read('assignment.json')
+    assert len(assignments) == 12
+    for scenario in scenarios:
+        pair = [a for a in assignments if a['scenario'] == scenario['id']]
+        assert {a['condition'] for a in pair} == {'patterns', 'prose'}
+        assert {a['blind_id'] for a in pair} == {scenario['id']+'-U', scenario['id']+'-V'}
+    return assignments, {s['id']: s for s in scenarios}
+
+
+def prepare():
+    assignments = []
+    for i, scenario in enumerate(read('scenarios.json')):
+        conditions = ['patterns', 'prose'] if i % 2 == 0 else ['prose', 'patterns']
+        labels = ['U', 'V'] if secrets.randbelow(2) == 0 else ['V', 'U']
+        for condition, label in zip(conditions, labels):
+            assignments.append({'run': f'r{len(assignments)+1:02d}',
+                                'scenario': scenario['id'], 'condition': condition,
+                                'blind_id': scenario['id']+'-'+label})
+    write(ROOT/'assignment.json', assignments)
+    inputs = ['protocol.md', 'pack-patterns.md', 'pack-prose.md', 'pack-equivalence.json',
+              'scenarios.json', 'instructions.txt', 'assignment.json', 'run.py']
+    paths = [ROOT / name for name in inputs]
+    paths += list((REPO/'research/candidates').glob('*.md'))
+    paths += [REPO/'scripts/research_bench.py', REPO/'research/walkthroughs/VC01-TransactionalAdministration.md',
+              REPO/'research/walkthroughs/VC02-OfflineFieldNotes.md']
+    write(ROOT/'freeze.json', {'corpus_commit': '78bd95156832c9c48c0e17947a406396c7660088',
+                              'prepared': '2026-09-26',
+                              'files': {str(p.relative_to(REPO)): sha(p.read_bytes()) for p in paths}})
+    validate()
+    print('Frozen 6 pairs; assignment mapping withheld from scoring view.')
+
+
+def run(start, stop):
+    assignments, scenarios = validate()
+    assert 0 <= start < stop <= 12
+    for assignment in assignments[start:stop]:
+        directory = ROOT/'raw'/assignment['run']
+        directory.mkdir(parents=True, exist_ok=False)  # no implicit reruns
+        scenario = scenarios[assignment['scenario']]
+        prompt = ((ROOT/'instructions.txt').read_text() + '\nREFERENCE\n' +
+                  (ROOT/f"pack-{assignment['condition']}.md").read_text() +
+                  '\nSCENARIO\n' + scenario['brief'])
+        (directory/'prompt.txt').write_text(prompt)
+        adapter = LLMConnectAdapter('http://127.0.0.1:18081', 'custodian-triage-balanced',
+                                    1400, diagnostics=True)
+        receipt = {'scenario': scenario['id'], 'condition': assignment['condition'],
+                   'prompt_sha256': sha(prompt.encode()), 'service_requests_attempted': 1,
+                   'endpoint': adapter.endpoint, 'config': adapter.config.to_dict(),
+                   'freeze_sha256': sha((ROOT/'freeze.json').read_bytes())}
+        write(directory/'started.json', {'status': 'attempting', **receipt})
+        then = time.monotonic()
+        try:
+            result = adapter.generate(SimpleNamespace(prompt=prompt, stage_id='decision-review'))
+            (directory/'answer.md').write_text(result.markdown)
+            receipt.update(status='completed', output_words=len(result.markdown.split()))
+        except Exception as exc:
+            receipt.update(status='failed', error_type=type(exc).__name__)
+            if adapter.rejected_content:
+                (directory/'rejected-response.md').write_text(adapter.rejected_content)
+        receipt.update(elapsed_seconds=round(time.monotonic()-then, 3), responses=adapter.responses)
+        write(directory/'receipt.json', receipt)
+        print(json.dumps({'session': assignment['run'], 'status': receipt['status']}), flush=True)
+
+
+def mask():
+    assignments, scenarios = validate()
+    target = ROOT/'masked'
+    target.mkdir(exist_ok=False)
+    for assignment in assignments:
+        source = ROOT/'raw'/assignment['run']
+        receipt = json.loads((source/'receipt.json').read_text())
+        body = ((source/'answer.md').read_text() if receipt['status']=='completed'
+                else 'UNAVAILABLE: generation failed; exclude from quality comparison.')
+        (target/(assignment['blind_id']+'.md')).write_text(body)
+    print('Masked views ready; read only masked answers and rubric before scoring.')
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('action', choices=['prepare', 'validate', 'run', 'mask'])
+    parser.add_argument('--start', type=int, default=0)
+    parser.add_argument('--stop', type=int, default=12)
+    args = parser.parse_args()
+    if args.action == 'prepare': prepare()
+    elif args.action == 'validate': validate(); print('Frozen inputs and pairing valid.')
+    elif args.action == 'run': run(args.start, args.stop)
+    else: mask()