"""Bounded, immutable 12-call presentation pilot; inference via llm-connect only.""" import argparse import hashlib import json from pathlib import Path import secrets import sys import time from types import SimpleNamespace ROOT = Path(__file__).resolve().parent REPO = ROOT.parents[2] sys.path.insert(0, str(REPO / 'scripts')) from research_bench import LLMConnectAdapter def sha(data): return hashlib.sha256(data).hexdigest() def read(name): return json.loads((ROOT / name).read_text()) def write(path, data): with path.open('x') as handle: json.dump(data, handle, indent=2) handle.write('\n') def validate(): manifest = read('freeze.json') for name, digest in manifest['files'].items(): if sha((REPO / name).read_bytes()) != digest: raise ValueError('Frozen input changed: ' + name) scenarios = read('scenarios.json') assert len(scenarios) == 6 assignments = read('assignment.json') assert len(assignments) == 12 for scenario in scenarios: pair = [a for a in assignments if a['scenario'] == scenario['id']] assert {a['condition'] for a in pair} == {'patterns', 'prose'} assert {a['blind_id'] for a in pair} == {scenario['id']+'-U', scenario['id']+'-V'} return assignments, {s['id']: s for s in scenarios} def prepare(): assignments = [] for i, scenario in enumerate(read('scenarios.json')): conditions = ['patterns', 'prose'] if i % 2 == 0 else ['prose', 'patterns'] labels = ['U', 'V'] if secrets.randbelow(2) == 0 else ['V', 'U'] for condition, label in zip(conditions, labels): assignments.append({'run': f'r{len(assignments)+1:02d}', 'scenario': scenario['id'], 'condition': condition, 'blind_id': scenario['id']+'-'+label}) write(ROOT/'assignment.json', assignments) inputs = ['protocol.md', 'pack-patterns.md', 'pack-prose.md', 'pack-equivalence.json', 'scenarios.json', 'instructions.txt', 'assignment.json', 'run.py'] paths = [ROOT / name for name in inputs] paths += list((REPO/'research/candidates').glob('*.md')) paths += [REPO/'scripts/research_bench.py', REPO/'research/walkthroughs/VC01-TransactionalAdministration.md', REPO/'research/walkthroughs/VC02-OfflineFieldNotes.md'] write(ROOT/'freeze.json', {'corpus_commit': '78bd95156832c9c48c0e17947a406396c7660088', 'prepared': '2026-09-26', 'files': {str(p.relative_to(REPO)): sha(p.read_bytes()) for p in paths}}) validate() print('Frozen 6 pairs; assignment mapping withheld from scoring view.') def run(start, stop): assignments, scenarios = validate() assert 0 <= start < stop <= 12 for assignment in assignments[start:stop]: directory = ROOT/'raw'/assignment['run'] directory.mkdir(parents=True, exist_ok=False) # no implicit reruns scenario = scenarios[assignment['scenario']] prompt = ((ROOT/'instructions.txt').read_text() + '\nREFERENCE\n' + (ROOT/f"pack-{assignment['condition']}.md").read_text() + '\nSCENARIO\n' + scenario['brief']) (directory/'prompt.txt').write_text(prompt) adapter = LLMConnectAdapter('http://127.0.0.1:18081', 'custodian-triage-balanced', 1400, diagnostics=True) receipt = {'scenario': scenario['id'], 'condition': assignment['condition'], 'prompt_sha256': sha(prompt.encode()), 'service_requests_attempted': 1, 'endpoint': adapter.endpoint, 'config': adapter.config.to_dict(), 'freeze_sha256': sha((ROOT/'freeze.json').read_bytes())} write(directory/'started.json', {'status': 'attempting', **receipt}) then = time.monotonic() try: result = adapter.generate(SimpleNamespace(prompt=prompt, stage_id='decision-review')) (directory/'answer.md').write_text(result.markdown) receipt.update(status='completed', output_words=len(result.markdown.split())) except Exception as exc: receipt.update(status='failed', error_type=type(exc).__name__) if adapter.rejected_content: (directory/'rejected-response.md').write_text(adapter.rejected_content) receipt.update(elapsed_seconds=round(time.monotonic()-then, 3), responses=adapter.responses) write(directory/'receipt.json', receipt) print(json.dumps({'session': assignment['run'], 'status': receipt['status']}), flush=True) def mask(): assignments, scenarios = validate() target = ROOT/'masked' target.mkdir(exist_ok=False) for assignment in assignments: source = ROOT/'raw'/assignment['run'] receipt = json.loads((source/'receipt.json').read_text()) body = ((source/'answer.md').read_text() if receipt['status']=='completed' else 'UNAVAILABLE: generation failed; exclude from quality comparison.') (target/(assignment['blind_id']+'.md')).write_text(body) print('Masked views ready; read only masked answers and rubric before scoring.') if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('action', choices=['prepare', 'validate', 'run', 'mask']) parser.add_argument('--start', type=int, default=0) parser.add_argument('--stop', type=int, default=12) args = parser.parse_args() if args.action == 'prepare': prepare() elif args.action == 'validate': validate(); print('Frozen inputs and pairing valid.') elif args.action == 'run': run(args.start, args.stop) else: mask()