frontend-patterns/research/evaluations/language-pilot-01/run.py

"""Bounded, immutable 12-call presentation pilot; inference via llm-connect only."""
import argparse
import hashlib
import json
from pathlib import Path
import secrets
import sys
import time
from types import SimpleNamespace

ROOT = Path(__file__).resolve().parent
REPO = ROOT.parents[2]
sys.path.insert(0, str(REPO / 'scripts'))
from research_bench import LLMConnectAdapter


def sha(data):
    return hashlib.sha256(data).hexdigest()


def read(name):
    return json.loads((ROOT / name).read_text())


def write(path, data):
    with path.open('x') as handle:
        json.dump(data, handle, indent=2)
        handle.write('\n')


def validate():
    manifest = read('freeze.json')
    for name, digest in manifest['files'].items():
        if sha((REPO / name).read_bytes()) != digest:
            raise ValueError('Frozen input changed: ' + name)
    scenarios = read('scenarios.json')
    assert len(scenarios) == 6
    assignments = read('assignment.json')
    assert len(assignments) == 12
    for scenario in scenarios:
        pair = [a for a in assignments if a['scenario'] == scenario['id']]
        assert {a['condition'] for a in pair} == {'patterns', 'prose'}
        assert {a['blind_id'] for a in pair} == {scenario['id']+'-U', scenario['id']+'-V'}
    return assignments, {s['id']: s for s in scenarios}


def prepare():
    assignments = []
    for i, scenario in enumerate(read('scenarios.json')):
        conditions = ['patterns', 'prose'] if i % 2 == 0 else ['prose', 'patterns']
        labels = ['U', 'V'] if secrets.randbelow(2) == 0 else ['V', 'U']
        for condition, label in zip(conditions, labels):
            assignments.append({'run': f'r{len(assignments)+1:02d}',
                                'scenario': scenario['id'], 'condition': condition,
                                'blind_id': scenario['id']+'-'+label})
    write(ROOT/'assignment.json', assignments)
    inputs = ['protocol.md', 'pack-patterns.md', 'pack-prose.md', 'pack-equivalence.json',
              'scenarios.json', 'instructions.txt', 'assignment.json', 'run.py']
    paths = [ROOT / name for name in inputs]
    paths += list((REPO/'research/candidates').glob('*.md'))
    paths += [REPO/'scripts/research_bench.py', REPO/'research/walkthroughs/VC01-TransactionalAdministration.md',
              REPO/'research/walkthroughs/VC02-OfflineFieldNotes.md']
    write(ROOT/'freeze.json', {'corpus_commit': '78bd95156832c9c48c0e17947a406396c7660088',
                              'prepared': '2026-09-26',
                              'files': {str(p.relative_to(REPO)): sha(p.read_bytes()) for p in paths}})
    validate()
    print('Frozen 6 pairs; assignment mapping withheld from scoring view.')


def run(start, stop):
    assignments, scenarios = validate()
    assert 0 <= start < stop <= 12
    for assignment in assignments[start:stop]:
        directory = ROOT/'raw'/assignment['run']
        directory.mkdir(parents=True, exist_ok=False)  # no implicit reruns
        scenario = scenarios[assignment['scenario']]
        prompt = ((ROOT/'instructions.txt').read_text() + '\nREFERENCE\n' +
                  (ROOT/f"pack-{assignment['condition']}.md").read_text() +
                  '\nSCENARIO\n' + scenario['brief'])
        (directory/'prompt.txt').write_text(prompt)
        adapter = LLMConnectAdapter('http://127.0.0.1:18081', 'custodian-triage-balanced',
                                    1400, diagnostics=True)
        receipt = {'scenario': scenario['id'], 'condition': assignment['condition'],
                   'prompt_sha256': sha(prompt.encode()), 'service_requests_attempted': 1,
                   'endpoint': adapter.endpoint, 'config': adapter.config.to_dict(),
                   'freeze_sha256': sha((ROOT/'freeze.json').read_bytes())}
        write(directory/'started.json', {'status': 'attempting', **receipt})
        then = time.monotonic()
        try:
            result = adapter.generate(SimpleNamespace(prompt=prompt, stage_id='decision-review'))
            (directory/'answer.md').write_text(result.markdown)
            receipt.update(status='completed', output_words=len(result.markdown.split()))
        except Exception as exc:
            receipt.update(status='failed', error_type=type(exc).__name__)
            if adapter.rejected_content:
                (directory/'rejected-response.md').write_text(adapter.rejected_content)
        receipt.update(elapsed_seconds=round(time.monotonic()-then, 3), responses=adapter.responses)
        write(directory/'receipt.json', receipt)
        print(json.dumps({'session': assignment['run'], 'status': receipt['status']}), flush=True)


def mask():
    assignments, scenarios = validate()
    target = ROOT/'masked'
    target.mkdir(exist_ok=False)
    for assignment in assignments:
        source = ROOT/'raw'/assignment['run']
        receipt = json.loads((source/'receipt.json').read_text())
        body = ((source/'answer.md').read_text() if receipt['status']=='completed'
                else 'UNAVAILABLE: generation failed; exclude from quality comparison.')
        (target/(assignment['blind_id']+'.md')).write_text(body)
    print('Masked views ready; read only masked answers and rubric before scoring.')


if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('action', choices=['prepare', 'validate', 'run', 'mask'])
    parser.add_argument('--start', type=int, default=0)
    parser.add_argument('--stop', type=int, default=12)
    args = parser.parse_args()
    if args.action == 'prepare': prepare()
    elif args.action == 'validate': validate(); print('Frozen inputs and pairing valid.')
    elif args.action == 'run': run(args.start, args.stop)
    else: mask()