76 lines
4 KiB
Python
76 lines
4 KiB
Python
|
"""Unmask frozen editorial judgments and verify all evidence before aggregation."""
|
||
|
|
import json
|
||
|
|
from pathlib import Path
|
||
|
|
from hashlib import sha256
|
||
|
|
|
||
|
|
ROOT = Path(__file__).resolve().parent
|
||
|
|
|
||
|
|
def read(name):
|
||
|
|
return json.loads((ROOT/name).read_text())
|
||
|
|
|
||
|
|
def digest(path):
|
||
|
|
return sha256(path.read_bytes()).hexdigest()
|
||
|
|
|
||
|
|
scores = read('editorial-scores-masked.json')
|
||
|
|
score_by_id = {a['answer']: a for a in scores['answers']}
|
||
|
|
assert len(score_by_id) == 12
|
||
|
|
rubric = {s['id']: s for s in read('scenarios.json')}
|
||
|
|
result = {'preregistration_commit':'6acbe6d', 'masked_scoring_commit':'3690af1',
|
||
|
|
'independent_human_review':'pending', 'interpretation':'exploratory descriptive counts only',
|
||
|
|
'score_sha256':digest(ROOT/'editorial-scores-masked.json'), 'conditions':{}, 'pairs':{}}
|
||
|
|
for condition in ['patterns','prose']:
|
||
|
|
result['conditions'][condition] = {'completed':0, 'failed':0, 'requirements_met':0,
|
||
|
|
'requirements_possible':0, 'criteria_met':dict.fromkeys(['C1','C2','R','Q'],0),
|
||
|
|
'unsupported_claims':0, 'calls':0, 'prompt_tokens':0, 'completion_tokens':0,
|
||
|
|
'total_tokens':0, 'elapsed_seconds':0, 'provider_reported_cost_usd':0,
|
||
|
|
'over_requested_word_limit':0, 'resolved_models':[], 'answers':[]}
|
||
|
|
for a in read('assignment.json'):
|
||
|
|
raw = ROOT/'raw'/a['run']; receipt = json.loads((raw/'receipt.json').read_text())
|
||
|
|
c = result['conditions'][a['condition']]
|
||
|
|
assert receipt['condition'] == a['condition'] and receipt['scenario'] == a['scenario']
|
||
|
|
assert receipt['freeze_sha256'] == digest(ROOT/'freeze.json')
|
||
|
|
assert receipt['prompt_sha256'] == digest(raw/'prompt.txt')
|
||
|
|
expected = ((ROOT/'instructions.txt').read_text()+'\nREFERENCE\n'+
|
||
|
|
(ROOT/f"pack-{a['condition']}.md").read_text()+'\nSCENARIO\n'+rubric[a['scenario']]['brief'])
|
||
|
|
assert expected == (raw/'prompt.txt').read_text(), 'Prompt contains unexpected input'
|
||
|
|
c['calls'] += receipt['service_requests_attempted']
|
||
|
|
c['elapsed_seconds'] += receipt['elapsed_seconds']
|
||
|
|
if receipt['status'] != 'completed':
|
||
|
|
c['failed'] += 1
|
||
|
|
continue
|
||
|
|
c['completed'] += 1
|
||
|
|
score = score_by_id[a['blind_id']]
|
||
|
|
assert digest(raw/'answer.md') == digest(ROOT/'masked'/(a['blind_id']+'.md')) == score['answer_sha256']
|
||
|
|
assert len(receipt['responses']) == 1
|
||
|
|
response = receipt['responses'][0]
|
||
|
|
assert response['finish_reason'] == 'stop'
|
||
|
|
assert response['content_sha256'] == digest(raw/'answer.md')
|
||
|
|
for key in ['prompt_tokens','completion_tokens','total_tokens']:
|
||
|
|
c[key] += response['usage'][key]
|
||
|
|
c['provider_reported_cost_usd'] += response['diagnostics']['provider_usage']['cost']
|
||
|
|
if response['model'] not in c['resolved_models']: c['resolved_models'].append(response['model'])
|
||
|
|
c['over_requested_word_limit'] += int(receipt['output_words'] > 220)
|
||
|
|
values = {k:v['score'] for k,v in score['criteria'].items()}
|
||
|
|
assert set(values)=={'C1','C2','R','Q'} and all(v in [0,1] for v in values.values())
|
||
|
|
met = sum(values.values()); unsafe=len(score['unsupported_claims'])
|
||
|
|
c['requirements_met'] += met; c['requirements_possible'] += 4
|
||
|
|
c['unsupported_claims'] += unsafe
|
||
|
|
for k,v in values.items(): c['criteria_met'][k] += v
|
||
|
|
record={'answer':a['blind_id'], 'run':a['run'], 'met':met, 'unsupported':unsafe, 'criteria':values}
|
||
|
|
c['answers'].append(record)
|
||
|
|
result['pairs'].setdefault(a['scenario'],{})[a['condition']]=record
|
||
|
|
for scenario,pair in result['pairs'].items():
|
||
|
|
p,b=pair['patterns'],pair['prose']
|
||
|
|
pm,bm=p['met'],b['met']; pu,bu=p['unsupported'],b['unsupported']
|
||
|
|
if pm==bm and pu==bu: outcome='tie'
|
||
|
|
elif pm>=bm and pu<=bu: outcome='patterns'
|
||
|
|
elif bm>=pm and bu<=pu: outcome='prose'
|
||
|
|
else: outcome='mixed'
|
||
|
|
pair['descriptive_advantage']=outcome
|
||
|
|
for c in result['conditions'].values():
|
||
|
|
c['missed_requirements']=c['requirements_possible']-c['requirements_met']
|
||
|
|
c['elapsed_seconds']=round(c['elapsed_seconds'],3)
|
||
|
|
c['provider_reported_cost_usd']=round(c['provider_reported_cost_usd'],7)
|
||
|
|
(ROOT/'summary.json').write_text(json.dumps(result,indent=2)+'\n')
|
||
|
|
print(json.dumps(result,indent=2))
|