tegwick
919aefbef5
Assistant: codex Assistant-Model: gpt-6-astra Assistant-Session: 01a09cbc-910e-77a0-b336-3130194350b3
130 lines
6.9 KiB
Python
130 lines
6.9 KiB
Python
"""Budgeted OpenRouter work-sample trials with deterministic verification."""
|
|
from datetime import datetime, timezone
|
|
from decimal import Decimal
|
|
import fcntl
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import time
|
|
from urllib.request import Request, urlopen
|
|
|
|
from .store import all_records, canonical, get, put
|
|
|
|
ENDPOINT = 'https://openrouter.ai/api/v1/chat/completions'
|
|
|
|
|
|
def money(value):
|
|
result = Decimal(str(value))
|
|
if not result.is_finite() or result < 0:
|
|
raise ValueError('invalid nonnegative dollar amount')
|
|
return result
|
|
|
|
|
|
def budget(root, campaign):
|
|
events = [(i,e) for i,e in all_records(root,'trial-events') if e['campaign']==campaign]
|
|
reservations = {i:e for i,e in events if e['kind']=='reserved'}
|
|
settled = {e['reservation_id']:e for _,e in events if e['kind']=='settled'}
|
|
spent = sum((money(e['actual_usd']) for e in settled.values()),Decimal(0))
|
|
pending = sum((money(e['reserved_usd']) for i,e in reservations.items() if i not in settled),Decimal(0))
|
|
return {'spent_usd':str(spent), 'pending_usd':str(pending), 'committed_usd':str(spent+pending)}
|
|
|
|
|
|
def verify(case, content):
|
|
try:
|
|
actual=json.loads(content)
|
|
except (ValueError,TypeError):
|
|
return {'accepted':False,'reason':'response is not JSON','actual':None}
|
|
# Canonical JSON comparison distinguishes booleans from numeric values.
|
|
accepted=canonical(actual)==canonical(case['expected'])
|
|
return {'accepted':accepted,'reason':'exact structured match' if accepted else 'structured mismatch', 'actual':actual}
|
|
|
|
|
|
def plan_trial(case, model):
|
|
prompt=case['instruction']+'\nInput:\n'+json.dumps(case['input'],sort_keys=True)
|
|
rates=model['max_usd_per_million']
|
|
payload={'model':model['id'],'messages':[{'role':'user','content':prompt}],
|
|
'max_tokens':2048,'stream':False,'usage':{'include':True},
|
|
'provider':{'allow_fallbacks':False,'require_parameters':True,
|
|
'max_price':{'prompt':float(money(rates['prompt'])),
|
|
'completion':float(money(rates['completion'])),'request':0}}}
|
|
# UTF-8 bytes plus a conservative chat framing allowance bound these short text prompts.
|
|
reserved=(Decimal(len(prompt.encode('utf-8'))+1024)*money(rates['prompt'])+
|
|
Decimal(payload['max_tokens'])*money(rates['completion']))/Decimal(1000000)
|
|
return payload,reserved
|
|
|
|
|
|
def execute(root, config, case, model, opener=urlopen):
|
|
root=Path(root)
|
|
if not os.environ.get('OPENROUTER_API_KEY'):
|
|
raise ValueError('OPENROUTER_API_KEY is required')
|
|
campaign=config['campaign']
|
|
limit=money(config['budget_usd'])
|
|
payload,reserved=plan_trial(case,model)
|
|
run_id=hashlib.sha256(canonical({'campaign':campaign,'case':case,'payload':payload})).hexdigest()
|
|
lockpath=root/'.runtime/trials.lock';lockpath.parent.mkdir(parents=True,exist_ok=True)
|
|
with lockpath.open('a') as lock:
|
|
fcntl.flock(lock,fcntl.LOCK_EX)
|
|
events = all_records(root, 'trial-events')
|
|
reservations = {i: e for i, e in events if e['kind'] == 'reserved'}
|
|
for _, event in events:
|
|
if event['campaign'] == campaign and event['kind'] == 'settled':
|
|
held = reservations[event['reservation_id']]
|
|
if money(event['actual_usd']) > money(held['reserved_usd']):
|
|
raise ValueError('campaign halted: provider cost exceeded reservation')
|
|
for ident,event in all_records(root,'trial-events'):
|
|
if event['campaign']==campaign and event.get('run_id')==run_id:
|
|
if event['kind']=='settled':
|
|
return get(root,'trials',event['trial_id'])
|
|
if any(e.get('run_id')==run_id for _,e in all_records(root,'trial-events')):
|
|
raise ValueError('trial already reserved; reconcile uncertain billing before retrying')
|
|
state=budget(root,campaign)
|
|
if money(state['pending_usd']):
|
|
raise ValueError('campaign has unresolved reservations; reconcile billing before continuing')
|
|
if money(state['committed_usd'])+reserved>limit:
|
|
raise ValueError('trial would exceed campaign budget')
|
|
stamp=datetime.now(timezone.utc).isoformat()
|
|
reservation=put(root,'trial-events',{'kind':'reserved','campaign':campaign,'run_id':run_id,
|
|
'reserved_usd':str(reserved),'budget_usd':str(limit),
|
|
'created_at':stamp,'model':model['id'],'case_id':case['id']})
|
|
request=Request(ENDPOINT,data=json.dumps(payload).encode(),method='POST',
|
|
headers={'Content-Type':'application/json','Authorization':'Bearer '+os.environ['OPENROUTER_API_KEY'],
|
|
'X-OpenRouter-Title':'Intelligence Radar field pilot'})
|
|
start=time.monotonic()
|
|
# No automatic inference retries: a timeout may already have incurred a charge.
|
|
with opener(request,timeout=90) as response:
|
|
raw=response.read(2*1024*1024+1)
|
|
if len(raw)>2*1024*1024:
|
|
raise ValueError('provider response exceeds retention limit; reservation retained')
|
|
value=json.loads(raw)
|
|
if 'error' in value:
|
|
raise ValueError('provider returned an error; reservation retained')
|
|
if value.get('usage',{}).get('cost') is None:
|
|
raise ValueError('provider omitted billing cost; reservation retained')
|
|
cost=money(value['usage']['cost'])
|
|
content=value['choices'][0]['message'].get('content')
|
|
trial={'schema_version':1,'campaign':campaign,'run_id':run_id,'case':case,
|
|
'requested_model':model['id'],'returned_model':value.get('model'),
|
|
'provider':value.get('provider'),'request':payload,'response':value,
|
|
'verification':verify(case,content),'actual_usd':str(cost),
|
|
'runtime_seconds':time.monotonic()-start,'completed_at':datetime.now(timezone.utc).isoformat(),
|
|
'reservation_id':reservation,'synthetic':False,
|
|
'sample_kind':'controlled repository work sample; real provider execution'}
|
|
ident=put(root,'trials',trial)
|
|
put(root,'trial-events',{'kind':'settled','campaign':campaign,'run_id':run_id,
|
|
'reservation_id':reservation,'trial_id':ident,'actual_usd':str(cost)})
|
|
if cost>reserved:
|
|
raise ValueError('provider cost exceeded reservation; campaign halted for review')
|
|
return trial
|
|
|
|
|
|
def run(root, config, suite, pilot=False):
|
|
rows=[]
|
|
selected=suite['cases'][:2] if pilot else suite['cases']
|
|
for case in selected:
|
|
for model in config['models']:
|
|
trial=execute(root,config,case,model)
|
|
rows.append({'case':case['id'],'model':model['id'],
|
|
'accepted':trial['verification']['accepted'],'actual_usd':trial['actual_usd']})
|
|
print(json.dumps(rows[-1]),flush=True)
|
|
return {'campaign':config['campaign'],'trials':rows,'budget':budget(root,config['campaign'])}
|