Prepare no-network trial plans for native recipient review
Assistant: codex Assistant-Model: gpt-6-astra Assistant-Session: 01a09cbc-910e-77a0-b336-3130194350b3
This commit is contained in:
parent
5f39eb7818
commit
33da93cd66
9 changed files with 709 additions and 4 deletions
|
|
@ -184,11 +184,13 @@ def main(argv=None):
|
|||
trials_command.add_argument('--config', type=Path, required=True)
|
||||
trials_command.add_argument('--suite', type=Path, required=True)
|
||||
trials_command.add_argument('--pilot', action='store_true')
|
||||
trials_command.add_argument('--plan', action='store_true', help='review request identities and budget without calls or writes')
|
||||
args = parser.parse_args(argv)
|
||||
try:
|
||||
if args.command == 'trials':
|
||||
from .trials import run
|
||||
print(json.dumps(run(args.root, json.loads(args.config.read_text()),
|
||||
from .trials import plan, run
|
||||
action = plan if args.plan else run
|
||||
print(json.dumps(action(args.root, json.loads(args.config.read_text()),
|
||||
json.loads(args.suite.read_text()), args.pilot), indent=2))
|
||||
elif args.command == 'export':
|
||||
from .bundles import export_bundle
|
||||
|
|
|
|||
|
|
@ -54,6 +54,54 @@ def plan_trial(case, model):
|
|||
return payload,reserved
|
||||
|
||||
|
||||
def trial_id(campaign, case, payload):
|
||||
return hashlib.sha256(canonical({'campaign':campaign,'case':case,'payload':payload})).hexdigest()
|
||||
|
||||
|
||||
def plan(root, config, suite, pilot=False):
|
||||
"""Render review inputs without credentials, network calls or ledger writes."""
|
||||
campaign = config['campaign']
|
||||
limit = money(config['budget_usd'])
|
||||
for label, items in [('case', suite['cases']), ('model', config['models'])]:
|
||||
ids = [item['id'] for item in items]
|
||||
if not ids or len(ids) != len(set(ids)):
|
||||
raise ValueError(f'{label} IDs must be nonempty and unique')
|
||||
events = [(i, e) for i, e in all_records(root, 'trial-events') if e['campaign'] == campaign]
|
||||
held = {i: e for i, e in events if e['kind'] == 'reserved'}
|
||||
settled = {e['reservation_id']: e for _, e in events if e['kind'] == 'settled'}
|
||||
spent = sum((money(e['actual_usd']) for e in settled.values()), Decimal(0))
|
||||
pending = sum((money(e['reserved_usd']) for i, e in held.items() if i not in settled), Decimal(0))
|
||||
existing = {e['run_id']: 'settled' if i in settled else 'reserved' for i, e in held.items()}
|
||||
rows = []
|
||||
new_total = Decimal(0)
|
||||
for case in suite['cases'][:2] if pilot else suite['cases']:
|
||||
for model in config['models']:
|
||||
payload, reserved = plan_trial(case, model)
|
||||
ident = trial_id(campaign, case, payload)
|
||||
status = existing.get(ident, 'new')
|
||||
if status == 'new':
|
||||
new_total += reserved
|
||||
rows.append({'case_id': case['id'], 'model': model['id'], 'run_id': ident,
|
||||
'status': status, 'reserved_usd': str(reserved),
|
||||
'request_sha256': hashlib.sha256(canonical(payload)).hexdigest()})
|
||||
blockers = []
|
||||
if pending:
|
||||
blockers.append('unresolved billing reservations')
|
||||
if any(money(e['actual_usd']) > money(held[i]['reserved_usd']) for i, e in settled.items()):
|
||||
blockers.append('campaign halted after provider cost exceeded reservation')
|
||||
total = spent + pending + new_total
|
||||
if total > limit:
|
||||
blockers.append('planned reservations exceed campaign budget')
|
||||
return {'schema_version': 1, 'campaign': campaign, 'pilot': pilot,
|
||||
'execution_authorized': False,
|
||||
'scope': 'review inputs only; requires admitted recipient, credential delivery and billing reconciliation',
|
||||
'config_sha256': hashlib.sha256(canonical(config)).hexdigest(),
|
||||
'suite_sha256': hashlib.sha256(canonical(suite)).hexdigest(),
|
||||
'budget_usd': str(limit), 'spent_usd': str(spent), 'pending_usd': str(pending),
|
||||
'new_reservations_usd': str(new_total), 'planned_committed_usd': str(total),
|
||||
'budget_blockers': blockers, 'trials': rows}
|
||||
|
||||
|
||||
def execute(root, config, case, model, opener=urlopen):
|
||||
root=Path(root)
|
||||
if not os.environ.get('OPENROUTER_API_KEY'):
|
||||
|
|
@ -61,7 +109,7 @@ def execute(root, config, case, model, opener=urlopen):
|
|||
campaign=config['campaign']
|
||||
limit=money(config['budget_usd'])
|
||||
payload,reserved=plan_trial(case,model)
|
||||
run_id=hashlib.sha256(canonical({'campaign':campaign,'case':case,'payload':payload})).hexdigest()
|
||||
run_id=trial_id(campaign, case, payload)
|
||||
lockpath=root/'.runtime/trials.lock';lockpath.parent.mkdir(parents=True,exist_ok=True)
|
||||
with lockpath.open('a') as lock:
|
||||
fcntl.flock(lock,fcntl.LOCK_EX)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue