import io import json import os from pathlib import Path import tempfile import unittest from unittest.mock import patch from intelligence_radar.trials import budget, execute, plan, verify class TrialTests(unittest.TestCase): def setUp(self): self.temp=tempfile.TemporaryDirectory();self.addCleanup(self.temp.cleanup) self.root=Path(self.temp.name) self.config={'campaign':'test','budget_usd':'10'} self.model={'id':'test/model','max_usd_per_million':{'prompt':'1','completion':'5'}} self.case={'id':'one','instruction':'Return JSON','input':{},'expected':{'answer':1}} self.env=patch.dict(os.environ,{'OPENROUTER_API_KEY':'test-not-a-secret'}) self.env.start();self.addCleanup(self.env.stop) def response(self,*args,**kwargs): return io.BytesIO(json.dumps({'id':'generation-test','model':'test/model','provider':'test', 'usage':{'cost':.001},'choices':[{'message':{'content':'{"answer":1}'}}]}).encode()) def test_billing_and_exact_once_success(self): result=execute(self.root,self.config,self.case,self.model,opener=self.response) self.assertTrue(result['verification']['accepted']) self.assertEqual(budget(self.root,'test')['spent_usd'],'0.001') same=execute(self.root,self.config,self.case,self.model,opener=lambda *a,**k:self.fail('duplicate call')) self.assertEqual(same,result) def test_budget_refuses_call_before_network(self): self.config['budget_usd']='0.00001' with self.assertRaisesRegex(ValueError,'budget'): execute(self.root,self.config,self.case,self.model,opener=lambda *a,**k:self.fail('budget bypass')) def test_timeout_retains_reservation_and_refuses_retry(self): def fail(*args,**kwargs):raise TimeoutError('timeout') with self.assertRaises(TimeoutError):execute(self.root,self.config,self.case,self.model,opener=fail) self.assertGreater(float(budget(self.root,'test')['pending_usd']),0) with self.assertRaisesRegex(ValueError,'already reserved'): execute(self.root,self.config,self.case,self.model,opener=self.response) def test_verifier_rejects_boolean_as_integer_and_non_json(self): self.assertFalse(verify(self.case,'{"answer":true}')['accepted']) self.assertFalse(verify(self.case,'```json\n{"answer":1}\n```')['accepted']) def test_uncertain_billing_blocks_other_cases(self): def fail(*args, **kwargs): raise TimeoutError('timeout') with self.assertRaises(TimeoutError): execute(self.root, self.config, self.case, self.model, opener=fail) other = dict(self.case, id='two') with self.assertRaisesRegex(ValueError, 'unresolved reservations'): execute(self.root, self.config, other, self.model, opener=lambda *a, **k: self.fail('unreconciled campaign made a call')) def test_overrun_halts_campaign_across_restarts(self): def expensive(*args, **kwargs): value = json.loads(self.response().read()) value['usage']['cost'] = 1 return io.BytesIO(json.dumps(value).encode()) with self.assertRaisesRegex(ValueError, 'exceeded reservation'): execute(self.root, self.config, self.case, self.model, opener=expensive) self.assertEqual(budget(self.root, 'test')['spent_usd'], '1') for case in [self.case, dict(self.case, id='two')]: with self.assertRaisesRegex(ValueError, 'campaign halted'): execute(self.root, self.config, case, self.model, opener=lambda *a, **k: self.fail('halted campaign made a call')) def test_plan_needs_no_key_and_writes_nothing(self): config = dict(self.config, models=[self.model]) with patch.dict(os.environ, {}, clear=True): result = plan(self.root, config, {'cases': [self.case]}) self.assertFalse(result['execution_authorized']) self.assertEqual(result['budget_blockers'], []) self.assertEqual(list(self.root.iterdir()), []) executed = execute(self.root, config, self.case, self.model, opener=self.response) self.assertEqual(result['trials'][0]['run_id'], executed['run_id']) after = plan(self.root, config, {'cases': [self.case]}) self.assertEqual(after['new_reservations_usd'], '0') self.assertEqual(after['planned_committed_usd'], '0.001') def test_plan_counts_pending_once_and_exposes_blocker(self): def fail(*args, **kwargs): raise TimeoutError('timeout') with self.assertRaises(TimeoutError): execute(self.root, self.config, self.case, self.model, opener=fail) result = plan(self.root, dict(self.config, models=[self.model]), {'cases': [self.case]}) self.assertEqual(result['budget_blockers'], ['unresolved billing reservations']) self.assertEqual(result['new_reservations_usd'], '0') self.assertEqual(result['planned_committed_usd'], budget(self.root, 'test')['pending_usd']) def test_plan_refuses_duplicates_and_marks_over_budget(self): config = dict(self.config, models=[self.model], budget_usd='0') with self.assertRaisesRegex(ValueError, 'unique'): plan(self.root, config, {'cases': [self.case, self.case]}) result = plan(self.root, config, {'cases': [self.case]}) self.assertIn('planned reservations exceed campaign budget', result['budget_blockers'])