95 lines
4.2 KiB
Python
95 lines
4.2 KiB
Python
|
import copy
|
||
|
|
import importlib.util
|
||
|
|
from pathlib import Path
|
||
|
|
|
||
|
|
import pytest
|
||
|
|
|
||
|
|
ROOT = Path(__file__).resolve().parents[1]
|
||
|
|
spec = importlib.util.spec_from_file_location('metered_review', ROOT / 'tools/prepare_metered_activation.py')
|
||
|
|
review = importlib.util.module_from_spec(spec)
|
||
|
|
spec.loader.exec_module(review)
|
||
|
|
|
||
|
|
PROVIDER = dict(model='claude-sonnet-5', context_tokens=1_000_000,
|
||
|
|
max_output_tokens=128_000, input_microusd_per_token=4,
|
||
|
|
output_microusd_per_token=10, source='fixture:provider-facts')
|
||
|
|
OWNER = {'messages_policy': dict(PROVIDER, max_output_tokens=32_000)}
|
||
|
|
SPEND = dict(max_liability_usd='5.74', eur_per_usd='0.87', per_run_eur='5')
|
||
|
|
|
||
|
|
|
||
|
|
def test_installed_200k_context_is_not_a_hard_input_bound():
|
||
|
|
owner = copy.deepcopy(OWNER)
|
||
|
|
owner['messages_policy']['context_tokens'] = 200_000
|
||
|
|
result = review.review_policy(owner, SPEND, PROVIDER)
|
||
|
|
assert result['maximum_request_microusd'] == 1_120_000
|
||
|
|
assert result['failures'] == ['input_reservation_below_provider_context']
|
||
|
|
assert not result['passed']
|
||
|
|
|
||
|
|
|
||
|
|
def test_full_context_fits_once_and_does_not_grant_authority():
|
||
|
|
result = review.review_policy(OWNER, SPEND, PROVIDER)
|
||
|
|
assert result['passed']
|
||
|
|
assert result['maximum_request_microusd'] == 4_320_000
|
||
|
|
assert result['maximum_request_eur_at_configured_fx'] == '3.7584'
|
||
|
|
assert result['full_output_requests_that_fit'] == 1
|
||
|
|
assert result['authority_granted'] is False
|
||
|
|
|
||
|
|
|
||
|
|
@pytest.mark.parametrize('key,value,reason', [
|
||
|
|
('model', 'other', 'model_mismatch'),
|
||
|
|
('input_microusd_per_token', 2, 'understated_input_microusd_per_token'),
|
||
|
|
('output_microusd_per_token', 9, 'understated_output_microusd_per_token'),
|
||
|
|
('max_output_tokens', 128001, 'output_exceeds_provider_limit'),
|
||
|
|
])
|
||
|
|
def test_invalid_provider_compatibility_refuses(key, value, reason):
|
||
|
|
owner = copy.deepcopy(OWNER)
|
||
|
|
owner['messages_policy'][key] = value
|
||
|
|
assert reason in review.review_policy(owner, SPEND, PROVIDER)['failures']
|
||
|
|
|
||
|
|
|
||
|
|
def test_no_capacity_and_invalid_fx_refuse():
|
||
|
|
assert 'no_full_output_request_fits' in review.review_policy(
|
||
|
|
OWNER, dict(SPEND, max_liability_usd='1'), PROVIDER)['failures']
|
||
|
|
assert 'parent_exceeds_eur_cap' in review.review_policy(
|
||
|
|
OWNER, dict(SPEND, eur_per_usd='1'), PROVIDER)['failures']
|
||
|
|
with pytest.raises(ValueError):
|
||
|
|
review.review_policy(OWNER, dict(SPEND, eur_per_usd='NaN'), PROVIDER)
|
||
|
|
|
||
|
|
|
||
|
|
def test_packet_has_six_distinct_unapproved_actions_and_unchanged_human_controls():
|
||
|
|
packet = review.build_packet(ROOT / 'catalog', 'glas-claude-agent-dev-anthropic',
|
||
|
|
OWNER, SPEND, PROVIDER)
|
||
|
|
actions = packet['actions']
|
||
|
|
assert len(actions) == 6
|
||
|
|
assert len({(a['catalog'], a['action']) for a in actions}) == 6
|
||
|
|
for row in actions:
|
||
|
|
request = row['request']
|
||
|
|
assert request['resource']['id'] == 'catalog:' + row['catalog']
|
||
|
|
assert request['action'] == row['action']
|
||
|
|
assert bool(request['context'].get('human_control')) == (row['catalog'] == 'glas-claude-agent-dev-anthropic')
|
||
|
|
assert 'approval_id' not in request['context']
|
||
|
|
assert packet['approval_ids'] == []
|
||
|
|
assert not packet['dispatch_enabled']
|
||
|
|
assert not packet['host_pins_verified']
|
||
|
|
|
||
|
|
|
||
|
|
def test_snapshot_must_match_the_recipient_pin(tmp_path):
|
||
|
|
snapshot = tmp_path / 'owner.json'
|
||
|
|
snapshot.write_text('{}')
|
||
|
|
with pytest.raises(ValueError, match='snapshot_file_pin_mismatch'):
|
||
|
|
review.load_pinned_snapshot(snapshot, '0' * 64)
|
||
|
|
|
||
|
|
|
||
|
|
def test_actual_cli_output_and_beta_must_fit_policy():
|
||
|
|
facts = dict(PROVIDER, observed_cli_max_tokens=64000,
|
||
|
|
required_request_betas=['mid-conversation-system-2026-04-07'])
|
||
|
|
result = review.review_policy(OWNER, SPEND, facts)
|
||
|
|
assert result['failures'] == ['output_limit_below_observed_cli_request',
|
||
|
|
'observed_cli_beta_not_admitted']
|
||
|
|
owner = copy.deepcopy(OWNER)
|
||
|
|
owner['messages_policy'].update(max_output_tokens=64000,
|
||
|
|
allowed_betas=['mid-conversation-system-2026-04-07'])
|
||
|
|
result = review.review_policy(owner, SPEND, facts)
|
||
|
|
assert result['passed']
|
||
|
|
assert result['maximum_request_microusd'] == 4640000
|
||
|
|
assert result['maximum_request_eur_at_configured_fx'] == '4.0368'
|