Screen research evidence and record bounded llm-connect refinement
Assistant: codex Assistant-Model: gpt-6-astra Assistant-Session: 01a0cffe-206f-7323-89d3-b20ca4cf4bf9
This commit is contained in:
parent
2ec23bb22c
commit
509e86d9c5
24 changed files with 688 additions and 9 deletions
57
tests/test_research_admission.py
Normal file
57
tests/test_research_admission.py
Normal file
|
|
@ -0,0 +1,57 @@
|
|||
import importlib.util
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
REPO = Path(__file__).resolve().parents[1]
|
||||
spec = importlib.util.spec_from_file_location('admission', REPO / 'scripts/research_admission.py')
|
||||
admission = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(admission)
|
||||
|
||||
|
||||
def block(value):
|
||||
return '```json\n' + json.dumps(value) + '\n```'
|
||||
|
||||
|
||||
def test_captured_carniato_failure_is_blocked():
|
||||
result = admission.audit(REPO / 'research/bench/infospaces/carniato-authority-03')
|
||||
assert result['structural_status'] == 'blocked'
|
||||
assert any('candidate contract' in f for f in result['findings'])
|
||||
assert any("'C01'" in f for f in result['findings'])
|
||||
assert any("'C02'" in f for f in result['findings'])
|
||||
assert result['admission'] == 'not_granted'
|
||||
|
||||
|
||||
def test_missing_known_use_evidence_and_recurrence_are_blocked():
|
||||
candidates = {'candidates': [{'id': 'P01', 'known_uses': [
|
||||
{'implementation': 'generic chat example'}], 'recurrence': 'supported'}]}
|
||||
result = admission.screen(block(candidates), block([]))
|
||||
assert any('known use lacks' in f for f in result['findings'])
|
||||
assert any('recurrence lacks' in f for f in result['findings'])
|
||||
|
||||
|
||||
def test_unknown_and_empty_findings_do_not_grant_admission():
|
||||
for candidates in ([], [{'id': 'P01', 'known_uses': [], 'recurrence': 'unknown'}]):
|
||||
result = admission.screen(block({'candidates': candidates}), block([]))
|
||||
assert result['structural_status'] == 'passed'
|
||||
assert result['admission'] == 'not_granted'
|
||||
assert result['semantic_review'] == 'required'
|
||||
|
||||
|
||||
def test_duplicates_and_malformed_contracts_fail_closed():
|
||||
item = {'id': 'P01', 'known_uses': [], 'recurrence': 'unknown'}
|
||||
for candidates, edges in [({'candidates': [item, item]}, []),
|
||||
({'candidates': [item]}, {}),
|
||||
({'candidates': [None]}, []),
|
||||
({'candidates': [item]}, [None])]:
|
||||
assert admission.screen(block(candidates), block(edges))['structural_status'] == 'blocked'
|
||||
|
||||
|
||||
def test_claim_endpoint_rejected_even_with_valid_candidate_ledger():
|
||||
candidate = {'id': 'P01', 'known_uses': [], 'recurrence': 'unknown'}
|
||||
edge = {'subject_candidate_id': 'P01', 'object_candidate_id': 'C01',
|
||||
'relation': 'requires', 'rationale': 'context',
|
||||
'source_claim_and_locator': 'C01 Passage A',
|
||||
'counterexample': 'local authority', 'observation_versus_hypothesis': 'hypothesis'}
|
||||
result = admission.screen(block({'candidates': [candidate]}), block([edge]))
|
||||
assert len(result['findings']) == 1
|
||||
assert 'invalid object_candidate_id' in result['findings'][0]
|
||||
|
|
@ -182,7 +182,7 @@ def test_llm_connect_service_contract(tmp_path, finish, content, complete):
|
|||
server.start()
|
||||
root = initialize(tmp_path)
|
||||
try:
|
||||
live = bench.LLMConnectAdapter(f'http://127.0.0.1:{server.port}', 'requested/model')
|
||||
live = bench.LLMConnectAdapter(f'http://127.0.0.1:{server.port}', 'requested/model', reasoning_effort='low')
|
||||
if complete:
|
||||
assert bench.execute(root, live)['status'] == 'completed'
|
||||
assert len(calls) == 4
|
||||
|
|
@ -195,9 +195,12 @@ def test_llm_connect_service_contract(tmp_path, finish, content, complete):
|
|||
assert receipt['responses'][0]['model'] == 'resolved/model'
|
||||
assert receipt['responses'][0]['usage']['total_tokens'] == 42
|
||||
assert receipt['responses'][0]['finish_reason'] == finish
|
||||
assert receipt['responses'][0]['content_characters'] == len(content)
|
||||
assert receipt['responses'][0]['content_sha256'] is not None
|
||||
assert receipt['service_requests_attempted'] == len(calls)
|
||||
assert calls[0][1].model_name == 'requested/model'
|
||||
assert calls[0][1].max_tokens == 4000
|
||||
assert calls[0][1].model_params['reasoning_effort'] == 'low'
|
||||
assert 'DO-NOT-PERSIST' not in receipt_text
|
||||
finally:
|
||||
server.stop()
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue