Reduce research calls with screened reuse and draft VC-02 walkthrough

Assistant: codex
Assistant-Model: gpt-6-astra
Assistant-Session: 01a0cffe-206f-7323-89d3-b20ca4cf4bf9

tests/test_research_bench.py

@@ -10,6 +10,7 @@ from infospace_bench import register_artifact
 from infospace_bench.workflow import FixtureAssistedGenerationAdapter
 
 REPO = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(REPO / "scripts"))
 spec = importlib.util.spec_from_file_location("research_bench", REPO / "scripts/research_bench.py")
 bench = importlib.util.module_from_spec(spec)
 spec.loader.exec_module(bench)
@@ -17,7 +18,9 @@ FIXTURES = REPO / "research/bench/fixtures"
 
 
 def initialize(tmp_path):
-    return bench.initialize(FIXTURES / "source.md", tmp_path, "synthetic-probe")
+    root = bench.initialize(FIXTURES / "source.md", tmp_path, "synthetic-probe")
+    (root / "profile/execution.json").unlink()  # Exercise historical four-call compatibility.
+    return root
 
 
 def adapter():
@@ -254,3 +257,147 @@ def test_current_llm_connect_drops_reasoning_effort():
     payload = {'max_tokens': 1000}
     merge_openai_chat_model_params(payload, {'reasoning_effort': 'low'})
     assert 'reasoning_effort' not in payload
+
+
+def conditional_root(tmp_path, slug='conditional'):
+    return bench.initialize(FIXTURES / 'source.md', tmp_path, slug)
+
+
+def structured_adapter(count=1, edges=None, malformed=False):
+    from infospace_bench.workflow import AssistedGenerationResult
+    candidates = {'candidates': [
+        {'id': f'P{i+1:02}', 'known_uses': [], 'recurrence': 'unknown'} for i in range(count)]}
+    return FixtureAssistedGenerationAdapter({
+        ('claims', '*'): AssistedGenerationResult('# Synthetic claims', provider='fixture'),
+        ('candidates', '*'): AssistedGenerationResult(
+            'broken' if malformed else '```json\n' + json.dumps(candidates) + '\n```', provider='fixture'),
+        ('relations', '*'): AssistedGenerationResult(
+            '```json\n' + json.dumps(edges or []) + '\n```', provider='fixture'),
+    })
+
+
+@pytest.mark.parametrize('count,expected', [(0, 2), (1, 2), (2, 3)])
+def test_conditional_stage_call_counts(tmp_path, count, expected):
+    root = conditional_root(tmp_path)
+    result = bench.execute(root, structured_adapter(count), max_calls=expected)
+    assert result['status'] == 'completed'
+    assert result['provider_calls_attempted'] == expected
+    assert result['deterministic_stages'] == 4 - expected
+    assert 'Editorial review required' in (root / 'output/evaluations/source-review.md').read_text()
+    assert result['editorial_corrections'] is None
+    assert result['review_status'] == 'unreviewed'
+
+
+def test_invalid_candidates_stop_before_relations(tmp_path):
+    root = conditional_root(tmp_path)
+    with pytest.raises(ValueError, match='candidates structural screen'):
+        bench.execute(root, structured_adapter(malformed=True))
+    receipt = json.loads((root / 'reports/research-run.json').read_text())
+    assert receipt['provider_calls_attempted'] == 2
+    assert receipt['rejected_assertions'] > 0
+    assert (root / 'reports/rejected-stage.json').exists()
+    assert not (root / 'artifacts/generated/relation-proposals.md').exists()
+
+
+def test_invalid_edges_stop_before_editorial_stage(tmp_path):
+    root = conditional_root(tmp_path)
+    with pytest.raises(ValueError, match='relations structural screen'):
+        bench.execute(root, structured_adapter(2, [{'subject_candidate_id': 'P01',
+            'object_candidate_id': 'C01', 'relation': 'requires'}]))
+    receipt = json.loads((root / 'reports/research-run.json').read_text())
+    assert receipt['provider_calls_attempted'] == 3
+    assert not (root / 'output/evaluations/source-review.md').exists()
+
+
+def test_explicit_reuse_makes_no_new_calls(tmp_path):
+    first = conditional_root(tmp_path, 'first')
+    bench.execute(first, structured_adapter())
+    second = conditional_root(tmp_path, 'second')
+    result = bench.execute(second, structured_adapter(), max_calls=0, reuse_from=first)
+    assert result['provider_calls_attempted'] == 0
+    assert result['cache_hits'] == 2
+    assert 'reused_from' in (second / 'artifacts/index.yaml').read_text()
+
+
+@pytest.mark.parametrize('change', ['prompt', 'source', 'fixture'])
+def test_changed_inputs_never_reuse_stale_output(tmp_path, change):
+    first = conditional_root(tmp_path, 'first')
+    bench.execute(first, structured_adapter())
+    source = FIXTURES / 'source.md'
+    if change == 'source':
+        source = tmp_path / 'different.md'
+        source.write_text((FIXTURES / 'source.md').read_text() + '\nNew constraint.\n')
+    second = bench.initialize(source, tmp_path, 'second')
+    if change == 'prompt':
+        template = second / 'profile/templates/claims.md'
+        template.write_text(template.read_text() + '\nChanged question.\n')
+    with pytest.raises(ValueError, match='call limit'):
+        bench.execute(second, structured_adapter(2 if change == 'fixture' else 1),
+                      max_calls=0, reuse_from=first)
+    receipt = json.loads((second / 'reports/research-run.json').read_text())
+    assert receipt['cache_hits'] == 0
+    assert receipt['provider_calls_attempted'] == 0
+
+
+def test_modified_cache_and_failed_run_are_not_reusable(tmp_path):
+    first = conditional_root(tmp_path, 'first')
+    bench.execute(first, structured_adapter())
+    cache = first / 'reports/stage-cache.json'
+    cache.write_text('{}')
+    second = conditional_root(tmp_path, 'second')
+    with pytest.raises(ValueError, match='matching cache digest'):
+        bench.execute(second, structured_adapter(), reuse_from=first)
+    assert not (second / 'reports/research-run.json').exists()
+
+
+def test_two_candidate_relation_survives_screen(tmp_path):
+    root = conditional_root(tmp_path)
+    edge = {'subject_candidate_id': 'P01', 'object_candidate_id': 'P02',
+            'relation': 'complements', 'rationale': 'Synthetic complement',
+            'source_claim_and_locator': 'C01 Scenario',
+            'counterexample': 'No persistence requirement',
+            'observation_versus_hypothesis': 'hypothesis'}
+    result = bench.execute(root, structured_adapter(2, [edge]))
+    assert result['provider_calls_attempted'] == 3
+    assert result['status'] == 'completed'
+
+
+def test_service_reuse_and_setting_invalidation(tmp_path):
+    from llm_connect.models import LLMResponse
+    from llm_connect.server import LLMServer
+    calls = []
+
+    class Service:
+        def validate_config(self, config):
+            return True
+
+        def execute_prompt(self, prompt, config):
+            calls.append(prompt)
+            return LLMResponse(content='```json\n{"candidates": []}\n```',
+                               model='test', usage={'total_tokens': 42})
+
+    server = LLMServer(Service(), port=0)
+    server.start()
+    endpoint = f'http://127.0.0.1:{server.port}'
+    try:
+        first = conditional_root(tmp_path, 'first')
+        r = bench.execute(first, bench.LLMConnectAdapter(endpoint, 'test'))
+        assert r['reported_tokens_this_run'] == 84
+        second = conditional_root(tmp_path, 'second')
+        r = bench.execute(second, bench.LLMConnectAdapter(endpoint, 'test'),
+                          max_calls=0, reuse_from=first)
+        assert r['reported_tokens_this_run'] == 0
+        assert r['service_requests_attempted'] == 0
+        assert len(calls) == 2
+        third = conditional_root(tmp_path, 'third')
+        with pytest.raises(ValueError, match='call limit'):
+            bench.execute(third, bench.LLMConnectAdapter(endpoint, 'different-model'),
+                          max_calls=0, reuse_from=first)
+        receipt = json.loads((first / 'reports/research-run.json').read_text())
+        receipt['status'] = 'failed'
+        (first / 'reports/research-run.json').write_text(json.dumps(receipt))
+        fourth = conditional_root(tmp_path, 'fourth')
+        with pytest.raises(ValueError, match='completed run'):
+            bench.execute(fourth, bench.LLMConnectAdapter(endpoint, 'test'), reuse_from=first)
+    finally:
+        server.stop()