Prevent passing run verdicts after failed scheduled realizations
Assistant: codex Assistant-Model: gpt-6-astra Assistant-Session: 01a0e76f-be98-7ae3-965d-e0b31290a4c4
This commit is contained in:
parent
3ce772466b
commit
e3aac44c59
3 changed files with 36 additions and 2 deletions
|
|
@ -155,3 +155,26 @@ def test_truncating_the_manifest_too_cannot_hide_a_missing_baseline_step(baselin
|
|||
candidate["verdicts"] = [v for v in candidate["verdicts"] if v["step_id"] != missing]
|
||||
candidate["observations"] = [o for o in candidate["observations"] if o["step_id"] != missing]
|
||||
assert not classify(baseline, candidate).safe_to_accept
|
||||
|
||||
|
||||
@pytest.mark.parametrize('mutation,expected', [(None, Verdict.INCONCLUSIVE), ('M17', Verdict.FAIL)])
|
||||
def test_failed_trailing_realization_cannot_leave_a_passing_receipt(tmp_path, mutation, expected):
|
||||
from testdriver import Step, SemanticAction
|
||||
from testdriver.storage import EvidenceStore
|
||||
world, driver, observer, asset, oracle = build(*([mutation] if mutation else []))
|
||||
asset.scenario = replace(
|
||||
asset.scenario,
|
||||
use_case=replace(asset.scenario.use_case, invariants=()),
|
||||
steps=(*asset.scenario.steps, Step('trailing-read', 'bob', SemanticAction(
|
||||
'read_resource', {'resource_id': 'absent'}, frozenset({'api'})))),
|
||||
)
|
||||
store = EvidenceStore(tmp_path)
|
||||
result = Runner(world, driver, observer, oracle).run(asset, evidence_store=store)
|
||||
assert result.verdict is expected
|
||||
assert store.load(result.run_id)['run_verdict'] == expected.value
|
||||
assert any(o.kind == 'realization' and o.step_id == 'trailing-read' and o.data['raised']
|
||||
for o in result.evidence.observations)
|
||||
assert not any(j.step_id == 'trailing-read' for j in result.judgments)
|
||||
if not mutation:
|
||||
assert all(j.verdict is Verdict.PASS for j in result.judgments)
|
||||
assert not classify(store.load(result.run_id), store.load(result.run_id)).safe_to_accept
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue