58 lines
2.5 KiB
Python
58 lines
2.5 KiB
Python
|
import importlib.util
|
||
|
|
from pathlib import Path
|
||
|
|
|
||
|
|
import pytest
|
||
|
|
|
||
|
|
spec = importlib.util.spec_from_file_location("supervision", Path(__file__).resolve().parents[1] / "scripts/summarize_agent_supervision.py")
|
||
|
|
module = importlib.util.module_from_spec(spec)
|
||
|
|
spec.loader.exec_module(module)
|
||
|
|
|
||
|
|
|
||
|
|
def record(*proposals):
|
||
|
|
return {"agent_id": "fixture", "scope": "synthetic", "mode": "supervised", "proposals": list(proposals)}
|
||
|
|
|
||
|
|
|
||
|
|
def proposal(identity, disposition="accepted_unchanged", outcome="succeeded", refinement=False):
|
||
|
|
approved = "original" if disposition == "accepted_unchanged" else "revised"
|
||
|
|
return dict(proposal_id=identity, disposition=disposition, outcome=outcome,
|
||
|
|
original_digest="original", approved_digest=approved, executed_digest=approved,
|
||
|
|
approval_ref="synthetic-approval", verification_ref="synthetic-verification",
|
||
|
|
refinement_or_rescue=refinement)
|
||
|
|
|
||
|
|
|
||
|
|
def test_no_sample_is_unknown_and_cannot_grant_authority():
|
||
|
|
report = module.summarize(record())
|
||
|
|
assert report["unchanged_acceptance_rate"] is None
|
||
|
|
assert report["unchanged_execution_success_rate"] is None
|
||
|
|
assert report["authority_granted"] is False
|
||
|
|
|
||
|
|
|
||
|
|
def test_revisions_rejections_and_rescue_do_not_earn_unchanged_success():
|
||
|
|
report = module.summarize(record(proposal("one"), proposal("two", "accepted_revised"),
|
||
|
|
proposal("three", refinement=True), proposal("four", "rejected", "not_executed")))
|
||
|
|
assert report["unchanged_acceptance_rate"] == 2 / 4
|
||
|
|
assert report["unchanged_execution_success_rate"] == 1 / 3
|
||
|
|
|
||
|
|
|
||
|
|
def test_approval_alone_is_not_execution_success():
|
||
|
|
report = module.summarize(record(proposal("one", outcome="not_executed")))
|
||
|
|
assert report["unchanged_acceptance_rate"] == 1
|
||
|
|
assert report["unchanged_execution_success_rate"] is None
|
||
|
|
|
||
|
|
|
||
|
|
def test_duplicate_trials_and_unapproved_revisions_are_rejected():
|
||
|
|
with pytest.raises(ValueError):
|
||
|
|
module.summarize(record(proposal("same"), proposal("same")))
|
||
|
|
wrong = proposal("one"); wrong["executed_digest"] = "unapproved"
|
||
|
|
with pytest.raises(ValueError):
|
||
|
|
module.summarize(record(wrong))
|
||
|
|
|
||
|
|
|
||
|
|
def test_unscored_history_remains_visible_without_manufacturing_a_rate():
|
||
|
|
report = module.summarize(record(proposal("historic", "unscored", "failed", True)))
|
||
|
|
assert report["counts"]["unscored"] == 1
|
||
|
|
assert report["counts"]["outcome_failed"] == 1
|
||
|
|
assert report["counts"]["refinement_or_rescue"] == 1
|
||
|
|
assert report["unchanged_acceptance_rate"] is None
|
||
|
|
assert report["unchanged_execution_success_rate"] is None
|