Advance supervised agent records and close verified Secret annotation guard
This commit is contained in:
parent
b2f6713721
commit
db91818e84
44 changed files with 6868 additions and 54 deletions
57
tests/test_agent_supervision.py
Normal file
57
tests/test_agent_supervision.py
Normal file
|
|
@ -0,0 +1,57 @@
|
|||
import importlib.util
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
spec = importlib.util.spec_from_file_location("supervision", Path(__file__).resolve().parents[1] / "scripts/summarize_agent_supervision.py")
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
|
||||
|
||||
def record(*proposals):
|
||||
return {"agent_id": "fixture", "scope": "synthetic", "mode": "supervised", "proposals": list(proposals)}
|
||||
|
||||
|
||||
def proposal(identity, disposition="accepted_unchanged", outcome="succeeded", refinement=False):
|
||||
approved = "original" if disposition == "accepted_unchanged" else "revised"
|
||||
return dict(proposal_id=identity, disposition=disposition, outcome=outcome,
|
||||
original_digest="original", approved_digest=approved, executed_digest=approved,
|
||||
approval_ref="synthetic-approval", verification_ref="synthetic-verification",
|
||||
refinement_or_rescue=refinement)
|
||||
|
||||
|
||||
def test_no_sample_is_unknown_and_cannot_grant_authority():
|
||||
report = module.summarize(record())
|
||||
assert report["unchanged_acceptance_rate"] is None
|
||||
assert report["unchanged_execution_success_rate"] is None
|
||||
assert report["authority_granted"] is False
|
||||
|
||||
|
||||
def test_revisions_rejections_and_rescue_do_not_earn_unchanged_success():
|
||||
report = module.summarize(record(proposal("one"), proposal("two", "accepted_revised"),
|
||||
proposal("three", refinement=True), proposal("four", "rejected", "not_executed")))
|
||||
assert report["unchanged_acceptance_rate"] == 2 / 4
|
||||
assert report["unchanged_execution_success_rate"] == 1 / 3
|
||||
|
||||
|
||||
def test_approval_alone_is_not_execution_success():
|
||||
report = module.summarize(record(proposal("one", outcome="not_executed")))
|
||||
assert report["unchanged_acceptance_rate"] == 1
|
||||
assert report["unchanged_execution_success_rate"] is None
|
||||
|
||||
|
||||
def test_duplicate_trials_and_unapproved_revisions_are_rejected():
|
||||
with pytest.raises(ValueError):
|
||||
module.summarize(record(proposal("same"), proposal("same")))
|
||||
wrong = proposal("one"); wrong["executed_digest"] = "unapproved"
|
||||
with pytest.raises(ValueError):
|
||||
module.summarize(record(wrong))
|
||||
|
||||
|
||||
def test_unscored_history_remains_visible_without_manufacturing_a_rate():
|
||||
report = module.summarize(record(proposal("historic", "unscored", "failed", True)))
|
||||
assert report["counts"]["unscored"] == 1
|
||||
assert report["counts"]["outcome_failed"] == 1
|
||||
assert report["counts"]["refinement_or_rescue"] == 1
|
||||
assert report["unchanged_acceptance_rate"] is None
|
||||
assert report["unchanged_execution_success_rate"] is None
|
||||
Loading…
Add table
Add a link
Reference in a new issue