82 lines
3 KiB
Python
82 lines
3 KiB
Python
|
"""Independent checks that new use cases discriminate defects and evidence loss."""
|
||
|
|
import json
|
||
|
|
|
||
|
|
import pytest
|
||
|
|
|
||
|
|
from testdriver import Oracle, Runner, Stratum, Verdict
|
||
|
|
from scenarios.delegated_approval import build as approval
|
||
|
|
from scenarios.tenant_lifecycle import build as tenant
|
||
|
|
|
||
|
|
|
||
|
|
def run(builder, defect=None):
|
||
|
|
world, driver, observer, asset, oracle = builder(defect)
|
||
|
|
return Runner(world, driver, observer, oracle).run(asset)
|
||
|
|
|
||
|
|
|
||
|
|
def test_approval_baseline():
|
||
|
|
assert run(approval).verdict is Verdict.PASS
|
||
|
|
|
||
|
|
|
||
|
|
@pytest.mark.parametrize("defect,claim", [
|
||
|
|
("premature-execution", "approval-premature"),
|
||
|
|
("self-approval", "approval-self"),
|
||
|
|
("stale-delegation", "approval-former"),
|
||
|
|
])
|
||
|
|
def test_approval_defects(defect, claim):
|
||
|
|
assert run(approval, defect).judgment(claim).verdict is Verdict.FAIL
|
||
|
|
|
||
|
|
|
||
|
|
def test_approval_denials_are_judged_not_skipped():
|
||
|
|
result = run(approval)
|
||
|
|
assert len(result.judgments) == 14
|
||
|
|
assert all(j.verdict is Verdict.PASS for j in result.judgments)
|
||
|
|
|
||
|
|
|
||
|
|
def test_tenant_baseline():
|
||
|
|
assert run(tenant).verdict is Verdict.PASS
|
||
|
|
|
||
|
|
|
||
|
|
@pytest.mark.parametrize("defect,claim", [
|
||
|
|
("foreign-read", "tenant-reads"),
|
||
|
|
("foreign-delete", "tenant-deny-a"),
|
||
|
|
("unscoped-delete", "tenant-delete"),
|
||
|
|
("resurrect-content", "tenant-recreate"),
|
||
|
|
])
|
||
|
|
def test_tenant_defects(defect, claim):
|
||
|
|
result = run(tenant, defect)
|
||
|
|
assert any(j.assertion_id == claim and j.verdict is Verdict.FAIL for j in result.judgments)
|
||
|
|
|
||
|
|
|
||
|
|
@pytest.mark.parametrize("builder,defect", [
|
||
|
|
(approval, None), (approval, "self-approval"),
|
||
|
|
(tenant, None), (tenant, "foreign-read"),
|
||
|
|
])
|
||
|
|
def test_verdicts_reproduce_from_serialized_s3_without_actors(builder, defect):
|
||
|
|
result = run(builder, defect)
|
||
|
|
pack = json.loads(result.evidence.to_json())
|
||
|
|
assertions = builder()[3].scenario.use_case
|
||
|
|
snapshots = {o["step_id"]: o["data"] for o in pack["observations"]
|
||
|
|
if o["kind"] == "state_snapshot"}
|
||
|
|
by_id = {a.id: a for a in (*assertions.claims, *assertions.invariants)}
|
||
|
|
for j in pack["verdicts"]:
|
||
|
|
replayed = Oracle().judge(by_id[j["assertion_id"]], snapshots[j["step_id"]], j["step_id"])
|
||
|
|
assert replayed.verdict.value == j["verdict"]
|
||
|
|
actors = set(builder()[0].cast.actors)
|
||
|
|
assert not actors.intersection(o.collector for o in result.evidence.observations
|
||
|
|
if o.stratum is not Stratum.SURFACE)
|
||
|
|
|
||
|
|
|
||
|
|
@pytest.mark.parametrize("builder", [approval, tenant])
|
||
|
|
def test_missing_evidence_is_inconclusive_for_every_assertion(builder):
|
||
|
|
case = builder()[3].scenario.use_case
|
||
|
|
for assertion in (*case.claims, *case.invariants):
|
||
|
|
assert Oracle().judge(assertion, {"unrelated": True}, None).verdict is Verdict.INCONCLUSIVE
|
||
|
|
|
||
|
|
|
||
|
|
@pytest.mark.parametrize("builder", [approval, tenant])
|
||
|
|
def test_replay_keeps_judgments_and_claims(builder):
|
||
|
|
case = builder()[3].scenario.use_case
|
||
|
|
before = (case.claims, case.invariants)
|
||
|
|
assert run(builder).judgments == run(builder).judgments
|
||
|
|
assert (case.claims, case.invariants) == before
|