Complete local generalisation tasks and document remaining experiment blockers
Assistant: codex Assistant-Model: gpt-6-astra Assistant-Session: 01a0e76f-be98-7ae3-965d-e0b31290a4c4
This commit is contained in:
parent
d54af628df
commit
cf682281d7
36 changed files with 5394 additions and 279 deletions
|
|
@ -173,3 +173,19 @@ def test_the_control_arm_is_not_a_straw_man():
|
|||
preserved = [m for m in MECHANICAL if m.preserves_test_ids]
|
||||
assert all(realized_ok(m.id, runtime=runtime) for m in preserved)
|
||||
assert len(preserved) >= 9
|
||||
|
||||
|
||||
def test_deciding_axis_has_ten_mutations_and_paired_structural_controls():
|
||||
from lab.mutations import EXTENDED_LAYOUTS
|
||||
from lab.http_api import render_resource_page
|
||||
from lab.mutations import build_lab
|
||||
assert sum(not m.preserves_test_ids for m in MECHANICAL) >= 10
|
||||
for dropped, preserved, _, _ in EXTENDED_LAYOUTS:
|
||||
dropped_app, _ = build_lab(dropped)
|
||||
preserved_app, _ = build_lab(preserved)
|
||||
# Version metadata differs; control markup must differ only in test ids.
|
||||
import re
|
||||
def shape(app):
|
||||
html = render_resource_page(app, "alice", "R")
|
||||
return re.sub(r'\s*data-td(?:-version)?="[^"]*"', "", html)
|
||||
assert shape(dropped_app) == shape(preserved_app)
|
||||
|
|
|
|||
|
|
@ -65,6 +65,12 @@ EXPECTED = {
|
|||
"M24": Classification.MECHANICAL_ADAPTATION,
|
||||
}
|
||||
|
||||
EXPECTED.update({
|
||||
f"M{number}": (Classification.UNCHANGED if number in (25, 32)
|
||||
else Classification.MECHANICAL_ADAPTATION)
|
||||
for number in range(25, 39)
|
||||
})
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mutation_id", sorted(EXPECTED))
|
||||
def test_response_matrix(baseline, mutation_id):
|
||||
|
|
|
|||
81
tests/test_generalisation.py
Normal file
81
tests/test_generalisation.py
Normal file
|
|
@ -0,0 +1,81 @@
|
|||
"""Independent checks that new use cases discriminate defects and evidence loss."""
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from testdriver import Oracle, Runner, Stratum, Verdict
|
||||
from scenarios.delegated_approval import build as approval
|
||||
from scenarios.tenant_lifecycle import build as tenant
|
||||
|
||||
|
||||
def run(builder, defect=None):
|
||||
world, driver, observer, asset, oracle = builder(defect)
|
||||
return Runner(world, driver, observer, oracle).run(asset)
|
||||
|
||||
|
||||
def test_approval_baseline():
|
||||
assert run(approval).verdict is Verdict.PASS
|
||||
|
||||
|
||||
@pytest.mark.parametrize("defect,claim", [
|
||||
("premature-execution", "approval-premature"),
|
||||
("self-approval", "approval-self"),
|
||||
("stale-delegation", "approval-former"),
|
||||
])
|
||||
def test_approval_defects(defect, claim):
|
||||
assert run(approval, defect).judgment(claim).verdict is Verdict.FAIL
|
||||
|
||||
|
||||
def test_approval_denials_are_judged_not_skipped():
|
||||
result = run(approval)
|
||||
assert len(result.judgments) == 14
|
||||
assert all(j.verdict is Verdict.PASS for j in result.judgments)
|
||||
|
||||
|
||||
def test_tenant_baseline():
|
||||
assert run(tenant).verdict is Verdict.PASS
|
||||
|
||||
|
||||
@pytest.mark.parametrize("defect,claim", [
|
||||
("foreign-read", "tenant-reads"),
|
||||
("foreign-delete", "tenant-deny-a"),
|
||||
("unscoped-delete", "tenant-delete"),
|
||||
("resurrect-content", "tenant-recreate"),
|
||||
])
|
||||
def test_tenant_defects(defect, claim):
|
||||
result = run(tenant, defect)
|
||||
assert any(j.assertion_id == claim and j.verdict is Verdict.FAIL for j in result.judgments)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("builder,defect", [
|
||||
(approval, None), (approval, "self-approval"),
|
||||
(tenant, None), (tenant, "foreign-read"),
|
||||
])
|
||||
def test_verdicts_reproduce_from_serialized_s3_without_actors(builder, defect):
|
||||
result = run(builder, defect)
|
||||
pack = json.loads(result.evidence.to_json())
|
||||
assertions = builder()[3].scenario.use_case
|
||||
snapshots = {o["step_id"]: o["data"] for o in pack["observations"]
|
||||
if o["kind"] == "state_snapshot"}
|
||||
by_id = {a.id: a for a in (*assertions.claims, *assertions.invariants)}
|
||||
for j in pack["verdicts"]:
|
||||
replayed = Oracle().judge(by_id[j["assertion_id"]], snapshots[j["step_id"]], j["step_id"])
|
||||
assert replayed.verdict.value == j["verdict"]
|
||||
actors = set(builder()[0].cast.actors)
|
||||
assert not actors.intersection(o.collector for o in result.evidence.observations
|
||||
if o.stratum is not Stratum.SURFACE)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("builder", [approval, tenant])
|
||||
def test_missing_evidence_is_inconclusive_for_every_assertion(builder):
|
||||
case = builder()[3].scenario.use_case
|
||||
for assertion in (*case.claims, *case.invariants):
|
||||
assert Oracle().judge(assertion, {"unrelated": True}, None).verdict is Verdict.INCONCLUSIVE
|
||||
|
||||
|
||||
@pytest.mark.parametrize("builder", [approval, tenant])
|
||||
def test_replay_keeps_judgments_and_claims(builder):
|
||||
case = builder()[3].scenario.use_case
|
||||
before = (case.claims, case.invariants)
|
||||
assert run(builder).judgments == run(builder).judgments
|
||||
assert (case.claims, case.invariants) == before
|
||||
|
|
@ -130,10 +130,3 @@ def test_seeded_authorization_defect_fails_the_run():
|
|||
# The claim set is untouched by the failure — there is no path to adapt it.
|
||||
by_id = {c.id: c for c in USE_CASE.claims}
|
||||
assert by_id["c-bob-revoked"].text == "Bob cannot read R after revocation"
|
||||
|
||||
|
||||
def test_defect_run_emits_an_energy_event():
|
||||
"""Energy events are captured; no score is computed (H-005 is dormant)."""
|
||||
import testdriver.energy as energy
|
||||
|
||||
assert not hasattr(energy, "score")
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue