"""A self-test that cannot fail is worth nothing. Every check in `checks.py` is fed a deliberately broken artefact here, and must complain. Without this file, the self-verification suite would be a row of green ticks with no evidence that any of them is load-bearing. Two kinds of break appear below, and the difference is worth noticing: * **structural breaks** — the guarantee is actually violated in a running system (actors sharing memory). The check must catch it from the evidence. * **artefact breaks** — the guarantee cannot be violated through the framework because the architecture forbids it, so the check is exercised against a hand-built pack instead. Where that applies it is stated explicitly, together with the separate assertion that the framework does refuse. """ from __future__ import annotations import copy import json import pytest from testdriver import Actor, Cast, Oracle, Runner, World from testdriver.runner import CollectorIndependenceError from lab.mutations import ObservationChannel, build_lab from testdriver import DirectDriver, StateObserver from scenarios.alice_bob_carol import USE_CASE, build from tests.selfverification.checks import ( check_actor_isolation, check_evidence_supports_every_verdict, check_intent_independence, check_no_actor_collected_judgment, check_runs_agree, check_verdicts_follow_from_judgment_evidence, ) from tests.selfverification.test_self_verification import SECRETS, run_and_serialize # --- td://self/actor-isolation ------------------------------------------- def test_shared_actor_memory_is_caught(): """A structural break: two actors handed the same private store. This is what an isolation bug actually looks like when the same agent technology executes several actors in one process — nobody writes `bob.memory = alice.memory`, but a shared default or a cached client achieves it by accident. """ lab, tokens = build_lab() shared: dict = {} cast = Cast() for name in ("alice", "bob", "carol"): actor = Actor(name, name.title(), credentials={"token": tokens[name]}) object.__setattr__(actor, "_memory", shared) # the leak cast.add(actor) _, _, _, asset, _ = build() world = World("w-leaky", lab, lab.version, cast=cast) driver = DirectDriver(lab, tokens) observer = StateObserver(ObservationChannel(lab), asset.scenario.watches) for actor_id, secret in SECRETS.items(): world.cast[actor_id].remember(f"private-{actor_id}", secret) result = Runner(world, driver, observer, Oracle()).run(asset) pack = json.loads(result.evidence.to_json()) memories = {a.id: {k: a.recall(k) for k in a.known_keys()} for a in world.cast} violations = check_actor_isolation(pack, SECRETS, memories) assert violations, "shared actor memory went undetected" assert any("holds" in v for v in violations) def test_a_secret_leaking_into_the_record_is_caught(): """An artefact break: the leak escapes into evidence even if memory is clean.""" pack, memories, _ = run_and_serialize() tampered = copy.deepcopy(pack) for obs in tampered["observations"]: if obs["collector"] == "alice": obs["data"]["stolen"] = SECRETS["bob"] break assert check_actor_isolation(tampered, SECRETS, memories) # --- td://self/oracle-independence --------------------------------------- def test_the_framework_refuses_to_let_an_actor_collect_judgment_evidence(): """The architecture prevents this, so it cannot be produced by a real run.""" world, driver, observer, asset, oracle = build() observer.name = "bob" with pytest.raises(CollectorIndependenceError): Runner(world, driver, observer, oracle).run(asset) def test_actor_collected_judgment_evidence_is_caught_in_the_record(): """...and if it ever did appear in a pack, the out-of-band check sees it. Both halves are needed. The first says the door is locked; this one says we would notice if someone came through the window. """ pack, _, actors = run_and_serialize() tampered = copy.deepcopy(pack) for obs in tampered["observations"]: if obs["stratum"] == "S3": obs["collector"] = "bob" break assert check_no_actor_collected_judgment(tampered, actors) def test_a_verdict_that_contradicts_its_evidence_is_caught(): """The strongest of the checks: verdicts must follow from S3 alone.""" pack, _, _ = run_and_serialize() tampered = copy.deepcopy(pack) tampered["verdicts"][0]["verdict"] = "FAIL" # a verdict nothing supports assertions = list(USE_CASE.claims) + list(USE_CASE.invariants) violations = check_verdicts_follow_from_judgment_evidence(tampered, assertions) assert violations assert "retained judgment evidence yields" in violations[0] def test_a_pass_silently_replacing_a_fail_is_caught(): """The specific corruption the project most needs to notice.""" pack, _, _ = run_and_serialize("M15") tampered = copy.deepcopy(pack) for verdict in tampered["verdicts"]: if verdict["verdict"] == "FAIL": verdict["verdict"] = "PASS" assertions = list(USE_CASE.claims) + list(USE_CASE.invariants) assert check_verdicts_follow_from_judgment_evidence(tampered, assertions) # --- td://self/evidence-reproducibility ----------------------------------- def test_a_verdict_without_a_snapshot_is_caught(): pack, _, _ = run_and_serialize() tampered = copy.deepcopy(pack) tampered["observations"] = [ o for o in tampered["observations"] if o["kind"] != "state_snapshot" ] assert check_evidence_supports_every_verdict(tampered) def test_a_verdict_without_a_record_of_how_the_step_ran_is_caught(): pack, _, _ = run_and_serialize() tampered = copy.deepcopy(pack) tampered["observations"] = [ o for o in tampered["observations"] if o["kind"] != "realization" ] violations = check_evidence_supports_every_verdict(tampered) assert any("how" in v for v in violations) def test_evidence_that_does_not_name_the_version_is_caught(): pack, _, _ = run_and_serialize() tampered = copy.deepcopy(pack) tampered["sut_version"] = "" assert check_evidence_supports_every_verdict(tampered) def test_disagreeing_runs_are_caught(): first, _, _ = run_and_serialize() second, _, _ = run_and_serialize("M15") assert check_runs_agree(first, second) # --- td://self/intent-independence ---------------------------------------- def test_implementation_derived_intent_is_caught(): pack, _, _ = run_and_serialize("M15") tampered = copy.deepcopy(pack) tampered["provenance_index"]["c-bob-revoked"] = "agent-from-implementation" violations = check_intent_independence(tampered) assert violations assert "derived from the implementation" in violations[0] def test_unrecorded_provenance_is_caught(): """Absent provenance is not a benign omission — it makes the verdict unauditable, which for this framework is the same as unusable.""" pack, _, _ = run_and_serialize() tampered = copy.deepcopy(pack) tampered["provenance_index"] = {} assert check_intent_independence(tampered)