test-driver/tests/selfverification/test_checks_can_fail.py

"""A self-test that cannot fail is worth nothing.

Every check in `checks.py` is fed a deliberately broken artefact here, and must
complain. Without this file, the self-verification suite would be a row of green
ticks with no evidence that any of them is load-bearing.

Two kinds of break appear below, and the difference is worth noticing:

* **structural breaks** — the guarantee is actually violated in a running
  system (actors sharing memory). The check must catch it from the evidence.
* **artefact breaks** — the guarantee cannot be violated through the framework
  because the architecture forbids it, so the check is exercised against a
  hand-built pack instead. Where that applies it is stated explicitly, together
  with the separate assertion that the framework does refuse.
"""

from __future__ import annotations

import copy
import json

import pytest

from testdriver import Actor, Cast, Oracle, Runner, World
from testdriver.runner import CollectorIndependenceError
from lab.mutations import ObservationChannel, build_lab
from testdriver import DirectDriver, StateObserver
from scenarios.alice_bob_carol import USE_CASE, build
from tests.selfverification.checks import (
    check_actor_isolation,
    check_evidence_supports_every_verdict,
    check_intent_independence,
    check_no_actor_collected_judgment,
    check_runs_agree,
    check_verdicts_follow_from_judgment_evidence,
)
from tests.selfverification.test_self_verification import SECRETS, run_and_serialize


# --- td://self/actor-isolation -------------------------------------------


def test_shared_actor_memory_is_caught():
    """A structural break: two actors handed the same private store.

    This is what an isolation bug actually looks like when the same agent
    technology executes several actors in one process — nobody writes
    `bob.memory = alice.memory`, but a shared default or a cached client
    achieves it by accident.
    """
    lab, tokens = build_lab()
    shared: dict = {}
    cast = Cast()
    for name in ("alice", "bob", "carol"):
        actor = Actor(name, name.title(), credentials={"token": tokens[name]})
        object.__setattr__(actor, "_memory", shared)  # the leak
        cast.add(actor)

    _, _, _, asset, _ = build()
    world = World("w-leaky", lab, lab.version, cast=cast)
    driver = DirectDriver(lab, tokens)
    observer = StateObserver(ObservationChannel(lab), asset.scenario.watches)

    for actor_id, secret in SECRETS.items():
        world.cast[actor_id].remember(f"private-{actor_id}", secret)

    result = Runner(world, driver, observer, Oracle()).run(asset)
    pack = json.loads(result.evidence.to_json())
    memories = {a.id: {k: a.recall(k) for k in a.known_keys()} for a in world.cast}

    violations = check_actor_isolation(pack, SECRETS, memories)
    assert violations, "shared actor memory went undetected"
    assert any("holds" in v for v in violations)


def test_a_secret_leaking_into_the_record_is_caught():
    """An artefact break: the leak escapes into evidence even if memory is clean."""
    pack, memories, _ = run_and_serialize()
    tampered = copy.deepcopy(pack)
    for obs in tampered["observations"]:
        if obs["collector"] == "alice":
            obs["data"]["stolen"] = SECRETS["bob"]
            break
    assert check_actor_isolation(tampered, SECRETS, memories)


# --- td://self/oracle-independence ---------------------------------------


def test_the_framework_refuses_to_let_an_actor_collect_judgment_evidence():
    """The architecture prevents this, so it cannot be produced by a real run."""
    world, driver, observer, asset, oracle = build()
    observer.name = "bob"
    with pytest.raises(CollectorIndependenceError):
        Runner(world, driver, observer, oracle).run(asset)


def test_actor_collected_judgment_evidence_is_caught_in_the_record():
    """...and if it ever did appear in a pack, the out-of-band check sees it.

    Both halves are needed. The first says the door is locked; this one says we
    would notice if someone came through the window.
    """
    pack, _, actors = run_and_serialize()
    tampered = copy.deepcopy(pack)
    for obs in tampered["observations"]:
        if obs["stratum"] == "S3":
            obs["collector"] = "bob"
            break
    assert check_no_actor_collected_judgment(tampered, actors)


def test_a_verdict_that_contradicts_its_evidence_is_caught():
    """The strongest of the checks: verdicts must follow from S3 alone."""
    pack, _, _ = run_and_serialize()
    tampered = copy.deepcopy(pack)
    tampered["verdicts"][0]["verdict"] = "FAIL"  # a verdict nothing supports
    assertions = list(USE_CASE.claims) + list(USE_CASE.invariants)
    violations = check_verdicts_follow_from_judgment_evidence(tampered, assertions)
    assert violations
    assert "retained judgment evidence yields" in violations[0]


def test_a_pass_silently_replacing_a_fail_is_caught():
    """The specific corruption the project most needs to notice."""
    pack, _, _ = run_and_serialize("M15")
    tampered = copy.deepcopy(pack)
    for verdict in tampered["verdicts"]:
        if verdict["verdict"] == "FAIL":
            verdict["verdict"] = "PASS"
    assertions = list(USE_CASE.claims) + list(USE_CASE.invariants)
    assert check_verdicts_follow_from_judgment_evidence(tampered, assertions)


# --- td://self/evidence-reproducibility -----------------------------------


def test_a_verdict_without_a_snapshot_is_caught():
    pack, _, _ = run_and_serialize()
    tampered = copy.deepcopy(pack)
    tampered["observations"] = [
        o for o in tampered["observations"] if o["kind"] != "state_snapshot"
    ]
    assert check_evidence_supports_every_verdict(tampered)


def test_a_verdict_without_a_record_of_how_the_step_ran_is_caught():
    pack, _, _ = run_and_serialize()
    tampered = copy.deepcopy(pack)
    tampered["observations"] = [
        o for o in tampered["observations"] if o["kind"] != "realization"
    ]
    violations = check_evidence_supports_every_verdict(tampered)
    assert any("how" in v for v in violations)


def test_evidence_that_does_not_name_the_version_is_caught():
    pack, _, _ = run_and_serialize()
    tampered = copy.deepcopy(pack)
    tampered["sut_version"] = ""
    assert check_evidence_supports_every_verdict(tampered)


def test_disagreeing_runs_are_caught():
    first, _, _ = run_and_serialize()
    second, _, _ = run_and_serialize("M15")
    assert check_runs_agree(first, second)


# --- td://self/intent-independence ----------------------------------------


def test_implementation_derived_intent_is_caught():
    pack, _, _ = run_and_serialize("M15")
    tampered = copy.deepcopy(pack)
    tampered["provenance_index"]["c-bob-revoked"] = "agent-from-implementation"
    violations = check_intent_independence(tampered)
    assert violations
    assert "derived from the implementation" in violations[0]


def test_unrecorded_provenance_is_caught():
    """Absent provenance is not a benign omission — it makes the verdict
    unauditable, which for this framework is the same as unusable."""
    pack, _, _ = run_and_serialize()
    tampered = copy.deepcopy(pack)
    tampered["provenance_index"] = {}
    assert check_intent_independence(tampered)