test-driver/tests/selfverification/test_checks_can_fail.py

"""A self-test that cannot fail is worth nothing.

Every check in `checks.py` is fed a deliberately broken artefact here, and must
complain. Without this file, the self-verification suite would be a row of green
ticks with no evidence that any of them is load-bearing.

Two kinds of break appear below, and the difference is worth noticing:

* **structural breaks** — the guarantee is actually violated in a running
  system (actors sharing memory). The check must catch it from the evidence.
* **artefact breaks** — the guarantee cannot be violated through the framework
  because the architecture forbids it, so the check is exercised against a
  hand-built pack instead. Where that applies it is stated explicitly, together
  with the separate assertion that the framework does refuse.
"""

from __future__ import annotations

import copy
import json

import pytest

from testdriver import Actor, Cast, Oracle, Runner, World
from testdriver.runner import CollectorIndependenceError
from lab.mutations import ObservationChannel, build_lab
from testdriver import DirectDriver, StateObserver
from scenarios.alice_bob_carol import USE_CASE, build
from tests.selfverification.checks import (
    check_actor_isolation,
    check_evidence_supports_every_verdict,
    check_intent_independence,
    check_isolation_was_examined,
    check_no_actor_collected_judgment,
    check_runs_agree,
    check_verdicts_follow_from_judgment_evidence,
)
from tests.selfverification.test_self_verification import SECRETS, run_and_serialize


# --- td://self/actor-isolation -------------------------------------------


def test_shared_actor_memory_is_caught():
    """A structural break: two actors handed the same private store.

    This is what an isolation bug actually looks like when the same agent
    technology executes several actors in one process — nobody writes
    `bob.memory = alice.memory`, but a shared default or a cached client
    achieves it by accident.
    """
    lab, tokens = build_lab()
    shared: dict = {}
    cast = Cast()
    for name in ("alice", "bob", "carol"):
        actor = Actor(name, name.title(), credentials={"token": tokens[name]})
        object.__setattr__(actor, "_memory", shared)  # the leak
        cast.add(actor)

    _, _, _, asset, _ = build()
    world = World("w-leaky", lab, lab.version, cast=cast)
    driver = DirectDriver(lab, tokens)
    observer = StateObserver(ObservationChannel(lab), asset.scenario.watches)

    for actor_id, secret in SECRETS.items():
        world.cast[actor_id].remember(f"private-{actor_id}", secret)

    result = Runner(world, driver, observer, Oracle()).run(asset)
    pack = json.loads(result.evidence.to_json())
    memories = {a.id: {k: a.recall(k) for k in a.known_keys()} for a in world.cast}

    violations = check_actor_isolation(pack, SECRETS, memories)
    assert violations, "shared actor memory went undetected"
    assert any("holds" in v for v in violations)


def test_a_secret_leaking_into_the_record_is_caught():
    """An artefact break: the leak escapes into evidence even if memory is clean."""
    pack, memories, _ = run_and_serialize()
    tampered = copy.deepcopy(pack)
    for obs in tampered["observations"]:
        if obs["collector"] == "alice":
            obs["data"]["stolen"] = SECRETS["bob"]
            break
    assert check_actor_isolation(tampered, SECRETS, memories)


# --- td://self/oracle-independence ---------------------------------------


def test_the_framework_refuses_to_let_an_actor_collect_judgment_evidence():
    """The architecture prevents this, so it cannot be produced by a real run."""
    world, driver, observer, asset, oracle = build()
    observer.name = "bob"
    with pytest.raises(CollectorIndependenceError):
        Runner(world, driver, observer, oracle).run(asset)


def test_actor_collected_judgment_evidence_is_caught_in_the_record():
    """...and if it ever did appear in a pack, the out-of-band check sees it.

    Both halves are needed. The first says the door is locked; this one says we
    would notice if someone came through the window.
    """
    pack, _, actors = run_and_serialize()
    tampered = copy.deepcopy(pack)
    for obs in tampered["observations"]:
        if obs["stratum"] == "S3":
            obs["collector"] = "bob"
            break
    assert check_no_actor_collected_judgment(tampered, actors)


def test_a_verdict_that_contradicts_its_evidence_is_caught():
    """The strongest of the checks: verdicts must follow from S3 alone."""
    pack, _, _ = run_and_serialize()
    tampered = copy.deepcopy(pack)
    tampered["verdicts"][0]["verdict"] = "FAIL"  # a verdict nothing supports
    assertions = list(USE_CASE.claims) + list(USE_CASE.invariants)
    violations = check_verdicts_follow_from_judgment_evidence(tampered, assertions)
    assert violations
    assert "retained judgment evidence yields" in violations[0]


def test_a_pass_silently_replacing_a_fail_is_caught():
    """The specific corruption the project most needs to notice."""
    pack, _, _ = run_and_serialize("M15")
    tampered = copy.deepcopy(pack)
    for verdict in tampered["verdicts"]:
        if verdict["verdict"] == "FAIL":
            verdict["verdict"] = "PASS"
    assertions = list(USE_CASE.claims) + list(USE_CASE.invariants)
    assert check_verdicts_follow_from_judgment_evidence(tampered, assertions)


# --- td://self/evidence-reproducibility -----------------------------------


def test_a_verdict_without_a_snapshot_is_caught():
    pack, _, _ = run_and_serialize()
    tampered = copy.deepcopy(pack)
    tampered["observations"] = [
        o for o in tampered["observations"] if o["kind"] != "state_snapshot"
    ]
    assert check_evidence_supports_every_verdict(tampered)


def test_a_verdict_without_a_record_of_how_the_step_ran_is_caught():
    pack, _, _ = run_and_serialize()
    tampered = copy.deepcopy(pack)
    tampered["observations"] = [
        o for o in tampered["observations"] if o["kind"] != "realization"
    ]
    violations = check_evidence_supports_every_verdict(tampered)
    assert any("how" in v for v in violations)


def test_evidence_that_does_not_name_the_version_is_caught():
    pack, _, _ = run_and_serialize()
    tampered = copy.deepcopy(pack)
    tampered["sut_version"] = ""
    assert check_evidence_supports_every_verdict(tampered)


def test_disagreeing_runs_are_caught():
    first, _, _ = run_and_serialize()
    second, _, _ = run_and_serialize("M15")
    assert check_runs_agree(first, second)


# --- td://self/intent-independence ----------------------------------------


def test_implementation_derived_intent_is_caught():
    pack, _, _ = run_and_serialize("M15")
    tampered = copy.deepcopy(pack)
    tampered["provenance_index"]["c-bob-revoked"] = "agent-from-implementation"
    violations = check_intent_independence(tampered)
    assert violations
    assert "derived from the implementation" in violations[0]


def test_unrecorded_provenance_is_caught():
    """Absent provenance is not a benign omission — it makes the verdict
    unauditable, which for this framework is the same as unusable."""
    pack, _, _ = run_and_serialize()
    tampered = copy.deepcopy(pack)
    tampered["provenance_index"] = {}
    assert check_intent_independence(tampered)


# --- F-0003 resolution: isolation observed on every run -------------------


def test_an_unexamined_run_is_caught():
    """A run that never looked at isolation must not read as isolated."""
    pack, _, _ = run_and_serialize()
    tampered = copy.deepcopy(pack)
    tampered["observations"] = [
        o for o in tampered["observations"] if o["kind"] != "actor_isolation"
    ]
    assert check_isolation_was_examined(tampered)


def test_a_leak_is_caught_without_the_test_planting_anything():
    """The regression F-0003 leaves behind.

    No secrets seeded by the harness, no scenario written to expose isolation.
    An actor holding another's automatic marker is caught by the ordinary run.
    """
    from testdriver import Oracle, Runner
    from scenarios.alice_bob_carol import build

    world, driver, observer, asset, oracle = build()
    world.cast["bob"].remember("overheard", world.cast["alice"].canary)
    result = Runner(world, driver, observer, oracle).run(asset)
    pack = json.loads(result.evidence.to_json())

    violations = check_isolation_was_examined(pack)
    assert violations
    assert "holds the private marker of 'alice'" in violations[0]