tegwick
5734b280c6
Checks written as plain functions over a serialized Evidence Pack, outside the framework - no Oracle, no Runner, no Verdict aggregation. 12 tests that they hold, 12 that they can fail. All four td://self identifiers covered. The substantive check is verdict reproducibility from S3 evidence alone, asserted on failing runs as well as passing ones. F-0003 (open): actor isolation leaves no trace in ordinary evidence - the self-test catches a shared memory store only because the harness plants per-actor canaries. Isolation is currently a property of a scenario written to expose it, not of runs in general. The mirror-image case is noted too: a guarantee enforced by construction cannot be verified by observing real runs, so four green self-tests are not four equivalent proofs. Carried to T10. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Assistant: claude-code Assistant-Model: opus Assistant-Process: 1629012@bnt-lap001 Assistant-Session: 78d4fb13-8a1e-474b-87a3-9b9261c49a39
110 lines
3.6 KiB
Python
110 lines
3.6 KiB
Python
"""The framework's foundational guarantees, verified from outside the framework.
|
|
|
|
These read an Evidence Pack the way an auditor would — as JSON, with no live
|
|
objects — and use plain pytest. No Oracle, no Runner, no Verdict aggregation.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
|
|
import pytest
|
|
|
|
from testdriver import Runner
|
|
from scenarios.alice_bob_carol import USE_CASE, build
|
|
from tests.selfverification.checks import (
|
|
SELF_CHECKS,
|
|
check_actor_isolation,
|
|
check_evidence_supports_every_verdict,
|
|
check_intent_independence,
|
|
check_no_actor_collected_judgment,
|
|
check_runs_agree,
|
|
check_verdicts_follow_from_judgment_evidence,
|
|
)
|
|
|
|
SECRETS = {
|
|
"alice": "alice-only-9f3c",
|
|
"bob": "bob-only-2a71",
|
|
"carol": "carol-only-55e0",
|
|
}
|
|
|
|
|
|
def run_and_serialize(*mutations: str):
|
|
"""Run, then throw away everything except the serialized evidence.
|
|
|
|
Deliberate: the checks must work from the artefact alone, exactly as they
|
|
would months later, with no access to the objects that produced it.
|
|
"""
|
|
world, driver, observer, asset, oracle = build(*mutations)
|
|
for actor_id, secret in SECRETS.items():
|
|
world.cast[actor_id].remember("private", secret)
|
|
result = Runner(world, driver, observer, oracle).run(asset)
|
|
pack = json.loads(result.evidence.to_json())
|
|
memories = {a.id: {k: a.recall(k) for k in a.known_keys()} for a in world.cast}
|
|
return pack, memories, list(world.cast.actors)
|
|
|
|
|
|
def test_the_registry_is_complete():
|
|
assert len(SELF_CHECKS) == 4
|
|
|
|
|
|
# --- td://self/actor-isolation -------------------------------------------
|
|
|
|
|
|
def test_actor_isolation():
|
|
pack, memories, _ = run_and_serialize()
|
|
assert check_actor_isolation(pack, SECRETS, memories) == []
|
|
|
|
|
|
# --- td://self/oracle-independence ---------------------------------------
|
|
|
|
|
|
def test_no_actor_collected_judgment_evidence():
|
|
pack, _, actors = run_and_serialize()
|
|
assert check_no_actor_collected_judgment(pack, actors) == []
|
|
|
|
|
|
@pytest.mark.parametrize("mutations", [(), ("M15",), ("M11",), ("M01", "M15")])
|
|
def test_verdicts_are_reproducible_from_judgment_evidence_alone(mutations):
|
|
"""Holds on passing and failing runs alike — a check that only works when
|
|
everything is green is not verifying independence, it is verifying luck."""
|
|
pack, _, _ = run_and_serialize(*mutations)
|
|
assertions = list(USE_CASE.claims) + list(USE_CASE.invariants)
|
|
assert check_verdicts_follow_from_judgment_evidence(pack, assertions) == []
|
|
|
|
|
|
# --- td://self/evidence-reproducibility -----------------------------------
|
|
|
|
|
|
def test_every_verdict_is_supported_by_retained_evidence():
|
|
pack, _, _ = run_and_serialize()
|
|
assert check_evidence_supports_every_verdict(pack) == []
|
|
|
|
|
|
def test_two_runs_from_the_same_seed_agree():
|
|
first, _, _ = run_and_serialize()
|
|
second, _, _ = run_and_serialize()
|
|
assert check_runs_agree(first, second) == []
|
|
assert first["run_id"] != second["run_id"]
|
|
|
|
|
|
def test_reproducibility_holds_on_a_failing_run():
|
|
first, _, _ = run_and_serialize("M15")
|
|
second, _, _ = run_and_serialize("M15")
|
|
assert check_runs_agree(first, second) == []
|
|
assert any(v["verdict"] == "FAIL" for v in first["verdicts"])
|
|
|
|
|
|
# --- td://self/intent-independence ----------------------------------------
|
|
|
|
|
|
def test_intent_independence():
|
|
pack, _, _ = run_and_serialize()
|
|
assert check_intent_independence(pack) == []
|
|
|
|
|
|
def test_intent_independence_holds_when_a_claim_fails():
|
|
"""The moment that matters: a FAIL is an accusation, and an accusation
|
|
resting on implementation-derived intent is worthless."""
|
|
pack, _, _ = run_and_serialize("M15")
|
|
assert check_intent_independence(pack) == []
|