T04: deterministic semantic kernel
Alice/Bob/Carol runs end to end, deterministically, replayable from seed. 16 tests pass, no third-party dependencies. - src/testdriver: intent, provenance, world, actions, drivers, observers, oracles, evidence, energy, scenario, runner - lab/minimal.py: the SUT, exposing the independent observation channel required by D-07 - evidence is stratified S1/S2/S3; Runner refuses to attribute S2/S3 to an actor; claims are frozen and provenance-checked at construction - missing evidence yields INCONCLUSIVE, which outranks PASS in the run verdict - EnergyEvents captured, no scoring (H-005 dormant) The observation channel records both stored state and an out-of-band enforcement probe; their disagreement is an invariant and is what detects an authorization defect that leaves the audit trail intact. A seeded RevokeIsCosmetic lab fails the run via both the claim and that invariant. Also closes TD-WP-0001-T02 (stack and commands now exist). Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Assistant: claude-code Assistant-Model: opus Assistant-Process: 1629012@bnt-lap001 Assistant-Session: 78d4fb13-8a1e-474b-87a3-9b9261c49a39
This commit is contained in:
parent
8da4c5bf7a
commit
04e9573b5a
42 changed files with 1533 additions and 20 deletions
Binary file not shown.
BIN
tests/__pycache__/test_kernel_guarantees.cpython-312.pyc
Normal file
BIN
tests/__pycache__/test_kernel_guarantees.cpython-312.pyc
Normal file
Binary file not shown.
Binary file not shown.
134
tests/test_kernel_guarantees.py
Normal file
134
tests/test_kernel_guarantees.py
Normal file
|
|
@ -0,0 +1,134 @@
|
|||
"""The kernel must enforce the properties the T02 design claims for it.
|
||||
|
||||
These are unit-level checks on the kernel's construction. The behavioural
|
||||
self-verification suite (actor isolation, oracle independence, evidence
|
||||
reproducibility, intent independence) is TD-WP-0002-T06 and lives separately.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import dataclasses
|
||||
|
||||
import pytest
|
||||
|
||||
from testdriver import (
|
||||
Claim, InadmissibleProvenance, Invariant, Oracle, Provenance, Runner,
|
||||
SemanticAction, StateObserver, SurfaceNotPermitted, Stratum, Verdict,
|
||||
)
|
||||
from lab.minimal import Denied, MinimalLab, ObservationChannel, build_baseline
|
||||
from scenarios.alice_bob_carol import build
|
||||
|
||||
|
||||
def test_implementation_derived_claims_are_rejected():
|
||||
"""D-06: a claim derived from watching the system cannot constrain it."""
|
||||
with pytest.raises(InadmissibleProvenance) as exc:
|
||||
Claim(
|
||||
"c-observed", "whatever the system currently does",
|
||||
Provenance.AGENT_FROM_IMPLEMENTATION,
|
||||
lambda obs: True, after_step="s1",
|
||||
)
|
||||
assert "derived from the implementation" in str(exc.value)
|
||||
|
||||
|
||||
def test_spec_and_human_provenance_are_admissible():
|
||||
for provenance in (Provenance.HUMAN, Provenance.SPEC, Provenance.AGENT_FROM_SPEC):
|
||||
Invariant("i-ok", "fine", provenance, lambda obs: True)
|
||||
|
||||
|
||||
def test_claims_are_frozen():
|
||||
"""D-02: claims are run inputs. No adaptation path may rewrite them."""
|
||||
claim = Claim("c", "text", Provenance.HUMAN, lambda obs: True, after_step="s1")
|
||||
with pytest.raises(dataclasses.FrozenInstanceError):
|
||||
claim.text = "something the implementation would prefer"
|
||||
|
||||
|
||||
def test_surface_substitution_is_refused():
|
||||
"""D-05: achieving an action through an unpermitted surface is not recovery."""
|
||||
action = SemanticAction(
|
||||
"grant_access", {"resource_id": "R"}, permitted_surfaces=frozenset({"browser"})
|
||||
)
|
||||
with pytest.raises(SurfaceNotPermitted):
|
||||
action.check_surface("api")
|
||||
|
||||
|
||||
def test_missing_evidence_yields_inconclusive_not_pass():
|
||||
"""An unjudgeable assertion must never default either way."""
|
||||
claim = Claim(
|
||||
"c-needs-missing", "depends on evidence nobody collected",
|
||||
Provenance.HUMAN, lambda obs: obs["probe_read:dave:R"], after_step="s1",
|
||||
)
|
||||
judgment = Oracle().judge(claim, {"probe_read:bob:R": True}, "s1")
|
||||
assert judgment.verdict is Verdict.INCONCLUSIVE
|
||||
assert "missing from snapshot" in judgment.detail["reason"]
|
||||
|
||||
|
||||
def test_inconclusive_outranks_pass_in_the_run_verdict():
|
||||
from testdriver.oracles import Judgment, overall
|
||||
|
||||
assert overall([
|
||||
Judgment("a", "", Verdict.PASS, None),
|
||||
Judgment("b", "", Verdict.INCONCLUSIVE, None),
|
||||
]) is Verdict.INCONCLUSIVE
|
||||
|
||||
|
||||
def test_actors_cannot_be_recorded_as_judgment_collectors():
|
||||
"""Wiring an actor in as an observer must fail loudly, not silently pass."""
|
||||
world, driver, observer, asset, oracle = build()
|
||||
observer.name = "alice" # an actor pretending to be the observer
|
||||
with pytest.raises(Exception) as exc:
|
||||
Runner(world, driver, observer, oracle).run(asset)
|
||||
assert "do not judge their own outcomes" in str(exc.value)
|
||||
|
||||
|
||||
# --- preview of the M05 authorization defect (built properly in T05) --------
|
||||
|
||||
|
||||
class RevokeIsCosmetic(MinimalLab):
|
||||
"""Revocation updates the record and the audit trail but not enforcement.
|
||||
|
||||
This is the shape of a real authorization defect: everything an operator
|
||||
would look at says access was removed, and access was not removed.
|
||||
"""
|
||||
|
||||
def _may_read(self, user_id: str, resource_id: str) -> bool:
|
||||
resource = self.resources.get(resource_id)
|
||||
if resource is None:
|
||||
return False
|
||||
if resource["owner"] == user_id:
|
||||
return True
|
||||
return any(
|
||||
r.event == "grant" and r.subject_id == user_id and r.resource_id == resource_id
|
||||
for r in self.audit
|
||||
)
|
||||
|
||||
|
||||
def test_seeded_authorization_defect_fails_the_run():
|
||||
"""The kernel must report FAIL, not adapt, when revocation does not revoke."""
|
||||
from testdriver import Actor, Cast, DirectDriver, World
|
||||
from scenarios.alice_bob_carol import USE_CASE
|
||||
from scenarios import alice_bob_carol as ref
|
||||
|
||||
lab = RevokeIsCosmetic(version="lab-0.1.0-M05-auth-defect")
|
||||
tokens = {u: lab.add_user(u) for u in ("alice", "bob", "carol")}
|
||||
cast = Cast()
|
||||
for name in tokens:
|
||||
cast.add(Actor(name, name.title(), credentials={"token": tokens[name]}))
|
||||
|
||||
_, _, _, baseline_asset, _ = ref.build()
|
||||
world = World("w-defect", lab, lab.version, cast=cast)
|
||||
driver = DirectDriver(lab, tokens)
|
||||
observer = StateObserver(ObservationChannel(lab), baseline_asset.scenario.watches)
|
||||
|
||||
result = Runner(world, driver, observer, Oracle()).run(baseline_asset)
|
||||
|
||||
assert result.verdict is Verdict.FAIL
|
||||
assert result.judgment("c-bob-revoked").verdict is Verdict.FAIL
|
||||
# The claim set is untouched by the failure — there is no path to adapt it.
|
||||
assert USE_CASE.claims[2].text == "Bob cannot read R after revocation"
|
||||
|
||||
|
||||
def test_defect_run_emits_an_energy_event():
|
||||
"""Energy events are captured; no score is computed (H-005 is dormant)."""
|
||||
import testdriver.energy as energy
|
||||
|
||||
assert not hasattr(energy, "score")
|
||||
70
tests/test_reference_scenario.py
Normal file
70
tests/test_reference_scenario.py
Normal file
|
|
@ -0,0 +1,70 @@
|
|||
"""The reference scenario must run deterministically and be replayable."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from testdriver import Runner, Stratum, Verdict
|
||||
from scenarios.alice_bob_carol import build
|
||||
|
||||
|
||||
def run_once(variant: str = "baseline"):
|
||||
world, driver, observer, asset, oracle = build(variant)
|
||||
return Runner(world, driver, observer, oracle).run(asset), world
|
||||
|
||||
|
||||
def test_reference_scenario_passes():
|
||||
result, _ = run_once()
|
||||
assert result.verdict is Verdict.PASS, [
|
||||
j.as_dict() for j in result.judgments if j.verdict is not Verdict.PASS
|
||||
]
|
||||
|
||||
|
||||
def test_every_claim_is_judged():
|
||||
result, _ = run_once()
|
||||
judged = {j.assertion_id for j in result.judgments}
|
||||
assert {"c-bob-reads", "c-carol-denied", "c-bob-revoked"} <= judged
|
||||
|
||||
|
||||
def test_invariants_are_evaluated_after_every_step():
|
||||
result, _ = run_once()
|
||||
per_step = [j for j in result.judgments if j.assertion_id == "i-audit-append-only"]
|
||||
assert len(per_step) == 3
|
||||
|
||||
|
||||
def test_run_is_replayable_from_known_initial_state():
|
||||
"""Two runs from the same seed produce identical judgments."""
|
||||
first, _ = run_once()
|
||||
second, _ = run_once()
|
||||
assert [(j.assertion_id, j.verdict) for j in first.judgments] == [
|
||||
(j.assertion_id, j.verdict) for j in second.judgments
|
||||
]
|
||||
assert first.run_id != second.run_id
|
||||
|
||||
|
||||
def test_evidence_is_stratified_and_serializable():
|
||||
result, _ = run_once()
|
||||
pack = result.evidence
|
||||
assert pack.of_stratum(Stratum.SURFACE)
|
||||
assert pack.of_stratum(Stratum.REALIZATION)
|
||||
assert pack.of_stratum(Stratum.JUDGMENT)
|
||||
parsed = json.loads(pack.to_json())
|
||||
assert parsed["run_id"] == result.run_id
|
||||
assert parsed["sut_version"] == "lab-0.1.0-baseline"
|
||||
|
||||
|
||||
def test_evidence_records_claim_provenance():
|
||||
"""A verdict must be auditable for the independence of the claim behind it."""
|
||||
result, _ = run_once()
|
||||
assert result.evidence.provenance_index["c-bob-revoked"] == "human"
|
||||
|
||||
|
||||
def test_actors_hold_isolated_credentials_and_memory():
|
||||
_, world = run_once()
|
||||
alice, bob = world.cast["alice"], world.cast["bob"]
|
||||
assert alice.credentials["token"] != bob.credentials["token"]
|
||||
alice.remember("secret", "only alice knows this")
|
||||
assert bob.recall("secret") is None
|
||||
assert bob.known_keys() == ()
|
||||
Loading…
Add table
Add a link
Reference in a new issue