tegwick
4ddb2f896c
lab/app.py (users, tenants, auth, resources, sharing, read/write, revoke, audit), lab/http_api.py (JSON API + browser UI, stdlib only), 20 labelled composable version-stamped mutations, ground-truth matrix. 48 tests pass. Detection against the reference scenario: MECHANICAL 0/10 flagged (correct), DEFECT 6/6, SEMANTIC 2/4 with both inert cases declared. - F-0002: M16 and M18 initially escaped detection entirely. A use case protects exactly what it asserts. Resolved by adding two claims already stated as intent in INTENT.md; the six-mutation catalogue would never have surfaced this. - test-id axis added: stable selectors survive most UI mutations, which would make H-001 trivially false. Mutations now vary on preserves_test_ids so the hypothesis is analysed split by that axis rather than rigged. - M12 (semantic deferred revoke) and M19 (defect race) are behaviourally identical and asserted as such - the discrimination problem as a test. lab/minimal.py removed; superseded by lab/app.py. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Assistant: claude-code Assistant-Model: opus Assistant-Process: 1629012@bnt-lap001 Assistant-Session: 78d4fb13-8a1e-474b-87a3-9b9261c49a39
70 lines
2.2 KiB
Python
70 lines
2.2 KiB
Python
"""The reference scenario must run deterministically and be replayable."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
|
|
import pytest
|
|
|
|
from testdriver import Runner, Stratum, Verdict
|
|
from scenarios.alice_bob_carol import build
|
|
|
|
|
|
def run_once(*mutations: str):
|
|
world, driver, observer, asset, oracle = build(*mutations)
|
|
return Runner(world, driver, observer, oracle).run(asset), world
|
|
|
|
|
|
def test_reference_scenario_passes():
|
|
result, _ = run_once()
|
|
assert result.verdict is Verdict.PASS, [
|
|
j.as_dict() for j in result.judgments if j.verdict is not Verdict.PASS
|
|
]
|
|
|
|
|
|
def test_every_claim_is_judged():
|
|
result, _ = run_once()
|
|
judged = {j.assertion_id for j in result.judgments}
|
|
assert {"c-bob-reads", "c-carol-denied", "c-bob-revoked"} <= judged
|
|
|
|
|
|
def test_invariants_are_evaluated_after_every_step():
|
|
result, _ = run_once()
|
|
per_step = [j for j in result.judgments if j.assertion_id == "i-audit-append-only"]
|
|
assert len(per_step) == 3
|
|
|
|
|
|
def test_run_is_replayable_from_known_initial_state():
|
|
"""Two runs from the same seed produce identical judgments."""
|
|
first, _ = run_once()
|
|
second, _ = run_once()
|
|
assert [(j.assertion_id, j.verdict) for j in first.judgments] == [
|
|
(j.assertion_id, j.verdict) for j in second.judgments
|
|
]
|
|
assert first.run_id != second.run_id
|
|
|
|
|
|
def test_evidence_is_stratified_and_serializable():
|
|
result, _ = run_once()
|
|
pack = result.evidence
|
|
assert pack.of_stratum(Stratum.SURFACE)
|
|
assert pack.of_stratum(Stratum.REALIZATION)
|
|
assert pack.of_stratum(Stratum.JUDGMENT)
|
|
parsed = json.loads(pack.to_json())
|
|
assert parsed["run_id"] == result.run_id
|
|
assert parsed["sut_version"] == "lab-0.2.0-baseline"
|
|
|
|
|
|
def test_evidence_records_claim_provenance():
|
|
"""A verdict must be auditable for the independence of the claim behind it."""
|
|
result, _ = run_once()
|
|
assert result.evidence.provenance_index["c-bob-revoked"] == "human"
|
|
|
|
|
|
def test_actors_hold_isolated_credentials_and_memory():
|
|
_, world = run_once()
|
|
alice, bob = world.cast["alice"], world.cast["bob"]
|
|
assert alice.credentials["token"] != bob.credentials["token"]
|
|
alice.remember("secret", "only alice knows this")
|
|
assert bob.recall("secret") is None
|
|
assert bob.known_keys() == ()
|