T04: deterministic semantic kernel
Alice/Bob/Carol runs end to end, deterministically, replayable from seed. 16 tests pass, no third-party dependencies. - src/testdriver: intent, provenance, world, actions, drivers, observers, oracles, evidence, energy, scenario, runner - lab/minimal.py: the SUT, exposing the independent observation channel required by D-07 - evidence is stratified S1/S2/S3; Runner refuses to attribute S2/S3 to an actor; claims are frozen and provenance-checked at construction - missing evidence yields INCONCLUSIVE, which outranks PASS in the run verdict - EnergyEvents captured, no scoring (H-005 dormant) The observation channel records both stored state and an out-of-band enforcement probe; their disagreement is an invariant and is what detects an authorization defect that leaves the audit trail intact. A seeded RevokeIsCosmetic lab fails the run via both the claim and that invariant. Also closes TD-WP-0001-T02 (stack and commands now exist). Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Assistant: claude-code Assistant-Model: opus Assistant-Process: 1629012@bnt-lap001 Assistant-Session: 78d4fb13-8a1e-474b-87a3-9b9261c49a39
This commit is contained in:
parent
8da4c5bf7a
commit
04e9573b5a
42 changed files with 1533 additions and 20 deletions
23
src/testdriver/__init__.py
Normal file
23
src/testdriver/__init__.py
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
"""test-driver — verification assets that mature alongside the software they protect."""
|
||||
|
||||
from .actions import SemanticAction, Surface, SurfaceNotPermitted
|
||||
from .drivers import DirectDriver, Realization
|
||||
from .energy import EnergyEvent, EnergyEventType
|
||||
from .evidence import EvidencePack, Observation, Stratum
|
||||
from .intent import Claim, Invariant, UseCase
|
||||
from .observers import StateObserver, Watch
|
||||
from .oracles import Judgment, Oracle, Verdict, overall
|
||||
from .provenance import InadmissibleProvenance, Provenance
|
||||
from .runner import CollectorIndependenceError, Runner, RunResult
|
||||
from .scenario import Scenario, Step, VerificationAsset
|
||||
from .world import Actor, ActorIsolationError, Cast, World
|
||||
|
||||
__all__ = [
|
||||
"Actor", "ActorIsolationError", "Cast", "Claim", "CollectorIndependenceError",
|
||||
"DirectDriver", "EnergyEvent", "EnergyEventType", "EvidencePack",
|
||||
"InadmissibleProvenance", "Invariant", "Judgment", "Observation", "Oracle",
|
||||
"Provenance", "Realization", "RunResult", "Runner", "Scenario",
|
||||
"SemanticAction", "StateObserver", "Step", "Stratum", "Surface",
|
||||
"SurfaceNotPermitted", "UseCase", "VerificationAsset", "Verdict", "Watch",
|
||||
"overall",
|
||||
]
|
||||
BIN
src/testdriver/__pycache__/__init__.cpython-312.pyc
Normal file
BIN
src/testdriver/__pycache__/__init__.cpython-312.pyc
Normal file
Binary file not shown.
BIN
src/testdriver/__pycache__/actions.cpython-312.pyc
Normal file
BIN
src/testdriver/__pycache__/actions.cpython-312.pyc
Normal file
Binary file not shown.
BIN
src/testdriver/__pycache__/drivers.cpython-312.pyc
Normal file
BIN
src/testdriver/__pycache__/drivers.cpython-312.pyc
Normal file
Binary file not shown.
BIN
src/testdriver/__pycache__/energy.cpython-312.pyc
Normal file
BIN
src/testdriver/__pycache__/energy.cpython-312.pyc
Normal file
Binary file not shown.
BIN
src/testdriver/__pycache__/evidence.cpython-312.pyc
Normal file
BIN
src/testdriver/__pycache__/evidence.cpython-312.pyc
Normal file
Binary file not shown.
BIN
src/testdriver/__pycache__/intent.cpython-312.pyc
Normal file
BIN
src/testdriver/__pycache__/intent.cpython-312.pyc
Normal file
Binary file not shown.
BIN
src/testdriver/__pycache__/observers.cpython-312.pyc
Normal file
BIN
src/testdriver/__pycache__/observers.cpython-312.pyc
Normal file
Binary file not shown.
BIN
src/testdriver/__pycache__/oracles.cpython-312.pyc
Normal file
BIN
src/testdriver/__pycache__/oracles.cpython-312.pyc
Normal file
Binary file not shown.
BIN
src/testdriver/__pycache__/provenance.cpython-312.pyc
Normal file
BIN
src/testdriver/__pycache__/provenance.cpython-312.pyc
Normal file
Binary file not shown.
BIN
src/testdriver/__pycache__/runner.cpython-312.pyc
Normal file
BIN
src/testdriver/__pycache__/runner.cpython-312.pyc
Normal file
Binary file not shown.
BIN
src/testdriver/__pycache__/scenario.cpython-312.pyc
Normal file
BIN
src/testdriver/__pycache__/scenario.cpython-312.pyc
Normal file
Binary file not shown.
BIN
src/testdriver/__pycache__/world.cpython-312.pyc
Normal file
BIN
src/testdriver/__pycache__/world.cpython-312.pyc
Normal file
Binary file not shown.
57
src/testdriver/actions.py
Normal file
57
src/testdriver/actions.py
Normal file
|
|
@ -0,0 +1,57 @@
|
|||
"""Semantic actions and the surfaces they may legitimately use.
|
||||
|
||||
A semantic action names *what* is being accomplished, never *how*. The `how` is
|
||||
a driver's business and is expected to change; the `what` is the stable identity
|
||||
that survives restructuring (H-001).
|
||||
|
||||
Every action declares the surfaces it is permitted to use — decision D-05. An
|
||||
actor that achieves `grant_access` through a surface the scenario did not permit
|
||||
has not recovered from a change; it has performed an unrequested
|
||||
surface-substitution, which is one of the catalogued security mutations. Without
|
||||
this check, an agent can route around a broken authorization control and score
|
||||
as a successful mechanical adaptation.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Callable, Mapping
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Surface:
|
||||
"""An interaction surface: an API, a browser UI, a CLI, a queue."""
|
||||
|
||||
id: str
|
||||
kind: str
|
||||
description: str = ""
|
||||
|
||||
|
||||
class SurfaceNotPermitted(Exception):
|
||||
"""An action was realized through a surface the scenario did not permit."""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class SemanticAction:
|
||||
"""Intent to change or inspect the world, expressed without mechanics.
|
||||
|
||||
`postcondition` is evaluated by an independent Observer (S2), never by the
|
||||
actor that performed the action. An actor never reports its own success.
|
||||
"""
|
||||
|
||||
name: str
|
||||
args: Mapping[str, Any] = field(default_factory=dict)
|
||||
permitted_surfaces: frozenset[str] = frozenset()
|
||||
postcondition: Callable[[Mapping[str, object]], bool] | None = None
|
||||
|
||||
def check_surface(self, surface_id: str) -> None:
|
||||
if self.permitted_surfaces and surface_id not in self.permitted_surfaces:
|
||||
raise SurfaceNotPermitted(
|
||||
f"action {self.name!r} was realized through surface "
|
||||
f"{surface_id!r}, which is not in "
|
||||
f"{sorted(self.permitted_surfaces)} (D-05)"
|
||||
)
|
||||
|
||||
def describe(self) -> str:
|
||||
rendered = ", ".join(f"{k}={v!r}" for k, v in sorted(self.args.items()))
|
||||
return f"{self.name}({rendered})"
|
||||
76
src/testdriver/drivers.py
Normal file
76
src/testdriver/drivers.py
Normal file
|
|
@ -0,0 +1,76 @@
|
|||
"""Drivers realize semantic actions against a surface.
|
||||
|
||||
A driver knows mechanics. It emits S1 evidence describing *how* it acted and
|
||||
reports which surface it used, but it never reports whether the action was
|
||||
correct — that is S2/S3 and belongs to the observer and the oracle.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Protocol
|
||||
|
||||
from .actions import SemanticAction, Surface
|
||||
from .world import Actor
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Realization:
|
||||
"""What a driver did, mechanically. Pure S1."""
|
||||
|
||||
surface_id: str
|
||||
mechanics: dict[str, Any]
|
||||
raised: str | None = None
|
||||
|
||||
|
||||
class Driver(Protocol):
|
||||
surface: Surface
|
||||
|
||||
def realize(self, actor: Actor, action: SemanticAction) -> Realization: ...
|
||||
|
||||
|
||||
class UnsupportedAction(Exception):
|
||||
"""The driver has no mechanical implementation for this semantic action."""
|
||||
|
||||
|
||||
class DirectDriver:
|
||||
"""Deterministic driver against the lab's enforcement path.
|
||||
|
||||
This is the T5 Deterministic end of the maturity continuum: a fixed mapping
|
||||
from semantic action to mechanics, with no discovery and no model.
|
||||
"""
|
||||
|
||||
def __init__(self, lab: Any, tokens: dict[str, str]) -> None:
|
||||
self._lab = lab
|
||||
self._tokens = tokens
|
||||
self.surface = Surface(
|
||||
id="api", kind="http-like", description="lab enforcement path"
|
||||
)
|
||||
|
||||
_MAPPING = {
|
||||
"create_resource": ("create_resource", ("resource_id", "content")),
|
||||
"grant_access": ("grant", ("resource_id", "subject_id", "permission")),
|
||||
"revoke_access": ("revoke", ("resource_id", "subject_id")),
|
||||
"read_resource": ("read_resource", ("resource_id",)),
|
||||
}
|
||||
|
||||
def realize(self, actor: Actor, action: SemanticAction) -> Realization:
|
||||
if action.name not in self._MAPPING:
|
||||
raise UnsupportedAction(action.name)
|
||||
action.check_surface(self.surface.id)
|
||||
|
||||
op, arg_names = self._MAPPING[action.name]
|
||||
args = {name: action.args[name] for name in arg_names if name in action.args}
|
||||
token = self._tokens[actor.id]
|
||||
|
||||
mechanics: dict[str, Any] = {
|
||||
"operation": op,
|
||||
"arguments": args,
|
||||
"actor": actor.id,
|
||||
}
|
||||
try:
|
||||
result = self._lab.request(token, op, **args)
|
||||
except Exception as exc: # the SUT refusing is data, not a framework error
|
||||
return Realization(self.surface.id, mechanics, raised=f"{type(exc).__name__}: {exc}")
|
||||
mechanics["result"] = result
|
||||
return Realization(self.surface.id, mechanics)
|
||||
45
src/testdriver/energy.py
Normal file
45
src/testdriver/energy.py
Normal file
|
|
@ -0,0 +1,45 @@
|
|||
"""Energy events — capture only.
|
||||
|
||||
H-005 is dormant by decision: verification energy is not testable at the current
|
||||
scale, and a scoring function producing a number nobody can check is worse than
|
||||
no number. Events are recorded from the first run because history cannot be
|
||||
reconstructed later; scores always can.
|
||||
|
||||
There is deliberately no score() function in this module.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from enum import Enum
|
||||
from typing import Any
|
||||
|
||||
|
||||
class EnergyEventType(str, Enum):
|
||||
DEFECT_DETECTED = "DEFECT_DETECTED"
|
||||
REGRESSION_CAUGHT = "REGRESSION_CAUGHT"
|
||||
MECHANICAL_ADAPTATION = "MECHANICAL_ADAPTATION"
|
||||
SEMANTIC_ADAPTATION = "SEMANTIC_ADAPTATION"
|
||||
TEST_DEFECT = "TEST_DEFECT"
|
||||
FALSE_POSITIVE = "FALSE_POSITIVE"
|
||||
DUPLICATE = "DUPLICATE"
|
||||
CRYSTALLIZED = "CRYSTALLIZED"
|
||||
USECASE_DEPRECATED = "USECASE_DEPRECATED"
|
||||
EXECUTED = "EXECUTED"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class EnergyEvent:
|
||||
"""Immutable. Energy is derived from event history, never stored as state."""
|
||||
|
||||
asset_id: str
|
||||
run_id: str
|
||||
event_type: EnergyEventType
|
||||
detail: dict[str, Any] = field(default_factory=dict)
|
||||
at: str = field(
|
||||
default_factory=lambda: datetime.now(timezone.utc).isoformat()
|
||||
)
|
||||
|
||||
def as_dict(self) -> dict[str, Any]:
|
||||
return {**asdict(self), "event_type": self.event_type.value}
|
||||
86
src/testdriver/evidence.py
Normal file
86
src/testdriver/evidence.py
Normal file
|
|
@ -0,0 +1,86 @@
|
|||
"""Stratified evidence — decision D-01.
|
||||
|
||||
S1 Surface how an action was performed (selectors, routes, payloads)
|
||||
S2 Realization whether it happened, and through which surface
|
||||
S3 Judgment whether that was correct
|
||||
|
||||
Each stratum has a different authority. Models may write S1. Nothing but an
|
||||
independent Observer writes S2 or S3.
|
||||
|
||||
See docs/TestDriverClassificationDesign.md, Part A.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from dataclasses import dataclass, field, asdict
|
||||
from datetime import datetime, timezone
|
||||
from enum import Enum
|
||||
from typing import Any
|
||||
|
||||
|
||||
class Stratum(str, Enum):
|
||||
SURFACE = "S1"
|
||||
REALIZATION = "S2"
|
||||
JUDGMENT = "S3"
|
||||
|
||||
|
||||
def _now() -> str:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Observation:
|
||||
"""One recorded fact, attributed to a stratum and a collector.
|
||||
|
||||
`collector` is never an actor for S2/S3 observations. The runner enforces
|
||||
this; see runner._assert_collector_independence.
|
||||
"""
|
||||
|
||||
id: str
|
||||
stratum: Stratum
|
||||
collector: str
|
||||
step_id: str | None
|
||||
kind: str
|
||||
data: dict[str, Any]
|
||||
at: str = field(default_factory=_now)
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class EvidencePack:
|
||||
"""Everything retained from one run.
|
||||
|
||||
The pack must be sufficient to replay the run and to diagnose a finding
|
||||
without the original process. An assertion that cannot be supported from the
|
||||
pack is an EVIDENCE_FAILURE, not a defect in the system under test.
|
||||
"""
|
||||
|
||||
run_id: str
|
||||
scenario_id: str
|
||||
use_case_id: str
|
||||
sut_version: str
|
||||
started_at: str = field(default_factory=_now)
|
||||
finished_at: str | None = None
|
||||
observations: list[Observation] = field(default_factory=list)
|
||||
verdicts: list[dict[str, Any]] = field(default_factory=list)
|
||||
energy_events: list[dict[str, Any]] = field(default_factory=list)
|
||||
provenance_index: dict[str, str] = field(default_factory=dict)
|
||||
|
||||
def record(self, observation: Observation) -> None:
|
||||
self.observations.append(observation)
|
||||
|
||||
def of_stratum(self, stratum: Stratum) -> list[Observation]:
|
||||
return [o for o in self.observations if o.stratum is stratum]
|
||||
|
||||
def latest(self, kind: str) -> Observation | None:
|
||||
for observation in reversed(self.observations):
|
||||
if observation.kind == kind:
|
||||
return observation
|
||||
return None
|
||||
|
||||
def to_json(self) -> str:
|
||||
payload = asdict(self)
|
||||
payload["observations"] = [
|
||||
{**asdict(o), "stratum": o.stratum.value} for o in self.observations
|
||||
]
|
||||
return json.dumps(payload, indent=2, sort_keys=True, default=str)
|
||||
63
src/testdriver/intent.py
Normal file
63
src/testdriver/intent.py
Normal file
|
|
@ -0,0 +1,63 @@
|
|||
"""The intent layer: what is supposed to be true.
|
||||
|
||||
Claims and invariants are *inputs* to a run and are frozen — decision D-02.
|
||||
There is deliberately no code path by which adaptation, retry, or a learned
|
||||
trajectory can modify them. That absence is what makes False Adaptation Rate = 0
|
||||
an architectural property rather than a tuning target.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Callable, Mapping
|
||||
|
||||
from .provenance import Provenance, require_admissible
|
||||
|
||||
# A predicate over the independent observations gathered during a run.
|
||||
# It receives the observation mapping and returns True when satisfied.
|
||||
Predicate = Callable[[Mapping[str, object]], bool]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Claim:
|
||||
"""A statement that must hold at a specific point in a scenario."""
|
||||
|
||||
id: str
|
||||
text: str
|
||||
provenance: Provenance
|
||||
predicate: Predicate
|
||||
after_step: str
|
||||
source_ref: str | None = None
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
require_admissible(self.provenance, f"Claim {self.id!r}")
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Invariant:
|
||||
"""A statement that must hold after *every* step, not merely at one point."""
|
||||
|
||||
id: str
|
||||
text: str
|
||||
provenance: Provenance
|
||||
predicate: Predicate
|
||||
source_ref: str | None = None
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
require_admissible(self.provenance, f"Invariant {self.id!r}")
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class UseCase:
|
||||
"""Purposeful behaviour, described independently of mechanics."""
|
||||
|
||||
id: str
|
||||
title: str
|
||||
narrative: str
|
||||
provenance: Provenance
|
||||
claims: tuple[Claim, ...] = field(default_factory=tuple)
|
||||
invariants: tuple[Invariant, ...] = field(default_factory=tuple)
|
||||
source_ref: str | None = None
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
require_admissible(self.provenance, f"UseCase {self.id!r}")
|
||||
47
src/testdriver/observers.py
Normal file
47
src/testdriver/observers.py
Normal file
|
|
@ -0,0 +1,47 @@
|
|||
"""Observers gather evidence independently of the actors.
|
||||
|
||||
An observer never asks an actor what happened. It reads the system directly
|
||||
through the independent observation channel required by decision D-07, and
|
||||
records both what the stored record says and what the enforcement path actually
|
||||
does. Disagreement between those two is meaningful in its own right: it is the
|
||||
signature of an authorization defect, where the audit trail says one thing and
|
||||
enforcement does another.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Sequence
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Watch:
|
||||
"""A (subject, resource) pair the scenario wants observed after every step."""
|
||||
|
||||
subject_id: str
|
||||
resource_id: str
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class StateObserver:
|
||||
"""Collects the S3 snapshot: what is true of the domain right now."""
|
||||
|
||||
channel: Any
|
||||
watches: Sequence[Watch] = field(default_factory=tuple)
|
||||
name: str = "state-observer"
|
||||
|
||||
def snapshot(self) -> dict[str, Any]:
|
||||
out: dict[str, Any] = {}
|
||||
resources = set()
|
||||
for watch in self.watches:
|
||||
key = f"{watch.subject_id}:{watch.resource_id}"
|
||||
out[f"probe_read:{key}"] = self.channel.probe_read(
|
||||
watch.subject_id, watch.resource_id
|
||||
)
|
||||
out[f"state_permission:{key}"] = self.channel.state_permission(
|
||||
watch.subject_id, watch.resource_id
|
||||
)
|
||||
resources.add(watch.resource_id)
|
||||
for resource_id in sorted(resources):
|
||||
out[f"audit:{resource_id}"] = self.channel.audit_events(resource_id)
|
||||
return out
|
||||
106
src/testdriver/oracles.py
Normal file
106
src/testdriver/oracles.py
Normal file
|
|
@ -0,0 +1,106 @@
|
|||
"""Oracles evaluate claims and invariants and produce verdicts.
|
||||
|
||||
An oracle reads only the independent observation snapshot. It has no access to
|
||||
the actor, to the driver, or to what either of them believes happened.
|
||||
|
||||
`INCONCLUSIVE` is a first-class outcome, not a failure mode of the framework. An
|
||||
oracle that cannot support a judgment from the retained evidence must say so
|
||||
rather than defaulting either way — silently defaulting to PASS hides defects,
|
||||
and silently defaulting to FAIL trains people to ignore results.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from enum import Enum
|
||||
from typing import Any, Mapping
|
||||
|
||||
from .intent import Claim, Invariant
|
||||
|
||||
|
||||
class Verdict(str, Enum):
|
||||
PASS = "PASS"
|
||||
FAIL = "FAIL"
|
||||
SUSPICIOUS = "SUSPICIOUS"
|
||||
INCONCLUSIVE = "INCONCLUSIVE"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Judgment:
|
||||
assertion_id: str
|
||||
text: str
|
||||
verdict: Verdict
|
||||
step_id: str | None
|
||||
detail: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
def as_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"assertion_id": self.assertion_id,
|
||||
"text": self.text,
|
||||
"verdict": self.verdict.value,
|
||||
"step_id": self.step_id,
|
||||
"detail": self.detail,
|
||||
}
|
||||
|
||||
|
||||
class Oracle:
|
||||
"""Deterministic evaluation of one assertion against an observation snapshot."""
|
||||
|
||||
def judge(
|
||||
self,
|
||||
assertion: Claim | Invariant,
|
||||
snapshot: Mapping[str, Any],
|
||||
step_id: str | None,
|
||||
) -> Judgment:
|
||||
if not snapshot:
|
||||
return Judgment(
|
||||
assertion.id,
|
||||
assertion.text,
|
||||
Verdict.INCONCLUSIVE,
|
||||
step_id,
|
||||
{"reason": "no observations were collected"},
|
||||
)
|
||||
try:
|
||||
satisfied = assertion.predicate(snapshot)
|
||||
except KeyError as missing:
|
||||
# The evidence needed to judge this assertion was not collected.
|
||||
# That is an evidence failure, never a pass and never a fail.
|
||||
return Judgment(
|
||||
assertion.id,
|
||||
assertion.text,
|
||||
Verdict.INCONCLUSIVE,
|
||||
step_id,
|
||||
{"reason": f"required observation {missing} missing from snapshot"},
|
||||
)
|
||||
except Exception as exc:
|
||||
return Judgment(
|
||||
assertion.id,
|
||||
assertion.text,
|
||||
Verdict.INCONCLUSIVE,
|
||||
step_id,
|
||||
{"reason": f"predicate raised {type(exc).__name__}: {exc}"},
|
||||
)
|
||||
return Judgment(
|
||||
assertion.id,
|
||||
assertion.text,
|
||||
Verdict.PASS if satisfied else Verdict.FAIL,
|
||||
step_id,
|
||||
)
|
||||
|
||||
|
||||
def overall(judgments: list[Judgment]) -> Verdict:
|
||||
"""Aggregate run verdict.
|
||||
|
||||
FAIL dominates; INCONCLUSIVE outranks PASS. A run containing an unjudgeable
|
||||
assertion has not passed, whatever else it did.
|
||||
"""
|
||||
verdicts = {j.verdict for j in judgments}
|
||||
if not judgments:
|
||||
return Verdict.INCONCLUSIVE
|
||||
if Verdict.FAIL in verdicts:
|
||||
return Verdict.FAIL
|
||||
if Verdict.INCONCLUSIVE in verdicts:
|
||||
return Verdict.INCONCLUSIVE
|
||||
if Verdict.SUSPICIOUS in verdicts:
|
||||
return Verdict.SUSPICIOUS
|
||||
return Verdict.PASS
|
||||
46
src/testdriver/provenance.py
Normal file
46
src/testdriver/provenance.py
Normal file
|
|
@ -0,0 +1,46 @@
|
|||
"""Claim provenance — decision D-06.
|
||||
|
||||
A claim may only be authored by a source causally independent of the
|
||||
implementation it constrains. Without this rule the framework's guarantee
|
||||
reduces to "the implementation agrees with itself", which is exactly the failure
|
||||
the project exists to prevent.
|
||||
|
||||
See docs/TestDriverClassificationDesign.md, Part B.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from enum import Enum
|
||||
|
||||
|
||||
class Provenance(str, Enum):
|
||||
"""Where an intent artifact came from."""
|
||||
|
||||
HUMAN = "human"
|
||||
SPEC = "spec"
|
||||
AGENT_FROM_SPEC = "agent-from-spec"
|
||||
AGENT_FROM_IMPLEMENTATION = "agent-from-implementation"
|
||||
|
||||
@property
|
||||
def admissible_as_claim(self) -> bool:
|
||||
"""Whether this provenance may back an assertion that can produce FAIL.
|
||||
|
||||
`agent-from-implementation` is not forbidden as an activity — it is
|
||||
genuinely useful for T0 exploration. It is forbidden as a *claim*.
|
||||
Such output enters as an exploratory hypothesis and requires an explicit
|
||||
human acceptance event before it can constrain the system.
|
||||
"""
|
||||
return self is not Provenance.AGENT_FROM_IMPLEMENTATION
|
||||
|
||||
|
||||
class InadmissibleProvenance(Exception):
|
||||
"""Raised when implementation-derived intent is used as a claim."""
|
||||
|
||||
|
||||
def require_admissible(provenance: Provenance, what: str) -> None:
|
||||
if not provenance.admissible_as_claim:
|
||||
raise InadmissibleProvenance(
|
||||
f"{what} has provenance {provenance.value!r}, which is derived from the "
|
||||
"implementation it would constrain. Promote it through an explicit human "
|
||||
"acceptance event before using it as a claim (D-06)."
|
||||
)
|
||||
190
src/testdriver/runner.py
Normal file
190
src/testdriver/runner.py
Normal file
|
|
@ -0,0 +1,190 @@
|
|||
"""The orchestrator: executes a scenario and assembles its evidence.
|
||||
|
||||
The runner is the only component that sees everything. Actors see their own
|
||||
credentials and memory; drivers see mechanics; observers see the system; oracles
|
||||
see the observation snapshot. Keeping those views separate is what makes the
|
||||
independence claim structural rather than procedural.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
from .actions import SurfaceNotPermitted
|
||||
from .drivers import Driver
|
||||
from .energy import EnergyEvent, EnergyEventType
|
||||
from .evidence import EvidencePack, Observation, Stratum
|
||||
from .observers import StateObserver
|
||||
from .oracles import Judgment, Oracle, Verdict, overall
|
||||
from .scenario import Scenario, VerificationAsset
|
||||
from .world import World
|
||||
|
||||
|
||||
class CollectorIndependenceError(Exception):
|
||||
"""An actor was about to be recorded as the collector of S2/S3 evidence."""
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class RunResult:
|
||||
run_id: str
|
||||
verdict: Verdict
|
||||
judgments: list[Judgment]
|
||||
evidence: EvidencePack
|
||||
|
||||
def judgment(self, assertion_id: str) -> Judgment:
|
||||
for j in self.judgments:
|
||||
if j.assertion_id == assertion_id:
|
||||
return j
|
||||
raise KeyError(assertion_id)
|
||||
|
||||
|
||||
class Runner:
|
||||
def __init__(
|
||||
self,
|
||||
world: World,
|
||||
driver: Driver,
|
||||
observer: StateObserver,
|
||||
oracle: Oracle | None = None,
|
||||
) -> None:
|
||||
self._world = world
|
||||
self._driver = driver
|
||||
self._observer = observer
|
||||
self._oracle = oracle or Oracle()
|
||||
|
||||
# -- independence guards ---------------------------------------------
|
||||
|
||||
def _assert_collector_independence(self, stratum: Stratum, collector: str) -> None:
|
||||
"""S2 and S3 evidence may never be attributed to an actor.
|
||||
|
||||
This is the check that makes oracle independence falsifiable rather than
|
||||
merely asserted: wiring an actor in as an observer fails loudly here.
|
||||
"""
|
||||
if stratum is Stratum.SURFACE:
|
||||
return
|
||||
if collector in self._world.cast.actors:
|
||||
raise CollectorIndependenceError(
|
||||
f"{stratum.value} evidence cannot be collected by actor "
|
||||
f"{collector!r}; actors do not judge their own outcomes"
|
||||
)
|
||||
|
||||
def _record(
|
||||
self,
|
||||
pack: EvidencePack,
|
||||
stratum: Stratum,
|
||||
collector: str,
|
||||
kind: str,
|
||||
data: dict[str, Any],
|
||||
step_id: str | None,
|
||||
) -> None:
|
||||
self._assert_collector_independence(stratum, collector)
|
||||
pack.record(
|
||||
Observation(
|
||||
id=f"obs-{len(pack.observations) + 1:04d}",
|
||||
stratum=stratum,
|
||||
collector=collector,
|
||||
step_id=step_id,
|
||||
kind=kind,
|
||||
data=data,
|
||||
)
|
||||
)
|
||||
|
||||
# -- execution --------------------------------------------------------
|
||||
|
||||
def run(self, asset: VerificationAsset) -> RunResult:
|
||||
scenario: Scenario = asset.scenario
|
||||
run_id = f"run-{uuid.uuid4().hex[:12]}"
|
||||
pack = EvidencePack(
|
||||
run_id=run_id,
|
||||
scenario_id=scenario.id,
|
||||
use_case_id=scenario.use_case.id,
|
||||
sut_version=self._world.sut_version,
|
||||
)
|
||||
pack.provenance_index = {
|
||||
scenario.use_case.id: scenario.use_case.provenance.value,
|
||||
**{c.id: c.provenance.value for c in scenario.use_case.claims},
|
||||
**{i.id: i.provenance.value for i in scenario.use_case.invariants},
|
||||
}
|
||||
pack.energy_events.append(
|
||||
EnergyEvent(asset.id, run_id, EnergyEventType.EXECUTED).as_dict()
|
||||
)
|
||||
|
||||
judgments: list[Judgment] = []
|
||||
claims_by_step: dict[str, list] = {}
|
||||
for claim in scenario.use_case.claims:
|
||||
claims_by_step.setdefault(claim.after_step, []).append(claim)
|
||||
|
||||
for step in scenario.steps:
|
||||
actor = self._world.cast[step.actor_id]
|
||||
|
||||
# --- S1: how it was done -------------------------------------
|
||||
try:
|
||||
realization = self._driver.realize(actor, step.action)
|
||||
except SurfaceNotPermitted as exc:
|
||||
# D-05: routing around a control is a finding, not a recovery.
|
||||
self._record(
|
||||
pack, Stratum.REALIZATION, self._observer.name,
|
||||
"surface_violation",
|
||||
{"step": step.id, "action": step.action.describe(), "error": str(exc)},
|
||||
step.id,
|
||||
)
|
||||
break
|
||||
|
||||
self._record(
|
||||
pack, Stratum.SURFACE, actor.id, "realization",
|
||||
{
|
||||
"action": step.action.describe(),
|
||||
"surface": realization.surface_id,
|
||||
"mechanics": realization.mechanics,
|
||||
"raised": realization.raised,
|
||||
},
|
||||
step.id,
|
||||
)
|
||||
|
||||
# --- S3: what is now true ------------------------------------
|
||||
snapshot = self._observer.snapshot()
|
||||
self._record(
|
||||
pack, Stratum.JUDGMENT, self._observer.name,
|
||||
"state_snapshot", dict(snapshot), step.id,
|
||||
)
|
||||
|
||||
# --- S2: did the action actually take effect ------------------
|
||||
refused = realization.raised is not None
|
||||
postcondition_met: bool | None = None
|
||||
if step.action.postcondition is not None:
|
||||
try:
|
||||
postcondition_met = step.action.postcondition(snapshot)
|
||||
except KeyError:
|
||||
postcondition_met = None
|
||||
self._record(
|
||||
pack, Stratum.REALIZATION, self._observer.name, "realization_check",
|
||||
{
|
||||
"step": step.id,
|
||||
"action": step.action.name,
|
||||
"surface_used": realization.surface_id,
|
||||
"refused_by_sut": refused,
|
||||
"refusal_expected": step.expect_refusal,
|
||||
"postcondition_met": postcondition_met,
|
||||
},
|
||||
step.id,
|
||||
)
|
||||
|
||||
# --- invariants after every step -----------------------------
|
||||
for invariant in scenario.use_case.invariants:
|
||||
judgments.append(self._oracle.judge(invariant, snapshot, step.id))
|
||||
|
||||
# --- claims attached to this step ----------------------------
|
||||
for claim in claims_by_step.get(step.id, ()):
|
||||
judgments.append(self._oracle.judge(claim, snapshot, step.id))
|
||||
|
||||
pack.verdicts = [j.as_dict() for j in judgments]
|
||||
pack.finished_at = datetime.now(timezone.utc).isoformat()
|
||||
|
||||
result_verdict = overall(judgments)
|
||||
if result_verdict is Verdict.FAIL:
|
||||
pack.energy_events.append(
|
||||
EnergyEvent(asset.id, run_id, EnergyEventType.DEFECT_DETECTED).as_dict()
|
||||
)
|
||||
return RunResult(run_id, result_verdict, judgments, pack)
|
||||
47
src/testdriver/scenario.py
Normal file
47
src/testdriver/scenario.py
Normal file
|
|
@ -0,0 +1,47 @@
|
|||
"""A scenario binds a use case to concrete actors, a world and a schedule."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Sequence
|
||||
|
||||
from .actions import SemanticAction
|
||||
from .intent import UseCase
|
||||
from .observers import Watch
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Step:
|
||||
"""One scheduled semantic action, attributed to one actor."""
|
||||
|
||||
id: str
|
||||
actor_id: str
|
||||
action: SemanticAction
|
||||
expect_refusal: bool = False
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Scenario:
|
||||
"""UseCase + Actors + World + Schedule + Surfaces + Variant."""
|
||||
|
||||
id: str
|
||||
use_case: UseCase
|
||||
steps: tuple[Step, ...]
|
||||
watches: tuple[Watch, ...] = field(default_factory=tuple)
|
||||
variant: str = "baseline"
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class VerificationAsset:
|
||||
"""A test as a durable thing with identity, maturity and lineage.
|
||||
|
||||
Maturity is the T0..T5 continuum. The kernel produces T5 assets: fully
|
||||
deterministic, no model involvement. Agentic assets (T1) arrive in T07, and
|
||||
crystallization walks an asset from T1 toward T5 in T09.
|
||||
"""
|
||||
|
||||
id: str
|
||||
scenario: Scenario
|
||||
maturity: str = "T5"
|
||||
parent_id: str | None = None
|
||||
adaptation_history: list[dict] = field(default_factory=list)
|
||||
79
src/testdriver/world.py
Normal file
79
src/testdriver/world.py
Normal file
|
|
@ -0,0 +1,79 @@
|
|||
"""The situation layer: who is acting, and in what state of the world.
|
||||
|
||||
Actor isolation is structural. An Actor holds its own credentials and private
|
||||
memory and has no reference to the Cast or to any sibling. The orchestrator may
|
||||
know the whole world; actors must not learn anything merely because they happen
|
||||
to be executed by the same process.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Iterator
|
||||
|
||||
|
||||
class ActorIsolationError(Exception):
|
||||
"""Raised when one actor is offered another actor's private state."""
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class Actor:
|
||||
"""An independent execution entity.
|
||||
|
||||
Deliberately holds no back-reference to the Cast or the World. An actor that
|
||||
can enumerate its siblings can leak knowledge it was never given, and no
|
||||
later check can reliably detect that it did.
|
||||
"""
|
||||
|
||||
id: str
|
||||
display_name: str
|
||||
credentials: dict[str, str] = field(default_factory=dict)
|
||||
session: dict[str, Any] = field(default_factory=dict)
|
||||
_memory: dict[str, Any] = field(default_factory=dict, repr=False)
|
||||
|
||||
def remember(self, key: str, value: Any) -> None:
|
||||
self._memory[key] = value
|
||||
|
||||
def recall(self, key: str, default: Any = None) -> Any:
|
||||
return self._memory.get(key, default)
|
||||
|
||||
def known_keys(self) -> tuple[str, ...]:
|
||||
return tuple(sorted(self._memory))
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class Cast:
|
||||
"""The set of actors in a scenario. Known to the orchestrator, not to actors."""
|
||||
|
||||
actors: dict[str, Actor] = field(default_factory=dict)
|
||||
|
||||
def add(self, actor: Actor) -> Actor:
|
||||
if actor.id in self.actors:
|
||||
raise ValueError(f"duplicate actor id {actor.id!r}")
|
||||
self.actors[actor.id] = actor
|
||||
return actor
|
||||
|
||||
def __getitem__(self, actor_id: str) -> Actor:
|
||||
return self.actors[actor_id]
|
||||
|
||||
def __iter__(self) -> Iterator[Actor]:
|
||||
return iter(self.actors.values())
|
||||
|
||||
def __len__(self) -> int:
|
||||
return len(self.actors)
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class World:
|
||||
"""Initial state plus the handle to the system under test.
|
||||
|
||||
`seed` is everything needed to rebuild the initial state, so that a run can
|
||||
be replayed from a known starting point rather than from wherever the
|
||||
previous run happened to leave things.
|
||||
"""
|
||||
|
||||
id: str
|
||||
sut: Any
|
||||
sut_version: str
|
||||
seed: dict[str, Any] = field(default_factory=dict)
|
||||
cast: Cast = field(default_factory=Cast)
|
||||
Loading…
Add table
Add a link
Reference in a new issue