T04: deterministic semantic kernel
Alice/Bob/Carol runs end to end, deterministically, replayable from seed. 16 tests pass, no third-party dependencies. - src/testdriver: intent, provenance, world, actions, drivers, observers, oracles, evidence, energy, scenario, runner - lab/minimal.py: the SUT, exposing the independent observation channel required by D-07 - evidence is stratified S1/S2/S3; Runner refuses to attribute S2/S3 to an actor; claims are frozen and provenance-checked at construction - missing evidence yields INCONCLUSIVE, which outranks PASS in the run verdict - EnergyEvents captured, no scoring (H-005 dormant) The observation channel records both stored state and an out-of-band enforcement probe; their disagreement is an invariant and is what detects an authorization defect that leaves the audit trail intact. A seeded RevokeIsCosmetic lab fails the run via both the claim and that invariant. Also closes TD-WP-0001-T02 (stack and commands now exist). Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Assistant: claude-code Assistant-Model: opus Assistant-Process: 1629012@bnt-lap001 Assistant-Session: 78d4fb13-8a1e-474b-87a3-9b9261c49a39
This commit is contained in:
parent
8da4c5bf7a
commit
04e9573b5a
42 changed files with 1533 additions and 20 deletions
106
src/testdriver/oracles.py
Normal file
106
src/testdriver/oracles.py
Normal file
|
|
@ -0,0 +1,106 @@
|
|||
"""Oracles evaluate claims and invariants and produce verdicts.
|
||||
|
||||
An oracle reads only the independent observation snapshot. It has no access to
|
||||
the actor, to the driver, or to what either of them believes happened.
|
||||
|
||||
`INCONCLUSIVE` is a first-class outcome, not a failure mode of the framework. An
|
||||
oracle that cannot support a judgment from the retained evidence must say so
|
||||
rather than defaulting either way — silently defaulting to PASS hides defects,
|
||||
and silently defaulting to FAIL trains people to ignore results.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from enum import Enum
|
||||
from typing import Any, Mapping
|
||||
|
||||
from .intent import Claim, Invariant
|
||||
|
||||
|
||||
class Verdict(str, Enum):
|
||||
PASS = "PASS"
|
||||
FAIL = "FAIL"
|
||||
SUSPICIOUS = "SUSPICIOUS"
|
||||
INCONCLUSIVE = "INCONCLUSIVE"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Judgment:
|
||||
assertion_id: str
|
||||
text: str
|
||||
verdict: Verdict
|
||||
step_id: str | None
|
||||
detail: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
def as_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"assertion_id": self.assertion_id,
|
||||
"text": self.text,
|
||||
"verdict": self.verdict.value,
|
||||
"step_id": self.step_id,
|
||||
"detail": self.detail,
|
||||
}
|
||||
|
||||
|
||||
class Oracle:
|
||||
"""Deterministic evaluation of one assertion against an observation snapshot."""
|
||||
|
||||
def judge(
|
||||
self,
|
||||
assertion: Claim | Invariant,
|
||||
snapshot: Mapping[str, Any],
|
||||
step_id: str | None,
|
||||
) -> Judgment:
|
||||
if not snapshot:
|
||||
return Judgment(
|
||||
assertion.id,
|
||||
assertion.text,
|
||||
Verdict.INCONCLUSIVE,
|
||||
step_id,
|
||||
{"reason": "no observations were collected"},
|
||||
)
|
||||
try:
|
||||
satisfied = assertion.predicate(snapshot)
|
||||
except KeyError as missing:
|
||||
# The evidence needed to judge this assertion was not collected.
|
||||
# That is an evidence failure, never a pass and never a fail.
|
||||
return Judgment(
|
||||
assertion.id,
|
||||
assertion.text,
|
||||
Verdict.INCONCLUSIVE,
|
||||
step_id,
|
||||
{"reason": f"required observation {missing} missing from snapshot"},
|
||||
)
|
||||
except Exception as exc:
|
||||
return Judgment(
|
||||
assertion.id,
|
||||
assertion.text,
|
||||
Verdict.INCONCLUSIVE,
|
||||
step_id,
|
||||
{"reason": f"predicate raised {type(exc).__name__}: {exc}"},
|
||||
)
|
||||
return Judgment(
|
||||
assertion.id,
|
||||
assertion.text,
|
||||
Verdict.PASS if satisfied else Verdict.FAIL,
|
||||
step_id,
|
||||
)
|
||||
|
||||
|
||||
def overall(judgments: list[Judgment]) -> Verdict:
|
||||
"""Aggregate run verdict.
|
||||
|
||||
FAIL dominates; INCONCLUSIVE outranks PASS. A run containing an unjudgeable
|
||||
assertion has not passed, whatever else it did.
|
||||
"""
|
||||
verdicts = {j.verdict for j in judgments}
|
||||
if not judgments:
|
||||
return Verdict.INCONCLUSIVE
|
||||
if Verdict.FAIL in verdicts:
|
||||
return Verdict.FAIL
|
||||
if Verdict.INCONCLUSIVE in verdicts:
|
||||
return Verdict.INCONCLUSIVE
|
||||
if Verdict.SUSPICIOUS in verdicts:
|
||||
return Verdict.SUSPICIOUS
|
||||
return Verdict.PASS
|
||||
Loading…
Add table
Add a link
Reference in a new issue