T08: the classifier, measured and attacked

False Adaptation Rate = 0/7 across the labelled catalogue and the three E-003
attacks. 11 of 12 mechanical mutations absorbed without a human, so the safety
result is not bought by escalating everything.

- classification.py: total function over three signals, rule order chosen so
  every rule that could excuse a regression sits after the rule that reports
  one. SAFE_TO_ACCEPT is a two-element closed set, asserted.
- CompositeDriver plus scenarios/full_journey.py: one asset crossing both
  surfaces, so UI mutations are visible as surface differences while the
  claims they do not touch stay green.
- E-003: surface substitution (new M23), concurrent mechanical+defect,
  evidence starvation, provenance laundering. All held.

F-0006 (CONCEPT_DRIFT, resolved): the T02 design listed SEMANTIC_CHANGE as an
outcome the table could produce. It cannot - M12 and M19 are behaviourally
identical, as the lab has asserted since T05. PRODUCT_DEFECT and
SEMANTIC_CHANGE collapse into one escalating outcome, BEHAVIOUR_CHANGED, and
the distinction becomes a human adjudication. INTENT_CHANGED survives but is
detected by the claim fingerprint moving, not inferred from behaviour.

Two classifier defects found and fixed rather than reported: claims downstream
of a failed realization now yield INCONCLUSIVE rather than FAIL (a false
accusation is the mirror image of a false adaptation), and the browser driver
records a page signature so surface change is detectable when the interaction
path is unchanged.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

Assistant: claude-code
Assistant-Model: opus
Assistant-Process: 1629012@bnt-lap001
Assistant-Session: 78d4fb13-8a1e-474b-87a3-9b9261c49a39

tests/test_classification.py

@@ -0,0 +1,195 @@
+"""The classifier, measured against the labelled catalogue — and attacked.
+
+`test_response_matrix` samples benign and hostile cases alike. The E-003 group
+below does something different: it *tries to make the framework unsafe*. An
+experiment that only samples cases chosen by the same person who wrote the
+implementation cannot establish a safety property.
+"""
+
+from __future__ import annotations
+
+import json
+
+import pytest
+
+from testdriver import Runner, Verdict
+from testdriver.classification import (
+    Classification, SAFE_TO_ACCEPT, classify,
+)
+from testdriver.provenance import InadmissibleProvenance, Provenance
+from testdriver.intent import Claim
+from lab.mutations import CATALOGUE
+from scenarios.full_journey import build_journey, journey_lab_server
+from tests.selfverification.checks import check_intent_independence
+
+
+def pack_for(*mutations: str, observer_factory=None) -> dict:
+    with journey_lab_server(*mutations) as (app, tokens, base_url):
+        world, driver, observer, asset, oracle = build_journey(app, tokens, base_url)
+        if observer_factory is not None:
+            observer = observer_factory(observer)
+        return json.loads(Runner(world, driver, observer, oracle).run(asset).evidence.to_json())
+
+
+@pytest.fixture(scope="module")
+def baseline() -> dict:
+    return pack_for()
+
+
+# --- the response matrix --------------------------------------------------
+
+EXPECTED = {
+    "M01": Classification.MECHANICAL_ADAPTATION,
+    "M02": Classification.MECHANICAL_ADAPTATION,
+    "M03": Classification.UNCHANGED,
+    "M04": Classification.MECHANICAL_ADAPTATION,
+    "M05": Classification.UNCHANGED,
+    "M06": Classification.MECHANICAL_ADAPTATION,
+    "M07": Classification.MECHANICAL_ADAPTATION,
+    "M08": Classification.MECHANICAL_ADAPTATION,
+    "M09": Classification.UNCHANGED,
+    "M10": Classification.UNCHANGED,
+    "M21": Classification.MECHANICAL_ADAPTATION,
+    "M22": Classification.AMBIGUOUS,
+    "M11": Classification.BEHAVIOUR_CHANGED,
+    "M12": Classification.BEHAVIOUR_CHANGED,
+    "M13": Classification.UNCHANGED,
+    "M14": Classification.UNCHANGED,
+    "M15": Classification.BEHAVIOUR_CHANGED,
+    "M16": Classification.BEHAVIOUR_CHANGED,
+    "M17": Classification.BEHAVIOUR_CHANGED,
+    "M18": Classification.BEHAVIOUR_CHANGED,
+    "M19": Classification.BEHAVIOUR_CHANGED,
+    "M20": Classification.BEHAVIOUR_CHANGED,
+    "M23": Classification.AMBIGUOUS,
+}
+
+
+@pytest.mark.parametrize("mutation_id", sorted(EXPECTED))
+def test_response_matrix(baseline, mutation_id):
+    assert classify(baseline, pack_for(mutation_id)).classification is EXPECTED[mutation_id]
+
+
+def test_baseline_classifies_as_unchanged(baseline):
+    assert classify(baseline, pack_for()).classification is Classification.UNCHANGED
+
+
+# --- the metric that matters ----------------------------------------------
+
+
+def test_false_adaptation_rate_is_zero(baseline):
+    """The project's existential safety metric.
+
+    Not "low". Zero. A non-zero result is a stop-and-redesign signal, because a
+    framework that normalizes a real defect once has spent the credibility it
+    exists to accumulate.
+    """
+    false_adaptations = [
+        mutation.id for mutation in CATALOGUE
+        if mutation.label == "DEFECT"
+        and classify(baseline, pack_for(mutation.id)).safe_to_accept
+    ]
+    assert false_adaptations == []
+
+
+def test_every_defect_is_escalated(baseline):
+    for mutation in CATALOGUE:
+        if mutation.label != "DEFECT":
+            continue
+        outcome = classify(baseline, pack_for(mutation.id))
+        assert not outcome.safe_to_accept, f"{mutation.id}: {outcome.reason}"
+
+
+def test_mechanical_changes_are_mostly_absorbed(baseline):
+    """Adaptation has to be *useful*, not merely safe.
+
+    A classifier that escalated everything would score a perfect False Adaptation
+    Rate and be worthless. This asserts the other side of the trade.
+    """
+    mechanical = [m for m in CATALOGUE if m.label == "MECHANICAL"]
+    absorbed = [
+        m.id for m in mechanical
+        if classify(baseline, pack_for(m.id)).safe_to_accept
+    ]
+    assert len(absorbed) >= len(mechanical) - 1
+
+
+def test_no_classification_is_outside_the_defined_space(baseline):
+    for mutation in CATALOGUE:
+        outcome = classify(baseline, pack_for(mutation.id))
+        assert isinstance(outcome.classification, Classification)
+        assert outcome.reason
+
+
+# --- E-003: attacking the boundary ---------------------------------------
+
+
+def test_attack_surface_substitution(baseline):
+    """M23: the control is gone from the UI; the API endpoint is still open.
+
+    The most plausible concrete route to a false adaptation. A driver that
+    'recovers' by calling the API has routed around a control, not adapted to a
+    change — and would score a successful mechanical adaptation while an
+    authorization control sat broken.
+    """
+    outcome = classify(baseline, pack_for("M23"))
+    assert not outcome.safe_to_accept
+    assert outcome.classification is Classification.AMBIGUOUS
+
+
+def test_attack_concurrent_mechanical_and_defect(baseline):
+    """Decision-table row 3. Coincidence is not exoneration.
+
+    This is the case a naive self-healing tool gets wrong: it sees a surface
+    change, explains the failure with it, and adapts.
+    """
+    for combination in (("M01", "M15"), ("M02", "M17"), ("M21", "M20")):
+        outcome = classify(baseline, pack_for(*combination))
+        assert outcome.classification is Classification.BEHAVIOUR_CHANGED
+        assert not outcome.safe_to_accept
+        assert "does not excuse it" in outcome.reason
+
+
+def test_attack_evidence_starvation(baseline):
+    """Cut the independent observation channel and the answer must be AMBIGUOUS.
+
+    Never a pass, never an adaptation. Evidence starvation is the condition under
+    which a framework is most tempted to fall back on the actor's own account of
+    what happened.
+    """
+    class Starved:
+        def __init__(self, real):
+            self._real, self._calls, self.name = real, 0, real.name
+            self.watches = real.watches
+
+        def snapshot(self):
+            self._calls += 1
+            return self._real.snapshot() if self._calls < 2 else {}
+
+    outcome = classify(baseline, pack_for(observer_factory=Starved))
+    assert outcome.classification is Classification.AMBIGUOUS
+    assert not outcome.safe_to_accept
+
+
+def test_attack_provenance_laundering():
+    """Intent derived from the implementation must not become a claim.
+
+    Rejected at authoring time, and caught again in the record if it ever got
+    past — belt and braces, because this one cannot be noticed by looking at
+    behaviour.
+    """
+    with pytest.raises(InadmissibleProvenance):
+        Claim("c-laundered", "whatever the system does",
+              Provenance.AGENT_FROM_IMPLEMENTATION, lambda obs: True, after_step="s1")
+
+    tampered = pack_for("M15")
+    tampered["provenance_index"]["c-bob-revoked"] = "agent-from-implementation"
+    assert check_intent_independence(tampered)
+
+
+def test_safe_to_accept_is_a_closed_set():
+    """Only two outcomes may proceed without a human. Widening this set is the
+    single easiest way to destroy the safety property, so it is asserted."""
+    assert SAFE_TO_ACCEPT == {
+        Classification.UNCHANGED, Classification.MECHANICAL_ADAPTATION,
+    }

tests/test_lab_ground_truth.py

@@ -28,7 +28,7 @@ def run_against(*mutations: str):
 
 def test_catalogue_is_large_enough_to_support_a_rate():
     """Six mutations cannot support precision or recall. Twenty can begin to."""
-    assert len(CATALOGUE) >= 20
+    assert len(CATALOGUE) >= 20  # 23 as of T08
 
 
 def test_every_mutation_is_labelled_and_reasoned():
@@ -82,10 +82,19 @@ EXPECTED_VERDICT = {
     "M18": Verdict.FAIL, "M19": Verdict.FAIL, "M20": Verdict.FAIL,
 }
 
-# The two SEMANTIC mutations the reference scenario cannot see, and why.
-KNOWN_INERT = {
+# Mutations the reference scenario cannot see, and why. Coverage is scoped to
+# what a scenario asserts (F-0002) *and* to the surfaces it touches: this
+# scenario is API-only, so a defect that lives in the UI is outside its reach.
+# Declaring them is mandatory — `test_out_of_scope_mutations_are_declared`
+# fails on any invisible mutation that is not named here.
+OUT_OF_SCOPE = {
     "M13": "only affects grants that omit a permission; the scenario passes READ explicitly",
     "M14": "a change of intent with no change of code; nothing observable moved",
+    "M21": "a UI mutation; this scenario never loads the UI",
+    "M22": "a UI mutation; this scenario never loads the UI",
+    "M23": "a UI-surface defect; this scenario is API-only. Covered by the "
+           "cross-surface journey in tests/test_classification.py, where it is "
+           "the E-003 surface-substitution attack.",
 }
 
 
@@ -105,26 +114,43 @@ def test_no_mechanical_mutation_changes_the_verdict():
             assert run_against(mutation.id).verdict is Verdict.PASS, mutation.id
 
 
-def test_every_defect_is_detected():
+def test_every_in_scope_defect_is_detected():
     """The floor of the whole project. A defect the lab cannot surface is a
     defect no later classifier can be measured against."""
     missed = [
         m.id for m in CATALOGUE
-        if m.label == "DEFECT" and run_against(m.id).verdict is Verdict.PASS
+        if m.label == "DEFECT"
+        and m.id not in OUT_OF_SCOPE
+        and run_against(m.id).verdict is Verdict.PASS
     ]
     assert missed == [], f"undetected seeded defects: {missed}"
 
 
-def test_inert_semantic_mutations_are_declared():
-    """A mutation the scenario cannot see must be named, not silently ignored."""
+def test_out_of_scope_mutations_are_declared():
+    """A mutation the scenario cannot see must be named, not silently ignored.
+
+    Applies to defects as much as to semantic changes — an undeclared invisible
+    defect is precisely how a suite comes to look greener than it is.
+    """
     for mutation in CATALOGUE:
-        if mutation.label != "SEMANTIC":
+        if run_against(mutation.id).verdict is not Verdict.PASS:
             continue
-        if run_against(mutation.id).verdict is Verdict.PASS:
-            assert mutation.id in KNOWN_INERT, (
-                f"{mutation.id} is invisible to the reference scenario and "
-                "undeclared — either cover it or record why not"
-            )
+        if mutation.label == "MECHANICAL":
+            continue  # passing is the correct outcome for these
+        assert mutation.id in OUT_OF_SCOPE, (
+            f"{mutation.id} ({mutation.label}) is invisible to the reference "
+            "scenario and undeclared — either cover it or record why not"
+        )
+
+
+def test_declared_out_of_scope_mutations_really_are_invisible():
+    """Stale declarations rot silently. If a mutation becomes visible, the
+    declaration must be removed rather than left as a standing excuse."""
+    for mutation_id in OUT_OF_SCOPE:
+        assert run_against(mutation_id).verdict is Verdict.PASS, (
+            f"{mutation_id} is declared out of scope but the reference scenario "
+            "now detects it — remove the declaration"
+        )
 
 
 def test_deferred_revoke_and_revoke_race_are_behaviourally_identical():