diff --git a/WORK-RECORDS.md b/WORK-RECORDS.md index 65aa4e3..9985b2b 100644 --- a/WORK-RECORDS.md +++ b/WORK-RECORDS.md @@ -19,7 +19,7 @@ | task | TD-WP-0002-T04 | done | — | workplans/TD-WP-0002-vertical-spike-crystallization.md | | task | TD-WP-0002-T05 | done | — | workplans/TD-WP-0002-vertical-spike-crystallization.md | | task | TD-WP-0002-T06 | done | — | workplans/TD-WP-0002-vertical-spike-crystallization.md | -| task | TD-WP-0002-T07 | todo | — | workplans/TD-WP-0002-vertical-spike-crystallization.md | +| task | TD-WP-0002-T07 | done | — | workplans/TD-WP-0002-vertical-spike-crystallization.md | | task | TD-WP-0002-T08 | todo | — | workplans/TD-WP-0002-vertical-spike-crystallization.md | | task | TD-WP-0002-T09 | todo | — | workplans/TD-WP-0002-vertical-spike-crystallization.md | | task | TD-WP-0002-T10 | todo | — | workplans/TD-WP-0002-vertical-spike-crystallization.md | diff --git a/docs/TestDriverClassificationDesign.md b/docs/TestDriverClassificationDesign.md index d65d619..08bebee 100644 --- a/docs/TestDriverClassificationDesign.md +++ b/docs/TestDriverClassificationDesign.md @@ -66,11 +66,21 @@ to the last accepted run of the same verification asset. | yes | yes | unchanged | `MECHANICAL_ADAPTATION` | | no | yes | changed to FAIL | `PRODUCT_DEFECT` | | yes | yes | changed to FAIL | `PRODUCT_DEFECT` | -| any | yes | unchanged, but the asset's declared claim set differs from the use case | `SEMANTIC_CHANGE` → human | +| any | yes | unchanged, but the asset's declared claim set differs from the use case | ~~`SEMANTIC_CHANGE` → human~~ → **`INTENT_CHANGED`**, see below | | any | no, and no legitimate surface affords it | — | `PRODUCT_DEFECT` | | any | no, but the action is expressible and the actor simply failed | — | `FRAMEWORK_LIMITATION` | | any | any | any oracle `INCONCLUSIVE`, or required evidence missing | `AMBIGUOUS` → escalate | +> **Revised at T08 — see `research/findings/F-0006-classifier-cannot-infer-intent.md`.** +> `SEMANTIC_CHANGE` was specified above as an outcome the table could produce. It +> cannot: a deliberate product decision and a defect are behaviourally identical +> (M12 and M19 in the lab), so no evidence separates them. `PRODUCT_DEFECT` and +> `SEMANTIC_CHANGE` are collapsed into one escalating outcome, +> **`BEHAVIOUR_CHANGED`**, and which of the two it is becomes a human +> adjudication. `INTENT_CHANGED` remains, but is detected by the *claim +> fingerprint* moving — a fact about the recorded use case, not an inference +> about behaviour. The row above described that, filed under the wrong heading. + Two rows carry the whole safety argument: - **Row 3** — a surface change occurring *alongside* a verdict change is classified diff --git a/lab/__pycache__/http_api.cpython-312.pyc b/lab/__pycache__/http_api.cpython-312.pyc index 91d5fde..153b125 100644 Binary files a/lab/__pycache__/http_api.cpython-312.pyc and b/lab/__pycache__/http_api.cpython-312.pyc differ diff --git a/lab/__pycache__/mutations.cpython-312.pyc b/lab/__pycache__/mutations.cpython-312.pyc index ef915da..50c2c1b 100644 Binary files a/lab/__pycache__/mutations.cpython-312.pyc and b/lab/__pycache__/mutations.cpython-312.pyc differ diff --git a/lab/http_api.py b/lab/http_api.py index 52b7222..dfb3b21 100644 --- a/lab/http_api.py +++ b/lab/http_api.py @@ -73,7 +73,11 @@ def _render(app: LabApp, user_id: str, resource_id: str) -> str: f'<{tag}{href}{role} id="share-submit" data-td="share-submit">' f"{share_label}" ) - if app.ui_share_control == "modal": + if app.ui_share_control == "removed": + # The control is gone from the UI. The API endpoint is still open — the + # trap for a driver that "recovers" by routing around the interface. + share_form = '

Sharing is unavailable.

' + elif app.ui_share_control == "modal": share_form = ( f'<{tag}{href}{role} id="open-share" data-td="open-share">{share_label}…' f"" diff --git a/lab/mutations.py b/lab/mutations.py index 0ea3e83..4016304 100644 --- a/lab/mutations.py +++ b/lab/mutations.py @@ -90,6 +90,9 @@ def _m10(app): _m(app, denied_status=401) # --- SEMANTIC ------------------------------------------------------------- # Intended behaviour changed. A human must decide; test-driver must not. +def _m23(app): _m(app, ui_share_control="removed") + + def _m21(app): _m(app, ui_labels="verbose", ui_test_ids="dropped") def _m22(app): _m(app, ui_field_names="renamed", ui_test_ids="dropped") @@ -169,6 +172,11 @@ CATALOGUE: tuple[Mutation, ...] = ( "generalises or merely pattern-matches known field names.", _m22, preserves_test_ids=False), + Mutation("M23", "Sharing control removed from the UI, API left open", "DEFECT", "ui", + "The surface-substitution trap. A driver that 'recovers' by calling " + "the API has routed around a control, not adapted to a change — the " + "most plausible concrete route to a false adaptation (E-003).", _m23), + Mutation("M11", "A share must be accepted before it takes effect", "SEMANTIC", "domain", "Bob genuinely cannot read until he accepts. The old claim 'Bob can " "read after the grant' is now wrong, and only a human may say so.", _m11), diff --git a/research/concepts/fitness-map.md b/research/concepts/fitness-map.md index 03c9b78..67e13ec 100644 --- a/research/concepts/fitness-map.md +++ b/research/concepts/fitness-map.md @@ -1,6 +1,6 @@ # Concept ↔ Implementation Fitness Map -**Updated:** 2026-08-22 (TD-WP-0002-T07) +**Updated:** 2026-08-22 (TD-WP-0002-T08) Traces each important concept to the implementation, experiment and evidence that support it. **Unsupported entries are the point of this map** — a concept with no @@ -31,11 +31,11 @@ were aspirational, not evidenced. | `C-use-case` | C1 | `intent.py` | — | — | Is a use case expressible without leaking mechanics? | | `C-actor-isolation` | C1 | `world.py` | E-001 | `td://self/actor-isolation` | **F-0003** — only observable when the scenario plants canaries. | | `C-semantic-action` | **C2** | `actions.py`, `agentic.py` | E-001 (partial) | T07 arm comparison | **F-0005** — supported only where stable identifiers are absent. Narrower than the concept model claims. | -| `C-oracle-independence` | C1 | `runner.py`, `oracles.py` | E-001, E-003 | — | Independence of components ≠ independence of belief. (H-004) | +| `C-oracle-independence` | **C2** | `runner.py`, `oracles.py` | E-001, E-003 | — | Independence of components ≠ independence of belief. (H-004) | | `C-evidence-pack` | C1 | `evidence.py` | — | `td://self/evidence-reproducibility` | Verdicts are reproducible from S3 alone, on passing and failing runs. | | `C-observation-channel` | C1 | `lab/app.py` | — | — | **D-07** — required of every system under test. Adoption cost unknown. | -| `C-adaptation` | C1 | — (T08) | E-001 | — | (H-002) | -| `C-classification` | C1 | — (T08) | E-001, E-003 | — | Decision table is total on paper; unexercised. | +| `C-adaptation` | **C2** | `classification.py`, `agentic.py` | E-001 | T08 matrix | 11/12 mechanical absorbed without a human. | +| `C-classification` | **C2** | `classification.py` | E-001, E-003 | T08 matrix, FAR 0/7 | **F-0006** — cannot infer SEMANTIC vs DEFECT; collapses to one escalating outcome. | | `C-crystallization` | C1 | — (T09) | E-002 | — | (H-003) | | `C-intent-provenance` | C1 | `provenance.py` | E-003 | `td://self/intent-independence` | Constrains provenance, not quality. Accepted residual. | | `C-lineage` | C0 | — | — | — | Parent pointer only in the spike. | diff --git a/research/experiments/E-001-mechanical-vs-defect.md b/research/experiments/E-001-mechanical-vs-defect.md index 3360d1c..58619af 100644 --- a/research/experiments/E-001-mechanical-vs-defect.md +++ b/research/experiments/E-001-mechanical-vs-defect.md @@ -35,4 +35,5 @@ tokens · wall time · retries. ## Status -`PLANNED`. Blocked on T05 (lab) and T07 (agentic driver). +`EXECUTED` 2026-08-22 (T08). Arm A run over 23 mutations; arm B over +the 12 mechanical ones. Results in H-001 and H-004. FAR 0/7. diff --git a/research/experiments/E-003-surface-substitution-attack.md b/research/experiments/E-003-surface-substitution-attack.md index c3677d9..1711a19 100644 --- a/research/experiments/E-003-surface-substitution-attack.md +++ b/research/experiments/E-003-surface-substitution-attack.md @@ -37,4 +37,6 @@ what E-001 reports. ## Status -`PLANNED`. Blocked on T05, T07, T08. +`EXECUTED` 2026-08-22 (T08). All four attacks held — see +`tests/test_classification.py::test_attack_*`. M23 was added to the catalogue +to make attack 1 real rather than hypothetical. diff --git a/research/findings/F-0006-classifier-cannot-infer-intent.md b/research/findings/F-0006-classifier-cannot-infer-intent.md new file mode 100644 index 0000000..b270f1a --- /dev/null +++ b/research/findings/F-0006-classifier-cannot-infer-intent.md @@ -0,0 +1,93 @@ +--- +id: F-0006 +type: framework-finding +class: CONCEPT_DRIFT +status: resolved +discovered: "2026-08-22" +resolved: "2026-08-22" +discovered_by: TD-WP-0002-T08 +workplan: TD-WP-0002 +task: TD-WP-0002-T08 +hypotheses: [H-004] +--- + +# F-0006 — The classifier cannot infer a semantic change, only escalate + +## What the design said + +`docs/TestDriverClassificationDesign.md` (T02) specifies a decision table with +four outcomes, one of which is `SEMANTIC_CHANGE`: + +> | any | yes | unchanged, but the asset's declared claim set differs from the use +> case | `SEMANTIC_CHANGE` → human | + +`TestDriverInitialMilestones.md` M6 states the same four-way path: +`IMPLEMENTATION CHANGE` / `INTENT CHANGE` / `PRODUCT DEFECT` / `AMBIGUOUS`. + +Both read as though a classifier could look at a run and determine that intended +behaviour had changed. + +## What building it showed + +It cannot, and the lab already contained the proof before the classifier existed. + +**M12** (revocation deferred by product decision, `SEMANTIC`) and **M19** +(revocation propagates after a delay, `DEFECT`) produce *identical* evidence: +same failing assertion, same step, same snapshot, same audit trail. +`test_deferred_revoke_and_revoke_race_are_behaviourally_identical` has asserted +this since T05. + +No quantity of evidence distinguishes them, because the difference is not in the +system. It is in whether someone decided the new behaviour was wanted. A +classifier that emitted `SEMANTIC_CHANGE` from evidence would be guessing, and +guessing in the one direction the project cannot afford: `SEMANTIC_CHANGE` +sounds benign, and "the team must have meant it" is exactly the reasoning that +normalizes a defect. + +## Resolution + +Path 2 — the concept is deliberately revised. + +The classifier's output space is now: + +| Outcome | Safe to accept | Meaning | +|---|---|---| +| `UNCHANGED` | yes | no observable difference | +| `MECHANICAL_ADAPTATION` | yes | surface moved, every claim still holds | +| `BEHAVIOUR_CHANGED` | **no** | a claim that held no longer holds | +| `INTENT_CHANGED` | **no** | the recorded claim set itself moved | +| `REALIZATION_FAILED` | **no** | the action could not be performed | +| `AMBIGUOUS` | **no** | the evidence does not support a conclusion | + +`BEHAVIOUR_CHANGED` replaces both `PRODUCT_DEFECT` and `SEMANTIC_CHANGE`. +Which of the two it is remains a real and important question — it is simply a +question for a human, recorded as an adjudication, not inferred from a run. + +**`INTENT_CHANGED` is detectable**, but note what makes it so: the *claim +fingerprint* changed, meaning a human already edited what is being asserted. +That is a fact about the recorded use case, not an inference about behaviour. +This is what the T02 table's fourth row was really describing; it was filed under +the wrong heading. + +## Why this makes the framework better, not weaker + +Calling a semantic change a defect is a **false alarm**: a human looks, says "we +meant that", and updates the claim — which then registers as `INTENT_CHANGED` +with full provenance. Cost: one review. + +Calling a defect a semantic change is a **false adaptation**: the thesis dies. + +Collapsing the two into one escalating outcome makes the framework err only in +the direction it can afford. The measured consequence is visible in the T08 +matrix: 2 of 4 `SEMANTIC` mutations are escalated as `BEHAVIOUR_CHANGED`, which +looks like imprecision and is in fact the design working. + +## Consequences + +- `docs/TestDriverClassificationDesign.md` decision table updated in place, with + the original row preserved and marked. +- `TestDriverInitialMilestones.md` M6's four-way path is superseded by this + finding; the milestone's *intent* (do not normalize defects) is unchanged and + is met. +- H-004's falsification condition is unaffected: False Adaptation Rate over + `DEFECT`-labelled mutations. Measured **0/7** at T08. diff --git a/research/hypotheses/H-004-independent-judgment.md b/research/hypotheses/H-004-independent-judgment.md index e4814a1..bac9938 100644 --- a/research/hypotheses/H-004-independent-judgment.md +++ b/research/hypotheses/H-004-independent-judgment.md @@ -1,7 +1,7 @@ --- id: H-004 title: Independent Judgment -status: PROPOSED +status: EXPERIMENTING created: "2026-08-22" experiments: [E-001, E-003] concepts: [C-oracle-independence, C-intent-provenance] @@ -41,6 +41,41 @@ The most plausible route to falsification is not a misclassification but via another surface and scoring as a successful recovery. D-05 exists to close that route; E-003 exists to attack it deliberately. +## Result (TD-WP-0002-T08) + +**False Adaptation Rate = 0/7.** No `DEFECT`-labelled mutation was classified as +safe to accept, including the three deliberate attacks in E-003. + +| Ground truth | Accepted without a human | +|---|---| +| MECHANICAL (12) | 11 | +| SEMANTIC (4) | 2 — both genuinely inert for this scenario | +| DEFECT (7) | **0** | + +E-003 attacks, all held: + +- **surface substitution** (M23, UI control removed, API left open) → `AMBIGUOUS`. + The driver did not route around; discovery failed loudly. +- **concurrent mechanical + defect** (M01+M15, M02+M17, M21+M20) → + `BEHAVIOUR_CHANGED`, with the reason explicitly noting that the coincident + surface change does not excuse it. +- **evidence starvation** → `AMBIGUOUS`, never a pass. +- **provenance laundering** → rejected at authoring, and caught in the record. + +Note what this does and does not establish. FAR = 0 follows largely from +*architecture* — claims are run inputs with no adaptation write path (D-02), and +`SAFE_TO_ACCEPT` is a two-element closed set. The experiment confirms the +architecture behaves as designed over 23 mutations; it does not establish that +the architecture is correct for mutations nobody thought of. That distinction +should survive into any external claim. + +The other side of the trade is asserted too: a classifier that escalated +everything would score a perfect FAR and be useless. +`test_mechanical_changes_are_mostly_absorbed` pins 11 of 12. + ## Status log - 2026-08-22 `PROPOSED`. No evidence. +- 2026-08-22 `EXPERIMENTING`. FAR 0/7 over the labelled set plus E-003 attacks. + Not promoted to SUPPORTED: 23 hand-written mutations is a small, self-chosen + sample. diff --git a/scenarios/__pycache__/full_journey.cpython-312.pyc b/scenarios/__pycache__/full_journey.cpython-312.pyc new file mode 100644 index 0000000..65e17f8 Binary files /dev/null and b/scenarios/__pycache__/full_journey.cpython-312.pyc differ diff --git a/scenarios/full_journey.py b/scenarios/full_journey.py new file mode 100644 index 0000000..e87f53a --- /dev/null +++ b/scenarios/full_journey.py @@ -0,0 +1,98 @@ +"""The reference journey, crossing two surfaces. + +Sharing happens in the browser UI, where a person would do it. Setup and +revocation go through the API. That mix is the realistic case and it is also what +makes classification measurable: a UI mutation must be visible as a *surface* +difference while the claims it does not touch stay green. + +The use case, the claims and the oracles are identical to +`scenarios/alice_bob_carol.py`. Only the realization path differs — which is the +whole point of a semantic action. +""" + +from __future__ import annotations + +from testdriver import ( + Actor, Cast, DirectDriver, Oracle, Scenario, SemanticAction, StateObserver, + Step, VerificationAsset, World, +) +from testdriver.agentic import DiscoveryRuntime +from testdriver.browser import BrowserDriver +from testdriver.drivers import CompositeDriver +from lab.mutations import ObservationChannel, build_lab +from scenarios.alice_bob_carol import RESOURCE, USE_CASE +from scenarios.browser_grant import lab_server # re-exported for callers + +API = frozenset({"api"}) +BROWSER = frozenset({"browser"}) + +__all__ = ["build_journey", "lab_server", "RESOURCE", "USE_CASE"] + + +def build_journey(app, tokens, base_url, runtime=None): + cast = Cast() + for name in ("alice", "bob", "carol"): + cast.add(Actor(name, name.title(), credentials={"token": tokens[name]})) + world = World(id="w-journey", sut=app, sut_version=app.version, cast=cast) + + from testdriver.observers import Watch + + scenario = Scenario( + id="sc-full-journey", + use_case=USE_CASE, + variant=f"journey/{'+'.join(app.applied_mutations) or 'baseline'}", + watches=(Watch("bob", RESOURCE), Watch("carol", RESOURCE)), + steps=( + Step("s1-create", "alice", SemanticAction( + "create_resource", + {"resource_id": RESOURCE, "content": "the secret"}, + permitted_surfaces=API, + postcondition=lambda obs: "audit:R" in obs, + )), + Step("s2-grant", "alice", SemanticAction( + "grant_access", + {"subject_id": "bob", "permission": "READ"}, + permitted_surfaces=BROWSER, + postcondition=lambda obs: obs["state_permission:bob:R"] == "READ", + )), + Step("s3-revoke", "alice", SemanticAction( + "revoke_access", + {"resource_id": RESOURCE, "subject_id": "bob"}, + permitted_surfaces=API, + postcondition=lambda obs: obs["state_permission:bob:R"] is None, + )), + ), + ) + + driver = CompositeDriver( + { + "api": DirectDriver(app, tokens), + "browser": BrowserDriver( + base_url, tokens, runtime or DiscoveryRuntime(), RESOURCE + ), + }, + default="api", + ) + observer = StateObserver(ObservationChannel(app), scenario.watches) + asset = VerificationAsset(id="va-full-journey", scenario=scenario, maturity="T2") + return world, driver, observer, asset, Oracle() + + +def journey_lab_server(*mutations: str): + """Like lab_server but without pre-creating the resource — s1 does that.""" + import threading + from contextlib import contextmanager + from lab.http_api import serve + + @contextmanager + def _run(): + app, tokens = build_lab(*mutations) + server = serve(app) + threading.Thread(target=server.serve_forever, daemon=True).start() + try: + yield app, tokens, f"http://127.0.0.1:{server.server_address[1]}" + finally: + server.shutdown() + server.server_close() + + return _run() diff --git a/src/testdriver/__pycache__/browser.cpython-312.pyc b/src/testdriver/__pycache__/browser.cpython-312.pyc index 30939fe..956759a 100644 Binary files a/src/testdriver/__pycache__/browser.cpython-312.pyc and b/src/testdriver/__pycache__/browser.cpython-312.pyc differ diff --git a/src/testdriver/__pycache__/classification.cpython-312.pyc b/src/testdriver/__pycache__/classification.cpython-312.pyc new file mode 100644 index 0000000..ea83da0 Binary files /dev/null and b/src/testdriver/__pycache__/classification.cpython-312.pyc differ diff --git a/src/testdriver/__pycache__/drivers.cpython-312.pyc b/src/testdriver/__pycache__/drivers.cpython-312.pyc index 6367df1..c735df0 100644 Binary files a/src/testdriver/__pycache__/drivers.cpython-312.pyc and b/src/testdriver/__pycache__/drivers.cpython-312.pyc differ diff --git a/src/testdriver/__pycache__/runner.cpython-312.pyc b/src/testdriver/__pycache__/runner.cpython-312.pyc index 09410a6..48c6c66 100644 Binary files a/src/testdriver/__pycache__/runner.cpython-312.pyc and b/src/testdriver/__pycache__/runner.cpython-312.pyc differ diff --git a/src/testdriver/browser.py b/src/testdriver/browser.py index 1cf8529..b1bd233 100644 --- a/src/testdriver/browser.py +++ b/src/testdriver/browser.py @@ -99,6 +99,7 @@ class BrowserDriver: ) document = Document.parse(body) + signature = page_signature(document) args = {"resource_id": self._resource_id, **dict(action.args)} try: @@ -110,7 +111,7 @@ class BrowserDriver: "action": action.name, "stage": "discovery", "runtime": getattr(self._runtime, "name", "?"), - "page_bytes": len(body), + "page_signature": signature, }, raised=f"RealizationFailed: {failure}", ) @@ -121,6 +122,7 @@ class BrowserDriver: "runtime": plan.metrics.runtime, "rationale": plan.rationale, "element_path": plan.element_path, + "page_signature": signature, "target": plan.action, "fields": sorted(plan.fields), "metrics": plan.metrics.as_dict(), @@ -136,6 +138,32 @@ class BrowserDriver: return Realization(self.surface.id, mechanics) +def page_signature(document: Document) -> str: + """A coarse, stable shape of the surface as the driver saw it. + + Element counts by tag plus the names of every control. Enough that a + reworded label, an extra confirmation step or a changed element type shows up + as a surface difference — without being so fine that incidental whitespace + makes every run look like an adaptation. + """ + from collections import Counter + + counts = Counter(element.tag for element in document.elements()) + controls = sorted( + element.attrs.get("name", "") + for element in document.elements("input", "select", "textarea") + ) + labels = sorted( + element.full_text.strip().lower() + for element in document.elements("button", "a", "label") + if element.full_text.strip() + ) + return json.dumps( + {"tags": dict(sorted(counts.items())), "controls": controls, "labels": labels}, + sort_keys=True, + ) + + def record_baseline_selectors(html: str) -> dict[str, str]: """Capture the control arm's selectors from a baseline page. diff --git a/src/testdriver/classification.py b/src/testdriver/classification.py new file mode 100644 index 0000000..4623111 --- /dev/null +++ b/src/testdriver/classification.py @@ -0,0 +1,245 @@ +"""Why did this run differ from the last accepted one? + +The centrepiece. Everything else in the framework exists so that this question +can be answered from evidence rather than guessed. + +The classifier is never asked *"is this failure acceptable?"* — it has no +authority to accept anything, and claims are run inputs it cannot reach (D-02). +It is asked only *"what kind of difference is this?"*, and the safe answers +outnumber the convenient ones. + +## What the T02 design got wrong + +`docs/TestDriverClassificationDesign.md` lists `SEMANTIC_CHANGE` as an outcome +the decision table can produce. Building it showed that it cannot — see F-0006. +A deliberate product decision and a defect are behaviourally identical (M12 and +M19 in the lab prove it), so no amount of evidence separates them. What the +classifier can honestly say is *"behaviour changed against intent that did not"*, +and hand that to a human. + +Intent change **is** detectable, but only when a human has actually changed the +intent: the claim fingerprint moves. That is a fact about the recorded use case, +not an inference about behaviour. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from enum import Enum +from typing import Any, Mapping + + +class Classification(str, Enum): + UNCHANGED = "UNCHANGED" + """Same surface, same verdicts. Nothing to report.""" + + MECHANICAL_ADAPTATION = "MECHANICAL_ADAPTATION" + """The surface moved; every protected claim still holds. Safe to accept.""" + + BEHAVIOUR_CHANGED = "BEHAVIOUR_CHANGED" + """A claim that held no longer holds. Never adapted to; always escalated. + + Whether this is a product defect or a deliberate change of intent is a + question about intent, not about evidence, and is resolved by a human. + """ + + INTENT_CHANGED = "INTENT_CHANGED" + """The recorded claim set itself changed. A human has already acted.""" + + REALIZATION_FAILED = "REALIZATION_FAILED" + """The action could not be performed at all. Not a verdict about the system.""" + + AMBIGUOUS = "AMBIGUOUS" + """The evidence does not support a conclusion. Escalate; never default.""" + + +#: Classifications that permit the run to be accepted without a human. +SAFE_TO_ACCEPT = frozenset({Classification.UNCHANGED, Classification.MECHANICAL_ADAPTATION}) + + +@dataclass(frozen=True, slots=True) +class Signals: + """The three strata, reduced to what classification actually needs.""" + + surface_differs: bool + realization_failed: bool + postcondition_met: bool | None + verdicts_regressed: bool + verdicts_inconclusive: bool + claims_differ: bool + evidence_incomplete: bool + + def as_dict(self) -> dict[str, Any]: + return { + "surface_differs": self.surface_differs, + "realization_failed": self.realization_failed, + "postcondition_met": self.postcondition_met, + "verdicts_regressed": self.verdicts_regressed, + "verdicts_inconclusive": self.verdicts_inconclusive, + "claims_differ": self.claims_differ, + "evidence_incomplete": self.evidence_incomplete, + } + + +@dataclass(frozen=True, slots=True) +class Outcome: + classification: Classification + reason: str + signals: Signals + regressions: tuple[str, ...] = () + + @property + def safe_to_accept(self) -> bool: + return self.classification in SAFE_TO_ACCEPT + + def as_dict(self) -> dict[str, Any]: + return { + "classification": self.classification.value, + "reason": self.reason, + "signals": self.signals.as_dict(), + "regressions": list(self.regressions), + } + + +# --- signal extraction ---------------------------------------------------- + + +def _verdict_map(pack: Mapping[str, Any]) -> dict[tuple[str, str], str]: + return {(v["assertion_id"], v["step_id"]): v["verdict"] for v in pack["verdicts"]} + + +def _surface_fingerprint(pack: Mapping[str, Any]) -> str: + """How the run was performed, reduced to a comparable shape. + + Deliberately excludes timings and free text: those change run to run and + would make every run look like a mechanical adaptation. + """ + parts = [] + for obs in pack["observations"]: + if obs["kind"] != "realization": + continue + mechanics = obs["data"].get("mechanics", {}) + parts.append({ + "step": obs["step_id"], + "surface": obs["data"].get("surface"), + "target": mechanics.get("target") or mechanics.get("operation"), + "fields": sorted(mechanics.get("fields") or mechanics.get("arguments") or []), + "element_path": mechanics.get("element_path"), + "page": mechanics.get("page_signature"), + }) + return json.dumps(parts, sort_keys=True) + + +def _claim_fingerprint(pack: Mapping[str, Any]) -> str: + return json.dumps(sorted(pack.get("provenance_index", {}).items())) + + +def extract_signals(baseline: Mapping[str, Any], candidate: Mapping[str, Any]) -> Signals: + before, after = _verdict_map(baseline), _verdict_map(candidate) + + regressed = any( + after.get(key) == "FAIL" and verdict != "FAIL" for key, verdict in before.items() + ) or any(v == "FAIL" for k, v in after.items() if k not in before) + + realization_failed = any( + obs["kind"] == "realization" and obs["data"].get("raised") + for obs in candidate["observations"] + ) + checks = [ + obs["data"] for obs in candidate["observations"] + if obs["kind"] == "realization_check" + ] + met: bool | None + if not checks: + met = None + elif any(c.get("postcondition_met") is False for c in checks): + met = False + elif any(c.get("postcondition_met") is None for c in checks): + met = None + else: + met = True + + return Signals( + surface_differs=_surface_fingerprint(baseline) != _surface_fingerprint(candidate), + realization_failed=realization_failed, + postcondition_met=met, + verdicts_regressed=regressed, + verdicts_inconclusive=any(v == "INCONCLUSIVE" for v in after.values()), + claims_differ=_claim_fingerprint(baseline) != _claim_fingerprint(candidate), + evidence_incomplete=not after or not candidate.get("sut_version"), + ) + + +def regressions(baseline: Mapping[str, Any], candidate: Mapping[str, Any]) -> tuple[str, ...]: + before, after = _verdict_map(baseline), _verdict_map(candidate) + return tuple(sorted( + f"{assertion}@{step}" + for (assertion, step), verdict in after.items() + if verdict == "FAIL" and before.get((assertion, step)) != "FAIL" + )) + + +# --- the decision table --------------------------------------------------- + + +def classify(baseline: Mapping[str, Any], candidate: Mapping[str, Any]) -> Outcome: + """Total over the signal space. Order matters, and the order is the safety argument. + + Every rule that could excuse a regression is placed *after* the rule that + reports one. A surface change occurring alongside a broken claim is a + regression that happens to coincide with a surface change — never a surface + change that happens to break a claim. + """ + signals = extract_signals(baseline, candidate) + failures = regressions(baseline, candidate) + + # 1. Evidence first. A conclusion drawn from incomplete evidence is worse + # than no conclusion, in either direction. + if signals.evidence_incomplete: + return Outcome(Classification.AMBIGUOUS, + "the run did not retain enough evidence to classify", signals) + if signals.verdicts_inconclusive: + return Outcome(Classification.AMBIGUOUS, + "at least one assertion could not be judged from the evidence", + signals, failures) + + # 2. Intent moving is a fact about the recorded use case, not an inference. + if signals.claims_differ: + return Outcome(Classification.INTENT_CHANGED, + "the recorded claim set differs from the baseline; " + "a human changed what is being asserted", signals, failures) + + # 3. A regression is reported before anything is allowed to explain it away. + # This is decision-table row 3: coincidence is not exoneration. + if signals.verdicts_regressed: + note = (" — a surface change occurred in the same run and does not excuse it" + if signals.surface_differs else "") + return Outcome(Classification.BEHAVIOUR_CHANGED, + f"claims that held at baseline no longer hold{note}", + signals, failures) + + # 4. An action the system accepted but did not perform is a behaviour + # change, observed independently of any claim. It is caught here even + # when no claim happens to cover it. + if signals.postcondition_met is False: + return Outcome(Classification.BEHAVIOUR_CHANGED, + "the action was accepted but its declared effect did not occur", + signals, failures) + + # 5. Could not act at all. Says nothing about whether the system is correct. + if signals.realization_failed: + return Outcome(Classification.REALIZATION_FAILED, + "the action could not be realized on this surface", signals) + if signals.postcondition_met is None: + return Outcome(Classification.AMBIGUOUS, + "the action's postcondition could not be evaluated", signals) + + # 6. Only now, with every claim intact and the action verified, may a + # surface difference be called an adaptation. + if signals.surface_differs: + return Outcome(Classification.MECHANICAL_ADAPTATION, + "the surface changed; every protected claim still holds", signals) + + return Outcome(Classification.UNCHANGED, "no observable difference from baseline", + signals) diff --git a/src/testdriver/drivers.py b/src/testdriver/drivers.py index bb8cdc8..eb869c0 100644 --- a/src/testdriver/drivers.py +++ b/src/testdriver/drivers.py @@ -74,3 +74,33 @@ class DirectDriver: return Realization(self.surface.id, mechanics, raised=f"{type(exc).__name__}: {exc}") mechanics["result"] = result return Realization(self.surface.id, mechanics) + + +class CompositeDriver: + """Routes each semantic action to the driver for the surface it permits. + + A real journey crosses surfaces: a share happens in the UI, an audit check + against the API. Binding one asset to one driver would force every step + through the same door and hide exactly the mechanical changes that only + appear on one of them. + + Routing is by the action's *permitted* surfaces (D-05), never by which driver + happens to be able to do it. A driver that could perform an action it was not + permitted to perform is surface substitution, not convenience. + """ + + def __init__(self, drivers: dict[str, Any], default: str) -> None: + self._drivers = drivers + self._default = default + self.surface = drivers[default].surface + + def realize(self, actor: Actor, action: SemanticAction) -> Realization: + permitted = action.permitted_surfaces or {self._default} + for surface_id in sorted(permitted): + driver = self._drivers.get(surface_id) + if driver is not None: + return driver.realize(actor, action) + raise UnsupportedAction( + f"no driver for any permitted surface of {action.name!r}: " + f"{sorted(permitted)}" + ) diff --git a/src/testdriver/runner.py b/src/testdriver/runner.py index 08fdff4..833eab0 100644 --- a/src/testdriver/runner.py +++ b/src/testdriver/runner.py @@ -112,6 +112,7 @@ class Runner: ) judgments: list[Judgment] = [] + scenario_sound = True claims_by_step: dict[str, list] = {} for claim in scenario.use_case.claims: claims_by_step.setdefault(claim.after_step, []).append(claim) @@ -171,13 +172,35 @@ class Runner: step.id, ) + # Only an action that did not *happen* makes the scenario unsound. + # An action that was accepted but did not take effect did happen — + # and that is a statement about the system, judged below, not a + # reason to stop judging. + if refused and not step.expect_refusal: + scenario_sound = False + # --- invariants after every step ----------------------------- + # Invariants are judged regardless: they are supposed to hold at all + # times, however far the scenario got. for invariant in scenario.use_case.invariants: judgments.append(self._oracle.judge(invariant, snapshot, step.id)) # --- claims attached to this step ---------------------------- + # Claims describe the state a *completed* scenario should reach. If a + # step did not happen, a claim about the state after it is not + # failing — it is unevaluable. Reporting FAIL there would accuse the + # system of a defect on the strength of the test's own inability to + # act, which is the mirror image of a false adaptation and just as + # dishonest. for claim in claims_by_step.get(step.id, ()): - judgments.append(self._oracle.judge(claim, snapshot, step.id)) + if scenario_sound: + judgments.append(self._oracle.judge(claim, snapshot, step.id)) + else: + judgments.append(Judgment( + claim.id, claim.text, Verdict.INCONCLUSIVE, step.id, + {"reason": "an earlier step in this scenario did not complete, " + "so the state this claim describes was never reached"}, + )) pack.verdicts = [j.as_dict() for j in judgments] pack.finished_at = datetime.now(timezone.utc).isoformat() diff --git a/tests/__pycache__/test_classification.cpython-312-pytest-7.4.4.pyc b/tests/__pycache__/test_classification.cpython-312-pytest-7.4.4.pyc new file mode 100644 index 0000000..e1a42df Binary files /dev/null and b/tests/__pycache__/test_classification.cpython-312-pytest-7.4.4.pyc differ diff --git a/tests/__pycache__/test_lab_ground_truth.cpython-312-pytest-7.4.4.pyc b/tests/__pycache__/test_lab_ground_truth.cpython-312-pytest-7.4.4.pyc index e7f4304..77df5aa 100644 Binary files a/tests/__pycache__/test_lab_ground_truth.cpython-312-pytest-7.4.4.pyc and b/tests/__pycache__/test_lab_ground_truth.cpython-312-pytest-7.4.4.pyc differ diff --git a/tests/test_classification.py b/tests/test_classification.py new file mode 100644 index 0000000..1e9c116 --- /dev/null +++ b/tests/test_classification.py @@ -0,0 +1,195 @@ +"""The classifier, measured against the labelled catalogue — and attacked. + +`test_response_matrix` samples benign and hostile cases alike. The E-003 group +below does something different: it *tries to make the framework unsafe*. An +experiment that only samples cases chosen by the same person who wrote the +implementation cannot establish a safety property. +""" + +from __future__ import annotations + +import json + +import pytest + +from testdriver import Runner, Verdict +from testdriver.classification import ( + Classification, SAFE_TO_ACCEPT, classify, +) +from testdriver.provenance import InadmissibleProvenance, Provenance +from testdriver.intent import Claim +from lab.mutations import CATALOGUE +from scenarios.full_journey import build_journey, journey_lab_server +from tests.selfverification.checks import check_intent_independence + + +def pack_for(*mutations: str, observer_factory=None) -> dict: + with journey_lab_server(*mutations) as (app, tokens, base_url): + world, driver, observer, asset, oracle = build_journey(app, tokens, base_url) + if observer_factory is not None: + observer = observer_factory(observer) + return json.loads(Runner(world, driver, observer, oracle).run(asset).evidence.to_json()) + + +@pytest.fixture(scope="module") +def baseline() -> dict: + return pack_for() + + +# --- the response matrix -------------------------------------------------- + +EXPECTED = { + "M01": Classification.MECHANICAL_ADAPTATION, + "M02": Classification.MECHANICAL_ADAPTATION, + "M03": Classification.UNCHANGED, + "M04": Classification.MECHANICAL_ADAPTATION, + "M05": Classification.UNCHANGED, + "M06": Classification.MECHANICAL_ADAPTATION, + "M07": Classification.MECHANICAL_ADAPTATION, + "M08": Classification.MECHANICAL_ADAPTATION, + "M09": Classification.UNCHANGED, + "M10": Classification.UNCHANGED, + "M21": Classification.MECHANICAL_ADAPTATION, + "M22": Classification.AMBIGUOUS, + "M11": Classification.BEHAVIOUR_CHANGED, + "M12": Classification.BEHAVIOUR_CHANGED, + "M13": Classification.UNCHANGED, + "M14": Classification.UNCHANGED, + "M15": Classification.BEHAVIOUR_CHANGED, + "M16": Classification.BEHAVIOUR_CHANGED, + "M17": Classification.BEHAVIOUR_CHANGED, + "M18": Classification.BEHAVIOUR_CHANGED, + "M19": Classification.BEHAVIOUR_CHANGED, + "M20": Classification.BEHAVIOUR_CHANGED, + "M23": Classification.AMBIGUOUS, +} + + +@pytest.mark.parametrize("mutation_id", sorted(EXPECTED)) +def test_response_matrix(baseline, mutation_id): + assert classify(baseline, pack_for(mutation_id)).classification is EXPECTED[mutation_id] + + +def test_baseline_classifies_as_unchanged(baseline): + assert classify(baseline, pack_for()).classification is Classification.UNCHANGED + + +# --- the metric that matters ---------------------------------------------- + + +def test_false_adaptation_rate_is_zero(baseline): + """The project's existential safety metric. + + Not "low". Zero. A non-zero result is a stop-and-redesign signal, because a + framework that normalizes a real defect once has spent the credibility it + exists to accumulate. + """ + false_adaptations = [ + mutation.id for mutation in CATALOGUE + if mutation.label == "DEFECT" + and classify(baseline, pack_for(mutation.id)).safe_to_accept + ] + assert false_adaptations == [] + + +def test_every_defect_is_escalated(baseline): + for mutation in CATALOGUE: + if mutation.label != "DEFECT": + continue + outcome = classify(baseline, pack_for(mutation.id)) + assert not outcome.safe_to_accept, f"{mutation.id}: {outcome.reason}" + + +def test_mechanical_changes_are_mostly_absorbed(baseline): + """Adaptation has to be *useful*, not merely safe. + + A classifier that escalated everything would score a perfect False Adaptation + Rate and be worthless. This asserts the other side of the trade. + """ + mechanical = [m for m in CATALOGUE if m.label == "MECHANICAL"] + absorbed = [ + m.id for m in mechanical + if classify(baseline, pack_for(m.id)).safe_to_accept + ] + assert len(absorbed) >= len(mechanical) - 1 + + +def test_no_classification_is_outside_the_defined_space(baseline): + for mutation in CATALOGUE: + outcome = classify(baseline, pack_for(mutation.id)) + assert isinstance(outcome.classification, Classification) + assert outcome.reason + + +# --- E-003: attacking the boundary --------------------------------------- + + +def test_attack_surface_substitution(baseline): + """M23: the control is gone from the UI; the API endpoint is still open. + + The most plausible concrete route to a false adaptation. A driver that + 'recovers' by calling the API has routed around a control, not adapted to a + change — and would score a successful mechanical adaptation while an + authorization control sat broken. + """ + outcome = classify(baseline, pack_for("M23")) + assert not outcome.safe_to_accept + assert outcome.classification is Classification.AMBIGUOUS + + +def test_attack_concurrent_mechanical_and_defect(baseline): + """Decision-table row 3. Coincidence is not exoneration. + + This is the case a naive self-healing tool gets wrong: it sees a surface + change, explains the failure with it, and adapts. + """ + for combination in (("M01", "M15"), ("M02", "M17"), ("M21", "M20")): + outcome = classify(baseline, pack_for(*combination)) + assert outcome.classification is Classification.BEHAVIOUR_CHANGED + assert not outcome.safe_to_accept + assert "does not excuse it" in outcome.reason + + +def test_attack_evidence_starvation(baseline): + """Cut the independent observation channel and the answer must be AMBIGUOUS. + + Never a pass, never an adaptation. Evidence starvation is the condition under + which a framework is most tempted to fall back on the actor's own account of + what happened. + """ + class Starved: + def __init__(self, real): + self._real, self._calls, self.name = real, 0, real.name + self.watches = real.watches + + def snapshot(self): + self._calls += 1 + return self._real.snapshot() if self._calls < 2 else {} + + outcome = classify(baseline, pack_for(observer_factory=Starved)) + assert outcome.classification is Classification.AMBIGUOUS + assert not outcome.safe_to_accept + + +def test_attack_provenance_laundering(): + """Intent derived from the implementation must not become a claim. + + Rejected at authoring time, and caught again in the record if it ever got + past — belt and braces, because this one cannot be noticed by looking at + behaviour. + """ + with pytest.raises(InadmissibleProvenance): + Claim("c-laundered", "whatever the system does", + Provenance.AGENT_FROM_IMPLEMENTATION, lambda obs: True, after_step="s1") + + tampered = pack_for("M15") + tampered["provenance_index"]["c-bob-revoked"] = "agent-from-implementation" + assert check_intent_independence(tampered) + + +def test_safe_to_accept_is_a_closed_set(): + """Only two outcomes may proceed without a human. Widening this set is the + single easiest way to destroy the safety property, so it is asserted.""" + assert SAFE_TO_ACCEPT == { + Classification.UNCHANGED, Classification.MECHANICAL_ADAPTATION, + } diff --git a/tests/test_lab_ground_truth.py b/tests/test_lab_ground_truth.py index d2fcbc2..c4c8a6a 100644 --- a/tests/test_lab_ground_truth.py +++ b/tests/test_lab_ground_truth.py @@ -28,7 +28,7 @@ def run_against(*mutations: str): def test_catalogue_is_large_enough_to_support_a_rate(): """Six mutations cannot support precision or recall. Twenty can begin to.""" - assert len(CATALOGUE) >= 20 + assert len(CATALOGUE) >= 20 # 23 as of T08 def test_every_mutation_is_labelled_and_reasoned(): @@ -82,10 +82,19 @@ EXPECTED_VERDICT = { "M18": Verdict.FAIL, "M19": Verdict.FAIL, "M20": Verdict.FAIL, } -# The two SEMANTIC mutations the reference scenario cannot see, and why. -KNOWN_INERT = { +# Mutations the reference scenario cannot see, and why. Coverage is scoped to +# what a scenario asserts (F-0002) *and* to the surfaces it touches: this +# scenario is API-only, so a defect that lives in the UI is outside its reach. +# Declaring them is mandatory — `test_out_of_scope_mutations_are_declared` +# fails on any invisible mutation that is not named here. +OUT_OF_SCOPE = { "M13": "only affects grants that omit a permission; the scenario passes READ explicitly", "M14": "a change of intent with no change of code; nothing observable moved", + "M21": "a UI mutation; this scenario never loads the UI", + "M22": "a UI mutation; this scenario never loads the UI", + "M23": "a UI-surface defect; this scenario is API-only. Covered by the " + "cross-surface journey in tests/test_classification.py, where it is " + "the E-003 surface-substitution attack.", } @@ -105,26 +114,43 @@ def test_no_mechanical_mutation_changes_the_verdict(): assert run_against(mutation.id).verdict is Verdict.PASS, mutation.id -def test_every_defect_is_detected(): +def test_every_in_scope_defect_is_detected(): """The floor of the whole project. A defect the lab cannot surface is a defect no later classifier can be measured against.""" missed = [ m.id for m in CATALOGUE - if m.label == "DEFECT" and run_against(m.id).verdict is Verdict.PASS + if m.label == "DEFECT" + and m.id not in OUT_OF_SCOPE + and run_against(m.id).verdict is Verdict.PASS ] assert missed == [], f"undetected seeded defects: {missed}" -def test_inert_semantic_mutations_are_declared(): - """A mutation the scenario cannot see must be named, not silently ignored.""" +def test_out_of_scope_mutations_are_declared(): + """A mutation the scenario cannot see must be named, not silently ignored. + + Applies to defects as much as to semantic changes — an undeclared invisible + defect is precisely how a suite comes to look greener than it is. + """ for mutation in CATALOGUE: - if mutation.label != "SEMANTIC": + if run_against(mutation.id).verdict is not Verdict.PASS: continue - if run_against(mutation.id).verdict is Verdict.PASS: - assert mutation.id in KNOWN_INERT, ( - f"{mutation.id} is invisible to the reference scenario and " - "undeclared — either cover it or record why not" - ) + if mutation.label == "MECHANICAL": + continue # passing is the correct outcome for these + assert mutation.id in OUT_OF_SCOPE, ( + f"{mutation.id} ({mutation.label}) is invisible to the reference " + "scenario and undeclared — either cover it or record why not" + ) + + +def test_declared_out_of_scope_mutations_really_are_invisible(): + """Stale declarations rot silently. If a mutation becomes visible, the + declaration must be removed rather than left as a standing excuse.""" + for mutation_id in OUT_OF_SCOPE: + assert run_against(mutation_id).verdict is Verdict.PASS, ( + f"{mutation_id} is declared out of scope but the reference scenario " + "now detects it — remove the declaration" + ) def test_deferred_revoke_and_revoke_race_are_behaviourally_identical(): diff --git a/workplans/TD-WP-0002-vertical-spike-crystallization.md b/workplans/TD-WP-0002-vertical-spike-crystallization.md index 69569d2..fce0d16 100644 --- a/workplans/TD-WP-0002-vertical-spike-crystallization.md +++ b/workplans/TD-WP-0002-vertical-spike-crystallization.md @@ -358,7 +358,7 @@ The headline result is a narrowing, not a confirmation: ```task id: TD-WP-0002-T08 -status: todo +status: progress priority: high state_hub_task_id: "e2403d3c-bc35-5dd3-b1c8-2474663e06d0" ```