diff --git a/WORK-RECORDS.md b/WORK-RECORDS.md index 0b2c90f..65aa4e3 100644 --- a/WORK-RECORDS.md +++ b/WORK-RECORDS.md @@ -18,7 +18,7 @@ | task | TD-WP-0002-T03 | done | — | workplans/TD-WP-0002-vertical-spike-crystallization.md | | task | TD-WP-0002-T04 | done | — | workplans/TD-WP-0002-vertical-spike-crystallization.md | | task | TD-WP-0002-T05 | done | — | workplans/TD-WP-0002-vertical-spike-crystallization.md | -| task | TD-WP-0002-T06 | todo | — | workplans/TD-WP-0002-vertical-spike-crystallization.md | +| task | TD-WP-0002-T06 | done | — | workplans/TD-WP-0002-vertical-spike-crystallization.md | | task | TD-WP-0002-T07 | todo | — | workplans/TD-WP-0002-vertical-spike-crystallization.md | | task | TD-WP-0002-T08 | todo | — | workplans/TD-WP-0002-vertical-spike-crystallization.md | | task | TD-WP-0002-T09 | todo | — | workplans/TD-WP-0002-vertical-spike-crystallization.md | diff --git a/lab/__pycache__/app.cpython-312.pyc b/lab/__pycache__/app.cpython-312.pyc index d0d6b76..1e1c01c 100644 Binary files a/lab/__pycache__/app.cpython-312.pyc and b/lab/__pycache__/app.cpython-312.pyc differ diff --git a/lab/__pycache__/http_api.cpython-312.pyc b/lab/__pycache__/http_api.cpython-312.pyc index e9bfbb7..91d5fde 100644 Binary files a/lab/__pycache__/http_api.cpython-312.pyc and b/lab/__pycache__/http_api.cpython-312.pyc differ diff --git a/lab/__pycache__/mutations.cpython-312.pyc b/lab/__pycache__/mutations.cpython-312.pyc index 000d0db..ef915da 100644 Binary files a/lab/__pycache__/mutations.cpython-312.pyc and b/lab/__pycache__/mutations.cpython-312.pyc differ diff --git a/lab/app.py b/lab/app.py index 04f5161..595e68f 100644 --- a/lab/app.py +++ b/lab/app.py @@ -96,6 +96,7 @@ class LabApp: ui_field_order: str = "natural" ui_button_element: str = "button" ui_test_ids: str = "stable" + ui_field_names: str = "canonical" ui_confirm_revoke: bool = False api_path_style: str = "long" tenant_sharing_announced: bool = False diff --git a/lab/http_api.py b/lab/http_api.py index f9f1bbf..52b7222 100644 --- a/lab/http_api.py +++ b/lab/http_api.py @@ -50,9 +50,10 @@ def _render(app: LabApp, user_id: str, resource_id: str) -> str: role = ' role="button"' if tag == "a" else "" href = ' href="#"' if tag == "a" else "" + subject_name = "recipient" if app.ui_field_names == "renamed" else "subject_id" subject_field = ( '' - '' + f'' ) permission_field = ( '' @@ -188,6 +189,9 @@ class LabHandler(BaseHTTPRequestHandler): self._send(404, {"error": "not found"}) return + if self.app.ui_field_names == "renamed" and "recipient" in body: + body["subject_id"] = body.pop("recipient") + if url.path.endswith("/grant"): self._dispatch("grant", resource_id=resource_id, **body) elif url.path.endswith("/revoke"): diff --git a/lab/mutations.py b/lab/mutations.py index 9a2c897..0ea3e83 100644 --- a/lab/mutations.py +++ b/lab/mutations.py @@ -90,6 +90,10 @@ def _m10(app): _m(app, denied_status=401) # --- SEMANTIC ------------------------------------------------------------- # Intended behaviour changed. A human must decide; test-driver must not. +def _m21(app): _m(app, ui_labels="verbose", ui_test_ids="dropped") +def _m22(app): _m(app, ui_field_names="renamed", ui_test_ids="dropped") + + def _m11(app): _m(app, require_share_acceptance=True) def _m12(app): _m(app, revoke_delay_seconds=3600.0) def _m13(app): _m(app, grant_permission_for=lambda p: p or "WRITE") @@ -154,6 +158,17 @@ CATALOGUE: tuple[Mutation, ...] = ( Mutation("M10", "Denials return 401 instead of 403", "MECHANICAL", "api", "Both mean refused. A driver keying on the exact code breaks.", _m10), + Mutation("M21", "Controls reworded and test ids dropped", "MECHANICAL", "ui", + "Two signals disturbed at once: the wording changed and the stable " + "identifiers are gone. Extends the deciding side of the test-id " + "axis, which was N=1 after T07's first run.", _m21, + preserves_test_ids=False), + Mutation("M22", "Form field names changed", "MECHANICAL", "ui", + "subject_id becomes recipient, test ids dropped. The API still means " + "the same thing. This is the case that probes whether discovery " + "generalises or merely pattern-matches known field names.", _m22, + preserves_test_ids=False), + Mutation("M11", "A share must be accepted before it takes effect", "SEMANTIC", "domain", "Bob genuinely cannot read until he accepts. The old claim 'Bob can " "read after the grant' is now wrong, and only a human may say so.", _m11), diff --git a/research/concepts/fitness-map.md b/research/concepts/fitness-map.md index d41bdb9..03c9b78 100644 --- a/research/concepts/fitness-map.md +++ b/research/concepts/fitness-map.md @@ -1,6 +1,6 @@ # Concept ↔ Implementation Fitness Map -**Updated:** 2026-08-22 (TD-WP-0002-T06) +**Updated:** 2026-08-22 (TD-WP-0002-T07) Traces each important concept to the implementation, experiment and evidence that support it. **Unsupported entries are the point of this map** — a concept with no @@ -13,8 +13,12 @@ Support levels follow `TestDriverImprovementLoop.md` §13: ## Current state +`C-semantic-action` is the first concept to reach `C2`: it has an experiment +behind it (the T07 two-arm comparison), and that experiment narrowed the claim +rather than confirming it. Everything else still rests on unit tests. + The deterministic kernel exists (T04) and its guarantees are covered by unit -tests. **Levels have not moved.** A passing unit test is not an experiment: it +tests. **Levels do not move for those.** A passing unit test is not an experiment: it shows the code does what its author intended, not that the concept holds under the mutations it claims to survive. Levels rise when E-001/E-002/E-003 produce evidence, not before. The implementation column below moves; the level column @@ -26,7 +30,7 @@ were aspirational, not evidenced. |---|---|---|---|---|---| | `C-use-case` | C1 | `intent.py` | — | — | Is a use case expressible without leaking mechanics? | | `C-actor-isolation` | C1 | `world.py` | E-001 | `td://self/actor-isolation` | **F-0003** — only observable when the scenario plants canaries. | -| `C-semantic-action` | C1 | `actions.py` | E-001 | — | Does identity survive restructuring better than a recorded sequence? (H-001) | +| `C-semantic-action` | **C2** | `actions.py`, `agentic.py` | E-001 (partial) | T07 arm comparison | **F-0005** — supported only where stable identifiers are absent. Narrower than the concept model claims. | | `C-oracle-independence` | C1 | `runner.py`, `oracles.py` | E-001, E-003 | — | Independence of components ≠ independence of belief. (H-004) | | `C-evidence-pack` | C1 | `evidence.py` | — | `td://self/evidence-reproducibility` | Verdicts are reproducible from S3 alone, on passing and failing runs. | | `C-observation-channel` | C1 | `lab/app.py` | — | — | **D-07** — required of every system under test. Adoption cost unknown. | diff --git a/research/findings/F-0004-stdlib-driver-instead-of-playwright.md b/research/findings/F-0004-stdlib-driver-instead-of-playwright.md new file mode 100644 index 0000000..b61722a --- /dev/null +++ b/research/findings/F-0004-stdlib-driver-instead-of-playwright.md @@ -0,0 +1,60 @@ +--- +id: F-0004 +type: framework-finding +class: FRAMEWORK_LIMITATION +status: open +discovered: "2026-08-22" +discovered_by: TD-WP-0002-T07 +workplan: TD-WP-0002 +task: TD-WP-0002-T07 +carried_to: TD-WP-0002-T10 +--- + +# F-0004 — The browser surface is driven without a browser + +## Decision + +`TD-WP-0002-T07` names Playwright as the browser driver. It was not used. +Instead, `src/testdriver/html.py` parses the lab's server-rendered HTML with +`html.parser`, and `src/testdriver/browser.py` drives it over real HTTP. + +Reasons, in order of weight: + +1. **The lab's surface has no JavaScript.** It is server-rendered forms. A + browser engine would not change what H-001 or H-002 can be measured against — + the mutations that matter are DOM restructuring, rewording and field renaming, + all of which a parser faces in full. +2. **Establishing Playwright is a real cost to the operator** — installation plus + a licence review, and the usual snag is the Chromium binaries the installer + pulls rather than the library licence itself. Blocking the vertical spike on + that was not worth it for a surface that gains nothing from it. +3. **The stack is deliberately boring.** Zero dependencies remains true. + +`Driver` is a Protocol, so a Playwright implementation sits alongside this one +without touching the kernel, the oracles or the evidence format. + +## What this costs — stated, not buried + +The driver cannot: + +- execute JavaScript, so **no single-page-application surface can be driven**; +- take screenshots, so that evidence type listed in `INTENT.md` is unavailable; +- read an accessibility tree, or observe visual layout at all. + +The third is the most consequential for the thesis. A real mechanical change +often moves a control *visually* while leaving the DOM largely intact, and this +driver is blind to that entire class. H-001's mutation set is therefore narrower +than the hypothesis's wording implies: it tests **structural** durability, not +**visual** durability. + +That should be said plainly whenever the H-001 result is quoted. + +## Carried to T10 + +Two questions, neither answerable now: + +1. Does a Playwright driver actually change the H-001 result, or merely widen the + surface it can reach? Worth one experiment, not a rewrite. +2. Is the screenshot evidence type in `INTENT.md` load-bearing, or was it listed + because screenshots are conventional in this space? If nothing has needed one + by T10, that is a candidate for compression. diff --git a/research/findings/F-0005-semantic-actions-earn-their-keep-narrowly.md b/research/findings/F-0005-semantic-actions-earn-their-keep-narrowly.md new file mode 100644 index 0000000..742c866 --- /dev/null +++ b/research/findings/F-0005-semantic-actions-earn-their-keep-narrowly.md @@ -0,0 +1,80 @@ +--- +id: F-0005 +type: framework-finding +class: CONCEPT_DRIFT +status: open +discovered: "2026-08-22" +discovered_by: TD-WP-0002-T07 +workplan: TD-WP-0002 +task: TD-WP-0002-T07 +hypotheses: [H-001] +carried_to: TD-WP-0002-T10 +--- + +# F-0005 — Semantic actions earn their keep more narrowly than claimed + +## The claim as written + +H-001, from `TestDriverImprovementLoop.md` § 3: + +> A semantic action survives implementation restructuring better than a recorded +> UI interaction sequence. + +`INTENT.md` treats this as foundational — semantic actions are "the bridge +between agentic exploration and deterministic crystallization". + +## The measurement + +Twelve mechanical mutations, two arms, same semantic action +(`grant_access(Bob, R, READ)`), same surface, same oracles. + +| | Discovery (agentic) | Recorded selectors (control) | +|---|---|---| +| test ids **preserved** (9 mutations) | 9/9 | **9/9** | +| test ids **dropped** (3 mutations) | 2/3 | 0/3 | + +The control arm is not a straw man: it uses stable `data-td` attributes, which is +what a well-instrumented application provides and what good practice recommends. +`test_the_control_arm_is_not_a_straw_man` asserts it keeps winning where those +attributes survive. + +## What this actually says + +**Where an application is well instrumented and keeps its identifiers, the +semantic action buys nothing.** Nine mutations, two arms, identical results. The +conventional approach is not merely adequate there — it is cheaper, faster and +deterministic. + +The semantic action earns its keep in exactly one circumstance: **when stable +identifiers are absent or are not carried forward through a change.** That is a +real and common circumstance — a rewrite rarely preserves test ids, and a large +share of applications never had them — but it is much narrower than "survives +implementation restructuring better", which reads as a general claim. + +## The second boundary: M22 + +Discovery fails on M22, where form field names change (`subject_id` → +`recipient`) with test ids dropped. The heuristic runtime scores candidates +partly on field names, so renaming them removes a signal it depends on. + +This is an honest limit rather than a bug. It marks where a scripted runtime +stops and where a model plausibly starts: the page still carries the label +"Person" next to the field, which a model could read and a keyword heuristic +cannot. **M22 is the first concrete piece of evidence that a live model would add +capability rather than merely cost** — worth more than a general argument that it +might. + +## Consequences + +1. **H-001 must never be quoted as a single rate.** Split by the test-id axis or + it is misleading. `research/hypotheses/H-001-semantic-action-stability.md` now + records the split. +2. **The concept model overstates this.** `INTENT.md` and the Concept Model + present semantic actions as generally superior. The evidence says + conditionally superior. Classified `CONCEPT_DRIFT` — the concept should be + revised to match the evidence (§ 5 path 2), not the other way round. +3. **Three mutations is still thin.** 2/3 and 0/3 are directionally clear and + statistically nothing. Any stronger statement needs more mutations on the + dropped-identifier side. Recorded rather than rounded up. +4. See also **F-0004**: this driver tests structural durability only. Visual + relayout, the other major mechanical change class, is untested entirely. diff --git a/research/hypotheses/H-001-semantic-action-stability.md b/research/hypotheses/H-001-semantic-action-stability.md index dcf2291..3e18733 100644 --- a/research/hypotheses/H-001-semantic-action-stability.md +++ b/research/hypotheses/H-001-semantic-action-stability.md @@ -1,7 +1,7 @@ --- id: H-001 title: Semantic Action Stability -status: PROPOSED +status: EXPERIMENTING created: "2026-08-22" experiments: [E-001] concepts: [C-semantic-action] @@ -36,6 +36,28 @@ The comparison is unfair if arm B is built naively — a brittle straw man makes H-001 trivially true and worthless. Arm B uses the most robust selector strategy reasonably available (roles, labels, test ids where the lab provides them). +## Result so far (TD-WP-0002-T07) + +Twelve mechanical mutations, both arms, same semantic action: + +| | Discovery (agentic) | Recorded selectors (control) | +|---|---|---| +| test ids preserved (9) | 9/9 | **9/9** | +| test ids dropped (3) | 2/3 | 0/3 | + +**Not falsified, but substantially narrowed.** Where stable identifiers survive, +the control arm matches the agentic arm exactly — the semantic action buys +nothing. It earns its keep only where identifiers are absent or not carried +forward. + +Three mutations on the deciding side is directionally clear and statistically +nothing. See `research/findings/F-0005-...` — the concept model overstates this +and should be revised to match the evidence. + +Scope caveat (F-0004): the driver tests **structural** durability only. Visual +relayout is untested. + ## Status log - 2026-08-22 `PROPOSED`. No evidence. +- 2026-08-22 `EXPERIMENTING`. Partial evidence from T07; awaiting E-001 in full. diff --git a/research/hypotheses/H-002-mechanical-adaptation.md b/research/hypotheses/H-002-mechanical-adaptation.md index 92d7191..0100494 100644 --- a/research/hypotheses/H-002-mechanical-adaptation.md +++ b/research/hypotheses/H-002-mechanical-adaptation.md @@ -1,7 +1,7 @@ --- id: H-002 title: Mechanical Adaptation -status: PROPOSED +status: EXPERIMENTING created: "2026-08-22" experiments: [E-001] concepts: [C-adaptation] @@ -33,8 +33,20 @@ at all. (`docs/TestDriverClassificationDesign.md` D-02). Any non-empty diff is both a falsification signal **and** a framework defect, since no write path should exist. +## Result so far (TD-WP-0002-T07) + +Discovery recovered from 11 of 12 mechanical mutations with **no claim or +invariant diff in any run**, as D-02 requires structurally. The single failure +(M22, field names renamed) failed *loudly* — a `RealizationFailed` recorded in +evidence, not a silent pass. That distinction is the one that matters: the +framework reported that it could not act, rather than reporting that nothing was +wrong. + +Full classification of recovery vs defect is T08. + ## Status log -- 2026-08-22 `PROPOSED`. Design decision D-02 makes the second falsification +- 2026-08-22 `PROPOSED`. +- 2026-08-22 `EXPERIMENTING`. Recovery demonstrated; classification pending T08. Design decision D-02 makes the second falsification branch structurally unreachable; the measurement is retained anyway, as an assertion that the architecture is what we believe it is. diff --git a/scenarios/__pycache__/browser_grant.cpython-312.pyc b/scenarios/__pycache__/browser_grant.cpython-312.pyc new file mode 100644 index 0000000..48a8f75 Binary files /dev/null and b/scenarios/__pycache__/browser_grant.cpython-312.pyc differ diff --git a/scenarios/browser_grant.py b/scenarios/browser_grant.py new file mode 100644 index 0000000..b202519 --- /dev/null +++ b/scenarios/browser_grant.py @@ -0,0 +1,88 @@ +"""The agentic realization scenario: one semantic action, on a real surface. + +Scope is deliberately one action. `grant_access(Bob, R, READ)` is realized +through the browser UI by an actor runtime that must *find* the control, while +everything else — setup, revocation, and every oracle — stays deterministic. + +That split is the design: agentic flexibility at the realization layer only, +deterministic truth everywhere else. +""" + +from __future__ import annotations + +import threading +from contextlib import contextmanager + +from testdriver import ( + Actor, Cast, DirectDriver, Oracle, Scenario, SemanticAction, StateObserver, + Step, VerificationAsset, World, +) +from testdriver.browser import BrowserDriver, Session, record_baseline_selectors +from testdriver.agentic import DiscoveryRuntime +from lab.http_api import serve +from lab.mutations import ObservationChannel, build_lab +from scenarios.alice_bob_carol import RESOURCE, USE_CASE + +BROWSER = frozenset({"browser"}) + + +@contextmanager +def lab_server(*mutations: str): + """A live lab on a free port. The surface is real HTTP, not a function call.""" + app, tokens = build_lab(*mutations) + app.request(tokens["alice"], "create_resource", + resource_id=RESOURCE, content="the secret") + server = serve(app) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + base_url = f"http://127.0.0.1:{server.server_address[1]}" + try: + yield app, tokens, base_url + finally: + server.shutdown() + server.server_close() + + +def baseline_recordings() -> dict[str, str]: + """Capture the control arm's selectors once, against the unmutated surface.""" + with lab_server() as (app, tokens, base_url): + session = Session(base_url, tokens["alice"]) + _, html = session.get(f"/resources/{RESOURCE}/view") + return record_baseline_selectors(html) + + +def build_agentic(app, tokens, base_url, runtime=None): + """One agentic step; deterministic oracles unchanged.""" + cast = Cast() + for name in ("alice", "bob", "carol"): + cast.add(Actor(name, name.title(), credentials={"token": tokens[name]})) + + world = World(id="w-browser", sut=app, sut_version=app.version, cast=cast) + + scenario = Scenario( + id="sc-grant-via-browser", + use_case=USE_CASE, + variant=f"browser/{'+'.join(app.applied_mutations) or 'baseline'}", + watches=(), # filled from the reference scenario below + steps=( + Step("s2-grant", "alice", SemanticAction( + "grant_access", + {"subject_id": "bob", "permission": "READ"}, + permitted_surfaces=BROWSER, + postcondition=lambda obs: obs["state_permission:bob:R"] == "READ", + )), + ), + ) + from scenarios.alice_bob_carol import build as build_reference + _, _, reference_observer, _, _ = build_reference() + scenario = Scenario( + id=scenario.id, use_case=scenario.use_case, steps=scenario.steps, + watches=reference_observer.watches, variant=scenario.variant, + ) + + driver = BrowserDriver(base_url, tokens, runtime or DiscoveryRuntime(), RESOURCE) + observer = StateObserver(ObservationChannel(app), scenario.watches) + asset = VerificationAsset( + id="va-grant-via-browser", scenario=scenario, maturity="T1" + ) + return world, driver, observer, asset, Oracle() diff --git a/src/testdriver/__pycache__/agentic.cpython-312.pyc b/src/testdriver/__pycache__/agentic.cpython-312.pyc new file mode 100644 index 0000000..836c867 Binary files /dev/null and b/src/testdriver/__pycache__/agentic.cpython-312.pyc differ diff --git a/src/testdriver/__pycache__/browser.cpython-312.pyc b/src/testdriver/__pycache__/browser.cpython-312.pyc new file mode 100644 index 0000000..30939fe Binary files /dev/null and b/src/testdriver/__pycache__/browser.cpython-312.pyc differ diff --git a/src/testdriver/__pycache__/html.cpython-312.pyc b/src/testdriver/__pycache__/html.cpython-312.pyc new file mode 100644 index 0000000..2151e4c Binary files /dev/null and b/src/testdriver/__pycache__/html.cpython-312.pyc differ diff --git a/src/testdriver/agentic.py b/src/testdriver/agentic.py new file mode 100644 index 0000000..62f5a94 --- /dev/null +++ b/src/testdriver/agentic.py @@ -0,0 +1,244 @@ +"""Actor runtimes: how a semantic action becomes a concrete interaction. + +A runtime is handed a *goal* and a *document* and must work out the path itself. +It is never handed a selector — a runtime given a selector has discovered +nothing and cannot demonstrate adaptation. + +Two runtimes live here, and the contrast between them is the H-001 experiment: + +* `DiscoveryRuntime` — the agentic arm. Scores candidate controls on semantic + signals (visible text, field names, form target) and **deliberately ignores + `data-td` test ids**. If it were allowed to use them it would be a recorded + selector wearing a different hat. +* `RecordedSelectorRuntime` — the control arm. Captures stable test-id selectors + against the baseline and replays them, which is the most robust form of the + conventional approach. It is a real control, not a straw man: where test ids + survive, it should win. + +Cost and nondeterminism are recorded from the first run. `DiscoveryRuntime` is +deterministic and free, so its token fields are zero — but the fields exist and +are populated from run one, because a live model runtime fills exactly the same +shape and the comparison is impossible to reconstruct after the fact. +""" + +from __future__ import annotations + +import time +from dataclasses import dataclass, field +from typing import Any, Protocol + +from .html import Document, Element + +# Vocabulary linking a semantic action to how a surface might name it. This is +# knowledge about *intent*, not about any particular implementation — which is +# why it lives with the runtime and survives the surface changing. +SYNONYMS: dict[str, tuple[str, ...]] = { + "grant_access": ("share", "give access", "grant", "invite", "add person"), + "revoke_access": ("revoke", "withdraw access", "remove access", "unshare"), +} + +REQUIRED_FIELDS: dict[str, tuple[str, ...]] = { + "grant_access": ("subject_id", "permission"), + "revoke_access": ("subject_id",), +} + + +@dataclass(slots=True) +class RealizationMetrics: + """The economics of one realization attempt. + + T4 in the assessment is an existential question — if agentic realization + costs more than asking an agent to rewrite the broken test, crystallization + is an aesthetic preference rather than a value proposition. These fields are + free to collect from run one and impossible to backfill. + """ + + runtime: str + wall_time_ms: float = 0.0 + candidates_considered: int = 0 + attempts: int = 0 + retries: int = 0 + tokens_in: int = 0 + tokens_out: int = 0 + model: str | None = None + + def as_dict(self) -> dict[str, Any]: + return { + "runtime": self.runtime, + "wall_time_ms": round(self.wall_time_ms, 3), + "candidates_considered": self.candidates_considered, + "attempts": self.attempts, + "retries": self.retries, + "tokens_in": self.tokens_in, + "tokens_out": self.tokens_out, + "model": self.model, + } + + +@dataclass(slots=True) +class Plan: + """A concrete interaction the driver can execute.""" + + method: str + action: str + fields: dict[str, str] + rationale: str + element_path: str + metrics: RealizationMetrics + + +class RealizationFailed(Exception): + """The runtime could not find a way to accomplish the goal on this surface.""" + + +class ActorRuntime(Protocol): + name: str + + def plan(self, document: Document, action_name: str, args: dict) -> Plan: ... + + +# --- the agentic arm ------------------------------------------------------ + + +def _visible_text(form: Element) -> str: + return form.full_text.lower() + + +def _field_names(form: Element) -> set[str]: + return { + element.attrs.get("name", "") + for element in form.walk() + if element.tag in ("input", "select", "textarea") + } - {""} + + +class DiscoveryRuntime: + """Finds a control by what it means, not where it is. + + Scores every form on three independent signals so that no single surface + change is fatal: + + 1. the visible wording matches a synonym of the goal; + 2. the form collects the fields the goal needs; + 3. the form's target names the operation. + + A mutation typically disturbs one signal. Requiring agreement across three is + what lets the same semantic action survive a control moving into a modal, a + DOM rewrite, or a rewording — while still failing loudly when the control is + genuinely gone, which is what keeps a removed authorization check from + reading as a recovery. + """ + + name = "discovery-runtime" + + def plan(self, document: Document, action_name: str, args: dict) -> Plan: + started = time.perf_counter() + synonyms = SYNONYMS.get(action_name, (action_name.replace("_", " "),)) + needed = set(REQUIRED_FIELDS.get(action_name, ())) + + scored: list[tuple[int, str, Element]] = [] + forms = document.forms() + for form in forms: + text = _visible_text(form) + names = _field_names(form) + target = form.attrs.get("action", "") + + reasons: list[str] = [] + score = 0 + if any(word in text for word in synonyms): + score += 3 + reasons.append("wording matches the goal") + if needed and needed <= names: + score += 3 + reasons.append("collects the required fields") + verb = action_name.split("_")[0] + if verb in target.lower(): + score += 2 + reasons.append("target names the operation") + if score: + scored.append((score, "; ".join(reasons), form)) + + metrics = RealizationMetrics( + runtime=self.name, + candidates_considered=len(forms), + attempts=1, + ) + if not scored: + metrics.wall_time_ms = (time.perf_counter() - started) * 1000 + raise RealizationFailed( + f"no control on this surface affords {action_name!r} " + f"(considered {len(forms)} candidates)" + ) + + scored.sort(key=lambda row: row[0], reverse=True) + score, rationale, form = scored[0] + + fields = { + name: str(args[name]) + for name in _field_names(form) + if name in args + } + missing = needed - set(fields) + if missing: + metrics.wall_time_ms = (time.perf_counter() - started) * 1000 + raise RealizationFailed( + f"control for {action_name!r} does not accept {sorted(missing)}" + ) + + metrics.wall_time_ms = (time.perf_counter() - started) * 1000 + return Plan( + method=form.attrs.get("method", "post").upper(), + action=form.attrs.get("action", ""), + fields=fields, + rationale=f"score {score}: {rationale}", + element_path=form.path(), + metrics=metrics, + ) + + +# --- the control arm ------------------------------------------------------ + + +class RecordedSelectorRuntime: + """Replays test-id selectors captured against a baseline surface. + + The conventional approach at its strongest: stable `data-td` attributes are + what a well-instrumented application provides and what good practice says to + use. Where a mutation preserves them this runtime is unbeatable; where a + rewrite drops them it has nothing left. That asymmetry is precisely what + H-001 is asking about. + """ + + name = "recorded-selector-runtime" + + def __init__(self, recordings: dict[str, str]) -> None: + self._recordings = recordings # action_name -> data-td value of the form + + def plan(self, document: Document, action_name: str, args: dict) -> Plan: + started = time.perf_counter() + recorded = self._recordings.get(action_name) + metrics = RealizationMetrics(runtime=self.name, attempts=1) + + candidates = [ + form for form in document.forms() + if form.attrs.get("data-td") == recorded + ] + metrics.candidates_considered = len(document.forms()) + metrics.wall_time_ms = (time.perf_counter() - started) * 1000 + + if recorded is None or not candidates: + raise RealizationFailed( + f"recorded selector [data-td={recorded!r}] matched nothing" + ) + form = candidates[0] + fields = { + name: str(args[name]) for name in _field_names(form) if name in args + } + return Plan( + method=form.attrs.get("method", "post").upper(), + action=form.attrs.get("action", ""), + fields=fields, + rationale=f"replayed recorded selector [data-td={recorded}]", + element_path=form.path(), + metrics=metrics, + ) diff --git a/src/testdriver/browser.py b/src/testdriver/browser.py new file mode 100644 index 0000000..1cf8529 --- /dev/null +++ b/src/testdriver/browser.py @@ -0,0 +1,156 @@ +"""A browser-surface driver over the lab's HTML UI. + +Each actor gets its **own** session — its own base URL binding, its own +credentials, its own cookie-equivalent. Nothing is shared between actors at +module or class level, because a shared client is exactly how isolation breaks in +practice and it never announces itself (F-0003). + +The driver executes a `Plan` produced by an `ActorRuntime`. It does not decide +what to click and it does not decide whether the outcome was correct: the first +belongs to the runtime, the second to the observer and the oracle. +""" + +from __future__ import annotations + +import json +import urllib.error +import urllib.parse +import urllib.request +from dataclasses import dataclass +from typing import Any + +from .actions import SemanticAction, Surface +from .agentic import ActorRuntime, RealizationFailed +from .drivers import Realization, UnsupportedAction +from .html import Document +from .world import Actor + + +@dataclass(slots=True) +class Session: + """One actor's private connection to the surface. Never shared.""" + + base_url: str + token: str + + def _open(self, method: str, path: str, body: bytes | None, content_type: str): + request = urllib.request.Request( + urllib.parse.urljoin(self.base_url, path), + data=body, + method=method, + headers={ + "Authorization": f"Bearer {self.token}", + **({"Content-Type": content_type} if body else {}), + }, + ) + try: + with urllib.request.urlopen(request, timeout=10) as response: + return response.status, response.read().decode() + except urllib.error.HTTPError as error: + return error.code, error.read().decode() + + def get(self, path: str) -> tuple[int, str]: + return self._open("GET", path, None, "") + + def post_form(self, path: str, fields: dict[str, str]) -> tuple[int, str]: + encoded = urllib.parse.urlencode(fields).encode() + return self._open("POST", path, encoded, "application/x-www-form-urlencoded") + + def post_json(self, path: str, payload: dict[str, Any]) -> tuple[int, str]: + return self._open("POST", path, json.dumps(payload).encode(), "application/json") + + +class BrowserDriver: + """Realizes semantic actions by finding and using controls on an HTML page.""" + + def __init__( + self, + base_url: str, + tokens: dict[str, str], + runtime: ActorRuntime, + resource_id: str, + ) -> None: + self._base_url = base_url + self._tokens = tokens + self._runtime = runtime + self._resource_id = resource_id + self.surface = Surface( + id="browser", kind="html", description="lab browser UI" + ) + + def _session_for(self, actor: Actor) -> Session: + # Constructed per call: no actor can inherit another's connection state. + return Session(self._base_url, self._tokens[actor.id]) + + def _view_path(self) -> str: + return f"/resources/{self._resource_id}/view" + + def realize(self, actor: Actor, action: SemanticAction) -> Realization: + action.check_surface(self.surface.id) + session = self._session_for(actor) + + status, body = session.get(self._view_path()) + if status != 200: + # A surface the actor cannot even load is not an adaptation problem. + return Realization( + self.surface.id, + {"action": action.name, "stage": "navigate", "status": status}, + raised=f"NavigationFailed: {status}", + ) + + document = Document.parse(body) + args = {"resource_id": self._resource_id, **dict(action.args)} + + try: + plan = self._runtime.plan(document, action.name, args) + except RealizationFailed as failure: + return Realization( + self.surface.id, + { + "action": action.name, + "stage": "discovery", + "runtime": getattr(self._runtime, "name", "?"), + "page_bytes": len(body), + }, + raised=f"RealizationFailed: {failure}", + ) + + mechanics: dict[str, Any] = { + "action": action.describe(), + "stage": "submit", + "runtime": plan.metrics.runtime, + "rationale": plan.rationale, + "element_path": plan.element_path, + "target": plan.action, + "fields": sorted(plan.fields), + "metrics": plan.metrics.as_dict(), + "actor": actor.id, + } + + status, response = session.post_form(plan.action, plan.fields) + mechanics["status"] = status + if status >= 400: + return Realization( + self.surface.id, mechanics, raised=f"Refused: {status} {response[:120]}" + ) + return Realization(self.surface.id, mechanics) + + +def record_baseline_selectors(html: str) -> dict[str, str]: + """Capture the control arm's selectors from a baseline page. + + Deliberately generous: it takes the strongest identifier the page offers. + A weak control arm would make H-001 trivially true and worthless. + """ + document = Document.parse(html) + recordings: dict[str, str] = {} + for form in document.forms(): + test_id = form.attrs.get("data-td", "") + target = form.attrs.get("action", "") + if not test_id: + continue + if target.endswith("/grant"): + recordings["grant_access"] = test_id + elif target.endswith("/revoke"): + recordings["revoke_access"] = test_id + return recordings diff --git a/src/testdriver/html.py b/src/testdriver/html.py new file mode 100644 index 0000000..af39544 --- /dev/null +++ b/src/testdriver/html.py @@ -0,0 +1,108 @@ +"""A minimal DOM for the browser surface — stdlib only. + +Not a browser. It parses server-rendered HTML into a queryable tree so that a +driver can *look for* a control rather than being told where one is. That +distinction is the whole point: a driver handed a selector has not discovered +anything, and cannot demonstrate adaptation. + +Chosen over Playwright deliberately (see F-0004): the lab's surface is +server-rendered forms with no JavaScript, so a browser engine would add a +dependency, a licence question and several hundred megabytes of binaries without +changing what H-001 and H-002 can be measured against. `Driver` is a Protocol, so +a Playwright implementation can sit alongside this one when a surface needs it. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from html.parser import HTMLParser +from typing import Iterator + +VOID_ELEMENTS = frozenset( + {"area", "base", "br", "col", "embed", "hr", "img", "input", + "link", "meta", "param", "source", "track", "wbr"} +) + + +@dataclass(slots=True) +class Element: + tag: str + attrs: dict[str, str] = field(default_factory=dict) + children: list["Element"] = field(default_factory=list) + text: str = "" + parent: "Element | None" = field(default=None, repr=False) + + def walk(self) -> Iterator["Element"]: + yield self + for child in self.children: + yield from child.walk() + + @property + def full_text(self) -> str: + return " ".join( + part for part in ( + [self.text] + [c.full_text for c in self.children] + ) if part + ).strip() + + def ancestors(self) -> Iterator["Element"]: + node = self.parent + while node is not None: + yield node + node = node.parent + + def path(self) -> str: + """A human-readable location, for evidence. Never used to find anything.""" + parts = [self.tag] + [a.tag for a in self.ancestors()] + return "/".join(reversed(parts)) + + +class _Builder(HTMLParser): + def __init__(self) -> None: + super().__init__(convert_charrefs=True) + self.root = Element("#document") + self._stack = [self.root] + + def handle_starttag(self, tag: str, attrs) -> None: + element = Element(tag, {k: (v or "") for k, v in attrs}, parent=self._stack[-1]) + self._stack[-1].children.append(element) + if tag not in VOID_ELEMENTS: + self._stack.append(element) + + def handle_startendtag(self, tag: str, attrs) -> None: + element = Element(tag, {k: (v or "") for k, v in attrs}, parent=self._stack[-1]) + self._stack[-1].children.append(element) + + def handle_endtag(self, tag: str) -> None: + for index in range(len(self._stack) - 1, 0, -1): + if self._stack[index].tag == tag: + del self._stack[index:] + return + + def handle_data(self, data: str) -> None: + stripped = data.strip() + if stripped: + self._stack[-1].text = (self._stack[-1].text + " " + stripped).strip() + + +@dataclass(slots=True) +class Document: + root: Element + source: str + + @classmethod + def parse(cls, html: str) -> "Document": + builder = _Builder() + builder.feed(html) + builder.close() + return cls(builder.root, html) + + def elements(self, *tags: str) -> list[Element]: + wanted = set(tags) + return [ + element for element in self.root.walk() + if element.tag != "#document" and (not wanted or element.tag in wanted) + ] + + def forms(self) -> list[Element]: + return self.elements("form") diff --git a/tests/__pycache__/test_agentic_realization.cpython-312-pytest-7.4.4.pyc b/tests/__pycache__/test_agentic_realization.cpython-312-pytest-7.4.4.pyc new file mode 100644 index 0000000..d2ac8c2 Binary files /dev/null and b/tests/__pycache__/test_agentic_realization.cpython-312-pytest-7.4.4.pyc differ diff --git a/tests/test_agentic_realization.py b/tests/test_agentic_realization.py new file mode 100644 index 0000000..552b318 --- /dev/null +++ b/tests/test_agentic_realization.py @@ -0,0 +1,175 @@ +"""Agentic realization of one semantic action, against a real HTTP surface. + +The agentic part is confined to *finding the control*. Setup, revocation and +every oracle remain deterministic, so a failure here is a failure to realize — +never a failure to judge. +""" + +from __future__ import annotations + +import pytest + +from testdriver import Runner, SemanticAction, Stratum, Verdict +from testdriver.agentic import ( + DiscoveryRuntime, RealizationFailed, RecordedSelectorRuntime, +) +from testdriver.actions import SurfaceNotPermitted +from testdriver.html import Document +from lab.mutations import BY_ID, CATALOGUE +from scenarios.browser_grant import baseline_recordings, build_agentic, lab_server + + +def realize(*mutations: str, runtime=None): + with lab_server(*mutations) as (app, tokens, base_url): + world, driver, observer, asset, oracle = build_agentic( + app, tokens, base_url, runtime + ) + result = Runner(world, driver, observer, oracle).run(asset) + surface = result.evidence.of_stratum(Stratum.SURFACE)[0] + return result, surface.data + + +def realized_ok(*mutations: str, runtime=None) -> bool: + _, surface = realize(*mutations, runtime=runtime) + return surface.get("raised") is None + + +# --- the realization itself ---------------------------------------------- + + +def test_an_agent_realizes_the_semantic_action_from_intent(): + result, surface = realize() + assert surface["raised"] is None + assert result.verdict is Verdict.PASS + assert surface["mechanics"]["runtime"] == "discovery-runtime" + + +def test_the_runtime_is_never_handed_a_selector(): + """Discovery must rest on meaning, not on identifiers it was given. + + If the runtime consulted `data-td` it would be a recorded selector wearing a + different hat, and H-001 would be measuring nothing. + """ + import inspect + + from testdriver import agentic + + source = inspect.getsource(agentic.DiscoveryRuntime) + assert "data-td" not in source + + +def test_deterministic_oracles_are_unchanged_by_agentic_realization(): + """The agent may find the button. It may not decide whether that was right.""" + result, _ = realize() + collectors = { + obs.collector for obs in result.evidence.observations + if obs.stratum is not Stratum.SURFACE + } + assert collectors == {"state-observer"} + + +def test_a_defect_still_fails_under_agentic_realization(): + result, surface = realize("M17") + assert surface["raised"] is None, "realization itself should succeed" + assert result.verdict is Verdict.FAIL + assert {j.assertion_id for j in result.judgments if j.verdict is Verdict.FAIL} == { + "c-carol-denied", "i-enforcement-matches-record", + } + + +def test_the_agentic_asset_judges_fewer_claims_than_the_full_reference(): + """A one-step scenario reaches fewer claims, and must not imply otherwise. + + M15 passes here purely because this asset never revokes. That is correct and + it is also exactly the kind of narrowing that silently overstates coverage if + nobody writes it down. + """ + result, _ = realize("M15") + judged = {j.assertion_id for j in result.judgments} + assert "c-bob-revoked" not in judged + assert result.verdict is Verdict.PASS + + +# --- isolation and cost --------------------------------------------------- + + +def test_each_actor_gets_its_own_session(): + """A shared client is how isolation breaks in practice (F-0003).""" + from testdriver.browser import BrowserDriver + from testdriver.world import Actor + + driver = BrowserDriver("http://127.0.0.1:1", {"a": "tok-a", "b": "tok-b"}, + DiscoveryRuntime(), "R") + first = driver._session_for(Actor("a", "A")) + second = driver._session_for(Actor("a", "A")) + other = driver._session_for(Actor("b", "B")) + assert first is not second, "sessions must not be cached across calls" + assert first.token != other.token + + +def test_cost_and_nondeterminism_are_recorded_from_the_first_run(): + """Free to collect now, impossible to backfill later.""" + _, surface = realize() + metrics = surface["mechanics"]["metrics"] + for field in ("runtime", "wall_time_ms", "candidates_considered", + "attempts", "retries", "tokens_in", "tokens_out", "model"): + assert field in metrics + assert metrics["candidates_considered"] > 0 + + +def test_runtime_identity_is_recorded_in_evidence(): + """Which agent, which configuration — auditable after the fact.""" + _, surface = realize() + assert surface["mechanics"]["metrics"]["runtime"] == "discovery-runtime" + assert "rationale" in surface["mechanics"] + + +# --- failing loudly ------------------------------------------------------- + + +def test_discovery_fails_loudly_when_the_control_is_absent(): + """A missing control is not something to route around. + + This is the guard that stops a removed authorization control from reading as + a successful adaptation. + """ + page = Document.parse("

Nothing here

") + with pytest.raises(RealizationFailed): + DiscoveryRuntime().plan(page, "grant_access", {"subject_id": "bob"}) + + +def test_the_browser_action_may_not_be_performed_through_the_api(): + """D-05: routing around the UI is surface substitution, not recovery.""" + action = SemanticAction( + "grant_access", {}, permitted_surfaces=frozenset({"browser"}) + ) + with pytest.raises(SurfaceNotPermitted): + action.check_surface("api") + + +# --- H-001: the two arms -------------------------------------------------- + +MECHANICAL = [m for m in CATALOGUE if m.label == "MECHANICAL"] + + +@pytest.mark.parametrize("mutation", MECHANICAL, ids=lambda m: m.id) +def test_discovery_recovers_except_where_field_names_move(mutation): + expected = mutation.id != "M22" + assert realized_ok(mutation.id) is expected + + +@pytest.mark.parametrize("mutation", MECHANICAL, ids=lambda m: m.id) +def test_recorded_selectors_survive_exactly_while_test_ids_do(mutation): + runtime = RecordedSelectorRuntime(baseline_recordings()) + assert realized_ok(mutation.id, runtime=runtime) is mutation.preserves_test_ids + + +def test_the_control_arm_is_not_a_straw_man(): + """Where test ids survive, the conventional approach loses nothing. + + Asserting this protects H-001 from being flattered by a weak control. + """ + runtime = RecordedSelectorRuntime(baseline_recordings()) + preserved = [m for m in MECHANICAL if m.preserves_test_ids] + assert all(realized_ok(m.id, runtime=runtime) for m in preserved) + assert len(preserved) >= 9 diff --git a/workplans/TD-WP-0002-vertical-spike-crystallization.md b/workplans/TD-WP-0002-vertical-spike-crystallization.md index 7c9c3ca..69569d2 100644 --- a/workplans/TD-WP-0002-vertical-spike-crystallization.md +++ b/workplans/TD-WP-0002-vertical-spike-crystallization.md @@ -307,7 +307,7 @@ fail*. All four `td://self/...` identifiers are covered. 72 tests pass overall. ```task id: TD-WP-0002-T07 -status: todo +status: done priority: high state_hub_task_id: "b58cf7ce-dc18-5218-8eac-179291626467" ``` @@ -324,6 +324,36 @@ run than simply asking an agent to rewrite the broken test, the crystallization argument is an aesthetic preference rather than a value proposition. This data is free to collect from run one and impossible to backfill. +**Done 2026-08-22.** `html.py` (stdlib DOM), `agentic.py` (two runtimes), +`browser.py` (per-actor sessions over real HTTP), `scenarios/browser_grant.py`. +107 tests pass. Two decisions taken with the operator: **stdlib HTML driver +instead of Playwright** (F-0004) and **a deterministic discovery runtime instead +of a live model**, both behind interfaces that let the alternatives drop in later. + +The headline result is a narrowing, not a confirmation: + +| | Discovery (agentic) | Recorded selectors (control) | +|---|---|---| +| test ids preserved (9 mutations) | 9/9 | **9/9** | +| test ids dropped (3 mutations) | 2/3 | 0/3 | + +- **F-0005 — semantic actions earn their keep more narrowly than claimed.** Where + an application keeps stable identifiers, the conventional approach matches the + agentic one exactly and is cheaper, faster and deterministic. The semantic + action wins only where identifiers are absent or not carried forward. Filed as + `CONCEPT_DRIFT`: the concept model overstates this and should be revised to + match the evidence. Three mutations on the deciding side is directionally clear + and statistically nothing — recorded rather than rounded up. +- **M22 marks where a scripted runtime stops.** Renaming form fields defeats the + heuristic, but the page still carries a "Person" label a model could read. This + is the first concrete evidence that a live model would add *capability* rather + than only cost — worth more than the general argument that it might. +- Two mutations (M21, M22) were added mid-task because the deciding side of the + test-id axis was N=1 after the first run. Extending the instrument when the + evidence shows it is too thin is the intended behaviour. +- Recovery happened with **no claim or invariant diff in any run**, and the one + failure failed *loudly* — `RealizationFailed` in evidence, not a silent pass. + ## Adaptation detection and the defect-vs-adaptation classifier ```task