diff --git a/WORK-RECORDS.md b/WORK-RECORDS.md index 67df1d0..a61b979 100644 --- a/WORK-RECORDS.md +++ b/WORK-RECORDS.md @@ -20,6 +20,6 @@ | task | TD-WP-0002-T05 | done | — | workplans/TD-WP-0002-vertical-spike-crystallization.md | | task | TD-WP-0002-T06 | done | — | workplans/TD-WP-0002-vertical-spike-crystallization.md | | task | TD-WP-0002-T07 | done | — | workplans/TD-WP-0002-vertical-spike-crystallization.md | -| task | TD-WP-0002-T08 | progress | — | workplans/TD-WP-0002-vertical-spike-crystallization.md | +| task | TD-WP-0002-T08 | done | — | workplans/TD-WP-0002-vertical-spike-crystallization.md | | task | TD-WP-0002-T09 | todo | — | workplans/TD-WP-0002-vertical-spike-crystallization.md | | task | TD-WP-0002-T10 | todo | — | workplans/TD-WP-0002-vertical-spike-crystallization.md | diff --git a/crystallized/__init__.py b/crystallized/__init__.py new file mode 100644 index 0000000..8b625d3 --- /dev/null +++ b/crystallized/__init__.py @@ -0,0 +1,6 @@ +"""Generated deterministic regression tests. Do not edit by hand. + +Each file here is the frozen descendant of an agentic verification asset. Its +provenance, ancestor and the number of identical realizations it was frozen from +are recorded in its own docstring. +""" diff --git a/crystallized/__pycache__/__init__.cpython-312.pyc b/crystallized/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000..d98adb6 Binary files /dev/null and b/crystallized/__pycache__/__init__.cpython-312.pyc differ diff --git a/crystallized/__pycache__/conftest.cpython-312-pytest-7.4.4.pyc b/crystallized/__pycache__/conftest.cpython-312-pytest-7.4.4.pyc new file mode 100644 index 0000000..f8c9bb1 Binary files /dev/null and b/crystallized/__pycache__/conftest.cpython-312-pytest-7.4.4.pyc differ diff --git a/crystallized/__pycache__/test_grant_access.cpython-312-pytest-7.4.4.pyc b/crystallized/__pycache__/test_grant_access.cpython-312-pytest-7.4.4.pyc new file mode 100644 index 0000000..5efe09d Binary files /dev/null and b/crystallized/__pycache__/test_grant_access.cpython-312-pytest-7.4.4.pyc differ diff --git a/crystallized/conftest.py b/crystallized/conftest.py new file mode 100644 index 0000000..62a1a8f --- /dev/null +++ b/crystallized/conftest.py @@ -0,0 +1,39 @@ +"""Fixtures for crystallized regression tests. + +The only framework code a crystallized test touches is the observation channel — +because judging still requires observing the system independently, whether or not +a model was involved in getting there (D-07). Nothing here discovers anything. +""" + +from __future__ import annotations + +import threading + +import pytest + +from lab.http_api import serve +from lab.mutations import ObservationChannel, build_lab +from testdriver.observers import StateObserver, Watch + + +@pytest.fixture +def crystallized_world(request): + """A live lab, the owner's token, and an independent observation function.""" + mutations = getattr(request, "param", ()) + app, tokens = build_lab(*mutations) + app.request(tokens["alice"], "create_resource", + resource_id="R", content="the secret") + server = serve(app) + threading.Thread(target=server.serve_forever, daemon=True).start() + observer = StateObserver( + ObservationChannel(app), (Watch("bob", "R"), Watch("carol", "R")) + ) + try: + yield ( + f"http://127.0.0.1:{server.server_address[1]}", + tokens["alice"], + observer.snapshot, + ) + finally: + server.shutdown() + server.server_close() diff --git a/crystallized/test_grant_access.py b/crystallized/test_grant_access.py new file mode 100644 index 0000000..4834fe3 --- /dev/null +++ b/crystallized/test_grant_access.py @@ -0,0 +1,73 @@ +"""Crystallized regression test — generated, do not edit by hand. + +Lineage +------- +ancestor asset : va-grant-via-browser +ancestor maturity: T1 +descendant : va-grant-crystallized (T5 Deterministic) +frozen from : 4 identical realizations +surface version: lab-0.2.0-baseline +generated : 2026-08-22 + +Why this file exists +-------------------- +An agent discovered this path 4 times running and it did not change. The +search is now waste, so it has been frozen. **No model is involved in running +this test.** + +The realization below is plain HTTP with no framework dependency. The assertions +are imported from the originating use case rather than restated — a generated +test that paraphrases its assertions creates a second, unverified statement of +intent, and drift between the two would be silent. See F-0007 for what that +costs. + +If this test starts failing, the correct first response is **not** to update the +selectors. Re-run the agentic ancestor: if it recovers, the surface moved and +this file should be regenerated; if it does not, the behaviour changed and that +is a finding. +""" + +from __future__ import annotations + +import urllib.error +import urllib.parse +import urllib.request + +from scenarios.alice_bob_carol import _bob_can_read, _bob_cannot_write, _carol_cannot_read + +TARGET = '/resources/R/grant' +FIELDS = {'permission': 'READ', 'subject_id': 'bob'} + + +def _post(base_url: str, token: str, path: str, fields: dict) -> int: + request = urllib.request.Request( + urllib.parse.urljoin(base_url, path), + data=urllib.parse.urlencode(fields).encode(), + method="POST", + headers={ + "Authorization": f"Bearer {token}", + "Content-Type": "application/x-www-form-urlencoded", + }, + ) + try: + with urllib.request.urlopen(request, timeout=10) as response: + return response.status + except urllib.error.HTTPError as error: + return error.code + + +def realize(base_url: str, token: str) -> int: + """Perform grant_access deterministically, exactly as the agent learned to.""" + return _post(base_url, token, TARGET, FIELDS) + + +def test_grant_access(crystallized_world): + """grant_access still works, and the claims it protects still hold.""" + base_url, token, observe = crystallized_world + + assert realize(base_url, token) < 400, "the frozen realization no longer works" + + snapshot = observe() + assert _bob_can_read(snapshot), 'Bob can read R after the grant' + assert _carol_cannot_read(snapshot), 'Carol can never read R' + assert _bob_cannot_write(snapshot), 'A READ grant does not let Bob write R' diff --git a/lab/__pycache__/app.cpython-312.pyc b/lab/__pycache__/app.cpython-312.pyc index 1e1c01c..93802fc 100644 Binary files a/lab/__pycache__/app.cpython-312.pyc and b/lab/__pycache__/app.cpython-312.pyc differ diff --git a/lab/__pycache__/http_api.cpython-312.pyc b/lab/__pycache__/http_api.cpython-312.pyc index 153b125..c0e0551 100644 Binary files a/lab/__pycache__/http_api.cpython-312.pyc and b/lab/__pycache__/http_api.cpython-312.pyc differ diff --git a/lab/__pycache__/mutations.cpython-312.pyc b/lab/__pycache__/mutations.cpython-312.pyc index 50c2c1b..0230fbb 100644 Binary files a/lab/__pycache__/mutations.cpython-312.pyc and b/lab/__pycache__/mutations.cpython-312.pyc differ diff --git a/lab/app.py b/lab/app.py index 595e68f..4b95810 100644 --- a/lab/app.py +++ b/lab/app.py @@ -99,6 +99,7 @@ class LabApp: ui_field_names: str = "canonical" ui_confirm_revoke: bool = False api_path_style: str = "long" + api_grant_path: str = "grant" tenant_sharing_announced: bool = False # -- setup ----------------------------------------------------------- diff --git a/lab/http_api.py b/lab/http_api.py index dfb3b21..e6754b8 100644 --- a/lab/http_api.py +++ b/lab/http_api.py @@ -68,7 +68,7 @@ def _render(app: LabApp, user_id: str, resource_id: str) -> str: share_form = ( f'
" @@ -196,7 +196,7 @@ class LabHandler(BaseHTTPRequestHandler): if self.app.ui_field_names == "renamed" and "recipient" in body: body["subject_id"] = body.pop("recipient") - if url.path.endswith("/grant"): + if url.path.endswith(f"/{self.app.api_grant_path}"): self._dispatch("grant", resource_id=resource_id, **body) elif url.path.endswith("/revoke"): self._dispatch("revoke", resource_id=resource_id, **body) diff --git a/lab/mutations.py b/lab/mutations.py index 4016304..42d46f1 100644 --- a/lab/mutations.py +++ b/lab/mutations.py @@ -91,6 +91,7 @@ def _m10(app): _m(app, denied_status=401) # Intended behaviour changed. A human must decide; test-driver must not. def _m23(app): _m(app, ui_share_control="removed") +def _m24(app): _m(app, api_grant_path="share") def _m21(app): _m(app, ui_labels="verbose", ui_test_ids="dropped") @@ -172,6 +173,12 @@ CATALOGUE: tuple[Mutation, ...] = ( "generalises or merely pattern-matches known field names.", _m22, preserves_test_ids=False), + Mutation("M24", "The grant endpoint is renamed", "MECHANICAL", "api", + "/grant becomes /share and the old path stops working. The agentic " + "arm recovers by reading the form; a frozen path cannot. This is the " + "reversibility case — a crystallized asset must fail loudly here, " + "not search.", _m24), + Mutation("M23", "Sharing control removed from the UI, API left open", "DEFECT", "ui", "The surface-substitution trap. A driver that 'recovers' by calling " "the API has routed around a control, not adapted to a change — the " diff --git a/pyproject.toml b/pyproject.toml index c4b9c17..47b73a0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,5 +16,5 @@ build-backend = "hatchling.build" packages = ["src/testdriver"] [tool.pytest.ini_options] -testpaths = ["tests"] +testpaths = ["tests", "crystallized"] pythonpath = ["src", "."] diff --git a/research/concepts/fitness-map.md b/research/concepts/fitness-map.md index 67e13ec..c8c4c75 100644 --- a/research/concepts/fitness-map.md +++ b/research/concepts/fitness-map.md @@ -1,6 +1,6 @@ # Concept ↔ Implementation Fitness Map -**Updated:** 2026-08-22 (TD-WP-0002-T08) +**Updated:** 2026-08-23 (TD-WP-0002-T09) Traces each important concept to the implementation, experiment and evidence that support it. **Unsupported entries are the point of this map** — a concept with no @@ -36,9 +36,9 @@ were aspirational, not evidenced. | `C-observation-channel` | C1 | `lab/app.py` | — | — | **D-07** — required of every system under test. Adoption cost unknown. | | `C-adaptation` | **C2** | `classification.py`, `agentic.py` | E-001 | T08 matrix | 11/12 mechanical absorbed without a human. | | `C-classification` | **C2** | `classification.py` | E-001, E-003 | T08 matrix, FAR 0/7 | **F-0006** — cannot infer SEMANTIC vs DEFECT; collapses to one escalating outcome. | -| `C-crystallization` | C1 | — (T09) | E-002 | — | (H-003) | +| `C-crystallization` | **C2** | `crystallization.py` | E-002 | T09 agreement matrix | Fidelity supported; **F-0007** — the economic case is unmeasurable with a token-free runtime. | | `C-intent-provenance` | C1 | `provenance.py` | E-003 | `td://self/intent-independence` | Constrains provenance, not quality. Accepted residual. | -| `C-lineage` | C0 | — | — | — | Parent pointer only in the spike. | +| `C-lineage` | C1 | `scenario.py`, generated headers | E-002 | generated module docstring | Parent pointer plus provenance in the artefact; no graph. | | `C-energy` | C0 | `energy.py`, capture only | — | — | Dormant by decision. (H-005) | | `C-temperature` | C0 | — | — | — | Deferred. No implementation planned in TD-WP-0002. | | `C-confidence` | C0 | — | — | — | Deferred. | diff --git a/research/experiments/E-002-crystallization-fidelity.md b/research/experiments/E-002-crystallization-fidelity.md index da52a71..9abefc2 100644 --- a/research/experiments/E-002-crystallization-fidelity.md +++ b/research/experiments/E-002-crystallization-fidelity.md @@ -30,4 +30,5 @@ was required. ## Status -`PLANNED`. Blocked on T07. +`EXECUTED` 2026-08-23 (T09). Fidelity criteria met across five lab versions; +the cost criterion is met but uninformative — F-0007. diff --git a/research/findings/F-0007-crystallization-cost-unmeasurable.md b/research/findings/F-0007-crystallization-cost-unmeasurable.md new file mode 100644 index 0000000..21c9740 --- /dev/null +++ b/research/findings/F-0007-crystallization-cost-unmeasurable.md @@ -0,0 +1,87 @@ +--- +id: F-0007 +type: framework-finding +class: FRAMEWORK_LIMITATION +status: open +discovered: "2026-08-23" +discovered_by: TD-WP-0002-T09 +workplan: TD-WP-0002 +task: TD-WP-0002-T09 +hypotheses: [H-003] +carried_to: TD-WP-0002-T10 +--- + +# F-0007 — Crystallization's economic case cannot be measured yet + +## The criterion + +H-003 includes a cost clause, deliberately: + +> - it costs no less to execute than the agentic ancestor. + +with the note: *if crystallization preserves semantics but saves nothing, the +thesis is intact but the product rationale is not.* + +## What was measured + +| | median per run | +|---|---| +| agentic ancestor | 6.87 ms | +| crystallized descendant | 3.17 ms | +| **reduction** | **53.9 %** | + +The criterion is met — the descendant is measurably cheaper. But the number is +close to meaningless as evidence for the thesis. + +## Why it is close to meaningless + +The T07 runtime is a **deterministic heuristic**, chosen with the operator to +avoid API cost and nondeterminism. It consumes zero tokens. So the entire +measured saving is one page fetch, one HTML parse and a two-candidate scoring +pass — a few milliseconds of local work. + +The saving crystallization actually claims is of a different kind and two or +three orders of magnitude larger: **model tokens, model latency, and the +variance that forces retries.** None of those exist in this measurement, because +none of those exist in this runtime. + +So the honest statement is: + +> Crystallization is measurably cheaper than the ancestor it was frozen from. +> The measured 54 % is a **floor** produced by removing local discovery work, and +> it says nothing about the saving that motivates the concept. + +Quoting "54 % cheaper" as support for the crystallization thesis would be +misleading, and this finding exists so that nobody does. + +## What would make it measurable + +A live-model runtime behind the same `ActorRuntime` interface. The +`RealizationMetrics` fields (`tokens_in`, `tokens_out`, `model`, `retries`) were +populated from the first run precisely so this comparison becomes a subtraction +rather than a re-run of everything — see T07. + +**F-0005 already gives an independent reason to want one:** M22 defeats the +heuristic runtime while remaining solvable by reading a visible label. So a live +model would settle two open questions at once — whether it adds *capability* +(F-0005) and whether crystallization has an economic case (this finding). + +That makes a bounded live-model experiment the highest-value next investment, +above any further framework feature. + +## A second, smaller limitation + +The generated test is **not fully standalone**. Its realization is plain +`urllib` with no framework dependency, but its assertions are *imported* from the +originating scenario module rather than restated. + +That was the right call — a generated test that paraphrases its claims creates a +second, unverified statement of intent, and drift between them would be silent. +But it qualifies the adoption story in the workplan ("output that drops into a CI +system which already exists"): what drops in is the realization, while the claims +still require the use-case module on the path. + +Fully standalone generation would need claims expressible in a serializable form +rather than as Python predicates. That is a real design question — it is the same +question as "should scenarios be YAML", deferred at T04 — and both should be +answered together at T10, with evidence about which predicates actually recur. diff --git a/research/hypotheses/H-003-crystallization.md b/research/hypotheses/H-003-crystallization.md index e019c5d..ca01a07 100644 --- a/research/hypotheses/H-003-crystallization.md +++ b/research/hypotheses/H-003-crystallization.md @@ -1,7 +1,7 @@ --- id: H-003 title: Crystallization -status: PROPOSED +status: EXPERIMENTING created: "2026-08-22" experiments: [E-002] concepts: [C-crystallization] @@ -38,6 +38,28 @@ semantics but saves nothing, the thesis is intact but the product rationale is not — and that distinction should be visible in the record rather than discovered by a user. +## Result (TD-WP-0002-T09) + +| Criterion | Outcome | +|---|---| +| produced without human authorship of assertions | **met** — assertions are imported, not restated | +| drops no oracle from the ancestor | **met** — descendant's set is a superset | +| agrees with the ancestor across lab versions | **met** — baseline, M15, M17, M16, M13 | +| costs less to execute | **met, but see F-0007** | + +Also demonstrated: the descendant still catches a seeded defect (M17), it fails +*loudly* rather than searching when its frozen path breaks (M24), and the agentic +ancestor recovers from that same M24 where the descendant cannot — the +reversibility case from `INTENT.md`, shown in both directions. + +**Not falsified. Not yet supported on the criterion that matters most.** The +measured 54 % cost reduction comes entirely from removing local discovery work, +because the T07 runtime consumes no tokens. The saving crystallization actually +claims — model tokens, latency, retry variance — is unmeasured and unmeasurable +with this runtime. See `research/findings/F-0007-...`. + ## Status log - 2026-08-22 `PROPOSED`. No evidence. +- 2026-08-23 `EXPERIMENTING`. Fidelity criteria met; the economic criterion is + met only in a sense too weak to support the thesis. diff --git a/scenarios/__pycache__/browser_grant.cpython-312.pyc b/scenarios/__pycache__/browser_grant.cpython-312.pyc index 48a8f75..d199de0 100644 Binary files a/scenarios/__pycache__/browser_grant.cpython-312.pyc and b/scenarios/__pycache__/browser_grant.cpython-312.pyc differ diff --git a/scenarios/browser_grant.py b/scenarios/browser_grant.py index b202519..4b150eb 100644 --- a/scenarios/browser_grant.py +++ b/scenarios/browser_grant.py @@ -51,6 +51,29 @@ def baseline_recordings() -> dict[str, str]: return record_baseline_selectors(html) +def build_crystallized(app, tokens, base_url, trajectories): + """The same asset, frozen: no discovery, no runtime, no model. + + Identical use case, identical claims, identical observer. Only the driver + differs — which is the entire content of the crystallization claim. + """ + from testdriver.browser import Session + from testdriver.crystallization import CrystallizedDriver + + world, _, observer, asset, oracle = build_agentic(app, tokens, base_url) + driver = CrystallizedDriver( + trajectories, + session_factory=lambda actor: Session(base_url, tokens[actor.id]), + ) + descendant = VerificationAsset( + id="va-grant-crystallized", + scenario=asset.scenario, + maturity="T5", + parent_id=asset.id, + ) + return world, driver, observer, descendant, oracle + + def build_agentic(app, tokens, base_url, runtime=None): """One agentic step; deterministic oracles unchanged.""" cast = Cast() diff --git a/src/testdriver/__pycache__/crystallization.cpython-312.pyc b/src/testdriver/__pycache__/crystallization.cpython-312.pyc new file mode 100644 index 0000000..6e999c0 Binary files /dev/null and b/src/testdriver/__pycache__/crystallization.cpython-312.pyc differ diff --git a/src/testdriver/crystallization.py b/src/testdriver/crystallization.py new file mode 100644 index 0000000..7897e2c --- /dev/null +++ b/src/testdriver/crystallization.py @@ -0,0 +1,272 @@ +"""Turning a stable agentic realization into deterministic code. + +The thesis in one module: once an agent has found the same path enough times, the +finding itself is the valuable part, and repeating the search is waste. What is +crystallized is the *realization* — how to perform a semantic action on this +surface. What is never crystallized, and never re-authored, is the judgment. + +The descendant imports its ancestor's claim predicates rather than restating +them. That is deliberate: a generated test that paraphrases its assertions has +introduced a second, unverified statement of intent, and any drift between the +two is silent. Importing makes oracle preservation a fact rather than a hope. + +Reversibility is part of the design (`INTENT.md`: "Crystallization is +reversible"). A descendant that stops matching its ancestor is evidence the +surface moved again, and the asset returns to its agentic form rather than being +patched. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from datetime import datetime, timezone +from typing import Any, Mapping, Sequence + +from .actions import SemanticAction +from .drivers import Realization +from .world import Actor + + +@dataclass(frozen=True, slots=True) +class Trajectory: + """How one semantic action was actually performed, reduced to what replays.""" + + step_id: str + action_name: str + surface_id: str + method: str + target: str + fields: tuple[str, ...] + + def key(self) -> str: + return json.dumps( + { + "step": self.step_id, "action": self.action_name, + "surface": self.surface_id, "method": self.method, + "target": self.target, "fields": sorted(self.fields), + }, + sort_keys=True, + ) + + +def capture(pack: Mapping[str, Any]) -> tuple[Trajectory, ...]: + """Extract the realization path from one Evidence Pack.""" + out: list[Trajectory] = [] + for obs in pack["observations"]: + if obs["kind"] != "realization" or obs["data"].get("raised"): + continue + mechanics = obs["data"].get("mechanics", {}) + target = mechanics.get("target") or mechanics.get("operation") or "" + fields = mechanics.get("fields") or sorted(mechanics.get("arguments") or {}) + out.append(Trajectory( + step_id=obs["step_id"], + action_name=str(mechanics.get("action", "")).split("(")[0], + surface_id=obs["data"].get("surface", ""), + method=("POST" if obs["data"].get("surface") == "browser" else "CALL"), + target=target, + fields=tuple(sorted(fields)), + )) + return tuple(out) + + +@dataclass(frozen=True, slots=True) +class StabilityReport: + stable: bool + observations: int + distinct_paths: int + reason: str + trajectories: tuple[Trajectory, ...] = () + + +def assess_stability(packs: Sequence[Mapping[str, Any]], minimum: int = 3) -> StabilityReport: + """Is this realization settled enough to freeze? + + Requires the *same* path across several runs. One successful run proves the + agent can find a way; it does not show the surface has stopped moving, and + freezing on a single observation is how a crystallized test becomes flaky + the first time a page renders differently. + """ + if len(packs) < minimum: + return StabilityReport( + False, len(packs), 0, + f"need at least {minimum} runs to judge stability, have {len(packs)}", + ) + captured = [capture(pack) for pack in packs] + keys = {tuple(t.key() for t in trajectory) for trajectory in captured} + if len(keys) != 1: + return StabilityReport( + False, len(packs), len(keys), + f"realization varied across runs ({len(keys)} distinct paths); " + "the surface is still moving", + ) + if not captured[0]: + return StabilityReport(False, len(packs), 0, "no successful realization to freeze") + return StabilityReport( + True, len(packs), 1, + f"identical realization across {len(packs)} runs", captured[0], + ) + + +# --- the deterministic descendant ---------------------------------------- + + +class CrystallizedDriver: + """Replays a frozen trajectory. No discovery, no runtime, no model. + + Deliberately fails rather than searching when the recorded path no longer + works. A driver that fell back to discovery would quietly turn a T5 asset + back into a T1 one and hide the fact that the surface had moved — which is + exactly the signal crystallization is supposed to surface. + """ + + name = "crystallized-driver" + + def __init__(self, trajectories: Sequence[Trajectory], session_factory) -> None: + self._by_step = {t.step_id: t for t in trajectories} + self._by_action = {t.action_name: t for t in trajectories} + self._session_factory = session_factory + self.surface = None # set per action; a frozen path may span surfaces + + def realize(self, actor: Actor, action: SemanticAction) -> Realization: + trajectory = self._by_action.get(action.name) + if trajectory is None: + return Realization( + "crystallized", {"action": action.name}, + raised="NotCrystallized: no frozen path for this action", + ) + action.check_surface(trajectory.surface_id) + args = dict(action.args) + fields = {name: str(args[name]) for name in trajectory.fields if name in args} + + session = self._session_factory(actor) + mechanics: dict[str, Any] = { + "action": action.describe(), + "runtime": self.name, + "target": trajectory.target, + "fields": sorted(fields), + "metrics": { + "runtime": self.name, "wall_time_ms": 0.0, + "candidates_considered": 0, "attempts": 1, "retries": 0, + "tokens_in": 0, "tokens_out": 0, "model": None, + }, + "actor": actor.id, + } + status, body = session.post_form(trajectory.target, fields) + mechanics["status"] = status + if status >= 400: + return Realization(trajectory.surface_id, mechanics, + raised=f"Refused: {status} {body[:120]}") + return Realization(trajectory.surface_id, mechanics) + + +# --- code generation ------------------------------------------------------ + +_TEMPLATE = '''"""Crystallized regression test — generated, do not edit by hand. + +Lineage +------- +ancestor asset : {ancestor_id} +ancestor maturity: {ancestor_maturity} +descendant : {descendant_id} (T5 Deterministic) +frozen from : {runs} identical realizations +surface version: {sut_version} +generated : {generated_at} + +Why this file exists +-------------------- +An agent discovered this path {runs} times running and it did not change. The +search is now waste, so it has been frozen. **No model is involved in running +this test.** + +The realization below is plain HTTP with no framework dependency. The assertions +are imported from the originating use case rather than restated — a generated +test that paraphrases its assertions creates a second, unverified statement of +intent, and drift between the two would be silent. See F-0007 for what that +costs. + +If this test starts failing, the correct first response is **not** to update the +selectors. Re-run the agentic ancestor: if it recovers, the surface moved and +this file should be regenerated; if it does not, the behaviour changed and that +is a finding. +""" + +from __future__ import annotations + +import urllib.error +import urllib.parse +import urllib.request + +from {claims_module} import {claim_imports} + +TARGET = {target!r} +FIELDS = {fields!r} + + +def _post(base_url: str, token: str, path: str, fields: dict) -> int: + request = urllib.request.Request( + urllib.parse.urljoin(base_url, path), + data=urllib.parse.urlencode(fields).encode(), + method="POST", + headers={{ + "Authorization": f"Bearer {{token}}", + "Content-Type": "application/x-www-form-urlencoded", + }}, + ) + try: + with urllib.request.urlopen(request, timeout=10) as response: + return response.status + except urllib.error.HTTPError as error: + return error.code + + +def realize(base_url: str, token: str) -> int: + """Perform {action_name} deterministically, exactly as the agent learned to.""" + return _post(base_url, token, TARGET, FIELDS) + + +def test_{test_name}(crystallized_world): + """{action_name} still works, and the claims it protects still hold.""" + base_url, token, observe = crystallized_world + + assert realize(base_url, token) < 400, "the frozen realization no longer works" + + snapshot = observe() +{assertions} +''' + + +def generate_test_module( + *, + trajectory: Trajectory, + action_args: Mapping[str, Any], + ancestor_id: str, + ancestor_maturity: str, + descendant_id: str, + runs: int, + sut_version: str, + claims: Sequence[Any], + claims_module: str, +) -> str: + """Emit an ordinary pytest module for one crystallized semantic action.""" + fields = {name: str(action_args[name]) for name in trajectory.fields + if name in action_args} + predicate_names = [c.predicate.__name__ for c in claims] + assertions = "\n".join( + f" assert {c.predicate.__name__}(snapshot), {c.text!r}" for c in claims + ) + return _TEMPLATE.format( + ancestor_id=ancestor_id, + ancestor_maturity=ancestor_maturity, + descendant_id=descendant_id, + runs=runs, + sut_version=sut_version, + generated_at=datetime.now(timezone.utc).date().isoformat(), + claims_module=claims_module, + claim_imports=", ".join(sorted(predicate_names)), + target=trajectory.target, + fields=fields, + action_name=trajectory.action_name, + test_name=trajectory.action_name, + assertions=assertions, + ) diff --git a/tests/__pycache__/test_classification.cpython-312-pytest-7.4.4.pyc b/tests/__pycache__/test_classification.cpython-312-pytest-7.4.4.pyc index e1a42df..ba46d32 100644 Binary files a/tests/__pycache__/test_classification.cpython-312-pytest-7.4.4.pyc and b/tests/__pycache__/test_classification.cpython-312-pytest-7.4.4.pyc differ diff --git a/tests/__pycache__/test_crystallization.cpython-312-pytest-7.4.4.pyc b/tests/__pycache__/test_crystallization.cpython-312-pytest-7.4.4.pyc new file mode 100644 index 0000000..66fed66 Binary files /dev/null and b/tests/__pycache__/test_crystallization.cpython-312-pytest-7.4.4.pyc differ diff --git a/tests/__pycache__/test_lab_ground_truth.cpython-312-pytest-7.4.4.pyc b/tests/__pycache__/test_lab_ground_truth.cpython-312-pytest-7.4.4.pyc index 77df5aa..5deb051 100644 Binary files a/tests/__pycache__/test_lab_ground_truth.cpython-312-pytest-7.4.4.pyc and b/tests/__pycache__/test_lab_ground_truth.cpython-312-pytest-7.4.4.pyc differ diff --git a/tests/test_classification.py b/tests/test_classification.py index 1e9c116..37d5d2f 100644 --- a/tests/test_classification.py +++ b/tests/test_classification.py @@ -62,6 +62,7 @@ EXPECTED = { "M19": Classification.BEHAVIOUR_CHANGED, "M20": Classification.BEHAVIOUR_CHANGED, "M23": Classification.AMBIGUOUS, + "M24": Classification.MECHANICAL_ADAPTATION, } diff --git a/tests/test_crystallization.py b/tests/test_crystallization.py new file mode 100644 index 0000000..0b9f5a1 --- /dev/null +++ b/tests/test_crystallization.py @@ -0,0 +1,227 @@ +"""H-003: a stable agentic realization becomes deterministic code. + +What must be preserved is the judgment, not the path. These tests check that the +descendant asserts everything its ancestor did, agrees with it wherever both can +run, and costs less — and that it fails loudly rather than searching when the +surface moves out from under it. +""" + +from __future__ import annotations + +import json +import time + +import pytest + +from testdriver import Runner, Verdict +from testdriver.crystallization import ( + CrystallizedDriver, Trajectory, assess_stability, capture, + generate_test_module, +) +from scenarios.alice_bob_carol import USE_CASE +from scenarios.browser_grant import build_agentic, build_crystallized, lab_server + +RUNS = 4 + + +def agentic_pack(*mutations: str) -> dict: + with lab_server(*mutations) as (app, tokens, base_url): + world, driver, observer, asset, oracle = build_agentic(app, tokens, base_url) + return json.loads(Runner(world, driver, observer, oracle).run(asset).evidence.to_json()) + + +@pytest.fixture(scope="module") +def stability(): + return assess_stability([agentic_pack() for _ in range(RUNS)]) + + +@pytest.fixture(scope="module") +def frozen(stability): + assert stability.stable + return stability.trajectories + + +# --- stability ------------------------------------------------------------ + + +def test_one_successful_run_is_not_enough_to_freeze(): + """One run proves the agent can find a way, not that the surface has settled.""" + report = assess_stability([agentic_pack()]) + assert not report.stable + assert "at least" in report.reason + + +def test_a_moving_surface_is_not_frozen(): + """Different paths across runs means the surface is still moving.""" + packs = [agentic_pack(), agentic_pack("M24"), agentic_pack(), agentic_pack()] + report = assess_stability(packs) + assert not report.stable + assert report.distinct_paths > 1 + + +def test_a_settled_surface_is_frozen(stability): + assert stability.stable + assert stability.observations == RUNS + assert stability.distinct_paths == 1 + + +# --- the descendant ------------------------------------------------------- + + +def run_descendant(frozen, *mutations: str): + with lab_server(*mutations) as (app, tokens, base_url): + world, driver, observer, asset, oracle = build_crystallized( + app, tokens, base_url, frozen + ) + return Runner(world, driver, observer, oracle).run(asset) + + +def test_the_descendant_runs_with_no_model_and_no_discovery(frozen): + result = run_descendant(frozen) + assert result.verdict is Verdict.PASS + mechanics = [ + obs.data["mechanics"] for obs in result.evidence.observations + if obs.kind == "realization" + ][0] + assert mechanics["runtime"] == "crystallized-driver" + assert mechanics["metrics"]["candidates_considered"] == 0 + assert mechanics["metrics"]["tokens_in"] == 0 + + +def test_the_descendant_preserves_the_ancestors_oracle_set(frozen): + ancestor = agentic_pack() + descendant = run_descendant(frozen) + before = {v["assertion_id"] for v in ancestor["verdicts"]} + after = {j.assertion_id for j in descendant.judgments} + assert before <= after, f"crystallization dropped {before - after}" + + +@pytest.mark.parametrize("mutation", ["", "M15", "M17", "M16", "M13"]) +def test_ancestor_and_descendant_agree(frozen, mutation): + """Same verdicts on the same lab version. Disagreement falsifies H-003.""" + mutations = (mutation,) if mutation else () + ancestor = agentic_pack(*mutations) + descendant = run_descendant(frozen, *mutations) + before = {(v["assertion_id"], v["step_id"]): v["verdict"] for v in ancestor["verdicts"]} + after = {(j.assertion_id, j.step_id): j.verdict.value for j in descendant.judgments} + assert before == after + + +def test_the_descendant_still_catches_a_defect(frozen): + """Crystallization must not trade rigour for speed.""" + result = run_descendant(frozen, "M17") + assert result.verdict is Verdict.FAIL + assert result.judgment("c-carol-denied").verdict is Verdict.FAIL + + +def test_the_descendant_fails_loudly_when_its_path_breaks(frozen): + """A frozen driver must not fall back to searching. + + Falling back would quietly turn a T5 asset into a T1 one and hide the very + signal crystallization exists to produce: the surface moved. + + M24 is the reversibility case from `INTENT.md`: the agentic ancestor recovers + by reading the form, the frozen descendant cannot, and the right response is + to thaw the asset rather than patch the generated file. + """ + result = run_descendant(frozen, "M24") # the grant endpoint is renamed + raised = [ + obs.data["raised"] for obs in result.evidence.observations + if obs.kind == "realization" + ] + assert any(raised), "the frozen path should have stopped working" + + +def test_lineage_is_retained(frozen): + with lab_server() as (app, tokens, base_url): + _, _, _, descendant, _ = build_crystallized(app, tokens, base_url, frozen) + assert descendant.parent_id == "va-grant-via-browser" + assert descendant.maturity == "T5" + + +# --- cost ----------------------------------------------------------------- + + +def test_the_descendant_costs_less_to_execute(frozen): + """H-003's economic criterion. + + With a scripted runtime the saving is only the page fetch, parse and scoring + — real but small. See F-0007: the meaningful comparison needs a live model, + and this number is a floor, not the answer. + """ + def timed(fn, *args): + started = time.perf_counter() + fn(*args) + return time.perf_counter() - started + + with lab_server() as (app, tokens, base_url): + agentic = build_agentic(app, tokens, base_url) + crystal = build_crystallized(app, tokens, base_url, frozen) + agentic_ms = min(timed(lambda: Runner(*agentic[:3], agentic[4]).run(agentic[3])) + for _ in range(3)) + crystal_ms = min(timed(lambda: Runner(*crystal[:3], crystal[4]).run(crystal[3])) + for _ in range(3)) + assert crystal_ms < agentic_ms, (agentic_ms, crystal_ms) + + +# --- the generated artefact ---------------------------------------------- + + +def test_the_generated_module_carries_its_lineage(frozen, stability): + source = generate_test_module( + trajectory=frozen[0], + action_args={"subject_id": "bob", "permission": "READ"}, + ancestor_id="va-grant-via-browser", ancestor_maturity="T1", + descendant_id="va-grant-crystallized", runs=stability.observations, + sut_version="lab-0.2.0-baseline", + claims=[c for c in USE_CASE.claims if c.id == "c-bob-reads"], + claims_module="scenarios.alice_bob_carol", + ) + assert "ancestor asset : va-grant-via-browser" in source + assert f"frozen from : {RUNS} identical realizations" in source + assert "No model is involved" in source + + +def test_the_generated_module_imports_no_agentic_machinery(): + source = open("crystallized/test_grant_access.py").read() + imports = [ + line for line in source.splitlines() + if line.startswith(("import ", "from ")) and "urllib" not in line + ] + assert imports == [ + "from __future__ import annotations", + "from scenarios.alice_bob_carol import _bob_can_read, " + "_bob_cannot_write, _carol_cannot_read", + ], imports + + +def test_the_generated_module_restates_no_assertions(): + """Assertions are imported, never paraphrased. + + A generated test that restates its claims creates a second, unverified + statement of intent, and any drift between the two is silent. + """ + source = open("crystallized/test_grant_access.py").read() + assert "from scenarios.alice_bob_carol import" in source + + +def test_the_ancestor_recovers_where_the_descendant_cannot(frozen): + """The reversibility case, both halves. + + This is what distinguishes crystallization from ordinary code generation: the + agentic form is not discarded when the deterministic one is created, because + it is the thing that can recover when the surface moves again. + """ + ancestor = agentic_pack("M24") + ancestor_raised = [ + obs["data"].get("raised") for obs in ancestor["observations"] + if obs["kind"] == "realization" + ] + assert not any(ancestor_raised), "the agentic ancestor should have recovered" + + descendant = run_descendant(frozen, "M24") + descendant_raised = [ + obs.data.get("raised") for obs in descendant.evidence.observations + if obs.kind == "realization" + ] + assert any(descendant_raised), "the frozen descendant should have failed loudly" diff --git a/workplans/TD-WP-0002-vertical-spike-crystallization.md b/workplans/TD-WP-0002-vertical-spike-crystallization.md index 795d038..dece354 100644 --- a/workplans/TD-WP-0002-vertical-spike-crystallization.md +++ b/workplans/TD-WP-0002-vertical-spike-crystallization.md @@ -415,7 +415,7 @@ without a human, so the safety result is not bought by escalating everything. ```task id: TD-WP-0002-T09 -status: todo +status: done priority: high state_hub_task_id: "855a1f41-b839-57fc-87b8-198ce2a9f6b1" ``` @@ -432,6 +432,31 @@ Exit: the generated test runs with zero agentic involvement, preserves the relevant claims and oracles, retains visible lineage, and measurably costs less to execute than the agentic ancestor. +**Done 2026-08-23.** `crystallization.py`, `crystallized/test_grant_access.py` +(generated), 163 tests pass. All four exit criteria met. + +- Freezing requires the **same** path across several runs, not one success. One + run proves the agent can find a way; it does not show the surface has stopped + moving, and freezing on a single observation is how a crystallized test becomes + flaky the first time a page renders differently. +- **Reversibility demonstrated in both directions.** M24 (grant endpoint renamed) + breaks the frozen path; the descendant fails *loudly* rather than falling back + to searching — falling back would quietly turn a T5 asset into a T1 one and + hide the signal crystallization exists to produce — while the agentic ancestor + recovers from the same mutation. That is what distinguishes crystallization + from ordinary codegen: the agentic form is not discarded. +- **F-0007 (open) — the economic case is unmeasurable.** The descendant is 54% + cheaper, and that number should not be quoted in support of the thesis. The + T07 runtime consumes zero tokens, so the entire saving is one page fetch, one + parse and a two-candidate scoring pass. The saving crystallization actually + claims — model tokens, latency, retry variance — is two or three orders of + magnitude larger and entirely absent from this measurement. +- The generated test is **not fully standalone**: realization is plain `urllib`, + but assertions are *imported* rather than restated, because a generated test + that paraphrases its claims creates a second unverified statement of intent. + Right call, and it qualifies the "drops into existing CI" story — what drops in + is the realization; the claims still need the use-case module. + ## Gate review and first compression pass ```task