Complete local generalisation tasks and document remaining experiment blockers

Assistant: codex
Assistant-Model: gpt-6-astra
Assistant-Session: 01a0e76f-be98-7ae3-965d-e0b31290a4c4

tools/measure_e001.py

@@ -0,0 +1,67 @@
+"""Reproduce E-001: PYTHONPATH=src:. python3 tools/measure_e001.py OUTPUT.json."""
+import json
+import sys
+from collections import defaultdict
+from datetime import datetime, timezone
+from pathlib import Path
+
+from lab.mutations import CATALOGUE
+from scenarios.browser_grant import baseline_recordings, build_agentic, lab_server
+from scenarios.full_journey import build_journey, journey_lab_server
+from testdriver import Runner, Stratum
+from testdriver.agentic import DiscoveryRuntime, RecordedSelectorRuntime
+from testdriver.classification import classify
+
+
+def journey(*mutations):
+    with journey_lab_server(*mutations) as (app, tokens, url):
+        world, driver, observer, asset, oracle = build_journey(app, tokens, url)
+        return json.loads(Runner(world, driver, observer, oracle).run(asset).evidence.to_json())
+
+
+def measure():
+    baseline = journey()
+    selectors = baseline_recordings()
+    rows = []
+    for mutation in CATALOGUE:
+        pack = journey(mutation.id)
+        outcome = classify(baseline, pack)
+        row = {"mutation": mutation.id, "label": mutation.label,
+               "preserves_test_ids": mutation.preserves_test_ids,
+               "classification": outcome.as_dict(), "arms": {}}
+        if mutation.label == "MECHANICAL":
+            for name, runtime in (("discovery", DiscoveryRuntime()),
+                                  ("recorded", RecordedSelectorRuntime(selectors))):
+                with lab_server(mutation.id) as (app, tokens, url):
+                    world, driver, observer, asset, oracle = build_agentic(app, tokens, url, runtime)
+                    result = Runner(world, driver, observer, oracle).run(asset)
+                    surface = result.evidence.of_stratum(Stratum.SURFACE)[0].data
+                    row["arms"][name] = {
+                        "realized": surface["raised"] is None,
+                        "verdict": result.verdict.value,
+                        "surface": surface,
+                        "judgments": [j.as_dict() for j in result.judgments],
+                    }
+        rows.append(row)
+    summary = defaultdict(lambda: {"total": 0, "discovery": 0, "recorded": 0})
+    for row in rows:
+        if not row["arms"]:
+            continue
+        group = summary["preserved" if row["preserves_test_ids"] else "dropped"]
+        group["total"] += 1
+        for arm in ("discovery", "recorded"):
+            group[arm] += int(row["arms"][arm]["realized"]
+                              and row["arms"][arm]["verdict"] == "PASS")
+    false_adaptations = [row["mutation"] for row in rows if row["label"] == "DEFECT"
+                        and row["classification"]["classification"]
+                        in ("UNCHANGED", "MECHANICAL_ADAPTATION")]
+    return {"measured_at": datetime.now(timezone.utc).isoformat(),
+            "scope": "Synthetic server-rendered HTML, deterministic runtimes; no browser engine or live model",
+            "summary": dict(summary), "false_adaptations": false_adaptations,
+            "defect_count": sum(row["label"] == "DEFECT" for row in rows), "rows": rows}
+
+
+if __name__ == "__main__":
+    result = measure()
+    Path(sys.argv[1]).write_text(json.dumps(result, indent=2) + "\n")
+    print(json.dumps({k: v for k, v in result.items() if k != "rows"}, indent=2))