"""Reproduce E-001: PYTHONPATH=src:. python3 tools/measure_e001.py OUTPUT.json.""" import json import sys from collections import defaultdict from datetime import datetime, timezone from pathlib import Path from lab.mutations import CATALOGUE from scenarios.browser_grant import baseline_recordings, build_agentic, lab_server from scenarios.full_journey import build_journey, journey_lab_server from testdriver import Runner, Stratum from testdriver.agentic import DiscoveryRuntime, RecordedSelectorRuntime from testdriver.classification import classify def journey(*mutations): with journey_lab_server(*mutations) as (app, tokens, url): world, driver, observer, asset, oracle = build_journey(app, tokens, url) return json.loads(Runner(world, driver, observer, oracle).run(asset).evidence.to_json()) def measure(): baseline = journey() selectors = baseline_recordings() rows = [] for mutation in CATALOGUE: pack = journey(mutation.id) outcome = classify(baseline, pack) row = {"mutation": mutation.id, "label": mutation.label, "preserves_test_ids": mutation.preserves_test_ids, "classification": outcome.as_dict(), "arms": {}} if mutation.label == "MECHANICAL": for name, runtime in (("discovery", DiscoveryRuntime()), ("recorded", RecordedSelectorRuntime(selectors))): with lab_server(mutation.id) as (app, tokens, url): world, driver, observer, asset, oracle = build_agentic(app, tokens, url, runtime) result = Runner(world, driver, observer, oracle).run(asset) surface = result.evidence.of_stratum(Stratum.SURFACE)[0].data row["arms"][name] = { "realized": surface["raised"] is None, "verdict": result.verdict.value, "surface": surface, "judgments": [j.as_dict() for j in result.judgments], } rows.append(row) summary = defaultdict(lambda: {"total": 0, "discovery": 0, "recorded": 0}) for row in rows: if not row["arms"]: continue group = summary["preserved" if row["preserves_test_ids"] else "dropped"] group["total"] += 1 for arm in ("discovery", "recorded"): group[arm] += int(row["arms"][arm]["realized"] and row["arms"][arm]["verdict"] == "PASS") false_adaptations = [row["mutation"] for row in rows if row["label"] == "DEFECT" and row["classification"]["classification"] in ("UNCHANGED", "MECHANICAL_ADAPTATION")] return {"measured_at": datetime.now(timezone.utc).isoformat(), "scope": "Synthetic server-rendered HTML, deterministic runtimes; no browser engine or live model", "summary": dict(summary), "false_adaptations": false_adaptations, "defect_count": sum(row["label"] == "DEFECT" for row in rows), "rows": rows} if __name__ == "__main__": result = measure() Path(sys.argv[1]).write_text(json.dumps(result, indent=2) + "\n") print(json.dumps({k: v for k, v in result.items() if k != "rows"}, indent=2))