68 lines
3.2 KiB
Python
68 lines
3.2 KiB
Python
|
"""Reproduce E-001: PYTHONPATH=src:. python3 tools/measure_e001.py OUTPUT.json."""
|
||
|
|
import json
|
||
|
|
import sys
|
||
|
|
from collections import defaultdict
|
||
|
|
from datetime import datetime, timezone
|
||
|
|
from pathlib import Path
|
||
|
|
|
||
|
|
from lab.mutations import CATALOGUE
|
||
|
|
from scenarios.browser_grant import baseline_recordings, build_agentic, lab_server
|
||
|
|
from scenarios.full_journey import build_journey, journey_lab_server
|
||
|
|
from testdriver import Runner, Stratum
|
||
|
|
from testdriver.agentic import DiscoveryRuntime, RecordedSelectorRuntime
|
||
|
|
from testdriver.classification import classify
|
||
|
|
|
||
|
|
|
||
|
|
def journey(*mutations):
|
||
|
|
with journey_lab_server(*mutations) as (app, tokens, url):
|
||
|
|
world, driver, observer, asset, oracle = build_journey(app, tokens, url)
|
||
|
|
return json.loads(Runner(world, driver, observer, oracle).run(asset).evidence.to_json())
|
||
|
|
|
||
|
|
|
||
|
|
def measure():
|
||
|
|
baseline = journey()
|
||
|
|
selectors = baseline_recordings()
|
||
|
|
rows = []
|
||
|
|
for mutation in CATALOGUE:
|
||
|
|
pack = journey(mutation.id)
|
||
|
|
outcome = classify(baseline, pack)
|
||
|
|
row = {"mutation": mutation.id, "label": mutation.label,
|
||
|
|
"preserves_test_ids": mutation.preserves_test_ids,
|
||
|
|
"classification": outcome.as_dict(), "arms": {}}
|
||
|
|
if mutation.label == "MECHANICAL":
|
||
|
|
for name, runtime in (("discovery", DiscoveryRuntime()),
|
||
|
|
("recorded", RecordedSelectorRuntime(selectors))):
|
||
|
|
with lab_server(mutation.id) as (app, tokens, url):
|
||
|
|
world, driver, observer, asset, oracle = build_agentic(app, tokens, url, runtime)
|
||
|
|
result = Runner(world, driver, observer, oracle).run(asset)
|
||
|
|
surface = result.evidence.of_stratum(Stratum.SURFACE)[0].data
|
||
|
|
row["arms"][name] = {
|
||
|
|
"realized": surface["raised"] is None,
|
||
|
|
"verdict": result.verdict.value,
|
||
|
|
"surface": surface,
|
||
|
|
"judgments": [j.as_dict() for j in result.judgments],
|
||
|
|
}
|
||
|
|
rows.append(row)
|
||
|
|
summary = defaultdict(lambda: {"total": 0, "discovery": 0, "recorded": 0})
|
||
|
|
for row in rows:
|
||
|
|
if not row["arms"]:
|
||
|
|
continue
|
||
|
|
group = summary["preserved" if row["preserves_test_ids"] else "dropped"]
|
||
|
|
group["total"] += 1
|
||
|
|
for arm in ("discovery", "recorded"):
|
||
|
|
group[arm] += int(row["arms"][arm]["realized"]
|
||
|
|
and row["arms"][arm]["verdict"] == "PASS")
|
||
|
|
false_adaptations = [row["mutation"] for row in rows if row["label"] == "DEFECT"
|
||
|
|
and row["classification"]["classification"]
|
||
|
|
in ("UNCHANGED", "MECHANICAL_ADAPTATION")]
|
||
|
|
return {"measured_at": datetime.now(timezone.utc).isoformat(),
|
||
|
|
"scope": "Synthetic server-rendered HTML, deterministic runtimes; no browser engine or live model",
|
||
|
|
"summary": dict(summary), "false_adaptations": false_adaptations,
|
||
|
|
"defect_count": sum(row["label"] == "DEFECT" for row in rows), "rows": rows}
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
result = measure()
|
||
|
|
Path(sys.argv[1]).write_text(json.dumps(result, indent=2) + "\n")
|
||
|
|
print(json.dumps({k: v for k, v in result.items() if k != "rows"}, indent=2))
|