Preserve oracle semantics in generated regression judgments

Assistant: codex
Assistant-Model: gpt-6-astra
Assistant-Session: 01a0e76f-be98-7ae3-965d-e0b31290a4c4
This commit is contained in:
tegwick 2026-09-28 15:57:45 +02:00
parent 10077edb8c
commit 3ce772466b
8 changed files with 224 additions and 46 deletions

View file

@ -7,7 +7,7 @@ ancestor maturity: T1
descendant : va-grant-crystallized (T5 Deterministic)
frozen from : 4 identical realizations
surface version: lab-0.2.0-baseline
generated : 2026-08-22
generated : 2026-09-28
Why this file exists
--------------------
@ -29,6 +29,7 @@ is a finding.
from __future__ import annotations
import unittest
import urllib.error
import urllib.parse
import urllib.request
@ -74,6 +75,36 @@ def authenticated_target(base_url, path):
return target, urllib.request.build_opener(_OriginRedirectHandler(origin))
def evaluate_predicate(predicate, snapshot):
"""Shared deterministic judgment semantics, embeddable without the framework."""
if not snapshot:
return "INCONCLUSIVE", "no observations were collected"
try:
satisfied = predicate(snapshot)
except KeyError as missing:
return "INCONCLUSIVE", f"required observation {missing} missing from snapshot"
except Exception as exc:
return "INCONCLUSIVE", f"predicate raised {type(exc).__name__}: {exc}"
if type(satisfied) is not bool:
return "INCONCLUSIVE", "predicate did not return a boolean"
return ("PASS" if satisfied else "FAIL"), None
def assert_predicates(predicates, snapshot):
"""Map oracle outcomes to pytest: FAIL dominates; INCONCLUSIVE is explicit skip."""
judgments = [(text, *evaluate_predicate(predicate, snapshot))
for predicate, text in predicates]
failures = [text for text, verdict, _ in judgments if verdict == "FAIL"]
if failures:
raise AssertionError("; ".join(failures))
unknown = [f"{text}: {reason}" for text, verdict, reason in judgments
if verdict == "INCONCLUSIVE"]
if unknown or not judgments:
raise unittest.SkipTest("INCONCLUSIVE: " + ("; ".join(unknown) or "no assertions"))
def _post(base_url: str, token: str, path: str, fields: dict) -> int:
target, opener = authenticated_target(base_url, path)
request = urllib.request.Request(
@ -104,6 +135,8 @@ def test_grant_access(crystallized_world):
assert realize(base_url, token) < 400, "the frozen realization no longer works"
snapshot = observe()
assert _bob_can_read(snapshot), 'Bob can read R after the grant'
assert _carol_cannot_read(snapshot), 'Carol can never read R'
assert _bob_cannot_write(snapshot), 'A READ grant does not let Bob write R'
assert_predicates([
(_bob_can_read, 'Bob can read R after the grant'),
(_carol_cannot_read, 'Carol can never read R'),
(_bob_cannot_write, 'A READ grant does not let Bob write R'),
], snapshot)