Preserve oracle semantics in generated regression judgments
Assistant: codex Assistant-Model: gpt-6-astra Assistant-Session: 01a0e76f-be98-7ae3-965d-e0b31290a4c4
This commit is contained in:
parent
10077edb8c
commit
3ce772466b
8 changed files with 224 additions and 46 deletions
|
|
@ -7,7 +7,7 @@ ancestor maturity: T1
|
|||
descendant : va-grant-crystallized (T5 Deterministic)
|
||||
frozen from : 4 identical realizations
|
||||
surface version: lab-0.2.0-baseline
|
||||
generated : 2026-08-22
|
||||
generated : 2026-09-28
|
||||
|
||||
Why this file exists
|
||||
--------------------
|
||||
|
|
@ -29,6 +29,7 @@ is a finding.
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
|
|
@ -74,6 +75,36 @@ def authenticated_target(base_url, path):
|
|||
return target, urllib.request.build_opener(_OriginRedirectHandler(origin))
|
||||
|
||||
|
||||
|
||||
def evaluate_predicate(predicate, snapshot):
|
||||
"""Shared deterministic judgment semantics, embeddable without the framework."""
|
||||
if not snapshot:
|
||||
return "INCONCLUSIVE", "no observations were collected"
|
||||
try:
|
||||
satisfied = predicate(snapshot)
|
||||
except KeyError as missing:
|
||||
return "INCONCLUSIVE", f"required observation {missing} missing from snapshot"
|
||||
except Exception as exc:
|
||||
return "INCONCLUSIVE", f"predicate raised {type(exc).__name__}: {exc}"
|
||||
if type(satisfied) is not bool:
|
||||
return "INCONCLUSIVE", "predicate did not return a boolean"
|
||||
return ("PASS" if satisfied else "FAIL"), None
|
||||
|
||||
|
||||
def assert_predicates(predicates, snapshot):
|
||||
"""Map oracle outcomes to pytest: FAIL dominates; INCONCLUSIVE is explicit skip."""
|
||||
judgments = [(text, *evaluate_predicate(predicate, snapshot))
|
||||
for predicate, text in predicates]
|
||||
failures = [text for text, verdict, _ in judgments if verdict == "FAIL"]
|
||||
if failures:
|
||||
raise AssertionError("; ".join(failures))
|
||||
unknown = [f"{text}: {reason}" for text, verdict, reason in judgments
|
||||
if verdict == "INCONCLUSIVE"]
|
||||
if unknown or not judgments:
|
||||
raise unittest.SkipTest("INCONCLUSIVE: " + ("; ".join(unknown) or "no assertions"))
|
||||
|
||||
|
||||
|
||||
def _post(base_url: str, token: str, path: str, fields: dict) -> int:
|
||||
target, opener = authenticated_target(base_url, path)
|
||||
request = urllib.request.Request(
|
||||
|
|
@ -104,6 +135,8 @@ def test_grant_access(crystallized_world):
|
|||
assert realize(base_url, token) < 400, "the frozen realization no longer works"
|
||||
|
||||
snapshot = observe()
|
||||
assert _bob_can_read(snapshot), 'Bob can read R after the grant'
|
||||
assert _carol_cannot_read(snapshot), 'Carol can never read R'
|
||||
assert _bob_cannot_write(snapshot), 'A READ grant does not let Bob write R'
|
||||
assert_predicates([
|
||||
(_bob_can_read, 'Bob can read R after the grant'),
|
||||
(_carol_cannot_read, 'Carol can never read R'),
|
||||
(_bob_cannot_write, 'A READ grant does not let Bob write R'),
|
||||
], snapshot)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue