Preserve oracle semantics in generated regression judgments

Assistant: codex
Assistant-Model: gpt-6-astra
Assistant-Session: 01a0e76f-be98-7ae3-965d-e0b31290a4c4

tests/test_generated_judgments.py

@@ -0,0 +1,106 @@
+"""Execute generated artifacts to check parity with deterministic oracle judgments."""
+from dataclasses import replace
+import sys
+import types
+import unittest
+
+import pytest
+
+from scenarios.alice_bob_carol import USE_CASE
+from testdriver import Oracle, Verdict
+from testdriver.crystallization import Trajectory, generate_test_module
+from testdriver.oracles import overall
+
+
+def yes(snapshot): return True
+
+def no(snapshot): return False
+
+def unknown(snapshot): return 'unknown'
+
+def absent(snapshot): return None
+
+def numeric(snapshot): return 1
+
+def missing(snapshot): return snapshot['missing']
+
+def broken(snapshot): raise RuntimeError('unavailable')
+
+
+class CannotCoerce:
+    def __bool__(self):
+        raise AssertionError('must not coerce')
+
+
+def opaque(snapshot): return CannotCoerce()
+
+
+def generated(predicates, monkeypatch):
+    module = types.ModuleType('_generated_claims_test')
+    for predicate in predicates:
+        setattr(module, predicate.__name__, predicate)
+    monkeypatch.setitem(sys.modules, module.__name__, module)
+    claims = [replace(USE_CASE.claims[0], id=f'claim-{i}', predicate=predicate)
+              for i, predicate in enumerate(predicates)]
+    source = generate_test_module(
+        trajectory=Trajectory('step', 'grant_access', 'browser', '/grant', ()),
+        action_args={}, ancestor_id='a', ancestor_maturity='T1', descendant_id='b',
+        runs=3, sut_version='test', claims=claims, claims_module=module.__name__)
+    namespace = {}
+    exec(compile(source, '<generated>', 'exec'), namespace)
+    namespace['realize'] = lambda *args: 200
+    return namespace['test_grant_access'], claims
+
+
+def outcome(test, snapshot):
+    try:
+        test(('unused', 'synthetic-token', lambda: snapshot))
+    except unittest.SkipTest as exc:
+        assert 'INCONCLUSIVE' in str(exc)
+        return Verdict.INCONCLUSIVE
+    except AssertionError:
+        return Verdict.FAIL
+    return Verdict.PASS
+
+
+@pytest.mark.parametrize('predicates', [(yes,), (no,), (unknown,), (absent,),
+                                      (numeric,), (missing,), (broken,), (opaque,),
+                                      (unknown, no), (no, unknown), (yes, unknown)])
+@pytest.mark.parametrize('snapshot', [{'observed': True}, {}])
+def test_generated_verdicts_match_oracle_including_failure_precedence(monkeypatch, predicates, snapshot):
+    test, claims = generated(predicates, monkeypatch)
+    expected = overall([Oracle().judge(c, snapshot, 'step') for c in claims])
+    assert outcome(test, snapshot) is expected
+
+
+@pytest.mark.parametrize('predicate', [yes, no, unknown, absent, numeric, missing, broken, opaque])
+def test_checked_in_descendant_preserves_the_same_semantics(monkeypatch, predicate):
+    import crystallized.test_grant_access as artifact
+    monkeypatch.setattr(artifact, 'realize', lambda *args: 200)
+    for name in ('_bob_can_read', '_carol_cannot_read', '_bob_cannot_write'):
+        monkeypatch.setattr(artifact, name, predicate)
+    claim = replace(USE_CASE.claims[0], predicate=predicate)
+    assert outcome(artifact.test_grant_access, {'observed': True}) is Oracle().judge(
+        claim, {'observed': True}, 'step').verdict
+
+
+def test_pytest_reports_inconclusive_as_skipped_and_never_as_passed(tmp_path):
+    import subprocess
+    (tmp_path / 'local_predicates.py').write_text(
+        "def yes(snapshot): return True\ndef no(snapshot): return False\n"
+        "def unknown(snapshot): return 'unknown'\n")
+    (tmp_path / 'conftest.py').write_text(
+        "import pytest\n@pytest.fixture\ndef crystallized_world():\n"
+        "    return ('unused', 'synthetic', lambda: {'observed': True})\n")
+    for predicate in (yes, no, unknown):
+        source = generate_test_module(
+            trajectory=Trajectory('step', 'grant_access', 'browser', '/grant', ()),
+            action_args={}, ancestor_id='a', ancestor_maturity='T1', descendant_id='b',
+            runs=3, sut_version='test', claims=[replace(USE_CASE.claims[0], predicate=predicate)],
+            claims_module='local_predicates')
+        (tmp_path / f'test_{predicate.__name__}.py').write_text(source + '\nrealize = lambda *args: 200\n')
+    completed = subprocess.run([sys.executable, '-m', 'pytest', '-q', '-rs', str(tmp_path)],
+                               cwd=tmp_path, text=True, capture_output=True, timeout=30)
+    assert completed.returncode == 1, completed.stdout + completed.stderr
+    assert '1 failed, 1 passed, 1 skipped' in completed.stdout
+    assert 'INCONCLUSIVE' in completed.stdout