"""Execute generated artifacts to check parity with deterministic oracle judgments.""" from dataclasses import replace import sys import types import unittest import pytest from scenarios.alice_bob_carol import USE_CASE from testdriver import Oracle, Verdict from testdriver.crystallization import Trajectory, generate_test_module from testdriver.oracles import overall def yes(snapshot): return True def no(snapshot): return False def unknown(snapshot): return 'unknown' def absent(snapshot): return None def numeric(snapshot): return 1 def missing(snapshot): return snapshot['missing'] def broken(snapshot): raise RuntimeError('unavailable') class CannotCoerce: def __bool__(self): raise AssertionError('must not coerce') def opaque(snapshot): return CannotCoerce() def generated(predicates, monkeypatch): module = types.ModuleType('_generated_claims_test') for predicate in predicates: setattr(module, predicate.__name__, predicate) monkeypatch.setitem(sys.modules, module.__name__, module) claims = [replace(USE_CASE.claims[0], id=f'claim-{i}', predicate=predicate) for i, predicate in enumerate(predicates)] source = generate_test_module( trajectory=Trajectory('step', 'grant_access', 'browser', '/grant', ()), action_args={}, ancestor_id='a', ancestor_maturity='T1', descendant_id='b', runs=3, sut_version='test', claims=claims, claims_module=module.__name__) namespace = {} exec(compile(source, '', 'exec'), namespace) namespace['realize'] = lambda *args: 200 return namespace['test_grant_access'], claims def outcome(test, snapshot): try: test(('unused', 'synthetic-token', lambda: snapshot)) except unittest.SkipTest as exc: assert 'INCONCLUSIVE' in str(exc) return Verdict.INCONCLUSIVE except AssertionError: return Verdict.FAIL return Verdict.PASS @pytest.mark.parametrize('predicates', [(yes,), (no,), (unknown,), (absent,), (numeric,), (missing,), (broken,), (opaque,), (unknown, no), (no, unknown), (yes, unknown)]) @pytest.mark.parametrize('snapshot', [{'observed': True}, {}]) def test_generated_verdicts_match_oracle_including_failure_precedence(monkeypatch, predicates, snapshot): test, claims = generated(predicates, monkeypatch) expected = overall([Oracle().judge(c, snapshot, 'step') for c in claims]) assert outcome(test, snapshot) is expected @pytest.mark.parametrize('predicate', [yes, no, unknown, absent, numeric, missing, broken, opaque]) def test_checked_in_descendant_preserves_the_same_semantics(monkeypatch, predicate): import crystallized.test_grant_access as artifact monkeypatch.setattr(artifact, 'realize', lambda *args: 200) for name in ('_bob_can_read', '_carol_cannot_read', '_bob_cannot_write'): monkeypatch.setattr(artifact, name, predicate) claim = replace(USE_CASE.claims[0], predicate=predicate) assert outcome(artifact.test_grant_access, {'observed': True}) is Oracle().judge( claim, {'observed': True}, 'step').verdict def test_pytest_reports_inconclusive_as_skipped_and_never_as_passed(tmp_path): import subprocess (tmp_path / 'local_predicates.py').write_text( "def yes(snapshot): return True\ndef no(snapshot): return False\n" "def unknown(snapshot): return 'unknown'\n") (tmp_path / 'conftest.py').write_text( "import pytest\n@pytest.fixture\ndef crystallized_world():\n" " return ('unused', 'synthetic', lambda: {'observed': True})\n") for predicate in (yes, no, unknown): source = generate_test_module( trajectory=Trajectory('step', 'grant_access', 'browser', '/grant', ()), action_args={}, ancestor_id='a', ancestor_maturity='T1', descendant_id='b', runs=3, sut_version='test', claims=[replace(USE_CASE.claims[0], predicate=predicate)], claims_module='local_predicates') (tmp_path / f'test_{predicate.__name__}.py').write_text(source + '\nrealize = lambda *args: 200\n') completed = subprocess.run([sys.executable, '-m', 'pytest', '-q', '-rs', str(tmp_path)], cwd=tmp_path, text=True, capture_output=True, timeout=30) assert completed.returncode == 1, completed.stdout + completed.stderr assert '1 failed, 1 passed, 1 skipped' in completed.stdout assert 'INCONCLUSIVE' in completed.stdout