107 lines
4.4 KiB
Python
107 lines
4.4 KiB
Python
|
"""Execute generated artifacts to check parity with deterministic oracle judgments."""
|
||
|
|
from dataclasses import replace
|
||
|
|
import sys
|
||
|
|
import types
|
||
|
|
import unittest
|
||
|
|
|
||
|
|
import pytest
|
||
|
|
|
||
|
|
from scenarios.alice_bob_carol import USE_CASE
|
||
|
|
from testdriver import Oracle, Verdict
|
||
|
|
from testdriver.crystallization import Trajectory, generate_test_module
|
||
|
|
from testdriver.oracles import overall
|
||
|
|
|
||
|
|
|
||
|
|
def yes(snapshot): return True
|
||
|
|
|
||
|
|
def no(snapshot): return False
|
||
|
|
|
||
|
|
def unknown(snapshot): return 'unknown'
|
||
|
|
|
||
|
|
def absent(snapshot): return None
|
||
|
|
|
||
|
|
def numeric(snapshot): return 1
|
||
|
|
|
||
|
|
def missing(snapshot): return snapshot['missing']
|
||
|
|
|
||
|
|
def broken(snapshot): raise RuntimeError('unavailable')
|
||
|
|
|
||
|
|
|
||
|
|
class CannotCoerce:
|
||
|
|
def __bool__(self):
|
||
|
|
raise AssertionError('must not coerce')
|
||
|
|
|
||
|
|
|
||
|
|
def opaque(snapshot): return CannotCoerce()
|
||
|
|
|
||
|
|
|
||
|
|
def generated(predicates, monkeypatch):
|
||
|
|
module = types.ModuleType('_generated_claims_test')
|
||
|
|
for predicate in predicates:
|
||
|
|
setattr(module, predicate.__name__, predicate)
|
||
|
|
monkeypatch.setitem(sys.modules, module.__name__, module)
|
||
|
|
claims = [replace(USE_CASE.claims[0], id=f'claim-{i}', predicate=predicate)
|
||
|
|
for i, predicate in enumerate(predicates)]
|
||
|
|
source = generate_test_module(
|
||
|
|
trajectory=Trajectory('step', 'grant_access', 'browser', '/grant', ()),
|
||
|
|
action_args={}, ancestor_id='a', ancestor_maturity='T1', descendant_id='b',
|
||
|
|
runs=3, sut_version='test', claims=claims, claims_module=module.__name__)
|
||
|
|
namespace = {}
|
||
|
|
exec(compile(source, '<generated>', 'exec'), namespace)
|
||
|
|
namespace['realize'] = lambda *args: 200
|
||
|
|
return namespace['test_grant_access'], claims
|
||
|
|
|
||
|
|
|
||
|
|
def outcome(test, snapshot):
|
||
|
|
try:
|
||
|
|
test(('unused', 'synthetic-token', lambda: snapshot))
|
||
|
|
except unittest.SkipTest as exc:
|
||
|
|
assert 'INCONCLUSIVE' in str(exc)
|
||
|
|
return Verdict.INCONCLUSIVE
|
||
|
|
except AssertionError:
|
||
|
|
return Verdict.FAIL
|
||
|
|
return Verdict.PASS
|
||
|
|
|
||
|
|
|
||
|
|
@pytest.mark.parametrize('predicates', [(yes,), (no,), (unknown,), (absent,),
|
||
|
|
(numeric,), (missing,), (broken,), (opaque,),
|
||
|
|
(unknown, no), (no, unknown), (yes, unknown)])
|
||
|
|
@pytest.mark.parametrize('snapshot', [{'observed': True}, {}])
|
||
|
|
def test_generated_verdicts_match_oracle_including_failure_precedence(monkeypatch, predicates, snapshot):
|
||
|
|
test, claims = generated(predicates, monkeypatch)
|
||
|
|
expected = overall([Oracle().judge(c, snapshot, 'step') for c in claims])
|
||
|
|
assert outcome(test, snapshot) is expected
|
||
|
|
|
||
|
|
|
||
|
|
@pytest.mark.parametrize('predicate', [yes, no, unknown, absent, numeric, missing, broken, opaque])
|
||
|
|
def test_checked_in_descendant_preserves_the_same_semantics(monkeypatch, predicate):
|
||
|
|
import crystallized.test_grant_access as artifact
|
||
|
|
monkeypatch.setattr(artifact, 'realize', lambda *args: 200)
|
||
|
|
for name in ('_bob_can_read', '_carol_cannot_read', '_bob_cannot_write'):
|
||
|
|
monkeypatch.setattr(artifact, name, predicate)
|
||
|
|
claim = replace(USE_CASE.claims[0], predicate=predicate)
|
||
|
|
assert outcome(artifact.test_grant_access, {'observed': True}) is Oracle().judge(
|
||
|
|
claim, {'observed': True}, 'step').verdict
|
||
|
|
|
||
|
|
|
||
|
|
def test_pytest_reports_inconclusive_as_skipped_and_never_as_passed(tmp_path):
|
||
|
|
import subprocess
|
||
|
|
(tmp_path / 'local_predicates.py').write_text(
|
||
|
|
"def yes(snapshot): return True\ndef no(snapshot): return False\n"
|
||
|
|
"def unknown(snapshot): return 'unknown'\n")
|
||
|
|
(tmp_path / 'conftest.py').write_text(
|
||
|
|
"import pytest\n@pytest.fixture\ndef crystallized_world():\n"
|
||
|
|
" return ('unused', 'synthetic', lambda: {'observed': True})\n")
|
||
|
|
for predicate in (yes, no, unknown):
|
||
|
|
source = generate_test_module(
|
||
|
|
trajectory=Trajectory('step', 'grant_access', 'browser', '/grant', ()),
|
||
|
|
action_args={}, ancestor_id='a', ancestor_maturity='T1', descendant_id='b',
|
||
|
|
runs=3, sut_version='test', claims=[replace(USE_CASE.claims[0], predicate=predicate)],
|
||
|
|
claims_module='local_predicates')
|
||
|
|
(tmp_path / f'test_{predicate.__name__}.py').write_text(source + '\nrealize = lambda *args: 200\n')
|
||
|
|
completed = subprocess.run([sys.executable, '-m', 'pytest', '-q', '-rs', str(tmp_path)],
|
||
|
|
cwd=tmp_path, text=True, capture_output=True, timeout=30)
|
||
|
|
assert completed.returncode == 1, completed.stdout + completed.stderr
|
||
|
|
assert '1 failed, 1 passed, 1 skipped' in completed.stdout
|
||
|
|
assert 'INCONCLUSIVE' in completed.stdout
|