61 lines
2.6 KiB
Python
61 lines
2.6 KiB
Python
|
"""Guard replication-03 identity, unchanged reused briefs and replication-02 bytes."""
|
||
|
|
from pathlib import Path
|
||
|
|
import shutil
|
||
|
|
import sys
|
||
|
|
|
||
|
|
import pytest
|
||
|
|
|
||
|
|
ROOT = Path(__file__).resolve().parents[1]
|
||
|
|
sys.path.insert(0, str(ROOT / 'scripts'))
|
||
|
|
import language_replication as base
|
||
|
|
import language_replication_03 as replication
|
||
|
|
|
||
|
|
|
||
|
|
@pytest.fixture
|
||
|
|
def packet(tmp_path):
|
||
|
|
shutil.copytree(ROOT / 'reference', tmp_path / 'reference')
|
||
|
|
(tmp_path / '.git').symlink_to(ROOT / '.git', target_is_directory=True)
|
||
|
|
for name in base.INPUTS:
|
||
|
|
target = tmp_path / replication.PACKET / name
|
||
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
||
|
|
shutil.copy(ROOT / replication.PACKET / name, target)
|
||
|
|
for relative in (replication.SOURCE / 'scenarios.json', Path('research/evaluations/language-pilot-01/scenarios.json')):
|
||
|
|
(tmp_path / relative).parent.mkdir(parents=True, exist_ok=True)
|
||
|
|
shutil.copy(ROOT / relative, tmp_path / relative)
|
||
|
|
for relative in replication.DEPENDENCIES:
|
||
|
|
(tmp_path / relative).parent.mkdir(parents=True, exist_ok=True)
|
||
|
|
shutil.copy(ROOT / relative, tmp_path / relative)
|
||
|
|
replication.prepare(tmp_path)
|
||
|
|
return tmp_path
|
||
|
|
|
||
|
|
|
||
|
|
def test_freeze_has_own_identity_and_no_scoring_material_in_prompts(packet):
|
||
|
|
frozen, assignments, scenarios = replication.check(packet)
|
||
|
|
assert frozen['packet'] == 'language-replication-03' and len(assignments) == 12
|
||
|
|
assert str(replication.SOURCE / 'scenarios.json') in frozen['files']
|
||
|
|
for assignment in assignments:
|
||
|
|
text = (packet / replication.PACKET / 'requests' / (assignment['run'] + '.txt')).read_text()
|
||
|
|
assert 'qualifies' not in text and 'reviewer_context' not in text and '"criteria"' not in text
|
||
|
|
with pytest.raises(ValueError, match='already frozen'):
|
||
|
|
replication.prepare(packet)
|
||
|
|
|
||
|
|
|
||
|
|
def test_changed_reused_brief_is_rejected(packet):
|
||
|
|
scenarios = base.read(packet / replication.PACKET / 'scenarios.json')
|
||
|
|
scenarios[0]['brief'] += ' Changed.'
|
||
|
|
with pytest.raises(ValueError, match='Reused brief'):
|
||
|
|
replication.scenarios_check(scenarios, packet)
|
||
|
|
|
||
|
|
|
||
|
|
def test_missing_criterion_examples_are_rejected(packet):
|
||
|
|
scenarios = base.read(packet / replication.PACKET / 'scenarios.json')
|
||
|
|
scenarios[0]['criteria'][0]['qualifies'] = []
|
||
|
|
with pytest.raises(ValueError, match='examples required'):
|
||
|
|
replication.scenarios_check(scenarios, packet)
|
||
|
|
|
||
|
|
|
||
|
|
def test_replication_02_freeze_still_verifies():
|
||
|
|
base.PACKET, base.DEPENDENCIES = replication.SOURCE, replication.DEPENDENCIES[:-1]
|
||
|
|
frozen, _, _ = base.check(ROOT)
|
||
|
|
assert frozen['packet'] == 'language-replication-02'
|