tegwick
f8c3ce3938
Owner decisions D1-D3 applied; execution not authorized. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Assistant: claude-code Assistant-Model: opus Assistant-Process: 182330@bnt-lap001 Assistant-Session: 242866cf-789d-4e24-ac5e-0a1d7b47a701
60 lines
2.6 KiB
Python
60 lines
2.6 KiB
Python
"""Guard replication-03 identity, unchanged reused briefs and replication-02 bytes."""
|
|
from pathlib import Path
|
|
import shutil
|
|
import sys
|
|
|
|
import pytest
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / 'scripts'))
|
|
import language_replication as base
|
|
import language_replication_03 as replication
|
|
|
|
|
|
@pytest.fixture
|
|
def packet(tmp_path):
|
|
shutil.copytree(ROOT / 'reference', tmp_path / 'reference')
|
|
(tmp_path / '.git').symlink_to(ROOT / '.git', target_is_directory=True)
|
|
for name in base.INPUTS:
|
|
target = tmp_path / replication.PACKET / name
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
shutil.copy(ROOT / replication.PACKET / name, target)
|
|
for relative in (replication.SOURCE / 'scenarios.json', Path('research/evaluations/language-pilot-01/scenarios.json')):
|
|
(tmp_path / relative).parent.mkdir(parents=True, exist_ok=True)
|
|
shutil.copy(ROOT / relative, tmp_path / relative)
|
|
for relative in replication.DEPENDENCIES:
|
|
(tmp_path / relative).parent.mkdir(parents=True, exist_ok=True)
|
|
shutil.copy(ROOT / relative, tmp_path / relative)
|
|
replication.prepare(tmp_path)
|
|
return tmp_path
|
|
|
|
|
|
def test_freeze_has_own_identity_and_no_scoring_material_in_prompts(packet):
|
|
frozen, assignments, scenarios = replication.check(packet)
|
|
assert frozen['packet'] == 'language-replication-03' and len(assignments) == 12
|
|
assert str(replication.SOURCE / 'scenarios.json') in frozen['files']
|
|
for assignment in assignments:
|
|
text = (packet / replication.PACKET / 'requests' / (assignment['run'] + '.txt')).read_text()
|
|
assert 'qualifies' not in text and 'reviewer_context' not in text and '"criteria"' not in text
|
|
with pytest.raises(ValueError, match='already frozen'):
|
|
replication.prepare(packet)
|
|
|
|
|
|
def test_changed_reused_brief_is_rejected(packet):
|
|
scenarios = base.read(packet / replication.PACKET / 'scenarios.json')
|
|
scenarios[0]['brief'] += ' Changed.'
|
|
with pytest.raises(ValueError, match='Reused brief'):
|
|
replication.scenarios_check(scenarios, packet)
|
|
|
|
|
|
def test_missing_criterion_examples_are_rejected(packet):
|
|
scenarios = base.read(packet / replication.PACKET / 'scenarios.json')
|
|
scenarios[0]['criteria'][0]['qualifies'] = []
|
|
with pytest.raises(ValueError, match='examples required'):
|
|
replication.scenarios_check(scenarios, packet)
|
|
|
|
|
|
def test_replication_02_freeze_still_verifies():
|
|
base.PACKET, base.DEPENDENCIES = replication.SOURCE, replication.DEPENDENCIES[:-1]
|
|
frozen, _, _ = base.check(ROOT)
|
|
assert frozen['packet'] == 'language-replication-02'
|