"""Guard the distinctions that make evolution reporting honest.""" from copy import deepcopy import importlib.util import json from pathlib import Path import subprocess import sys import pytest ROOT = Path(__file__).resolve().parents[1] spec = importlib.util.spec_from_file_location('evolution', ROOT / 'scripts/research_evolution.py') evolution = importlib.util.module_from_spec(spec) spec.loader.exec_module(evolution) @pytest.fixture def data(): return evolution.load() def test_current_records_and_report_are_consistent(data): evolution.validate(data) baseline = json.loads((ROOT / evolution.DATA / 'snapshots/baseline-2026-09-26.json').read_text()) evolution.validate_snapshot(baseline) assert evolution.render(data, baseline) == (ROOT / evolution.DATA / 'EvolutionReport.md').read_text() assert evolution.comparison_windows(data) == json.loads((ROOT / evolution.DATA / 'ComparisonWindows.json').read_text()) @pytest.mark.parametrize('mutation,match', [ (lambda d: d['profiles']['profiles'][0]['review'].update(last_substantive_review_date='2099-01-01'), 'Review date exceeds'), (lambda d: d['profiles']['profiles'][0]['review'].update(substantive_review_ref=None), 'needs provenance'), (lambda d: d['profiles']['profiles'][0]['review'].update(substantive_review_scope=''), 'needs provenance'), (lambda d: d['profiles']['profiles'][0].update(sha256='0'*64), 'Stale profile'), (lambda d: d['profiles']['profiles'].pop(), 'inventory mismatch'), (lambda d: d['profiles']['profiles'][1].update(id='CAND-0001'), 'Duplicate candidate'), (lambda d: d['profiles']['profiles'][0]['review'].update(independent_human=True), 'human validation'), (lambda d: d['profiles']['profiles'][0]['evidence'][0].update(ref='../missing.md'), 'Missing/unsafe'), (lambda d: d['ledger']['events'][0].update(affected_patterns=['FEP-9999']), 'Unknown affected'), (lambda d: d['ledger']['events'][0].update(supersedes=['LEARN-9999']), 'Supersession'), (lambda d: d['ledger']['events'][0]['effort'].update(research_minutes=-1), 'Invalid effort'), (lambda d: d['ledger']['events'][0].update(recording='prospective'), 'preregistered protocol'), ]) def test_rejects_misleading_or_untraceable_records(data, mutation, match): mutation(data) with pytest.raises(ValueError, match=match): evolution.validate(data) def test_unknown_effort_and_retrospective_sample_cannot_create_a_trend(data): data['ledger']['events'] = [e for e in data['ledger']['events'] if e['recording'] == 'retrospective'] for entry in data['ledger']['events']: entry['effort']['research_minutes'] = None report = evolution.render(data) assert 'Learning rate and diminishing returns: **not measurable' in report assert f"0/{len(data['ledger']['events'])}" in report assert 'historical only' in report assert 'no revised numeric count' in report assert 'Consequential outcome/hour' not in report # Even retrospectively supplied time cannot enter a prospective comparison group. for entry in data['ledger']['events']: entry['effort']['research_minutes'] = 10 entry['comparison_group'] = 'same-label' assert 'Consequential outcome/hour' not in evolution.render(data) def test_rates_separate_corroboration_unknown_effort_and_streams(data): events = data['ledger']['events'][:4] data['ledger']['events'] = events for entry in events: entry.update(recording='prospective', comparison_group='controlled-mechanism', stream='D', protocol_ref='templates/LearningCycle.md') entry['effort']['research_minutes'] = 30 events[1]['disposition'] = 'corroboration' events[2]['effort']['research_minutes'] = None events[3]['stream'] = 'F' evolution.validate(data) report = evolution.render(data) assert '| revision | 30 | 2.00 |' in report assert '| corroboration | 30 | 0.00 |' in report assert '| revision | unknown | unknown |' in report assert '### Stream F / controlled-mechanism' in report assert 'Fewer than three declared cycles' in report assert 'does not demonstrate a plateau' in report events[0]['effort']['research_minutes'] = 0 assert '| revision | 0 | unknown |' in evolution.render(data) @pytest.mark.parametrize('minutes', [None, 0]) def test_three_declared_cycles_trigger_review_without_fabricating_effort(data, minutes): entries = deepcopy(data['ledger']['events'][:3]) for entry in entries: entry.update(recording='prospective', comparison_group='test-window', stream='B', protocol_ref='templates/LearningCycle.md') entry['effort']['research_minutes'] = 10 entries[1]['effort']['research_minutes'] = minutes entries[1]['disposition'] = 'inconclusive' data['ledger']['events'] = entries window, = evolution.comparison_windows(data)['windows'] assert window['qualitative_review_due'] assert window['cycles_until_review_trigger'] == 0 assert not window['effort_complete'] assert not window['effort_comparison_ready'] assert window['positive_effort_cycle_ids'] == ['LEARN-0001', 'LEARN-0003'] assert window['unknown_effort_cycle_ids'] == (['LEARN-0002'] if minutes is None else []) assert window['zero_effort_cycle_ids'] == (['LEARN-0002'] if minutes == 0 else []) assert window['outcomes']['inconclusive'] == 1 # One more measured cycle cannot silently drop the intervening effort gap. extra = deepcopy(entries[0]) extra['id'] = 'LEARN-0004' data['ledger']['events'].append(extra) assert not evolution.comparison_windows(data)['windows'][0]['effort_comparison_ready'] entries[1]['effort']['research_minutes'] = 10 assert evolution.comparison_windows(data)['windows'][0]['effort_comparison_ready'] assert 'subject to protocol comparability review' in evolution.render(data) def test_window_audit_retains_order_protocols_and_stream_separation(data): entries = deepcopy(data['ledger']['events'][:4]) for entry in entries[:3]: entry.update(recording='prospective', comparison_group='same-label', stream='B', protocol_ref='templates/LearningCycle.md') entry['effort']['research_minutes'] = 10 entries[1]['protocol_ref'] = 'research/evolution/cycles/2026-09-27-navigation.md' entries[2]['stream'] = 'F' entries[3]['comparison_group'] = 'same-label' # Retrospective data stays excluded. entries[3]['effort']['research_minutes'] = 10 data['ledger']['events'] = list(reversed(entries)) audit = evolution.comparison_windows(data) b, f = audit['windows'] assert b['cycle_ids'] == ['LEARN-0001', 'LEARN-0002'] assert b['protocol_refs'] == ['templates/LearningCycle.md', 'research/evolution/cycles/2026-09-27-navigation.md'] assert b['cycles_until_review_trigger'] == 1 assert b['effort_complete'] and not b['effort_comparison_ready'] assert f['cycle_ids'] == ['LEARN-0003'] assert not f['qualitative_review_due'] def test_check_rejects_stale_window_audit(tmp_path, monkeypatch, capsys): for ref in ['EvolutionReport.md', 'ComparisonWindows.json', 'snapshots/baseline-2026-09-26.json']: path = tmp_path / evolution.DATA / ref path.parent.mkdir(parents=True, exist_ok=True) path.write_bytes((ROOT / evolution.DATA / ref).read_bytes()) (tmp_path / evolution.DATA / 'ComparisonWindows.json').write_text('{}\n') monkeypatch.setattr(evolution, 'ROOT', tmp_path) monkeypatch.setattr(sys, 'argv', ['research_evolution.py', 'check']) with pytest.raises(SystemExit) as exc: evolution.main() assert exc.value.code == 1 assert 'Stale comparison windows' in capsys.readouterr().err def test_snapshot_delta_does_not_count_additions_as_maturity_changes(data): old = deepcopy(data) old['profiles']['profiles'].pop() old['profiles']['profiles'][0]['sha256'] = 'old-content' baseline = {'id': 'test', 'source_revision': 'test-revision', 'assessment': old} report = evolution.render(data, baseline) assert 'Added: 1; removed: 0; shared candidates with changed content: 1' in report assert 'maturity transitions: 0' in report def test_snapshot_pins_committed_candidates_and_refuses_overwrite(tmp_path, data): subprocess.run(['git', 'init', '-q', str(tmp_path)], check=True) for p in data['profiles']['profiles']: path = tmp_path / p['path'] path.parent.mkdir(parents=True, exist_ok=True) path.write_bytes((ROOT / p['path']).read_bytes()) subprocess.run(['git', 'add', '.'], cwd=tmp_path, check=True) subprocess.run(['git', '-c', 'user.name=Test', '-c', 'user.email=test@example.invalid', 'commit', '-qm', 'fixture'], cwd=tmp_path, check=True) (tmp_path / evolution.DATA / 'snapshots').mkdir(parents=True) first = evolution.snapshot(data, 'test-baseline', tmp_path) assert json.loads(first.read_text())['assessment'] == data with pytest.raises(FileExistsError): evolution.snapshot(data, 'test-baseline', tmp_path) data['profiles']['profiles'][0]['sha256'] = 'uncommitted' with pytest.raises(ValueError, match='Commit candidate changes'): evolution.snapshot(data, 'second', tmp_path) def test_corrupt_baseline_cannot_masquerade_as_pinned_history(): baseline = json.loads((ROOT / evolution.DATA / 'snapshots/baseline-2026-09-26.json').read_text()) baseline['assessment']['profiles']['profiles'][0]['sha256'] = '0'*64 with pytest.raises(ValueError, match='pinned source'): evolution.validate_snapshot(baseline)