frontend-patterns/tests/test_research_evolution.py

"""Guard the distinctions that make evolution reporting honest."""
from copy import deepcopy
import importlib.util
import json
from pathlib import Path
import subprocess
import sys

import pytest

ROOT = Path(__file__).resolve().parents[1]
spec = importlib.util.spec_from_file_location('evolution', ROOT / 'scripts/research_evolution.py')
evolution = importlib.util.module_from_spec(spec)
spec.loader.exec_module(evolution)


@pytest.fixture
def data():
    return evolution.load()


def test_current_records_and_report_are_consistent(data):
    evolution.validate(data)
    baseline = json.loads((ROOT / evolution.DATA / 'snapshots/baseline-2026-09-26.json').read_text())
    evolution.validate_snapshot(baseline)
    assert evolution.render(data, baseline) == (ROOT / evolution.DATA / 'EvolutionReport.md').read_text()
    assert evolution.comparison_windows(data) == json.loads((ROOT / evolution.DATA / 'ComparisonWindows.json').read_text())


@pytest.mark.parametrize('mutation,match', [
    (lambda d: d['profiles']['profiles'][0]['review'].update(last_substantive_review_date='2099-01-01'), 'Review date exceeds'),
    (lambda d: d['profiles']['profiles'][0]['review'].update(substantive_review_ref=None), 'needs provenance'),
    (lambda d: d['profiles']['profiles'][0]['review'].update(substantive_review_scope=''), 'needs provenance'),
    (lambda d: d['profiles']['profiles'][0].update(sha256='0'*64), 'Stale profile'),
    (lambda d: d['profiles']['profiles'].pop(), 'inventory mismatch'),
    (lambda d: d['profiles']['profiles'][1].update(id='CAND-0001'), 'Duplicate candidate'),
    (lambda d: d['profiles']['profiles'][0]['review'].update(independent_human=True), 'human validation'),
    (lambda d: d['profiles']['profiles'][0]['evidence'][0].update(ref='../missing.md'), 'Missing/unsafe'),
    (lambda d: d['ledger']['events'][0].update(affected_patterns=['FEP-9999']), 'Unknown affected'),
    (lambda d: d['ledger']['events'][0].update(supersedes=['LEARN-9999']), 'Supersession'),
    (lambda d: d['ledger']['events'][0]['effort'].update(research_minutes=-1), 'Invalid effort'),
    (lambda d: d['ledger']['events'][0].update(recording='prospective'), 'preregistered protocol'),
])
def test_rejects_misleading_or_untraceable_records(data, mutation, match):
    mutation(data)
    with pytest.raises(ValueError, match=match):
        evolution.validate(data)


def test_unknown_effort_and_retrospective_sample_cannot_create_a_trend(data):
    data['ledger']['events'] = [e for e in data['ledger']['events'] if e['recording'] == 'retrospective']
    for entry in data['ledger']['events']:
        entry['effort']['research_minutes'] = None
    report = evolution.render(data)
    assert 'Learning rate and diminishing returns: **not measurable' in report
    assert f"0/{len(data['ledger']['events'])}" in report
    assert 'historical only' in report
    assert 'no revised numeric count' in report
    assert 'Consequential outcome/hour' not in report
    # Even retrospectively supplied time cannot enter a prospective comparison group.
    for entry in data['ledger']['events']:
        entry['effort']['research_minutes'] = 10
        entry['comparison_group'] = 'same-label'
    assert 'Consequential outcome/hour' not in evolution.render(data)


def test_rates_separate_corroboration_unknown_effort_and_streams(data):
    events = data['ledger']['events'][:4]
    data['ledger']['events'] = events
    for entry in events:
        entry.update(recording='prospective', comparison_group='controlled-mechanism', stream='D', protocol_ref='templates/LearningCycle.md')
        entry['effort']['research_minutes'] = 30
    events[1]['disposition'] = 'corroboration'
    events[2]['effort']['research_minutes'] = None
    events[3]['stream'] = 'F'
    evolution.validate(data)
    report = evolution.render(data)
    assert '| revision | 30 | 2.00 |' in report
    assert '| corroboration | 30 | 0.00 |' in report
    assert '| revision | unknown | unknown |' in report
    assert '### Stream F / controlled-mechanism' in report
    assert 'Fewer than three declared cycles' in report
    assert 'does not demonstrate a plateau' in report
    events[0]['effort']['research_minutes'] = 0
    assert '| revision | 0 | unknown |' in evolution.render(data)


@pytest.mark.parametrize('minutes', [None, 0])
def test_three_declared_cycles_trigger_review_without_fabricating_effort(data, minutes):
    entries = deepcopy(data['ledger']['events'][:3])
    for entry in entries:
        entry.update(recording='prospective', comparison_group='test-window', stream='B', protocol_ref='templates/LearningCycle.md')
        entry['effort']['research_minutes'] = 10
    entries[1]['effort']['research_minutes'] = minutes
    entries[1]['disposition'] = 'inconclusive'
    data['ledger']['events'] = entries
    window, = evolution.comparison_windows(data)['windows']
    assert window['qualitative_review_due']
    assert window['cycles_until_review_trigger'] == 0
    assert not window['effort_complete']
    assert not window['effort_comparison_ready']
    assert window['positive_effort_cycle_ids'] == ['LEARN-0001', 'LEARN-0003']
    assert window['unknown_effort_cycle_ids'] == (['LEARN-0002'] if minutes is None else [])
    assert window['zero_effort_cycle_ids'] == (['LEARN-0002'] if minutes == 0 else [])
    assert window['outcomes']['inconclusive'] == 1
    # One more measured cycle cannot silently drop the intervening effort gap.
    extra = deepcopy(entries[0])
    extra['id'] = 'LEARN-0004'
    data['ledger']['events'].append(extra)
    assert not evolution.comparison_windows(data)['windows'][0]['effort_comparison_ready']
    entries[1]['effort']['research_minutes'] = 10
    assert evolution.comparison_windows(data)['windows'][0]['effort_comparison_ready']
    assert 'subject to protocol comparability review' in evolution.render(data)


def test_window_audit_retains_order_protocols_and_stream_separation(data):
    entries = deepcopy(data['ledger']['events'][:4])
    for entry in entries[:3]:
        entry.update(recording='prospective', comparison_group='same-label', stream='B', protocol_ref='templates/LearningCycle.md')
        entry['effort']['research_minutes'] = 10
    entries[1]['protocol_ref'] = 'research/evolution/cycles/2026-09-27-navigation.md'
    entries[2]['stream'] = 'F'
    entries[3]['comparison_group'] = 'same-label'  # Retrospective data stays excluded.
    entries[3]['effort']['research_minutes'] = 10
    data['ledger']['events'] = list(reversed(entries))
    audit = evolution.comparison_windows(data)
    b, f = audit['windows']
    assert b['cycle_ids'] == ['LEARN-0001', 'LEARN-0002']
    assert b['protocol_refs'] == ['templates/LearningCycle.md', 'research/evolution/cycles/2026-09-27-navigation.md']
    assert b['cycles_until_review_trigger'] == 1
    assert b['effort_complete'] and not b['effort_comparison_ready']
    assert f['cycle_ids'] == ['LEARN-0003']
    assert not f['qualitative_review_due']


def test_check_rejects_stale_window_audit(tmp_path, monkeypatch, capsys):
    for ref in ['EvolutionReport.md', 'ComparisonWindows.json', 'snapshots/baseline-2026-09-26.json']:
        path = tmp_path / evolution.DATA / ref
        path.parent.mkdir(parents=True, exist_ok=True)
        path.write_bytes((ROOT / evolution.DATA / ref).read_bytes())
    (tmp_path / evolution.DATA / 'ComparisonWindows.json').write_text('{}\n')
    monkeypatch.setattr(evolution, 'ROOT', tmp_path)
    monkeypatch.setattr(sys, 'argv', ['research_evolution.py', 'check'])
    with pytest.raises(SystemExit) as exc:
        evolution.main()
    assert exc.value.code == 1
    assert 'Stale comparison windows' in capsys.readouterr().err


def test_snapshot_delta_does_not_count_additions_as_maturity_changes(data):
    old = deepcopy(data)
    old['profiles']['profiles'].pop()
    old['profiles']['profiles'][0]['sha256'] = 'old-content'
    baseline = {'id': 'test', 'source_revision': 'test-revision', 'assessment': old}
    report = evolution.render(data, baseline)
    assert 'Added: 1; removed: 0; shared candidates with changed content: 1' in report
    assert 'maturity transitions: 0' in report


def test_snapshot_pins_committed_candidates_and_refuses_overwrite(tmp_path, data):
    subprocess.run(['git', 'init', '-q', str(tmp_path)], check=True)
    for p in data['profiles']['profiles']:
        path = tmp_path / p['path']
        path.parent.mkdir(parents=True, exist_ok=True)
        path.write_bytes((ROOT / p['path']).read_bytes())
    subprocess.run(['git', 'add', '.'], cwd=tmp_path, check=True)
    subprocess.run(['git', '-c', 'user.name=Test', '-c', 'user.email=test@example.invalid', 'commit', '-qm', 'fixture'], cwd=tmp_path, check=True)
    (tmp_path / evolution.DATA / 'snapshots').mkdir(parents=True)
    first = evolution.snapshot(data, 'test-baseline', tmp_path)
    assert json.loads(first.read_text())['assessment'] == data
    with pytest.raises(FileExistsError):
        evolution.snapshot(data, 'test-baseline', tmp_path)
    data['profiles']['profiles'][0]['sha256'] = 'uncommitted'
    with pytest.raises(ValueError, match='Commit candidate changes'):
        evolution.snapshot(data, 'second', tmp_path)


def test_corrupt_baseline_cannot_masquerade_as_pinned_history():
    baseline = json.loads((ROOT / evolution.DATA / 'snapshots/baseline-2026-09-26.json').read_text())
    baseline['assessment']['profiles']['profiles'][0]['sha256'] = '0'*64
    with pytest.raises(ValueError, match='pinned source'):
        evolution.validate_snapshot(baseline)