diff --git a/WORK-RECORDS.md b/WORK-RECORDS.md index ecdf416..b2b6cce 100644 --- a/WORK-RECORDS.md +++ b/WORK-RECORDS.md @@ -13,7 +13,7 @@ | workplan | FEP-WP-0003 | backlog | — | workplans/FEP-WP-0003-contrasting-schools.md | | workplan | FEP-WP-0004 | backlog | — | workplans/FEP-WP-0004-reference-surface.md | | workplan | FEP-WP-0005 | finished | — | workplans/FEP-WP-0005-research-bench-setup.md | -| workplan | FEP-WP-0006 | backlog | — | workplans/FEP-WP-0006-language-evolution.md | +| workplan | FEP-WP-0006 | blocked | — | workplans/FEP-WP-0006-language-evolution.md | | workplan | FEP-WP-0007 | finished | — | workplans/FEP-WP-0007-language-v01-preview.md | | workplan | FEP-WP-0008 | blocked | — | workplans/FEP-WP-0008-use-driven-improvement.md | | workplan | FEP-WP-0009 | backlog | — | workplans/FEP-WP-0009-m1-independent-validation.md | diff --git a/research/evolution/ComparisonWindows.json b/research/evolution/ComparisonWindows.json new file mode 100644 index 0000000..2290854 --- /dev/null +++ b/research/evolution/ComparisonWindows.json @@ -0,0 +1,199 @@ +{ + "schema_version": 1, + "basis": "Declared prospective groups; manually review shared stream, evidence tier, question granularity, evaluation and effort accounting before comparing returns.", + "review_trigger_cycles": 3, + "limits": [ + "Group labels and cycle counts do not establish protocol comparability or a learning trend.", + "Unknown or zero research time prevents effort comparison of the complete declared window; never estimate missing time or silently omit its cycle.", + "A qualitative review may retain cycles with unknown effort; corroboration and inconclusive results remain visible.", + "Readiness is a review trigger, not statistical evidence, a plateau declaration or a maturity judgment." + ], + "windows": [ + { + "stream": "B", + "comparison_group": "B-independent-navigation-source-v1", + "cycle_ids": [ + "LEARN-0014", + "LEARN-0017" + ], + "protocol_refs": [ + "research/evolution/cycles/2026-09-27-zulip-navigation.md", + "research/evolution/cycles/2026-09-27-zulip-target-lookup.md" + ], + "cycles_until_review_trigger": 1, + "qualitative_review_due": false, + "positive_effort_cycle_ids": [ + "LEARN-0014" + ], + "unknown_effort_cycle_ids": [ + "LEARN-0017" + ], + "zero_effort_cycle_ids": [], + "effort_complete": false, + "effort_comparison_ready": false, + "outcomes": { + "corroboration": 1, + "inconclusive": 0, + "merge": 0, + "rejection": 0, + "revision": 1 + } + }, + { + "stream": "B", + "comparison_group": "B-navigation-mechanism-v1", + "cycle_ids": [ + "LEARN-0011" + ], + "protocol_refs": [ + "research/evolution/cycles/2026-09-27-navigation.md" + ], + "cycles_until_review_trigger": 2, + "qualitative_review_due": false, + "positive_effort_cycle_ids": [ + "LEARN-0011" + ], + "unknown_effort_cycle_ids": [], + "zero_effort_cycle_ids": [], + "effort_complete": true, + "effort_comparison_ready": false, + "outcomes": { + "corroboration": 1, + "inconclusive": 0, + "merge": 0, + "rejection": 0, + "revision": 0 + } + }, + { + "stream": "D", + "comparison_group": "D-sqlite-git-recovery-v1", + "cycle_ids": [ + "LEARN-0018" + ], + "protocol_refs": [ + "research/evolution/cycles/2026-09-28-storage-receiver.md" + ], + "cycles_until_review_trigger": 2, + "qualitative_review_due": false, + "positive_effort_cycle_ids": [ + "LEARN-0018" + ], + "unknown_effort_cycle_ids": [], + "zero_effort_cycle_ids": [], + "effort_complete": true, + "effort_comparison_ready": false, + "outcomes": { + "corroboration": 1, + "inconclusive": 0, + "merge": 0, + "rejection": 0, + "revision": 0 + } + }, + { + "stream": "F", + "comparison_group": "F-navigation-focus-v1", + "cycle_ids": [ + "LEARN-0012" + ], + "protocol_refs": [ + "research/evolution/cycles/2026-09-27-navigation-focus.md" + ], + "cycles_until_review_trigger": 2, + "qualitative_review_due": false, + "positive_effort_cycle_ids": [ + "LEARN-0012" + ], + "unknown_effort_cycle_ids": [], + "zero_effort_cycle_ids": [], + "effort_complete": true, + "effort_comparison_ready": false, + "outcomes": { + "corroboration": 1, + "inconclusive": 0, + "merge": 0, + "rejection": 0, + "revision": 0 + } + }, + { + "stream": "F", + "comparison_group": "F-navigation-ownership-v1", + "cycle_ids": [ + "LEARN-0013" + ], + "protocol_refs": [ + "research/evolution/cycles/2026-09-27-navigation-ownership.md" + ], + "cycles_until_review_trigger": 2, + "qualitative_review_due": false, + "positive_effort_cycle_ids": [ + "LEARN-0013" + ], + "unknown_effort_cycle_ids": [], + "zero_effort_cycle_ids": [], + "effort_complete": true, + "effort_comparison_ready": false, + "outcomes": { + "corroboration": 0, + "inconclusive": 0, + "merge": 0, + "rejection": 0, + "revision": 1 + } + }, + { + "stream": "cross-stream", + "comparison_group": "CD-acceptance-projection-static-v1", + "cycle_ids": [ + "LEARN-0016" + ], + "protocol_refs": [ + "research/evolution/cycles/2026-09-27-zulip-acceptance.md" + ], + "cycles_until_review_trigger": 2, + "qualitative_review_due": false, + "positive_effort_cycle_ids": [ + "LEARN-0016" + ], + "unknown_effort_cycle_ids": [], + "zero_effort_cycle_ids": [], + "effort_complete": true, + "effort_comparison_ready": false, + "outcomes": { + "corroboration": 1, + "inconclusive": 0, + "merge": 0, + "rejection": 0, + "revision": 0 + } + }, + { + "stream": "cross-stream", + "comparison_group": "CD-authority-projection-documentary-v1", + "cycle_ids": [ + "LEARN-0015" + ], + "protocol_refs": [ + "research/evolution/cycles/2026-09-27-authority-projection.md" + ], + "cycles_until_review_trigger": 2, + "qualitative_review_due": false, + "positive_effort_cycle_ids": [ + "LEARN-0015" + ], + "unknown_effort_cycle_ids": [], + "zero_effort_cycle_ids": [], + "effort_complete": true, + "effort_comparison_ready": false, + "outcomes": { + "corroboration": 0, + "inconclusive": 0, + "merge": 0, + "rejection": 0, + "revision": 1 + } + } + ] +} diff --git a/research/evolution/EvolutionReport.md b/research/evolution/EvolutionReport.md index 5c2541b..c8621f4 100644 --- a/research/evolution/EvolutionReport.md +++ b/research/evolution/EvolutionReport.md @@ -59,62 +59,78 @@ Ten selected retrospective outcomes (not a historical census), followed by prosp Cycles with recorded research effort: 7/18. Unknown effort is neither zero nor estimated from commits, tokens or assertion counts. +### Comparison readiness + +Declared groups require a review of shared evidence tier, question granularity, evaluation and effort accounting. A matching label is not proof of comparability. [Machine-readable window audit](../../research/evolution/ComparisonWindows.json). + +| Stream / declared group | Cycles | Positive effort | Unknown / zero effort | Qualitative review trigger | Complete-window effort comparison | +| --- | ---: | ---: | ---: | --- | --- | +| B / B-independent-navigation-source-v1 | 2 | 1 | 1 / 0 | pending | not ready | +| B / B-navigation-mechanism-v1 | 1 | 1 | 0 / 0 | pending | not ready | +| D / D-sqlite-git-recovery-v1 | 1 | 1 | 0 / 0 | pending | not ready | +| F / F-navigation-focus-v1 | 1 | 1 | 0 / 0 | pending | not ready | +| F / F-navigation-ownership-v1 | 1 | 1 | 0 / 0 | pending | not ready | +| cross-stream / CD-acceptance-projection-static-v1 | 1 | 1 | 0 / 0 | pending | not ready | +| cross-stream / CD-authority-projection-documentary-v1 | 1 | 1 | 0 / 0 | pending | not ready | + +Unknown or zero effort is not dropped to create a favorable window. Qualitative review can retain those cycles, but effort comparisons need an explicitly reviewed window with measured positive research time. No trend or plateau is inferred. + ### Stream B / B-independent-navigation-source-v1 -| Cycle | Outcome | Research minutes | Consequential outcome/hour | -| --- | --- | ---: | ---: | -| LEARN-0014 | revision | 6.345 | 9.46 | -| LEARN-0017 | corroboration | unknown | unknown | +| Cycle | Protocol | Outcome | Research minutes | Consequential outcome/hour | +| --- | --- | --- | ---: | ---: | +| LEARN-0014 | [protocol](../../research/evolution/cycles/2026-09-27-zulip-navigation.md) | revision | 6.345 | 9.46 | +| LEARN-0017 | [protocol](../../research/evolution/cycles/2026-09-27-zulip-target-lookup.md) | corroboration | unknown | unknown | -One bounded question counts at most one consequential outcome. Corroboration and inconclusive work remain visible separately. Fewer than three comparable cycles: no trend assessment. +One bounded question counts at most one consequential outcome. Corroboration and inconclusive work remain visible separately. Fewer than three declared cycles: qualitative review trigger pending; no trend assessment. ### Stream B / B-navigation-mechanism-v1 -| Cycle | Outcome | Research minutes | Consequential outcome/hour | -| --- | --- | ---: | ---: | -| LEARN-0011 | corroboration | 6.177 | 0.00 | +| Cycle | Protocol | Outcome | Research minutes | Consequential outcome/hour | +| --- | --- | --- | ---: | ---: | +| LEARN-0011 | [protocol](../../research/evolution/cycles/2026-09-27-navigation.md) | corroboration | 6.177 | 0.00 | -One bounded question counts at most one consequential outcome. Corroboration and inconclusive work remain visible separately. Fewer than three comparable cycles: no trend assessment. +One bounded question counts at most one consequential outcome. Corroboration and inconclusive work remain visible separately. Fewer than three declared cycles: qualitative review trigger pending; no trend assessment. ### Stream D / D-sqlite-git-recovery-v1 -| Cycle | Outcome | Research minutes | Consequential outcome/hour | -| --- | --- | ---: | ---: | -| LEARN-0018 | corroboration | 4.123 | 0.00 | +| Cycle | Protocol | Outcome | Research minutes | Consequential outcome/hour | +| --- | --- | --- | ---: | ---: | +| LEARN-0018 | [protocol](../../research/evolution/cycles/2026-09-28-storage-receiver.md) | corroboration | 4.123 | 0.00 | -One bounded question counts at most one consequential outcome. Corroboration and inconclusive work remain visible separately. Fewer than three comparable cycles: no trend assessment. +One bounded question counts at most one consequential outcome. Corroboration and inconclusive work remain visible separately. Fewer than three declared cycles: qualitative review trigger pending; no trend assessment. ### Stream F / F-navigation-focus-v1 -| Cycle | Outcome | Research minutes | Consequential outcome/hour | -| --- | --- | ---: | ---: | -| LEARN-0012 | corroboration | 5.138 | 0.00 | +| Cycle | Protocol | Outcome | Research minutes | Consequential outcome/hour | +| --- | --- | --- | ---: | ---: | +| LEARN-0012 | [protocol](../../research/evolution/cycles/2026-09-27-navigation-focus.md) | corroboration | 5.138 | 0.00 | -One bounded question counts at most one consequential outcome. Corroboration and inconclusive work remain visible separately. Fewer than three comparable cycles: no trend assessment. +One bounded question counts at most one consequential outcome. Corroboration and inconclusive work remain visible separately. Fewer than three declared cycles: qualitative review trigger pending; no trend assessment. ### Stream F / F-navigation-ownership-v1 -| Cycle | Outcome | Research minutes | Consequential outcome/hour | -| --- | --- | ---: | ---: | -| LEARN-0013 | revision | 13.918 | 4.31 | +| Cycle | Protocol | Outcome | Research minutes | Consequential outcome/hour | +| --- | --- | --- | ---: | ---: | +| LEARN-0013 | [protocol](../../research/evolution/cycles/2026-09-27-navigation-ownership.md) | revision | 13.918 | 4.31 | -One bounded question counts at most one consequential outcome. Corroboration and inconclusive work remain visible separately. Fewer than three comparable cycles: no trend assessment. +One bounded question counts at most one consequential outcome. Corroboration and inconclusive work remain visible separately. Fewer than three declared cycles: qualitative review trigger pending; no trend assessment. ### Stream cross-stream / CD-acceptance-projection-static-v1 -| Cycle | Outcome | Research minutes | Consequential outcome/hour | -| --- | --- | ---: | ---: | -| LEARN-0016 | corroboration | 4.182 | 0.00 | +| Cycle | Protocol | Outcome | Research minutes | Consequential outcome/hour | +| --- | --- | --- | ---: | ---: | +| LEARN-0016 | [protocol](../../research/evolution/cycles/2026-09-27-zulip-acceptance.md) | corroboration | 4.182 | 0.00 | -One bounded question counts at most one consequential outcome. Corroboration and inconclusive work remain visible separately. Fewer than three comparable cycles: no trend assessment. +One bounded question counts at most one consequential outcome. Corroboration and inconclusive work remain visible separately. Fewer than three declared cycles: qualitative review trigger pending; no trend assessment. ### Stream cross-stream / CD-authority-projection-documentary-v1 -| Cycle | Outcome | Research minutes | Consequential outcome/hour | -| --- | --- | ---: | ---: | -| LEARN-0015 | revision | 3.128 | 19.18 | +| Cycle | Protocol | Outcome | Research minutes | Consequential outcome/hour | +| --- | --- | --- | ---: | ---: | +| LEARN-0015 | [protocol](../../research/evolution/cycles/2026-09-27-authority-projection.md) | revision | 3.128 | 19.18 | -One bounded question counts at most one consequential outcome. Corroboration and inconclusive work remain visible separately. Fewer than three comparable cycles: no trend assessment. +One bounded question counts at most one consequential outcome. Corroboration and inconclusive work remain visible separately. Fewer than three declared cycles: qualitative review trigger pending; no trend assessment. ## Coherence and maintainability @@ -124,7 +140,7 @@ Profiles with unknown last substantive review date: 0. Assessment dates record t Coherence metrics (ambiguous selections, unjustified relations and overlap resolved): not measured. Conflict Copy / Explicit Reconciliation overlap remains an explicit open question. -Review backlog and evidence gaps remain in [FEP-WP-0002-seed-language-validation](../../workplans/FEP-WP-0002-seed-language-validation.md), [FEP-WP-0003-contrasting-schools](../../workplans/FEP-WP-0003-contrasting-schools.md), [FEP-WP-0006-language-evolution](../../workplans/FEP-WP-0006-language-evolution.md). +Review backlog and evidence gaps remain in [FEP-WP-0009-m1-independent-validation](../../workplans/FEP-WP-0009-m1-independent-validation.md), [FEP-WP-0003-contrasting-schools](../../workplans/FEP-WP-0003-contrasting-schools.md), [FEP-WP-0006-language-evolution](../../workplans/FEP-WP-0006-language-evolution.md), [FEP-WP-0008-use-driven-improvement](../../workplans/FEP-WP-0008-use-driven-improvement.md). ## Pattern profiles diff --git a/research/evolution/README.md b/research/evolution/README.md index 7afcd9f..ac8d63b 100644 --- a/research/evolution/README.md +++ b/research/evolution/README.md @@ -66,6 +66,9 @@ renew them. Historical snapshots retain their original schema-v1 assessments. undermine their scope. Never promote automatically from metric thresholds. Commands: `make evolution-report`, `make evolution-check`, `make evolution-test`. +Generation writes both EvolutionReport.md and the machine-readable +[ComparisonWindows.json](ComparisonWindows.json); checking rejects stale output +in either artifact. The check rejects stale candidate hashes, missing references, inventory drift, invalid provenance, unknown identities, corrupted baseline candidate hashes and stale generated output. It does not @@ -112,6 +115,21 @@ baseline or the selected retrospective entries. Prefer more independent validati when beliefs stabilize; broaden when new contexts expose missing decisions; repair structure when ambiguity prevents use. Low novelty alone is not a stopping rule. +The window audit groups prospective entries by declared stream and comparison-group +label, orders them by date/identity, and retains links to every registered protocol. +Its three-cycle qualitative review trigger does not verify comparability: review +evidence tier, question granularity, evaluation and accounting against those protocols +first. Retrospective entries never join a window, even if later assigned effort or +a matching label. Outcomes include corroboration and inconclusive work. + +The audit distinguishes positive research time, unknown time and recorded zero time. +Unknown or zero effort permits qualitative discussion with limits but prevents +effort comparison of the complete declared window. Do not silently discard those +cycles or reconstruct their time from commits. A narrower window needs an explicit +review explaining its boundary and retained exclusions. Readiness does not calculate +a trend, declare a plateau or complete WP-0006 T03; that task needs an evidence-bearing +review of useful comparable research, not a generated count. + Keep both the unchanged candidate cohort and additions/removals visible when comparing snapshots. A growing inventory must not hide weakening evidence for old claims. Learning continues after milestones; finite workplans carry the next bounded diff --git a/research/evolution/context.json b/research/evolution/context.json index aa7e362..be8c46e 100644 --- a/research/evolution/context.json +++ b/research/evolution/context.json @@ -64,8 +64,9 @@ "adjudicated_unsupported_claim_count": null }, "open_work": [ - "workplans/FEP-WP-0002-seed-language-validation.md", + "workplans/FEP-WP-0009-m1-independent-validation.md", "workplans/FEP-WP-0003-contrasting-schools.md", - "workplans/FEP-WP-0006-language-evolution.md" + "workplans/FEP-WP-0006-language-evolution.md", + "workplans/FEP-WP-0008-use-driven-improvement.md" ] } diff --git a/scripts/research_evolution.py b/scripts/research_evolution.py index 200ecde..42b6745 100644 --- a/scripts/research_evolution.py +++ b/scripts/research_evolution.py @@ -105,6 +105,49 @@ def cell(value): return str(value).replace('|', '\\|').replace('\n', ' ') +def comparison_windows(data): + """Audit declared prospective groups; labels do not prove comparability.""" + groups = defaultdict(list) + for event in data['ledger']['events']: + if event['recording'] == 'prospective': + groups[(event['stream'], event['comparison_group'])].append(event) + windows = [] + for (stream, group), entries in sorted(groups.items()): + entries = sorted(entries, key=lambda e: (e['date'], e['id'])) + unknown = [e['id'] for e in entries if e['effort']['research_minutes'] is None] + zero = [e['id'] for e in entries if e['effort']['research_minutes'] == 0] + measured = [e['id'] for e in entries + if e['effort']['research_minutes'] is not None and e['effort']['research_minutes'] > 0] + due = len(entries) >= 3 + windows.append({ + 'stream': stream, + 'comparison_group': group, + 'cycle_ids': [e['id'] for e in entries], + 'protocol_refs': list(dict.fromkeys(e['protocol_ref'] for e in entries)), + 'cycles_until_review_trigger': max(0, 3 - len(entries)), + 'qualitative_review_due': due, + 'positive_effort_cycle_ids': measured, + 'unknown_effort_cycle_ids': unknown, + 'zero_effort_cycle_ids': zero, + 'effort_complete': not unknown and not zero, + 'effort_comparison_ready': due and not unknown and not zero, + 'outcomes': {kind: sum(e['disposition'] == kind for e in entries) + for kind in sorted(DISPOSITIONS)}, + }) + return { + 'schema_version': 1, + 'basis': 'Declared prospective groups; manually review shared stream, evidence tier, question granularity, evaluation and effort accounting before comparing returns.', + 'review_trigger_cycles': 3, + 'limits': [ + 'Group labels and cycle counts do not establish protocol comparability or a learning trend.', + 'Unknown or zero research time prevents effort comparison of the complete declared window; never estimate missing time or silently omit its cycle.', + 'A qualitative review may retain cycles with unknown effort; corroboration and inconclusive results remain visible.', + 'Readiness is a review trigger, not statistical evidence, a plateau declaration or a maturity judgment.', + ], + 'windows': windows, + } + + def render(data, baseline=None): profiles = data['profiles']['profiles'] events = data['ledger']['events'] @@ -147,23 +190,32 @@ def render(data, baseline=None): lines += [f'| {k} | {counts[k]} |' for k in sorted(DISPOSITIONS)] lines += ['', f'Cycles with recorded research effort: {sum(e["effort"]["research_minutes"] is not None for e in events)}/{len(events)}. ' 'Unknown effort is neither zero nor estimated from commits, tokens or assertion counts.', ''] - groups = defaultdict(list) - for e in events: - if e['recording'] == 'prospective': - groups[(e['stream'], e['comparison_group'])].append(e) - if not groups: + audit = comparison_windows(data) + if not audit['windows']: lines += ['No prospective comparison windows yet. Learning rate and diminishing returns: **not measurable from this baseline**.', ''] - for (stream, group), entries in sorted(groups.items()): - entries = sorted(entries, key=lambda e: (e['date'], e['id'])) - lines += [f'### Stream {stream} / {cell(group)}', '', - '| Cycle | Outcome | Research minutes | Consequential outcome/hour |', '| --- | --- | ---: | ---: |'] + else: + lines += ['### Comparison readiness', '', + 'Declared groups require a review of shared evidence tier, question granularity, evaluation and effort accounting. A matching label is not proof of comparability. ' + + link('research/evolution/ComparisonWindows.json', 'Machine-readable window audit') + '.', '', + '| Stream / declared group | Cycles | Positive effort | Unknown / zero effort | Qualitative review trigger | Complete-window effort comparison |', + '| --- | ---: | ---: | ---: | --- | --- |'] + for window in audit['windows']: + lines.append(f'| {window["stream"]} / {cell(window["comparison_group"])} | {len(window["cycle_ids"])} | ' + f'{len(window["positive_effort_cycle_ids"])} | {len(window["unknown_effort_cycle_ids"])} / {len(window["zero_effort_cycle_ids"])} | ' + f'{"due" if window["qualitative_review_due"] else "pending"} | {"ready for protocol review" if window["effort_comparison_ready"] else "not ready"} |') + lines += ['', 'Unknown or zero effort is not dropped to create a favorable window. Qualitative review can retain those cycles, but effort comparisons need an explicitly reviewed window with measured positive research time. No trend or plateau is inferred.', ''] + by_id = {e['id']: e for e in events} + for window in audit['windows']: + entries = [by_id[ident] for ident in window['cycle_ids']] + lines += [f'### Stream {window["stream"]} / {cell(window["comparison_group"])}', '', + '| Cycle | Protocol | Outcome | Research minutes | Consequential outcome/hour |', '| --- | --- | --- | ---: | ---: |'] for e in entries: minutes = e['effort']['research_minutes'] consequential = e['disposition'] in {'revision', 'rejection', 'merge'} rate = f'{60/minutes:.2f}' if consequential and minutes else ('0.00' if minutes else 'unknown') - lines.append(f'| {e["id"]} | {e["disposition"]} | {minutes if minutes is not None else "unknown"} | {rate} |') + lines.append(f'| {e["id"]} | {link(e["protocol_ref"], "protocol")} | {e["disposition"]} | {minutes if minutes is not None else "unknown"} | {rate} |') lines += ['', 'One bounded question counts at most one consequential outcome. Corroboration and inconclusive work remain visible separately. ' - + ('Fewer than three comparable cycles: no trend assessment.' if len(entries) < 3 else 'At least three cycles: eligible for a qualitative review of returns; this does not demonstrate a plateau.'), ''] + + ('Fewer than three declared cycles: qualitative review trigger pending; no trend assessment.' if not window['qualitative_review_due'] else 'At least three declared cycles: qualitative review trigger reached, subject to protocol comparability review; this does not demonstrate a plateau.'), ''] lines += ['## Coherence and maintainability', '', 'Profile hashes and local references passed validation when this report was generated. That checks traceability, not source truth or external URL freshness.', '', f'Profiles with unknown last substantive review date: {sum(p["review"]["last_substantive_review_date"] is None for p in profiles)}. ' @@ -228,12 +280,17 @@ def main(): validate_snapshot(baseline) output = render(data, baseline) path = ROOT / DATA / 'EvolutionReport.md' + audit_path = ROOT / DATA / 'ComparisonWindows.json' + audit_output = json.dumps(comparison_windows(data), indent=2, ensure_ascii=False) + '\n' if args.command == 'check': require(path.read_text() == output, 'Stale report: run make evolution-report') + require(audit_path.read_text() == audit_output, 'Stale comparison windows: run make evolution-report') print('Evolution records and generated report are consistent.') else: path.write_text(output) + audit_path.write_text(audit_output) print(path.relative_to(ROOT)) + print(audit_path.relative_to(ROOT)) except (ValueError, KeyError, OSError) as exc: parser.exit(1, f'Evolution validation failed: {exc}\n') diff --git a/templates/LearningCycle.md b/templates/LearningCycle.md index 60de69b..b4c0233 100644 --- a/templates/LearningCycle.md +++ b/templates/LearningCycle.md @@ -6,6 +6,8 @@ Status: planned; register before execution. One bounded question per cycle. - Stream (A–F or cross-stream): - Comparison group and shared protocol (question size, evidence tier, evaluation, and effort accounting must match; use a new group when they change): +- Prior cycle/protocol references and why this question is useful (a group label or + reaching the three-cycle review trigger is not justification): - Question and affected CAND identities: - Previous belief and expected decision: - Observation that would contradict the belief: @@ -22,6 +24,8 @@ Status: planned; register before execution. One bounded question per cycle. - Remaining uncertainty and live workplan/task: - Superseded ledger entries, if any: - Active research minutes / execution minutes (unknown if not measured): +- Accounting gaps or recorded zero time and their effect on comparison (retain the + cycle; never reconstruct missing effort or silently remove it from a window): - Reported tokens / cost USD, with receipts (unknown if not measured): - Reviewer name, role and independence: diff --git a/tests/test_research_evolution.py b/tests/test_research_evolution.py index ed8b175..a2d9c59 100644 --- a/tests/test_research_evolution.py +++ b/tests/test_research_evolution.py @@ -4,6 +4,7 @@ import importlib.util import json from pathlib import Path import subprocess +import sys import pytest @@ -23,6 +24,7 @@ def test_current_records_and_report_are_consistent(data): baseline = json.loads((ROOT / evolution.DATA / 'snapshots/baseline-2026-09-26.json').read_text()) evolution.validate_snapshot(baseline) assert evolution.render(data, baseline) == (ROOT / evolution.DATA / 'EvolutionReport.md').read_text() + assert evolution.comparison_windows(data) == json.loads((ROOT / evolution.DATA / 'ComparisonWindows.json').read_text()) @pytest.mark.parametrize('mutation,match', [ @@ -73,14 +75,76 @@ def test_rates_separate_corroboration_unknown_effort_and_streams(data): events[3]['stream'] = 'F' evolution.validate(data) report = evolution.render(data) - assert '| LEARN-0001 | revision | 30 | 2.00 |' in report - assert '| LEARN-0002 | corroboration | 30 | 0.00 |' in report - assert '| LEARN-0003 | revision | unknown | unknown |' in report + assert '| revision | 30 | 2.00 |' in report + assert '| corroboration | 30 | 0.00 |' in report + assert '| revision | unknown | unknown |' in report assert '### Stream F / controlled-mechanism' in report - assert 'Fewer than three comparable cycles' in report + assert 'Fewer than three declared cycles' in report assert 'does not demonstrate a plateau' in report events[0]['effort']['research_minutes'] = 0 - assert '| LEARN-0001 | revision | 0 | unknown |' in evolution.render(data) + assert '| revision | 0 | unknown |' in evolution.render(data) + + +@pytest.mark.parametrize('minutes', [None, 0]) +def test_three_declared_cycles_trigger_review_without_fabricating_effort(data, minutes): + entries = deepcopy(data['ledger']['events'][:3]) + for entry in entries: + entry.update(recording='prospective', comparison_group='test-window', stream='B', protocol_ref='templates/LearningCycle.md') + entry['effort']['research_minutes'] = 10 + entries[1]['effort']['research_minutes'] = minutes + entries[1]['disposition'] = 'inconclusive' + data['ledger']['events'] = entries + window, = evolution.comparison_windows(data)['windows'] + assert window['qualitative_review_due'] + assert window['cycles_until_review_trigger'] == 0 + assert not window['effort_complete'] + assert not window['effort_comparison_ready'] + assert window['positive_effort_cycle_ids'] == ['LEARN-0001', 'LEARN-0003'] + assert window['unknown_effort_cycle_ids'] == (['LEARN-0002'] if minutes is None else []) + assert window['zero_effort_cycle_ids'] == (['LEARN-0002'] if minutes == 0 else []) + assert window['outcomes']['inconclusive'] == 1 + # One more measured cycle cannot silently drop the intervening effort gap. + extra = deepcopy(entries[0]) + extra['id'] = 'LEARN-0004' + data['ledger']['events'].append(extra) + assert not evolution.comparison_windows(data)['windows'][0]['effort_comparison_ready'] + entries[1]['effort']['research_minutes'] = 10 + assert evolution.comparison_windows(data)['windows'][0]['effort_comparison_ready'] + assert 'subject to protocol comparability review' in evolution.render(data) + + +def test_window_audit_retains_order_protocols_and_stream_separation(data): + entries = deepcopy(data['ledger']['events'][:4]) + for entry in entries[:3]: + entry.update(recording='prospective', comparison_group='same-label', stream='B', protocol_ref='templates/LearningCycle.md') + entry['effort']['research_minutes'] = 10 + entries[1]['protocol_ref'] = 'research/evolution/cycles/2026-09-27-navigation.md' + entries[2]['stream'] = 'F' + entries[3]['comparison_group'] = 'same-label' # Retrospective data stays excluded. + entries[3]['effort']['research_minutes'] = 10 + data['ledger']['events'] = list(reversed(entries)) + audit = evolution.comparison_windows(data) + b, f = audit['windows'] + assert b['cycle_ids'] == ['LEARN-0001', 'LEARN-0002'] + assert b['protocol_refs'] == ['templates/LearningCycle.md', 'research/evolution/cycles/2026-09-27-navigation.md'] + assert b['cycles_until_review_trigger'] == 1 + assert b['effort_complete'] and not b['effort_comparison_ready'] + assert f['cycle_ids'] == ['LEARN-0003'] + assert not f['qualitative_review_due'] + + +def test_check_rejects_stale_window_audit(tmp_path, monkeypatch, capsys): + for ref in ['EvolutionReport.md', 'ComparisonWindows.json', 'snapshots/baseline-2026-09-26.json']: + path = tmp_path / evolution.DATA / ref + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes((ROOT / evolution.DATA / ref).read_bytes()) + (tmp_path / evolution.DATA / 'ComparisonWindows.json').write_text('{}\n') + monkeypatch.setattr(evolution, 'ROOT', tmp_path) + monkeypatch.setattr(sys, 'argv', ['research_evolution.py', 'check']) + with pytest.raises(SystemExit) as exc: + evolution.main() + assert exc.value.code == 1 + assert 'Stale comparison windows' in capsys.readouterr().err def test_snapshot_delta_does_not_count_additions_as_maturity_changes(data): diff --git a/workplans/FEP-WP-0006-language-evolution.md b/workplans/FEP-WP-0006-language-evolution.md index 4b1177f..fcd1a64 100644 --- a/workplans/FEP-WP-0006-language-evolution.md +++ b/workplans/FEP-WP-0006-language-evolution.md @@ -4,11 +4,11 @@ type: workplan title: "Make ongoing language learning observable" domain: infotech repo: frontend-patterns -status: backlog +status: blocked owner: codex topic_slug: canon created: "2026-09-26" -updated: "2026-10-03" +updated: "2026-10-04" state_hub_workstream_id: "528d24c1-c2c4-5914-a065-992b0dd877c8" --- @@ -16,7 +16,7 @@ state_hub_workstream_id: "528d24c1-c2c4-5914-a065-992b0dd877c8" User-approved baseline, learning ledger and generated evolution report. Finite investigations support an ongoing language; no composite maturity score or automatic -promotion. M1/M2 and independent human validation remain open in WP-0002/WP-0003. +promotion. M1/M2 and independent human validation remain open in WP-0009/WP-0003. ## Establish the versioned evidence baseline @@ -50,8 +50,7 @@ experiment receipts. Intent/protocol/program now describe continuous learning. ```task id: FEP-WP-0006-T03 status: wait -depends_on: [FEP-WP-0003-T01] -blocking_reason: "Registers a bounded cycle before FEP-WP-0003-T01's next investigation." +blocking_reason: "Needs useful comparable prospective evidence and an explicit protocol/returns review from WP-0003 or consumer-led WP-0008; largest declared group has two cycles, one with unknown effort." priority: medium state_hub_task_id: "2c5c4bae-a0a9-54bf-ac86-49b2803fdd8c" ``` @@ -220,3 +219,30 @@ when useful research under existing WP-0003 T01 or consumer-led WP-0008 supplies comparable window with honest effort accounting. Do not restart the paused navigation fixture or manufacture repeated studies merely to close T03. No new task/workplan; no diminishing-returns, independent-human or maturity claim. + +## Resumed measurement audit — 2026-10-04 + +The user explicitly resumed this workplan from backlog. Reviewed the current ledger, +registered cycle protocols and reporting implementation. T01/T02 and substantive +review-date reconciliation remain complete. Eight prospective cycles occupy seven +declared groups; B-independent-navigation-source-v1 has LEARN-0014/0017, with +LEARN-0017's effort honestly unknown. No complete three-cycle window exists. + +Implemented a deterministic [window audit](../research/evolution/ComparisonWindows.json) +alongside the evolution report: ordered cycle IDs, protocol references, outcome +counts, positive/unknown/zero effort, qualitative review triggers and complete-window +effort readiness. A matching group label no longer appears to establish comparability. +Three declared entries trigger protocol review; missing effort still permits scoped +qualitative review but cannot silently produce an effort comparison. Regression cases +cover unknown/zero time, nonconsequential outcomes, mixed streams, retrospective +exclusion, ordering and stale generated audit rejection. The cycle template now asks +why a follow-up is useful and requires accounting gaps to stay visible. + +This is measurement infrastructure, not a new empirical learning cycle; no ledger +outcome, candidate freshness, maturity or missing historical time is invented. Updated +the report's open-work references to the live M1 owner WP-0009 and consumer-led WP-0008. +Removed the stale exclusive dependency on parked WP-0003 T01: either owner can supply +useful comparable evidence under its existing scope. No downstream research gate is +waived. T03 remains wait and the resumed workplan is blocked on that evidence; the +paused navigation fixture stays paused. Existing work records retain the next useful +research and returns review; no replacement backlog is created.