Audit prospective comparison readiness and retain effort gaps
Assistant: codex Assistant-Model: gpt-6.1-sol Assistant-Session: 01a1045b-933d-7aa1-b2db-104557487a79
This commit is contained in:
parent
280c1d38bf
commit
d8d3a22b54
9 changed files with 439 additions and 54 deletions
|
|
@ -13,7 +13,7 @@
|
|||
| workplan | FEP-WP-0003 | backlog | — | workplans/FEP-WP-0003-contrasting-schools.md |
|
||||
| workplan | FEP-WP-0004 | backlog | — | workplans/FEP-WP-0004-reference-surface.md |
|
||||
| workplan | FEP-WP-0005 | finished | — | workplans/FEP-WP-0005-research-bench-setup.md |
|
||||
| workplan | FEP-WP-0006 | backlog | — | workplans/FEP-WP-0006-language-evolution.md |
|
||||
| workplan | FEP-WP-0006 | blocked | — | workplans/FEP-WP-0006-language-evolution.md |
|
||||
| workplan | FEP-WP-0007 | finished | — | workplans/FEP-WP-0007-language-v01-preview.md |
|
||||
| workplan | FEP-WP-0008 | blocked | — | workplans/FEP-WP-0008-use-driven-improvement.md |
|
||||
| workplan | FEP-WP-0009 | backlog | — | workplans/FEP-WP-0009-m1-independent-validation.md |
|
||||
|
|
|
|||
199
research/evolution/ComparisonWindows.json
Normal file
199
research/evolution/ComparisonWindows.json
Normal file
|
|
@ -0,0 +1,199 @@
|
|||
{
|
||||
"schema_version": 1,
|
||||
"basis": "Declared prospective groups; manually review shared stream, evidence tier, question granularity, evaluation and effort accounting before comparing returns.",
|
||||
"review_trigger_cycles": 3,
|
||||
"limits": [
|
||||
"Group labels and cycle counts do not establish protocol comparability or a learning trend.",
|
||||
"Unknown or zero research time prevents effort comparison of the complete declared window; never estimate missing time or silently omit its cycle.",
|
||||
"A qualitative review may retain cycles with unknown effort; corroboration and inconclusive results remain visible.",
|
||||
"Readiness is a review trigger, not statistical evidence, a plateau declaration or a maturity judgment."
|
||||
],
|
||||
"windows": [
|
||||
{
|
||||
"stream": "B",
|
||||
"comparison_group": "B-independent-navigation-source-v1",
|
||||
"cycle_ids": [
|
||||
"LEARN-0014",
|
||||
"LEARN-0017"
|
||||
],
|
||||
"protocol_refs": [
|
||||
"research/evolution/cycles/2026-09-27-zulip-navigation.md",
|
||||
"research/evolution/cycles/2026-09-27-zulip-target-lookup.md"
|
||||
],
|
||||
"cycles_until_review_trigger": 1,
|
||||
"qualitative_review_due": false,
|
||||
"positive_effort_cycle_ids": [
|
||||
"LEARN-0014"
|
||||
],
|
||||
"unknown_effort_cycle_ids": [
|
||||
"LEARN-0017"
|
||||
],
|
||||
"zero_effort_cycle_ids": [],
|
||||
"effort_complete": false,
|
||||
"effort_comparison_ready": false,
|
||||
"outcomes": {
|
||||
"corroboration": 1,
|
||||
"inconclusive": 0,
|
||||
"merge": 0,
|
||||
"rejection": 0,
|
||||
"revision": 1
|
||||
}
|
||||
},
|
||||
{
|
||||
"stream": "B",
|
||||
"comparison_group": "B-navigation-mechanism-v1",
|
||||
"cycle_ids": [
|
||||
"LEARN-0011"
|
||||
],
|
||||
"protocol_refs": [
|
||||
"research/evolution/cycles/2026-09-27-navigation.md"
|
||||
],
|
||||
"cycles_until_review_trigger": 2,
|
||||
"qualitative_review_due": false,
|
||||
"positive_effort_cycle_ids": [
|
||||
"LEARN-0011"
|
||||
],
|
||||
"unknown_effort_cycle_ids": [],
|
||||
"zero_effort_cycle_ids": [],
|
||||
"effort_complete": true,
|
||||
"effort_comparison_ready": false,
|
||||
"outcomes": {
|
||||
"corroboration": 1,
|
||||
"inconclusive": 0,
|
||||
"merge": 0,
|
||||
"rejection": 0,
|
||||
"revision": 0
|
||||
}
|
||||
},
|
||||
{
|
||||
"stream": "D",
|
||||
"comparison_group": "D-sqlite-git-recovery-v1",
|
||||
"cycle_ids": [
|
||||
"LEARN-0018"
|
||||
],
|
||||
"protocol_refs": [
|
||||
"research/evolution/cycles/2026-09-28-storage-receiver.md"
|
||||
],
|
||||
"cycles_until_review_trigger": 2,
|
||||
"qualitative_review_due": false,
|
||||
"positive_effort_cycle_ids": [
|
||||
"LEARN-0018"
|
||||
],
|
||||
"unknown_effort_cycle_ids": [],
|
||||
"zero_effort_cycle_ids": [],
|
||||
"effort_complete": true,
|
||||
"effort_comparison_ready": false,
|
||||
"outcomes": {
|
||||
"corroboration": 1,
|
||||
"inconclusive": 0,
|
||||
"merge": 0,
|
||||
"rejection": 0,
|
||||
"revision": 0
|
||||
}
|
||||
},
|
||||
{
|
||||
"stream": "F",
|
||||
"comparison_group": "F-navigation-focus-v1",
|
||||
"cycle_ids": [
|
||||
"LEARN-0012"
|
||||
],
|
||||
"protocol_refs": [
|
||||
"research/evolution/cycles/2026-09-27-navigation-focus.md"
|
||||
],
|
||||
"cycles_until_review_trigger": 2,
|
||||
"qualitative_review_due": false,
|
||||
"positive_effort_cycle_ids": [
|
||||
"LEARN-0012"
|
||||
],
|
||||
"unknown_effort_cycle_ids": [],
|
||||
"zero_effort_cycle_ids": [],
|
||||
"effort_complete": true,
|
||||
"effort_comparison_ready": false,
|
||||
"outcomes": {
|
||||
"corroboration": 1,
|
||||
"inconclusive": 0,
|
||||
"merge": 0,
|
||||
"rejection": 0,
|
||||
"revision": 0
|
||||
}
|
||||
},
|
||||
{
|
||||
"stream": "F",
|
||||
"comparison_group": "F-navigation-ownership-v1",
|
||||
"cycle_ids": [
|
||||
"LEARN-0013"
|
||||
],
|
||||
"protocol_refs": [
|
||||
"research/evolution/cycles/2026-09-27-navigation-ownership.md"
|
||||
],
|
||||
"cycles_until_review_trigger": 2,
|
||||
"qualitative_review_due": false,
|
||||
"positive_effort_cycle_ids": [
|
||||
"LEARN-0013"
|
||||
],
|
||||
"unknown_effort_cycle_ids": [],
|
||||
"zero_effort_cycle_ids": [],
|
||||
"effort_complete": true,
|
||||
"effort_comparison_ready": false,
|
||||
"outcomes": {
|
||||
"corroboration": 0,
|
||||
"inconclusive": 0,
|
||||
"merge": 0,
|
||||
"rejection": 0,
|
||||
"revision": 1
|
||||
}
|
||||
},
|
||||
{
|
||||
"stream": "cross-stream",
|
||||
"comparison_group": "CD-acceptance-projection-static-v1",
|
||||
"cycle_ids": [
|
||||
"LEARN-0016"
|
||||
],
|
||||
"protocol_refs": [
|
||||
"research/evolution/cycles/2026-09-27-zulip-acceptance.md"
|
||||
],
|
||||
"cycles_until_review_trigger": 2,
|
||||
"qualitative_review_due": false,
|
||||
"positive_effort_cycle_ids": [
|
||||
"LEARN-0016"
|
||||
],
|
||||
"unknown_effort_cycle_ids": [],
|
||||
"zero_effort_cycle_ids": [],
|
||||
"effort_complete": true,
|
||||
"effort_comparison_ready": false,
|
||||
"outcomes": {
|
||||
"corroboration": 1,
|
||||
"inconclusive": 0,
|
||||
"merge": 0,
|
||||
"rejection": 0,
|
||||
"revision": 0
|
||||
}
|
||||
},
|
||||
{
|
||||
"stream": "cross-stream",
|
||||
"comparison_group": "CD-authority-projection-documentary-v1",
|
||||
"cycle_ids": [
|
||||
"LEARN-0015"
|
||||
],
|
||||
"protocol_refs": [
|
||||
"research/evolution/cycles/2026-09-27-authority-projection.md"
|
||||
],
|
||||
"cycles_until_review_trigger": 2,
|
||||
"qualitative_review_due": false,
|
||||
"positive_effort_cycle_ids": [
|
||||
"LEARN-0015"
|
||||
],
|
||||
"unknown_effort_cycle_ids": [],
|
||||
"zero_effort_cycle_ids": [],
|
||||
"effort_complete": true,
|
||||
"effort_comparison_ready": false,
|
||||
"outcomes": {
|
||||
"corroboration": 0,
|
||||
"inconclusive": 0,
|
||||
"merge": 0,
|
||||
"rejection": 0,
|
||||
"revision": 1
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -59,62 +59,78 @@ Ten selected retrospective outcomes (not a historical census), followed by prosp
|
|||
|
||||
Cycles with recorded research effort: 7/18. Unknown effort is neither zero nor estimated from commits, tokens or assertion counts.
|
||||
|
||||
### Comparison readiness
|
||||
|
||||
Declared groups require a review of shared evidence tier, question granularity, evaluation and effort accounting. A matching label is not proof of comparability. [Machine-readable window audit](../../research/evolution/ComparisonWindows.json).
|
||||
|
||||
| Stream / declared group | Cycles | Positive effort | Unknown / zero effort | Qualitative review trigger | Complete-window effort comparison |
|
||||
| --- | ---: | ---: | ---: | --- | --- |
|
||||
| B / B-independent-navigation-source-v1 | 2 | 1 | 1 / 0 | pending | not ready |
|
||||
| B / B-navigation-mechanism-v1 | 1 | 1 | 0 / 0 | pending | not ready |
|
||||
| D / D-sqlite-git-recovery-v1 | 1 | 1 | 0 / 0 | pending | not ready |
|
||||
| F / F-navigation-focus-v1 | 1 | 1 | 0 / 0 | pending | not ready |
|
||||
| F / F-navigation-ownership-v1 | 1 | 1 | 0 / 0 | pending | not ready |
|
||||
| cross-stream / CD-acceptance-projection-static-v1 | 1 | 1 | 0 / 0 | pending | not ready |
|
||||
| cross-stream / CD-authority-projection-documentary-v1 | 1 | 1 | 0 / 0 | pending | not ready |
|
||||
|
||||
Unknown or zero effort is not dropped to create a favorable window. Qualitative review can retain those cycles, but effort comparisons need an explicitly reviewed window with measured positive research time. No trend or plateau is inferred.
|
||||
|
||||
### Stream B / B-independent-navigation-source-v1
|
||||
|
||||
| Cycle | Outcome | Research minutes | Consequential outcome/hour |
|
||||
| --- | --- | ---: | ---: |
|
||||
| LEARN-0014 | revision | 6.345 | 9.46 |
|
||||
| LEARN-0017 | corroboration | unknown | unknown |
|
||||
| Cycle | Protocol | Outcome | Research minutes | Consequential outcome/hour |
|
||||
| --- | --- | --- | ---: | ---: |
|
||||
| LEARN-0014 | [protocol](../../research/evolution/cycles/2026-09-27-zulip-navigation.md) | revision | 6.345 | 9.46 |
|
||||
| LEARN-0017 | [protocol](../../research/evolution/cycles/2026-09-27-zulip-target-lookup.md) | corroboration | unknown | unknown |
|
||||
|
||||
One bounded question counts at most one consequential outcome. Corroboration and inconclusive work remain visible separately. Fewer than three comparable cycles: no trend assessment.
|
||||
One bounded question counts at most one consequential outcome. Corroboration and inconclusive work remain visible separately. Fewer than three declared cycles: qualitative review trigger pending; no trend assessment.
|
||||
|
||||
### Stream B / B-navigation-mechanism-v1
|
||||
|
||||
| Cycle | Outcome | Research minutes | Consequential outcome/hour |
|
||||
| --- | --- | ---: | ---: |
|
||||
| LEARN-0011 | corroboration | 6.177 | 0.00 |
|
||||
| Cycle | Protocol | Outcome | Research minutes | Consequential outcome/hour |
|
||||
| --- | --- | --- | ---: | ---: |
|
||||
| LEARN-0011 | [protocol](../../research/evolution/cycles/2026-09-27-navigation.md) | corroboration | 6.177 | 0.00 |
|
||||
|
||||
One bounded question counts at most one consequential outcome. Corroboration and inconclusive work remain visible separately. Fewer than three comparable cycles: no trend assessment.
|
||||
One bounded question counts at most one consequential outcome. Corroboration and inconclusive work remain visible separately. Fewer than three declared cycles: qualitative review trigger pending; no trend assessment.
|
||||
|
||||
### Stream D / D-sqlite-git-recovery-v1
|
||||
|
||||
| Cycle | Outcome | Research minutes | Consequential outcome/hour |
|
||||
| --- | --- | ---: | ---: |
|
||||
| LEARN-0018 | corroboration | 4.123 | 0.00 |
|
||||
| Cycle | Protocol | Outcome | Research minutes | Consequential outcome/hour |
|
||||
| --- | --- | --- | ---: | ---: |
|
||||
| LEARN-0018 | [protocol](../../research/evolution/cycles/2026-09-28-storage-receiver.md) | corroboration | 4.123 | 0.00 |
|
||||
|
||||
One bounded question counts at most one consequential outcome. Corroboration and inconclusive work remain visible separately. Fewer than three comparable cycles: no trend assessment.
|
||||
One bounded question counts at most one consequential outcome. Corroboration and inconclusive work remain visible separately. Fewer than three declared cycles: qualitative review trigger pending; no trend assessment.
|
||||
|
||||
### Stream F / F-navigation-focus-v1
|
||||
|
||||
| Cycle | Outcome | Research minutes | Consequential outcome/hour |
|
||||
| --- | --- | ---: | ---: |
|
||||
| LEARN-0012 | corroboration | 5.138 | 0.00 |
|
||||
| Cycle | Protocol | Outcome | Research minutes | Consequential outcome/hour |
|
||||
| --- | --- | --- | ---: | ---: |
|
||||
| LEARN-0012 | [protocol](../../research/evolution/cycles/2026-09-27-navigation-focus.md) | corroboration | 5.138 | 0.00 |
|
||||
|
||||
One bounded question counts at most one consequential outcome. Corroboration and inconclusive work remain visible separately. Fewer than three comparable cycles: no trend assessment.
|
||||
One bounded question counts at most one consequential outcome. Corroboration and inconclusive work remain visible separately. Fewer than three declared cycles: qualitative review trigger pending; no trend assessment.
|
||||
|
||||
### Stream F / F-navigation-ownership-v1
|
||||
|
||||
| Cycle | Outcome | Research minutes | Consequential outcome/hour |
|
||||
| --- | --- | ---: | ---: |
|
||||
| LEARN-0013 | revision | 13.918 | 4.31 |
|
||||
| Cycle | Protocol | Outcome | Research minutes | Consequential outcome/hour |
|
||||
| --- | --- | --- | ---: | ---: |
|
||||
| LEARN-0013 | [protocol](../../research/evolution/cycles/2026-09-27-navigation-ownership.md) | revision | 13.918 | 4.31 |
|
||||
|
||||
One bounded question counts at most one consequential outcome. Corroboration and inconclusive work remain visible separately. Fewer than three comparable cycles: no trend assessment.
|
||||
One bounded question counts at most one consequential outcome. Corroboration and inconclusive work remain visible separately. Fewer than three declared cycles: qualitative review trigger pending; no trend assessment.
|
||||
|
||||
### Stream cross-stream / CD-acceptance-projection-static-v1
|
||||
|
||||
| Cycle | Outcome | Research minutes | Consequential outcome/hour |
|
||||
| --- | --- | ---: | ---: |
|
||||
| LEARN-0016 | corroboration | 4.182 | 0.00 |
|
||||
| Cycle | Protocol | Outcome | Research minutes | Consequential outcome/hour |
|
||||
| --- | --- | --- | ---: | ---: |
|
||||
| LEARN-0016 | [protocol](../../research/evolution/cycles/2026-09-27-zulip-acceptance.md) | corroboration | 4.182 | 0.00 |
|
||||
|
||||
One bounded question counts at most one consequential outcome. Corroboration and inconclusive work remain visible separately. Fewer than three comparable cycles: no trend assessment.
|
||||
One bounded question counts at most one consequential outcome. Corroboration and inconclusive work remain visible separately. Fewer than three declared cycles: qualitative review trigger pending; no trend assessment.
|
||||
|
||||
### Stream cross-stream / CD-authority-projection-documentary-v1
|
||||
|
||||
| Cycle | Outcome | Research minutes | Consequential outcome/hour |
|
||||
| --- | --- | ---: | ---: |
|
||||
| LEARN-0015 | revision | 3.128 | 19.18 |
|
||||
| Cycle | Protocol | Outcome | Research minutes | Consequential outcome/hour |
|
||||
| --- | --- | --- | ---: | ---: |
|
||||
| LEARN-0015 | [protocol](../../research/evolution/cycles/2026-09-27-authority-projection.md) | revision | 3.128 | 19.18 |
|
||||
|
||||
One bounded question counts at most one consequential outcome. Corroboration and inconclusive work remain visible separately. Fewer than three comparable cycles: no trend assessment.
|
||||
One bounded question counts at most one consequential outcome. Corroboration and inconclusive work remain visible separately. Fewer than three declared cycles: qualitative review trigger pending; no trend assessment.
|
||||
|
||||
## Coherence and maintainability
|
||||
|
||||
|
|
@ -124,7 +140,7 @@ Profiles with unknown last substantive review date: 0. Assessment dates record t
|
|||
|
||||
Coherence metrics (ambiguous selections, unjustified relations and overlap resolved): not measured. Conflict Copy / Explicit Reconciliation overlap remains an explicit open question.
|
||||
|
||||
Review backlog and evidence gaps remain in [FEP-WP-0002-seed-language-validation](../../workplans/FEP-WP-0002-seed-language-validation.md), [FEP-WP-0003-contrasting-schools](../../workplans/FEP-WP-0003-contrasting-schools.md), [FEP-WP-0006-language-evolution](../../workplans/FEP-WP-0006-language-evolution.md).
|
||||
Review backlog and evidence gaps remain in [FEP-WP-0009-m1-independent-validation](../../workplans/FEP-WP-0009-m1-independent-validation.md), [FEP-WP-0003-contrasting-schools](../../workplans/FEP-WP-0003-contrasting-schools.md), [FEP-WP-0006-language-evolution](../../workplans/FEP-WP-0006-language-evolution.md), [FEP-WP-0008-use-driven-improvement](../../workplans/FEP-WP-0008-use-driven-improvement.md).
|
||||
|
||||
## Pattern profiles
|
||||
|
||||
|
|
|
|||
|
|
@ -66,6 +66,9 @@ renew them. Historical snapshots retain their original schema-v1 assessments.
|
|||
undermine their scope. Never promote automatically from metric thresholds.
|
||||
|
||||
Commands: `make evolution-report`, `make evolution-check`, `make evolution-test`.
|
||||
Generation writes both EvolutionReport.md and the machine-readable
|
||||
[ComparisonWindows.json](ComparisonWindows.json); checking rejects stale output
|
||||
in either artifact.
|
||||
The check rejects stale candidate hashes, missing references, inventory drift,
|
||||
invalid provenance, unknown identities, corrupted baseline candidate hashes and stale
|
||||
generated output. It does not
|
||||
|
|
@ -112,6 +115,21 @@ baseline or the selected retrospective entries. Prefer more independent validati
|
|||
when beliefs stabilize; broaden when new contexts expose missing decisions; repair
|
||||
structure when ambiguity prevents use. Low novelty alone is not a stopping rule.
|
||||
|
||||
The window audit groups prospective entries by declared stream and comparison-group
|
||||
label, orders them by date/identity, and retains links to every registered protocol.
|
||||
Its three-cycle qualitative review trigger does not verify comparability: review
|
||||
evidence tier, question granularity, evaluation and accounting against those protocols
|
||||
first. Retrospective entries never join a window, even if later assigned effort or
|
||||
a matching label. Outcomes include corroboration and inconclusive work.
|
||||
|
||||
The audit distinguishes positive research time, unknown time and recorded zero time.
|
||||
Unknown or zero effort permits qualitative discussion with limits but prevents
|
||||
effort comparison of the complete declared window. Do not silently discard those
|
||||
cycles or reconstruct their time from commits. A narrower window needs an explicit
|
||||
review explaining its boundary and retained exclusions. Readiness does not calculate
|
||||
a trend, declare a plateau or complete WP-0006 T03; that task needs an evidence-bearing
|
||||
review of useful comparable research, not a generated count.
|
||||
|
||||
Keep both the unchanged candidate cohort and additions/removals visible when
|
||||
comparing snapshots. A growing inventory must not hide weakening evidence for old
|
||||
claims. Learning continues after milestones; finite workplans carry the next bounded
|
||||
|
|
|
|||
|
|
@ -64,8 +64,9 @@
|
|||
"adjudicated_unsupported_claim_count": null
|
||||
},
|
||||
"open_work": [
|
||||
"workplans/FEP-WP-0002-seed-language-validation.md",
|
||||
"workplans/FEP-WP-0009-m1-independent-validation.md",
|
||||
"workplans/FEP-WP-0003-contrasting-schools.md",
|
||||
"workplans/FEP-WP-0006-language-evolution.md"
|
||||
"workplans/FEP-WP-0006-language-evolution.md",
|
||||
"workplans/FEP-WP-0008-use-driven-improvement.md"
|
||||
]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -105,6 +105,49 @@ def cell(value):
|
|||
return str(value).replace('|', '\\|').replace('\n', ' ')
|
||||
|
||||
|
||||
def comparison_windows(data):
|
||||
"""Audit declared prospective groups; labels do not prove comparability."""
|
||||
groups = defaultdict(list)
|
||||
for event in data['ledger']['events']:
|
||||
if event['recording'] == 'prospective':
|
||||
groups[(event['stream'], event['comparison_group'])].append(event)
|
||||
windows = []
|
||||
for (stream, group), entries in sorted(groups.items()):
|
||||
entries = sorted(entries, key=lambda e: (e['date'], e['id']))
|
||||
unknown = [e['id'] for e in entries if e['effort']['research_minutes'] is None]
|
||||
zero = [e['id'] for e in entries if e['effort']['research_minutes'] == 0]
|
||||
measured = [e['id'] for e in entries
|
||||
if e['effort']['research_minutes'] is not None and e['effort']['research_minutes'] > 0]
|
||||
due = len(entries) >= 3
|
||||
windows.append({
|
||||
'stream': stream,
|
||||
'comparison_group': group,
|
||||
'cycle_ids': [e['id'] for e in entries],
|
||||
'protocol_refs': list(dict.fromkeys(e['protocol_ref'] for e in entries)),
|
||||
'cycles_until_review_trigger': max(0, 3 - len(entries)),
|
||||
'qualitative_review_due': due,
|
||||
'positive_effort_cycle_ids': measured,
|
||||
'unknown_effort_cycle_ids': unknown,
|
||||
'zero_effort_cycle_ids': zero,
|
||||
'effort_complete': not unknown and not zero,
|
||||
'effort_comparison_ready': due and not unknown and not zero,
|
||||
'outcomes': {kind: sum(e['disposition'] == kind for e in entries)
|
||||
for kind in sorted(DISPOSITIONS)},
|
||||
})
|
||||
return {
|
||||
'schema_version': 1,
|
||||
'basis': 'Declared prospective groups; manually review shared stream, evidence tier, question granularity, evaluation and effort accounting before comparing returns.',
|
||||
'review_trigger_cycles': 3,
|
||||
'limits': [
|
||||
'Group labels and cycle counts do not establish protocol comparability or a learning trend.',
|
||||
'Unknown or zero research time prevents effort comparison of the complete declared window; never estimate missing time or silently omit its cycle.',
|
||||
'A qualitative review may retain cycles with unknown effort; corroboration and inconclusive results remain visible.',
|
||||
'Readiness is a review trigger, not statistical evidence, a plateau declaration or a maturity judgment.',
|
||||
],
|
||||
'windows': windows,
|
||||
}
|
||||
|
||||
|
||||
def render(data, baseline=None):
|
||||
profiles = data['profiles']['profiles']
|
||||
events = data['ledger']['events']
|
||||
|
|
@ -147,23 +190,32 @@ def render(data, baseline=None):
|
|||
lines += [f'| {k} | {counts[k]} |' for k in sorted(DISPOSITIONS)]
|
||||
lines += ['', f'Cycles with recorded research effort: {sum(e["effort"]["research_minutes"] is not None for e in events)}/{len(events)}. '
|
||||
'Unknown effort is neither zero nor estimated from commits, tokens or assertion counts.', '']
|
||||
groups = defaultdict(list)
|
||||
for e in events:
|
||||
if e['recording'] == 'prospective':
|
||||
groups[(e['stream'], e['comparison_group'])].append(e)
|
||||
if not groups:
|
||||
audit = comparison_windows(data)
|
||||
if not audit['windows']:
|
||||
lines += ['No prospective comparison windows yet. Learning rate and diminishing returns: **not measurable from this baseline**.', '']
|
||||
for (stream, group), entries in sorted(groups.items()):
|
||||
entries = sorted(entries, key=lambda e: (e['date'], e['id']))
|
||||
lines += [f'### Stream {stream} / {cell(group)}', '',
|
||||
'| Cycle | Outcome | Research minutes | Consequential outcome/hour |', '| --- | --- | ---: | ---: |']
|
||||
else:
|
||||
lines += ['### Comparison readiness', '',
|
||||
'Declared groups require a review of shared evidence tier, question granularity, evaluation and effort accounting. A matching label is not proof of comparability. '
|
||||
+ link('research/evolution/ComparisonWindows.json', 'Machine-readable window audit') + '.', '',
|
||||
'| Stream / declared group | Cycles | Positive effort | Unknown / zero effort | Qualitative review trigger | Complete-window effort comparison |',
|
||||
'| --- | ---: | ---: | ---: | --- | --- |']
|
||||
for window in audit['windows']:
|
||||
lines.append(f'| {window["stream"]} / {cell(window["comparison_group"])} | {len(window["cycle_ids"])} | '
|
||||
f'{len(window["positive_effort_cycle_ids"])} | {len(window["unknown_effort_cycle_ids"])} / {len(window["zero_effort_cycle_ids"])} | '
|
||||
f'{"due" if window["qualitative_review_due"] else "pending"} | {"ready for protocol review" if window["effort_comparison_ready"] else "not ready"} |')
|
||||
lines += ['', 'Unknown or zero effort is not dropped to create a favorable window. Qualitative review can retain those cycles, but effort comparisons need an explicitly reviewed window with measured positive research time. No trend or plateau is inferred.', '']
|
||||
by_id = {e['id']: e for e in events}
|
||||
for window in audit['windows']:
|
||||
entries = [by_id[ident] for ident in window['cycle_ids']]
|
||||
lines += [f'### Stream {window["stream"]} / {cell(window["comparison_group"])}', '',
|
||||
'| Cycle | Protocol | Outcome | Research minutes | Consequential outcome/hour |', '| --- | --- | --- | ---: | ---: |']
|
||||
for e in entries:
|
||||
minutes = e['effort']['research_minutes']
|
||||
consequential = e['disposition'] in {'revision', 'rejection', 'merge'}
|
||||
rate = f'{60/minutes:.2f}' if consequential and minutes else ('0.00' if minutes else 'unknown')
|
||||
lines.append(f'| {e["id"]} | {e["disposition"]} | {minutes if minutes is not None else "unknown"} | {rate} |')
|
||||
lines.append(f'| {e["id"]} | {link(e["protocol_ref"], "protocol")} | {e["disposition"]} | {minutes if minutes is not None else "unknown"} | {rate} |')
|
||||
lines += ['', 'One bounded question counts at most one consequential outcome. Corroboration and inconclusive work remain visible separately. '
|
||||
+ ('Fewer than three comparable cycles: no trend assessment.' if len(entries) < 3 else 'At least three cycles: eligible for a qualitative review of returns; this does not demonstrate a plateau.'), '']
|
||||
+ ('Fewer than three declared cycles: qualitative review trigger pending; no trend assessment.' if not window['qualitative_review_due'] else 'At least three declared cycles: qualitative review trigger reached, subject to protocol comparability review; this does not demonstrate a plateau.'), '']
|
||||
lines += ['## Coherence and maintainability', '',
|
||||
'Profile hashes and local references passed validation when this report was generated. That checks traceability, not source truth or external URL freshness.', '',
|
||||
f'Profiles with unknown last substantive review date: {sum(p["review"]["last_substantive_review_date"] is None for p in profiles)}. '
|
||||
|
|
@ -228,12 +280,17 @@ def main():
|
|||
validate_snapshot(baseline)
|
||||
output = render(data, baseline)
|
||||
path = ROOT / DATA / 'EvolutionReport.md'
|
||||
audit_path = ROOT / DATA / 'ComparisonWindows.json'
|
||||
audit_output = json.dumps(comparison_windows(data), indent=2, ensure_ascii=False) + '\n'
|
||||
if args.command == 'check':
|
||||
require(path.read_text() == output, 'Stale report: run make evolution-report')
|
||||
require(audit_path.read_text() == audit_output, 'Stale comparison windows: run make evolution-report')
|
||||
print('Evolution records and generated report are consistent.')
|
||||
else:
|
||||
path.write_text(output)
|
||||
audit_path.write_text(audit_output)
|
||||
print(path.relative_to(ROOT))
|
||||
print(audit_path.relative_to(ROOT))
|
||||
except (ValueError, KeyError, OSError) as exc:
|
||||
parser.exit(1, f'Evolution validation failed: {exc}\n')
|
||||
|
||||
|
|
|
|||
|
|
@ -6,6 +6,8 @@ Status: planned; register before execution. One bounded question per cycle.
|
|||
- Stream (A–F or cross-stream):
|
||||
- Comparison group and shared protocol (question size, evidence tier, evaluation,
|
||||
and effort accounting must match; use a new group when they change):
|
||||
- Prior cycle/protocol references and why this question is useful (a group label or
|
||||
reaching the three-cycle review trigger is not justification):
|
||||
- Question and affected CAND identities:
|
||||
- Previous belief and expected decision:
|
||||
- Observation that would contradict the belief:
|
||||
|
|
@ -22,6 +24,8 @@ Status: planned; register before execution. One bounded question per cycle.
|
|||
- Remaining uncertainty and live workplan/task:
|
||||
- Superseded ledger entries, if any:
|
||||
- Active research minutes / execution minutes (unknown if not measured):
|
||||
- Accounting gaps or recorded zero time and their effect on comparison (retain the
|
||||
cycle; never reconstruct missing effort or silently remove it from a window):
|
||||
- Reported tokens / cost USD, with receipts (unknown if not measured):
|
||||
- Reviewer name, role and independence:
|
||||
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import importlib.util
|
|||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
|
||||
|
|
@ -23,6 +24,7 @@ def test_current_records_and_report_are_consistent(data):
|
|||
baseline = json.loads((ROOT / evolution.DATA / 'snapshots/baseline-2026-09-26.json').read_text())
|
||||
evolution.validate_snapshot(baseline)
|
||||
assert evolution.render(data, baseline) == (ROOT / evolution.DATA / 'EvolutionReport.md').read_text()
|
||||
assert evolution.comparison_windows(data) == json.loads((ROOT / evolution.DATA / 'ComparisonWindows.json').read_text())
|
||||
|
||||
|
||||
@pytest.mark.parametrize('mutation,match', [
|
||||
|
|
@ -73,14 +75,76 @@ def test_rates_separate_corroboration_unknown_effort_and_streams(data):
|
|||
events[3]['stream'] = 'F'
|
||||
evolution.validate(data)
|
||||
report = evolution.render(data)
|
||||
assert '| LEARN-0001 | revision | 30 | 2.00 |' in report
|
||||
assert '| LEARN-0002 | corroboration | 30 | 0.00 |' in report
|
||||
assert '| LEARN-0003 | revision | unknown | unknown |' in report
|
||||
assert '| revision | 30 | 2.00 |' in report
|
||||
assert '| corroboration | 30 | 0.00 |' in report
|
||||
assert '| revision | unknown | unknown |' in report
|
||||
assert '### Stream F / controlled-mechanism' in report
|
||||
assert 'Fewer than three comparable cycles' in report
|
||||
assert 'Fewer than three declared cycles' in report
|
||||
assert 'does not demonstrate a plateau' in report
|
||||
events[0]['effort']['research_minutes'] = 0
|
||||
assert '| LEARN-0001 | revision | 0 | unknown |' in evolution.render(data)
|
||||
assert '| revision | 0 | unknown |' in evolution.render(data)
|
||||
|
||||
|
||||
@pytest.mark.parametrize('minutes', [None, 0])
|
||||
def test_three_declared_cycles_trigger_review_without_fabricating_effort(data, minutes):
|
||||
entries = deepcopy(data['ledger']['events'][:3])
|
||||
for entry in entries:
|
||||
entry.update(recording='prospective', comparison_group='test-window', stream='B', protocol_ref='templates/LearningCycle.md')
|
||||
entry['effort']['research_minutes'] = 10
|
||||
entries[1]['effort']['research_minutes'] = minutes
|
||||
entries[1]['disposition'] = 'inconclusive'
|
||||
data['ledger']['events'] = entries
|
||||
window, = evolution.comparison_windows(data)['windows']
|
||||
assert window['qualitative_review_due']
|
||||
assert window['cycles_until_review_trigger'] == 0
|
||||
assert not window['effort_complete']
|
||||
assert not window['effort_comparison_ready']
|
||||
assert window['positive_effort_cycle_ids'] == ['LEARN-0001', 'LEARN-0003']
|
||||
assert window['unknown_effort_cycle_ids'] == (['LEARN-0002'] if minutes is None else [])
|
||||
assert window['zero_effort_cycle_ids'] == (['LEARN-0002'] if minutes == 0 else [])
|
||||
assert window['outcomes']['inconclusive'] == 1
|
||||
# One more measured cycle cannot silently drop the intervening effort gap.
|
||||
extra = deepcopy(entries[0])
|
||||
extra['id'] = 'LEARN-0004'
|
||||
data['ledger']['events'].append(extra)
|
||||
assert not evolution.comparison_windows(data)['windows'][0]['effort_comparison_ready']
|
||||
entries[1]['effort']['research_minutes'] = 10
|
||||
assert evolution.comparison_windows(data)['windows'][0]['effort_comparison_ready']
|
||||
assert 'subject to protocol comparability review' in evolution.render(data)
|
||||
|
||||
|
||||
def test_window_audit_retains_order_protocols_and_stream_separation(data):
|
||||
entries = deepcopy(data['ledger']['events'][:4])
|
||||
for entry in entries[:3]:
|
||||
entry.update(recording='prospective', comparison_group='same-label', stream='B', protocol_ref='templates/LearningCycle.md')
|
||||
entry['effort']['research_minutes'] = 10
|
||||
entries[1]['protocol_ref'] = 'research/evolution/cycles/2026-09-27-navigation.md'
|
||||
entries[2]['stream'] = 'F'
|
||||
entries[3]['comparison_group'] = 'same-label' # Retrospective data stays excluded.
|
||||
entries[3]['effort']['research_minutes'] = 10
|
||||
data['ledger']['events'] = list(reversed(entries))
|
||||
audit = evolution.comparison_windows(data)
|
||||
b, f = audit['windows']
|
||||
assert b['cycle_ids'] == ['LEARN-0001', 'LEARN-0002']
|
||||
assert b['protocol_refs'] == ['templates/LearningCycle.md', 'research/evolution/cycles/2026-09-27-navigation.md']
|
||||
assert b['cycles_until_review_trigger'] == 1
|
||||
assert b['effort_complete'] and not b['effort_comparison_ready']
|
||||
assert f['cycle_ids'] == ['LEARN-0003']
|
||||
assert not f['qualitative_review_due']
|
||||
|
||||
|
||||
def test_check_rejects_stale_window_audit(tmp_path, monkeypatch, capsys):
|
||||
for ref in ['EvolutionReport.md', 'ComparisonWindows.json', 'snapshots/baseline-2026-09-26.json']:
|
||||
path = tmp_path / evolution.DATA / ref
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_bytes((ROOT / evolution.DATA / ref).read_bytes())
|
||||
(tmp_path / evolution.DATA / 'ComparisonWindows.json').write_text('{}\n')
|
||||
monkeypatch.setattr(evolution, 'ROOT', tmp_path)
|
||||
monkeypatch.setattr(sys, 'argv', ['research_evolution.py', 'check'])
|
||||
with pytest.raises(SystemExit) as exc:
|
||||
evolution.main()
|
||||
assert exc.value.code == 1
|
||||
assert 'Stale comparison windows' in capsys.readouterr().err
|
||||
|
||||
|
||||
def test_snapshot_delta_does_not_count_additions_as_maturity_changes(data):
|
||||
|
|
|
|||
|
|
@ -4,11 +4,11 @@ type: workplan
|
|||
title: "Make ongoing language learning observable"
|
||||
domain: infotech
|
||||
repo: frontend-patterns
|
||||
status: backlog
|
||||
status: blocked
|
||||
owner: codex
|
||||
topic_slug: canon
|
||||
created: "2026-09-26"
|
||||
updated: "2026-10-03"
|
||||
updated: "2026-10-04"
|
||||
state_hub_workstream_id: "528d24c1-c2c4-5914-a065-992b0dd877c8"
|
||||
---
|
||||
|
||||
|
|
@ -16,7 +16,7 @@ state_hub_workstream_id: "528d24c1-c2c4-5914-a065-992b0dd877c8"
|
|||
|
||||
User-approved baseline, learning ledger and generated evolution report. Finite
|
||||
investigations support an ongoing language; no composite maturity score or automatic
|
||||
promotion. M1/M2 and independent human validation remain open in WP-0002/WP-0003.
|
||||
promotion. M1/M2 and independent human validation remain open in WP-0009/WP-0003.
|
||||
|
||||
## Establish the versioned evidence baseline
|
||||
|
||||
|
|
@ -50,8 +50,7 @@ experiment receipts. Intent/protocol/program now describe continuous learning.
|
|||
```task
|
||||
id: FEP-WP-0006-T03
|
||||
status: wait
|
||||
depends_on: [FEP-WP-0003-T01]
|
||||
blocking_reason: "Registers a bounded cycle before FEP-WP-0003-T01's next investigation."
|
||||
blocking_reason: "Needs useful comparable prospective evidence and an explicit protocol/returns review from WP-0003 or consumer-led WP-0008; largest declared group has two cycles, one with unknown effort."
|
||||
priority: medium
|
||||
state_hub_task_id: "2c5c4bae-a0a9-54bf-ac86-49b2803fdd8c"
|
||||
```
|
||||
|
|
@ -220,3 +219,30 @@ when useful research under existing WP-0003 T01 or consumer-led WP-0008 supplies
|
|||
comparable window with honest effort accounting. Do not restart the paused navigation
|
||||
fixture or manufacture repeated studies merely to close T03. No new task/workplan;
|
||||
no diminishing-returns, independent-human or maturity claim.
|
||||
|
||||
## Resumed measurement audit — 2026-10-04
|
||||
|
||||
The user explicitly resumed this workplan from backlog. Reviewed the current ledger,
|
||||
registered cycle protocols and reporting implementation. T01/T02 and substantive
|
||||
review-date reconciliation remain complete. Eight prospective cycles occupy seven
|
||||
declared groups; B-independent-navigation-source-v1 has LEARN-0014/0017, with
|
||||
LEARN-0017's effort honestly unknown. No complete three-cycle window exists.
|
||||
|
||||
Implemented a deterministic [window audit](../research/evolution/ComparisonWindows.json)
|
||||
alongside the evolution report: ordered cycle IDs, protocol references, outcome
|
||||
counts, positive/unknown/zero effort, qualitative review triggers and complete-window
|
||||
effort readiness. A matching group label no longer appears to establish comparability.
|
||||
Three declared entries trigger protocol review; missing effort still permits scoped
|
||||
qualitative review but cannot silently produce an effort comparison. Regression cases
|
||||
cover unknown/zero time, nonconsequential outcomes, mixed streams, retrospective
|
||||
exclusion, ordering and stale generated audit rejection. The cycle template now asks
|
||||
why a follow-up is useful and requires accounting gaps to stay visible.
|
||||
|
||||
This is measurement infrastructure, not a new empirical learning cycle; no ledger
|
||||
outcome, candidate freshness, maturity or missing historical time is invented. Updated
|
||||
the report's open-work references to the live M1 owner WP-0009 and consumer-led WP-0008.
|
||||
Removed the stale exclusive dependency on parked WP-0003 T01: either owner can supply
|
||||
useful comparable evidence under its existing scope. No downstream research gate is
|
||||
waived. T03 remains wait and the resumed workplan is blocked on that evidence; the
|
||||
paused navigation fixture stays paused. Existing work records retain the next useful
|
||||
research and returns review; no replacement backlog is created.
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue