Audit prospective comparison readiness and retain effort gaps
Assistant: codex Assistant-Model: gpt-6.1-sol Assistant-Session: 01a1045b-933d-7aa1-b2db-104557487a79
This commit is contained in:
parent
280c1d38bf
commit
d8d3a22b54
9 changed files with 439 additions and 54 deletions
|
|
@ -105,6 +105,49 @@ def cell(value):
|
|||
return str(value).replace('|', '\\|').replace('\n', ' ')
|
||||
|
||||
|
||||
def comparison_windows(data):
|
||||
"""Audit declared prospective groups; labels do not prove comparability."""
|
||||
groups = defaultdict(list)
|
||||
for event in data['ledger']['events']:
|
||||
if event['recording'] == 'prospective':
|
||||
groups[(event['stream'], event['comparison_group'])].append(event)
|
||||
windows = []
|
||||
for (stream, group), entries in sorted(groups.items()):
|
||||
entries = sorted(entries, key=lambda e: (e['date'], e['id']))
|
||||
unknown = [e['id'] for e in entries if e['effort']['research_minutes'] is None]
|
||||
zero = [e['id'] for e in entries if e['effort']['research_minutes'] == 0]
|
||||
measured = [e['id'] for e in entries
|
||||
if e['effort']['research_minutes'] is not None and e['effort']['research_minutes'] > 0]
|
||||
due = len(entries) >= 3
|
||||
windows.append({
|
||||
'stream': stream,
|
||||
'comparison_group': group,
|
||||
'cycle_ids': [e['id'] for e in entries],
|
||||
'protocol_refs': list(dict.fromkeys(e['protocol_ref'] for e in entries)),
|
||||
'cycles_until_review_trigger': max(0, 3 - len(entries)),
|
||||
'qualitative_review_due': due,
|
||||
'positive_effort_cycle_ids': measured,
|
||||
'unknown_effort_cycle_ids': unknown,
|
||||
'zero_effort_cycle_ids': zero,
|
||||
'effort_complete': not unknown and not zero,
|
||||
'effort_comparison_ready': due and not unknown and not zero,
|
||||
'outcomes': {kind: sum(e['disposition'] == kind for e in entries)
|
||||
for kind in sorted(DISPOSITIONS)},
|
||||
})
|
||||
return {
|
||||
'schema_version': 1,
|
||||
'basis': 'Declared prospective groups; manually review shared stream, evidence tier, question granularity, evaluation and effort accounting before comparing returns.',
|
||||
'review_trigger_cycles': 3,
|
||||
'limits': [
|
||||
'Group labels and cycle counts do not establish protocol comparability or a learning trend.',
|
||||
'Unknown or zero research time prevents effort comparison of the complete declared window; never estimate missing time or silently omit its cycle.',
|
||||
'A qualitative review may retain cycles with unknown effort; corroboration and inconclusive results remain visible.',
|
||||
'Readiness is a review trigger, not statistical evidence, a plateau declaration or a maturity judgment.',
|
||||
],
|
||||
'windows': windows,
|
||||
}
|
||||
|
||||
|
||||
def render(data, baseline=None):
|
||||
profiles = data['profiles']['profiles']
|
||||
events = data['ledger']['events']
|
||||
|
|
@ -147,23 +190,32 @@ def render(data, baseline=None):
|
|||
lines += [f'| {k} | {counts[k]} |' for k in sorted(DISPOSITIONS)]
|
||||
lines += ['', f'Cycles with recorded research effort: {sum(e["effort"]["research_minutes"] is not None for e in events)}/{len(events)}. '
|
||||
'Unknown effort is neither zero nor estimated from commits, tokens or assertion counts.', '']
|
||||
groups = defaultdict(list)
|
||||
for e in events:
|
||||
if e['recording'] == 'prospective':
|
||||
groups[(e['stream'], e['comparison_group'])].append(e)
|
||||
if not groups:
|
||||
audit = comparison_windows(data)
|
||||
if not audit['windows']:
|
||||
lines += ['No prospective comparison windows yet. Learning rate and diminishing returns: **not measurable from this baseline**.', '']
|
||||
for (stream, group), entries in sorted(groups.items()):
|
||||
entries = sorted(entries, key=lambda e: (e['date'], e['id']))
|
||||
lines += [f'### Stream {stream} / {cell(group)}', '',
|
||||
'| Cycle | Outcome | Research minutes | Consequential outcome/hour |', '| --- | --- | ---: | ---: |']
|
||||
else:
|
||||
lines += ['### Comparison readiness', '',
|
||||
'Declared groups require a review of shared evidence tier, question granularity, evaluation and effort accounting. A matching label is not proof of comparability. '
|
||||
+ link('research/evolution/ComparisonWindows.json', 'Machine-readable window audit') + '.', '',
|
||||
'| Stream / declared group | Cycles | Positive effort | Unknown / zero effort | Qualitative review trigger | Complete-window effort comparison |',
|
||||
'| --- | ---: | ---: | ---: | --- | --- |']
|
||||
for window in audit['windows']:
|
||||
lines.append(f'| {window["stream"]} / {cell(window["comparison_group"])} | {len(window["cycle_ids"])} | '
|
||||
f'{len(window["positive_effort_cycle_ids"])} | {len(window["unknown_effort_cycle_ids"])} / {len(window["zero_effort_cycle_ids"])} | '
|
||||
f'{"due" if window["qualitative_review_due"] else "pending"} | {"ready for protocol review" if window["effort_comparison_ready"] else "not ready"} |')
|
||||
lines += ['', 'Unknown or zero effort is not dropped to create a favorable window. Qualitative review can retain those cycles, but effort comparisons need an explicitly reviewed window with measured positive research time. No trend or plateau is inferred.', '']
|
||||
by_id = {e['id']: e for e in events}
|
||||
for window in audit['windows']:
|
||||
entries = [by_id[ident] for ident in window['cycle_ids']]
|
||||
lines += [f'### Stream {window["stream"]} / {cell(window["comparison_group"])}', '',
|
||||
'| Cycle | Protocol | Outcome | Research minutes | Consequential outcome/hour |', '| --- | --- | --- | ---: | ---: |']
|
||||
for e in entries:
|
||||
minutes = e['effort']['research_minutes']
|
||||
consequential = e['disposition'] in {'revision', 'rejection', 'merge'}
|
||||
rate = f'{60/minutes:.2f}' if consequential and minutes else ('0.00' if minutes else 'unknown')
|
||||
lines.append(f'| {e["id"]} | {e["disposition"]} | {minutes if minutes is not None else "unknown"} | {rate} |')
|
||||
lines.append(f'| {e["id"]} | {link(e["protocol_ref"], "protocol")} | {e["disposition"]} | {minutes if minutes is not None else "unknown"} | {rate} |')
|
||||
lines += ['', 'One bounded question counts at most one consequential outcome. Corroboration and inconclusive work remain visible separately. '
|
||||
+ ('Fewer than three comparable cycles: no trend assessment.' if len(entries) < 3 else 'At least three cycles: eligible for a qualitative review of returns; this does not demonstrate a plateau.'), '']
|
||||
+ ('Fewer than three declared cycles: qualitative review trigger pending; no trend assessment.' if not window['qualitative_review_due'] else 'At least three declared cycles: qualitative review trigger reached, subject to protocol comparability review; this does not demonstrate a plateau.'), '']
|
||||
lines += ['## Coherence and maintainability', '',
|
||||
'Profile hashes and local references passed validation when this report was generated. That checks traceability, not source truth or external URL freshness.', '',
|
||||
f'Profiles with unknown last substantive review date: {sum(p["review"]["last_substantive_review_date"] is None for p in profiles)}. '
|
||||
|
|
@ -228,12 +280,17 @@ def main():
|
|||
validate_snapshot(baseline)
|
||||
output = render(data, baseline)
|
||||
path = ROOT / DATA / 'EvolutionReport.md'
|
||||
audit_path = ROOT / DATA / 'ComparisonWindows.json'
|
||||
audit_output = json.dumps(comparison_windows(data), indent=2, ensure_ascii=False) + '\n'
|
||||
if args.command == 'check':
|
||||
require(path.read_text() == output, 'Stale report: run make evolution-report')
|
||||
require(audit_path.read_text() == audit_output, 'Stale comparison windows: run make evolution-report')
|
||||
print('Evolution records and generated report are consistent.')
|
||||
else:
|
||||
path.write_text(output)
|
||||
audit_path.write_text(audit_output)
|
||||
print(path.relative_to(ROOT))
|
||||
print(audit_path.relative_to(ROOT))
|
||||
except (ValueError, KeyError, OSError) as exc:
|
||||
parser.exit(1, f'Evolution validation failed: {exc}\n')
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue