295 lines
16 KiB
Python
295 lines
16 KiB
Python
|
"""Deterministic task-fit ranking with explicit gaps and empirical outcome bounds."""
|
||
|
|
from datetime import date, datetime
|
||
|
|
import hashlib
|
||
|
|
import json
|
||
|
|
import math
|
||
|
|
from pathlib import Path
|
||
|
|
|
||
|
|
from .store import all_records, canonical, get, put
|
||
|
|
|
||
|
|
|
||
|
|
def number(value, name, low=0, high=None):
|
||
|
|
if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value):
|
||
|
|
raise ValueError(f'{name} must be a finite number')
|
||
|
|
if value < low or (high is not None and value > high):
|
||
|
|
raise ValueError(f'{name} outside allowed range')
|
||
|
|
return value
|
||
|
|
|
||
|
|
|
||
|
|
def unique(items, label):
|
||
|
|
ids = [x['id'] for x in items]
|
||
|
|
if len(ids) != len(set(ids)):
|
||
|
|
raise ValueError(f'duplicate {label} IDs')
|
||
|
|
|
||
|
|
|
||
|
|
def task_key(task):
|
||
|
|
# Includes acceptance, constraints, environment, workload and capability requirements.
|
||
|
|
return hashlib.sha256(canonical(task)).hexdigest()
|
||
|
|
|
||
|
|
|
||
|
|
def wilson(successes, total, z):
|
||
|
|
p = successes / total
|
||
|
|
center = p + z*z / (2*total)
|
||
|
|
delta = z * math.sqrt(p*(1-p)/total + z*z/(4*total*total))
|
||
|
|
denominator = 1 + z*z/total
|
||
|
|
return [max(0, (center-delta)/denominator), min(1, (center+delta)/denominator)]
|
||
|
|
|
||
|
|
|
||
|
|
def select(task, benchmarks, policy):
|
||
|
|
capabilities = task['capabilities']
|
||
|
|
if not capabilities or sum(capabilities.values()) <= 0:
|
||
|
|
raise ValueError('task requires positive capability weights')
|
||
|
|
for k, v in capabilities.items():
|
||
|
|
number(v, k, high=1)
|
||
|
|
weights = policy['selection_weights']
|
||
|
|
if not math.isclose(sum(weights.values()), 1):
|
||
|
|
raise ValueError('selection policy weights must sum to one')
|
||
|
|
candidates = []
|
||
|
|
for b in benchmarks:
|
||
|
|
if b.get('aggregate', False):
|
||
|
|
continue
|
||
|
|
if not b.get('version') or not b.get('limitations'):
|
||
|
|
raise ValueError('benchmarks require version and limitations')
|
||
|
|
relevance = sum(v for k, v in capabilities.items() if k in b['constructs']) / sum(capabilities.values())
|
||
|
|
health = b['health']
|
||
|
|
base = weights['task_relevance'] * relevance
|
||
|
|
for key, weight in weights.items():
|
||
|
|
if key != 'task_relevance':
|
||
|
|
base += weight * number(health[key], key, high=1)
|
||
|
|
base -= number(b.get('known_task_bias', 0), 'known_task_bias', high=1)
|
||
|
|
candidates.append(dict(b, relevance=relevance, selection_score=base))
|
||
|
|
chosen = []
|
||
|
|
while candidates and len(chosen) < policy['selection_count']:
|
||
|
|
def marginal(b):
|
||
|
|
overlap = max(len(set(b['constructs']) & set(s['constructs'])) /
|
||
|
|
max(1, len(set(b['constructs']) | set(s['constructs']))) for s in chosen) if chosen else 0
|
||
|
|
return b['selection_score'] - policy['redundancy_penalty'] * overlap
|
||
|
|
candidates.sort(key=lambda b: (-marginal(b), b['id']))
|
||
|
|
picked = candidates.pop(0)
|
||
|
|
picked['marginal_score'] = marginal(picked)
|
||
|
|
chosen.append(picked)
|
||
|
|
total = sum(b['relevance'] for b in chosen)
|
||
|
|
for b in chosen:
|
||
|
|
b['weight'] = b['relevance'] / total if total else 0
|
||
|
|
return chosen
|
||
|
|
|
||
|
|
|
||
|
|
def evidence_ok(root, item, cutoff, max_age, synthetic):
|
||
|
|
observed = datetime.fromisoformat(item['observed_at'].replace('Z', '+00:00'))
|
||
|
|
if observed.tzinfo is None:
|
||
|
|
raise ValueError('evidence timestamps require timezone')
|
||
|
|
age = (cutoff - observed.date()).days
|
||
|
|
if age < 0 or age > max_age:
|
||
|
|
return False
|
||
|
|
if synthetic:
|
||
|
|
return item.get('evidence_kind') == 'synthetic'
|
||
|
|
if not item.get('capture_id'):
|
||
|
|
return False
|
||
|
|
capture = get(root, 'captures', item['capture_id'])
|
||
|
|
if not datetime.fromisoformat(capture['retrieved_at']).date() <= observed.date() <= cutoff:
|
||
|
|
return False
|
||
|
|
artifact = Path(root) / 'data/artifacts' / capture['content_sha256']
|
||
|
|
if hashlib.sha256(artifact.read_bytes()).hexdigest() != capture['content_sha256']:
|
||
|
|
raise ValueError('capture artifact integrity mismatch')
|
||
|
|
return item.get('review_status') == 'reviewed' and bool(item.get('reviewer'))
|
||
|
|
|
||
|
|
|
||
|
|
def estimate_cost(task, price):
|
||
|
|
if price is None:
|
||
|
|
return None
|
||
|
|
workload = task['workload']
|
||
|
|
rates = price['usd_per_million']
|
||
|
|
cost = 0
|
||
|
|
for key in ('input', 'cached_input', 'cache_write', 'output'):
|
||
|
|
tokens = number(workload.get(key, 0), key)
|
||
|
|
if tokens and key not in rates:
|
||
|
|
return None
|
||
|
|
cost += tokens * number(rates.get(key, 0), key + ' rate') / 1_000_000
|
||
|
|
# Output includes billable reasoning; uncached/cache buckets are disjoint.
|
||
|
|
cost += number(workload['tool_usd'], 'tool_usd')
|
||
|
|
cost *= number(workload['attempts'], 'attempts', low=1)
|
||
|
|
cost += number(workload['verification_usd'], 'verification_usd')
|
||
|
|
return {'scenario_usd': cost, 'basis': 'declared workload; not a probability-derived estimate',
|
||
|
|
'price_observed_at': price['observed_at']}
|
||
|
|
|
||
|
|
|
||
|
|
def advise(root, task, catalog, policy, as_of, synthetic=False, saved_outcomes=None):
|
||
|
|
task, catalog, policy = (json.loads(canonical(x)) for x in (task, catalog, policy))
|
||
|
|
cutoff = date.fromisoformat(as_of)
|
||
|
|
number(policy['topnotch_ratio'], 'topnotch ratio', high=1)
|
||
|
|
number(policy['redundancy_penalty'], 'redundancy penalty', high=1)
|
||
|
|
number(policy['minimum_outcomes'], 'minimum outcomes', low=1)
|
||
|
|
number(policy['max_evidence_age_days'], 'maximum evidence age')
|
||
|
|
number(policy['wilson_z'], 'Wilson z', low=0.01)
|
||
|
|
for v in policy['thresholds'].values():
|
||
|
|
number(v, 'sufficiency threshold', high=1)
|
||
|
|
if date.fromisoformat(policy['effective_from']) > cutoff:
|
||
|
|
raise ValueError('policy not effective at evidence cutoff')
|
||
|
|
if synthetic != bool(catalog.get('synthetic', False)):
|
||
|
|
raise ValueError('synthetic catalogs require --synthetic and cannot mix with live catalogs')
|
||
|
|
if policy['selection_count'] != 5:
|
||
|
|
raise ValueError('advisor requires five measurement candidates')
|
||
|
|
for key in ('objective', 'acceptance', 'environment'):
|
||
|
|
if not isinstance(task[key], str) or not task[key].strip():
|
||
|
|
raise ValueError(f'task {key} required')
|
||
|
|
if task['risk'] not in policy['thresholds']:
|
||
|
|
raise ValueError('unknown task risk')
|
||
|
|
for collection in ('benchmarks', 'profiles'):
|
||
|
|
unique(catalog[collection], collection)
|
||
|
|
selected = select(task, catalog['benchmarks'], policy)
|
||
|
|
complete_selection = len(selected) == 5 and all(b['relevance'] > 0 and b['version'] != 'pending-verification' for b in selected)
|
||
|
|
gaps = [] if complete_selection else ['Fewer than five relevant distinct measurements; ranking withheld.']
|
||
|
|
max_age = policy['max_evidence_age_days']
|
||
|
|
rows = []
|
||
|
|
outcomes = [r for _, r in (all_records(root, 'outcomes') if saved_outcomes is None else saved_outcomes)]
|
||
|
|
for profile in catalog['profiles']:
|
||
|
|
for field in ('model', 'harness', 'effort', 'provider', 'deployment'):
|
||
|
|
if not profile.get(field):
|
||
|
|
raise ValueError(f'profile {field} required')
|
||
|
|
reasons = []
|
||
|
|
available = profile.get('availability')
|
||
|
|
if not available or not evidence_ok(root, available, cutoff, max_age, synthetic) or available['available'] is not True:
|
||
|
|
reasons.append('availability unknown, stale, or unavailable')
|
||
|
|
for key, required in task['constraints'].items():
|
||
|
|
actual = profile.get('attributes', {}).get(key)
|
||
|
|
if key.startswith('min_'):
|
||
|
|
if actual is None or number(actual, key) < number(required, key):
|
||
|
|
reasons.append(f'constraint {key} not satisfied')
|
||
|
|
elif actual != required:
|
||
|
|
reasons.append(f'constraint {key} not satisfied')
|
||
|
|
scores = []
|
||
|
|
for b in selected:
|
||
|
|
matches = [r for r in catalog['results'] if r['profile_id'] == profile['id'] and
|
||
|
|
r['benchmark_id'] == b['id'] and r['benchmark_version'] == b['version'] and
|
||
|
|
evidence_ok(root, r, cutoff, max_age, synthetic)]
|
||
|
|
if len(matches) != 1:
|
||
|
|
scores.append(None)
|
||
|
|
continue
|
||
|
|
scale = b['scale']
|
||
|
|
lower = number(scale['minimum'], 'scale minimum')
|
||
|
|
upper = number(scale['maximum'], 'scale maximum')
|
||
|
|
if upper <= lower:
|
||
|
|
raise ValueError('benchmark scale maximum must exceed minimum')
|
||
|
|
raw = number(matches[0]['raw_score'], 'raw score', low=lower, high=upper)
|
||
|
|
scores.append((raw-lower)/(upper-lower))
|
||
|
|
fit = sum(b['weight'] * score for b, score in zip(selected, scores)) if complete_selection and all(s is not None for s in scores) else None
|
||
|
|
prices = [p for p in catalog['prices'] if p['profile_id'] == profile['id'] and
|
||
|
|
p['currency'] == 'USD' and p['valid_from'] <= as_of <= p['valid_until'] and
|
||
|
|
evidence_ok(root, p, cutoff, max_age, synthetic)]
|
||
|
|
price = prices[0] if len(prices) == 1 else None
|
||
|
|
cost = estimate_cost(task, price)
|
||
|
|
eligible_outcomes = [o for o in outcomes if o['task_key'] == task_key(task) and
|
||
|
|
o['profile_hash'] == hashlib.sha256(canonical(profile)).hexdigest() and
|
||
|
|
o['completed_at'][:10] <= as_of and o['synthetic'] == synthetic]
|
||
|
|
n = len(eligible_outcomes)
|
||
|
|
interval = wilson(sum(o['accepted'] for o in eligible_outcomes), n, policy['wilson_z']) if n else None
|
||
|
|
sufficient = n >= policy['minimum_outcomes'] and interval[0] >= policy['thresholds'][task['risk']]
|
||
|
|
if task['risk'] == 'high_assurance':
|
||
|
|
sufficient = sufficient and all(o['independent_verification'] for o in eligible_outcomes)
|
||
|
|
rows.append({'id': profile['id'], 'eligible': not reasons, 'excluded_reasons': reasons,
|
||
|
|
'fitness_proxy': fit, 'missing_measurements': [b['id'] for b, s in zip(selected, scores) if s is None],
|
||
|
|
'cost': cost, 'price_status': 'usable' if price else 'missing_stale_or_conflicting',
|
||
|
|
'calibration': {'trials': n, 'success_interval_95': interval},
|
||
|
|
'sufficient': bool(sufficient and not reasons)})
|
||
|
|
ranked = sorted([r for r in rows if r['eligible'] and r['fitness_proxy'] is not None],
|
||
|
|
key=lambda r: (-r['fitness_proxy'], r['id']))
|
||
|
|
frontier = ranked[0]['id'] if ranked else None
|
||
|
|
topnotch = [r for r in ranked if r['fitness_proxy'] >= ranked[0]['fitness_proxy'] * policy['topnotch_ratio']]
|
||
|
|
sufficient = [r for r in rows if r['sufficient']]
|
||
|
|
def cheapest(items):
|
||
|
|
# Missing prices prevent a defensible claim of cheapest in the eligible set.
|
||
|
|
if not items or any(r['cost'] is None for r in items):
|
||
|
|
return None
|
||
|
|
return min(items, key=lambda r: (r['cost']['scenario_usd'], r['id']))['id']
|
||
|
|
if not ranked:
|
||
|
|
gaps.append('No eligible profile has complete benchmark evidence.')
|
||
|
|
if not sufficient:
|
||
|
|
gaps.append('no_evidenced_sufficient_model')
|
||
|
|
return {'schema_version': 1, 'algorithm_version': 'advisor-0.2', 'as_of': as_of, 'synthetic': synthetic,
|
||
|
|
'task_key': task_key(task), 'policy_id': policy['id'], 'selected_benchmarks': selected,
|
||
|
|
'profiles': rows, 'frontier': frontier, 'topnotch': [r['id'] for r in topnotch],
|
||
|
|
'efficient': cheapest(topnotch), 'sufficient': [r['id'] for r in sufficient],
|
||
|
|
'budget': cheapest(sufficient), 'evidence_gaps': gaps,
|
||
|
|
'ranking_scope': 'eligible profiles with complete selected evidence; unknown competitors are not proven inferior',
|
||
|
|
'confidence': 'curator-assigned benchmark health; empirical bounds only for exactly matching task and profile; fitness is a proxy'}
|
||
|
|
|
||
|
|
|
||
|
|
def recommend(root, task, catalog, policy, as_of, synthetic=False):
|
||
|
|
saved = all_records(root, 'outcomes')
|
||
|
|
report = advise(root, task, catalog, policy, as_of, synthetic, saved)
|
||
|
|
snapshot = {'task': task, 'catalog': catalog, 'policy': policy,
|
||
|
|
'outcomes': saved, 'report': report}
|
||
|
|
return put(root, 'recommendations', snapshot), report
|
||
|
|
|
||
|
|
|
||
|
|
def _outcome(root, value):
|
||
|
|
recommendation = get(root, 'recommendations', value['recommendation_id'])
|
||
|
|
task = recommendation['task']
|
||
|
|
profiles = [p for p in recommendation['catalog']['profiles'] if p['id'] == value['profile_id']]
|
||
|
|
if len(profiles) != 1:
|
||
|
|
raise ValueError('outcome profile absent from recommendation')
|
||
|
|
for key in ('accepted', 'independent_verification'):
|
||
|
|
if type(value[key]) is not bool:
|
||
|
|
raise ValueError(f'{key} must be boolean')
|
||
|
|
for key in ('actual_usd', 'runtime_seconds'):
|
||
|
|
number(value[key], key)
|
||
|
|
completed = datetime.fromisoformat(value['completed_at'].replace('Z', '+00:00'))
|
||
|
|
if not completed.tzinfo or completed.date() < date.fromisoformat(recommendation['report']['as_of']):
|
||
|
|
raise ValueError('invalid outcome timestamp')
|
||
|
|
if not value.get('run_id') or not value.get('verifier_evidence'):
|
||
|
|
raise ValueError('run_id and verifier_evidence required')
|
||
|
|
result = dict(value, task_key=task_key(task), synthetic=recommendation['report']['synthetic'],
|
||
|
|
profile_hash=hashlib.sha256(canonical(profiles[0])).hexdigest())
|
||
|
|
for ident, existing in all_records(root, 'outcomes'):
|
||
|
|
if existing['run_id'] == value['run_id']:
|
||
|
|
if existing != result:
|
||
|
|
raise ValueError('run_id already recorded with different outcome')
|
||
|
|
return ident
|
||
|
|
return put(root, 'outcomes', result)
|
||
|
|
|
||
|
|
|
||
|
|
def replay(root, ident):
|
||
|
|
snapshot = get(root, 'recommendations', ident)
|
||
|
|
report = snapshot['report']
|
||
|
|
if report.get('algorithm_version') != 'advisor-0.2':
|
||
|
|
raise ValueError('unsupported historical advisor version')
|
||
|
|
rebuilt = advise(root, snapshot['task'], snapshot['catalog'], snapshot['policy'],
|
||
|
|
report['as_of'], report['synthetic'], snapshot['outcomes'])
|
||
|
|
if rebuilt != report:
|
||
|
|
raise ValueError('recommendation replay mismatch')
|
||
|
|
return rebuilt
|
||
|
|
|
||
|
|
|
||
|
|
def outcome(root, value):
|
||
|
|
import fcntl
|
||
|
|
lockpath = Path(root) / '.runtime/outcomes.lock'
|
||
|
|
lockpath.parent.mkdir(parents=True, exist_ok=True)
|
||
|
|
with lockpath.open('a') as lock:
|
||
|
|
fcntl.flock(lock, fcntl.LOCK_EX)
|
||
|
|
return _outcome(root, value)
|
||
|
|
|
||
|
|
|
||
|
|
def render_report(root, ident):
|
||
|
|
report = get(root, 'recommendations', ident)['report']
|
||
|
|
lines = [f"# Capability comparison — {report['as_of']}", '',
|
||
|
|
'Synthetic demonstration.' if report['synthetic'] else 'Source-backed comparison; evidence gaps remain explicit.', '',
|
||
|
|
f"Recommendation: `{ident}`", f"Policy: `{report['policy_id']}`", '',
|
||
|
|
'| Profile | Eligible | Fitness proxy | Scenario USD | Outcome trials |',
|
||
|
|
'| --- | --- | --- | --- | --- |']
|
||
|
|
for row in report['profiles']:
|
||
|
|
cost = f"{row['cost']['scenario_usd']:.4f}" if row['cost'] else 'unknown'
|
||
|
|
fit = f"{row['fitness_proxy']:.4f}" if row['fitness_proxy'] is not None else 'unknown'
|
||
|
|
lines.append(f"| {row['id']} | {row['eligible']} | {fit} | {cost} | {row['calibration']['trials']} |")
|
||
|
|
lines += ['', f"Frontier: {report['frontier'] or 'undetermined'}",
|
||
|
|
f"Efficient: {report['efficient'] or 'undetermined'}",
|
||
|
|
f"Budget: {report['budget'] or 'undetermined'}", '', '## Evidence gaps', '']
|
||
|
|
lines.extend('- '+gap for gap in report['evidence_gaps'])
|
||
|
|
for row in report['profiles']:
|
||
|
|
if row['excluded_reasons']:
|
||
|
|
lines.append(f"- {row['id']}: " + '; '.join(row['excluded_reasons']))
|
||
|
|
if row['missing_measurements']:
|
||
|
|
lines.append(f"- {row['id']}: missing " + ', '.join(row['missing_measurements']))
|
||
|
|
lines += ['', 'Cost is a declared token/tool/retry/verification scenario, not a measured success-cost distribution.',
|
||
|
|
'Source review does not prove requester access or task success. See saved inputs for capture references.', '']
|
||
|
|
return '\n'.join(lines)
|