coulomb/intelligence-radar
5
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Build reproducible capability advisor and scheduled source capture

Browse source 
Assistant: codex
Assistant-Model: gpt-6-astra
Assistant-Session: 01a09c77-b1a9-7eb3-b501-6cb24b458c97
This commit is contained in:
tegwick 2026-09-13 23:07:41 +02:00
parent a1edefd7c5
commit 29ceb77938
68 changed files with 2431 additions and 85 deletions
Download patch file Download diff file Expand all files Collapse all files

323
examples/advisor-catalog.json Normal file
View file

@ -0,0 +1,323 @@
{
"synthetic": true,
"benchmarks": [
{
"id": "fixture-1",
"version": "1",
"constructs": [
"coding"
],
"health": {
"ecological_validity": 0.8,
"measurement_reliability": 0.8,
"discriminative_power": 0.8,
"freshness": 0.8,
"execution_profile_similarity": 0.8,
"reproducibility": 0.8,
"contamination_resistance": 0.8
},
"limitations": "Synthetic instrument for behavior tests only.",
"scale": {
"minimum": 0,
"maximum": 100
}
},
{
"id": "fixture-2",
"version": "1",
"constructs": [
"tool_use"
],
"health": {
"ecological_validity": 0.8,
"measurement_reliability": 0.8,
"discriminative_power": 0.8,
"freshness": 0.8,
"execution_profile_similarity": 0.8,
"reproducibility": 0.8,
"contamination_resistance": 0.8
},
"limitations": "Synthetic instrument for behavior tests only.",
"scale": {
"minimum": 0,
"maximum": 100
}
},
{
"id": "fixture-3",
"version": "1",
"constructs": [
"reasoning"
],
"health": {
"ecological_validity": 0.8,
"measurement_reliability": 0.8,
"discriminative_power": 0.8,
"freshness": 0.8,
"execution_profile_similarity": 0.8,
"reproducibility": 0.8,
"contamination_resistance": 0.8
},
"limitations": "Synthetic instrument for behavior tests only.",
"scale": {
"minimum": 0,
"maximum": 100
}
},
{
"id": "fixture-4",
"version": "1",
"constructs": [
"web_research"
],
"health": {
"ecological_validity": 0.8,
"measurement_reliability": 0.8,
"discriminative_power": 0.8,
"freshness": 0.8,
"execution_profile_similarity": 0.8,
"reproducibility": 0.8,
"contamination_resistance": 0.8
},
"limitations": "Synthetic instrument for behavior tests only.",
"scale": {
"minimum": 0,
"maximum": 100
}
},
{
"id": "fixture-5",
"version": "1",
"constructs": [
"artifact_creation"
],
"health": {
"ecological_validity": 0.8,
"measurement_reliability": 0.8,
"discriminative_power": 0.8,
"freshness": 0.8,
"execution_profile_similarity": 0.8,
"reproducibility": 0.8,
"contamination_resistance": 0.8
},
"limitations": "Synthetic instrument for behavior tests only.",
"scale": {
"minimum": 0,
"maximum": 100
}
}
],
"profiles": [
{
"id": "frontier-fixture",
"model": "frontier-fixture",
"harness": "fixture-harness-v1",
"effort": "default",
"provider": "fixture",
"deployment": "fixture-local",
"attributes": {
"requester_access": true,
"min_context": 200000
},
"availability": {
"evidence_kind": "synthetic",
"observed_at": "2026-09-13T10:00:00Z",
"available": true
}
},
{
"id": "efficient-fixture",
"model": "efficient-fixture",
"harness": "fixture-harness-v1",
"effort": "default",
"provider": "fixture",
"deployment": "fixture-local",
"attributes": {
"requester_access": true,
"min_context": 200000
},
"availability": {
"evidence_kind": "synthetic",
"observed_at": "2026-09-13T10:00:00Z",
"available": true
}
},
{
"id": "budget-fixture",
"model": "budget-fixture",
"harness": "fixture-harness-v1",
"effort": "default",
"provider": "fixture",
"deployment": "fixture-local",
"attributes": {
"requester_access": true,
"min_context": 200000
},
"availability": {
"evidence_kind": "synthetic",
"observed_at": "2026-09-13T10:00:00Z",
"available": true
}
}
],
"results": [
{
"evidence_kind": "synthetic",
"observed_at": "2026-09-13T10:00:00Z",
"profile_id": "frontier-fixture",
"benchmark_id": "fixture-1",
"benchmark_version": "1",
"raw_score": 90
},
{
"evidence_kind": "synthetic",
"observed_at": "2026-09-13T10:00:00Z",
"profile_id": "frontier-fixture",
"benchmark_id": "fixture-2",
"benchmark_version": "1",
"raw_score": 90
},
{
"evidence_kind": "synthetic",
"observed_at": "2026-09-13T10:00:00Z",
"profile_id": "frontier-fixture",
"benchmark_id": "fixture-3",
"benchmark_version": "1",
"raw_score": 90
},
{
"evidence_kind": "synthetic",
"observed_at": "2026-09-13T10:00:00Z",
"profile_id": "frontier-fixture",
"benchmark_id": "fixture-4",
"benchmark_version": "1",
"raw_score": 90
},
{
"evidence_kind": "synthetic",
"observed_at": "2026-09-13T10:00:00Z",
"profile_id": "frontier-fixture",
"benchmark_id": "fixture-5",
"benchmark_version": "1",
"raw_score": 90
},
{
"evidence_kind": "synthetic",
"observed_at": "2026-09-13T10:00:00Z",
"profile_id": "efficient-fixture",
"benchmark_id": "fixture-1",
"benchmark_version": "1",
"raw_score": 85
},
{
"evidence_kind": "synthetic",
"observed_at": "2026-09-13T10:00:00Z",
"profile_id": "efficient-fixture",
"benchmark_id": "fixture-2",
"benchmark_version": "1",
"raw_score": 85
},
{
"evidence_kind": "synthetic",
"observed_at": "2026-09-13T10:00:00Z",
"profile_id": "efficient-fixture",
"benchmark_id": "fixture-3",
"benchmark_version": "1",
"raw_score": 85
},
{
"evidence_kind": "synthetic",
"observed_at": "2026-09-13T10:00:00Z",
"profile_id": "efficient-fixture",
"benchmark_id": "fixture-4",
"benchmark_version": "1",
"raw_score": 85
},
{
"evidence_kind": "synthetic",
"observed_at": "2026-09-13T10:00:00Z",
"profile_id": "efficient-fixture",
"benchmark_id": "fixture-5",
"benchmark_version": "1",
"raw_score": 85
},
{
"evidence_kind": "synthetic",
"observed_at": "2026-09-13T10:00:00Z",
"profile_id": "budget-fixture",
"benchmark_id": "fixture-1",
"benchmark_version": "1",
"raw_score": 70
},
{
"evidence_kind": "synthetic",
"observed_at": "2026-09-13T10:00:00Z",
"profile_id": "budget-fixture",
"benchmark_id": "fixture-2",
"benchmark_version": "1",
"raw_score": 70
},
{
"evidence_kind": "synthetic",
"observed_at": "2026-09-13T10:00:00Z",
"profile_id": "budget-fixture",
"benchmark_id": "fixture-3",
"benchmark_version": "1",
"raw_score": 70
},
{
"evidence_kind": "synthetic",
"observed_at": "2026-09-13T10:00:00Z",
"profile_id": "budget-fixture",
"benchmark_id": "fixture-4",
"benchmark_version": "1",
"raw_score": 70
},
{
"evidence_kind": "synthetic",
"observed_at": "2026-09-13T10:00:00Z",
"profile_id": "budget-fixture",
"benchmark_id": "fixture-5",
"benchmark_version": "1",
"raw_score": 70
}
],
"prices": [
{
"evidence_kind": "synthetic",
"observed_at": "2026-09-13T10:00:00Z",
"profile_id": "frontier-fixture",
"currency": "USD",
"valid_from": "2026-09-13",
"valid_until": "2026-10-01",
"usd_per_million": {
"input": 10,
"output": 50
}
},
{
"evidence_kind": "synthetic",
"observed_at": "2026-09-13T10:00:00Z",
"profile_id": "efficient-fixture",
"currency": "USD",
"valid_from": "2026-09-13",
"valid_until": "2026-10-01",
"usd_per_million": {
"input": 2,
"output": 10
}
},
{
"evidence_kind": "synthetic",
"observed_at": "2026-09-13T10:00:00Z",
"profile_id": "budget-fixture",
"currency": "USD",
"valid_from": "2026-09-13",
"valid_until": "2026-10-01",
"usd_per_million": {
"input": 0.2,
"output": 1.0
}
}
]
}

24
examples/source-backed-task.json Normal file
View file

@ -0,0 +1,24 @@
{
"objective": "Compare candidate API deployments for repository issue resolution.",
"acceptance": "Repository acceptance tests pass and an independent reviewer accepts the change.",
"environment": "Proposed Claude Platform API integration; requester access and tool harness not yet evidenced.",
"risk": "routine",
"capabilities": {
"coding": 1,
"tool_use": 1,
"reasoning": 1,
"web_research": 1,
"artifact_creation": 1
},
"constraints": {
"requester_access": true,
"min_context": 32000
},
"workload": {
"input": 10000,
"output": 2000,
"tool_usd": 0.01,
"attempts": 1,
"verification_usd": 0.02
}
}

24
examples/task.json Normal file
View file

@ -0,0 +1,24 @@
{
"objective": "Resolve a representative repository issue with a sourced change explanation.",
"acceptance": "Repository acceptance tests pass and an independent reviewer accepts the change.",
"environment": "Fixture sandbox with shell and browser",
"risk": "routine",
"capabilities": {
"coding": 1,
"tool_use": 1,
"reasoning": 1,
"web_research": 1,
"artifact_creation": 1
},
"constraints": {
"requester_access": true,
"min_context": 32000
},
"workload": {
"input": 10000,
"output": 2000,
"tool_usd": 0.01,
"attempts": 1,
"verification_usd": 0.02
}
}