tegwick
29ceb77938
Assistant: codex Assistant-Model: gpt-6-astra Assistant-Session: 01a09c77-b1a9-7eb3-b501-6cb24b458c97
323 lines
8.2 KiB
JSON
323 lines
8.2 KiB
JSON
{
|
|
"synthetic": true,
|
|
"benchmarks": [
|
|
{
|
|
"id": "fixture-1",
|
|
"version": "1",
|
|
"constructs": [
|
|
"coding"
|
|
],
|
|
"health": {
|
|
"ecological_validity": 0.8,
|
|
"measurement_reliability": 0.8,
|
|
"discriminative_power": 0.8,
|
|
"freshness": 0.8,
|
|
"execution_profile_similarity": 0.8,
|
|
"reproducibility": 0.8,
|
|
"contamination_resistance": 0.8
|
|
},
|
|
"limitations": "Synthetic instrument for behavior tests only.",
|
|
"scale": {
|
|
"minimum": 0,
|
|
"maximum": 100
|
|
}
|
|
},
|
|
{
|
|
"id": "fixture-2",
|
|
"version": "1",
|
|
"constructs": [
|
|
"tool_use"
|
|
],
|
|
"health": {
|
|
"ecological_validity": 0.8,
|
|
"measurement_reliability": 0.8,
|
|
"discriminative_power": 0.8,
|
|
"freshness": 0.8,
|
|
"execution_profile_similarity": 0.8,
|
|
"reproducibility": 0.8,
|
|
"contamination_resistance": 0.8
|
|
},
|
|
"limitations": "Synthetic instrument for behavior tests only.",
|
|
"scale": {
|
|
"minimum": 0,
|
|
"maximum": 100
|
|
}
|
|
},
|
|
{
|
|
"id": "fixture-3",
|
|
"version": "1",
|
|
"constructs": [
|
|
"reasoning"
|
|
],
|
|
"health": {
|
|
"ecological_validity": 0.8,
|
|
"measurement_reliability": 0.8,
|
|
"discriminative_power": 0.8,
|
|
"freshness": 0.8,
|
|
"execution_profile_similarity": 0.8,
|
|
"reproducibility": 0.8,
|
|
"contamination_resistance": 0.8
|
|
},
|
|
"limitations": "Synthetic instrument for behavior tests only.",
|
|
"scale": {
|
|
"minimum": 0,
|
|
"maximum": 100
|
|
}
|
|
},
|
|
{
|
|
"id": "fixture-4",
|
|
"version": "1",
|
|
"constructs": [
|
|
"web_research"
|
|
],
|
|
"health": {
|
|
"ecological_validity": 0.8,
|
|
"measurement_reliability": 0.8,
|
|
"discriminative_power": 0.8,
|
|
"freshness": 0.8,
|
|
"execution_profile_similarity": 0.8,
|
|
"reproducibility": 0.8,
|
|
"contamination_resistance": 0.8
|
|
},
|
|
"limitations": "Synthetic instrument for behavior tests only.",
|
|
"scale": {
|
|
"minimum": 0,
|
|
"maximum": 100
|
|
}
|
|
},
|
|
{
|
|
"id": "fixture-5",
|
|
"version": "1",
|
|
"constructs": [
|
|
"artifact_creation"
|
|
],
|
|
"health": {
|
|
"ecological_validity": 0.8,
|
|
"measurement_reliability": 0.8,
|
|
"discriminative_power": 0.8,
|
|
"freshness": 0.8,
|
|
"execution_profile_similarity": 0.8,
|
|
"reproducibility": 0.8,
|
|
"contamination_resistance": 0.8
|
|
},
|
|
"limitations": "Synthetic instrument for behavior tests only.",
|
|
"scale": {
|
|
"minimum": 0,
|
|
"maximum": 100
|
|
}
|
|
}
|
|
],
|
|
"profiles": [
|
|
{
|
|
"id": "frontier-fixture",
|
|
"model": "frontier-fixture",
|
|
"harness": "fixture-harness-v1",
|
|
"effort": "default",
|
|
"provider": "fixture",
|
|
"deployment": "fixture-local",
|
|
"attributes": {
|
|
"requester_access": true,
|
|
"min_context": 200000
|
|
},
|
|
"availability": {
|
|
"evidence_kind": "synthetic",
|
|
"observed_at": "2026-09-13T10:00:00Z",
|
|
"available": true
|
|
}
|
|
},
|
|
{
|
|
"id": "efficient-fixture",
|
|
"model": "efficient-fixture",
|
|
"harness": "fixture-harness-v1",
|
|
"effort": "default",
|
|
"provider": "fixture",
|
|
"deployment": "fixture-local",
|
|
"attributes": {
|
|
"requester_access": true,
|
|
"min_context": 200000
|
|
},
|
|
"availability": {
|
|
"evidence_kind": "synthetic",
|
|
"observed_at": "2026-09-13T10:00:00Z",
|
|
"available": true
|
|
}
|
|
},
|
|
{
|
|
"id": "budget-fixture",
|
|
"model": "budget-fixture",
|
|
"harness": "fixture-harness-v1",
|
|
"effort": "default",
|
|
"provider": "fixture",
|
|
"deployment": "fixture-local",
|
|
"attributes": {
|
|
"requester_access": true,
|
|
"min_context": 200000
|
|
},
|
|
"availability": {
|
|
"evidence_kind": "synthetic",
|
|
"observed_at": "2026-09-13T10:00:00Z",
|
|
"available": true
|
|
}
|
|
}
|
|
],
|
|
"results": [
|
|
{
|
|
"evidence_kind": "synthetic",
|
|
"observed_at": "2026-09-13T10:00:00Z",
|
|
"profile_id": "frontier-fixture",
|
|
"benchmark_id": "fixture-1",
|
|
"benchmark_version": "1",
|
|
"raw_score": 90
|
|
},
|
|
{
|
|
"evidence_kind": "synthetic",
|
|
"observed_at": "2026-09-13T10:00:00Z",
|
|
"profile_id": "frontier-fixture",
|
|
"benchmark_id": "fixture-2",
|
|
"benchmark_version": "1",
|
|
"raw_score": 90
|
|
},
|
|
{
|
|
"evidence_kind": "synthetic",
|
|
"observed_at": "2026-09-13T10:00:00Z",
|
|
"profile_id": "frontier-fixture",
|
|
"benchmark_id": "fixture-3",
|
|
"benchmark_version": "1",
|
|
"raw_score": 90
|
|
},
|
|
{
|
|
"evidence_kind": "synthetic",
|
|
"observed_at": "2026-09-13T10:00:00Z",
|
|
"profile_id": "frontier-fixture",
|
|
"benchmark_id": "fixture-4",
|
|
"benchmark_version": "1",
|
|
"raw_score": 90
|
|
},
|
|
{
|
|
"evidence_kind": "synthetic",
|
|
"observed_at": "2026-09-13T10:00:00Z",
|
|
"profile_id": "frontier-fixture",
|
|
"benchmark_id": "fixture-5",
|
|
"benchmark_version": "1",
|
|
"raw_score": 90
|
|
},
|
|
{
|
|
"evidence_kind": "synthetic",
|
|
"observed_at": "2026-09-13T10:00:00Z",
|
|
"profile_id": "efficient-fixture",
|
|
"benchmark_id": "fixture-1",
|
|
"benchmark_version": "1",
|
|
"raw_score": 85
|
|
},
|
|
{
|
|
"evidence_kind": "synthetic",
|
|
"observed_at": "2026-09-13T10:00:00Z",
|
|
"profile_id": "efficient-fixture",
|
|
"benchmark_id": "fixture-2",
|
|
"benchmark_version": "1",
|
|
"raw_score": 85
|
|
},
|
|
{
|
|
"evidence_kind": "synthetic",
|
|
"observed_at": "2026-09-13T10:00:00Z",
|
|
"profile_id": "efficient-fixture",
|
|
"benchmark_id": "fixture-3",
|
|
"benchmark_version": "1",
|
|
"raw_score": 85
|
|
},
|
|
{
|
|
"evidence_kind": "synthetic",
|
|
"observed_at": "2026-09-13T10:00:00Z",
|
|
"profile_id": "efficient-fixture",
|
|
"benchmark_id": "fixture-4",
|
|
"benchmark_version": "1",
|
|
"raw_score": 85
|
|
},
|
|
{
|
|
"evidence_kind": "synthetic",
|
|
"observed_at": "2026-09-13T10:00:00Z",
|
|
"profile_id": "efficient-fixture",
|
|
"benchmark_id": "fixture-5",
|
|
"benchmark_version": "1",
|
|
"raw_score": 85
|
|
},
|
|
{
|
|
"evidence_kind": "synthetic",
|
|
"observed_at": "2026-09-13T10:00:00Z",
|
|
"profile_id": "budget-fixture",
|
|
"benchmark_id": "fixture-1",
|
|
"benchmark_version": "1",
|
|
"raw_score": 70
|
|
},
|
|
{
|
|
"evidence_kind": "synthetic",
|
|
"observed_at": "2026-09-13T10:00:00Z",
|
|
"profile_id": "budget-fixture",
|
|
"benchmark_id": "fixture-2",
|
|
"benchmark_version": "1",
|
|
"raw_score": 70
|
|
},
|
|
{
|
|
"evidence_kind": "synthetic",
|
|
"observed_at": "2026-09-13T10:00:00Z",
|
|
"profile_id": "budget-fixture",
|
|
"benchmark_id": "fixture-3",
|
|
"benchmark_version": "1",
|
|
"raw_score": 70
|
|
},
|
|
{
|
|
"evidence_kind": "synthetic",
|
|
"observed_at": "2026-09-13T10:00:00Z",
|
|
"profile_id": "budget-fixture",
|
|
"benchmark_id": "fixture-4",
|
|
"benchmark_version": "1",
|
|
"raw_score": 70
|
|
},
|
|
{
|
|
"evidence_kind": "synthetic",
|
|
"observed_at": "2026-09-13T10:00:00Z",
|
|
"profile_id": "budget-fixture",
|
|
"benchmark_id": "fixture-5",
|
|
"benchmark_version": "1",
|
|
"raw_score": 70
|
|
}
|
|
],
|
|
"prices": [
|
|
{
|
|
"evidence_kind": "synthetic",
|
|
"observed_at": "2026-09-13T10:00:00Z",
|
|
"profile_id": "frontier-fixture",
|
|
"currency": "USD",
|
|
"valid_from": "2026-09-13",
|
|
"valid_until": "2026-10-01",
|
|
"usd_per_million": {
|
|
"input": 10,
|
|
"output": 50
|
|
}
|
|
},
|
|
{
|
|
"evidence_kind": "synthetic",
|
|
"observed_at": "2026-09-13T10:00:00Z",
|
|
"profile_id": "efficient-fixture",
|
|
"currency": "USD",
|
|
"valid_from": "2026-09-13",
|
|
"valid_until": "2026-10-01",
|
|
"usd_per_million": {
|
|
"input": 2,
|
|
"output": 10
|
|
}
|
|
},
|
|
{
|
|
"evidence_kind": "synthetic",
|
|
"observed_at": "2026-09-13T10:00:00Z",
|
|
"profile_id": "budget-fixture",
|
|
"currency": "USD",
|
|
"valid_from": "2026-09-13",
|
|
"valid_until": "2026-10-01",
|
|
"usd_per_million": {
|
|
"input": 0.2,
|
|
"output": 1.0
|
|
}
|
|
}
|
|
]
|
|
}
|