coulomb/frontend-patterns
5
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Report inconclusive presentation pilot and prepare human adjudication

Browse source 
Assistant: codex
Assistant-Model: gpt-6-astra
Assistant-Session: 01a0cffe-206f-7323-89d3-b20ca4cf4bf9
This commit is contained in:
tegwick 2026-09-26 20:07:55 +02:00
parent 3690af12a5
commit 629bc2e880
10 changed files with 932 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

369
research/evaluations/language-pilot-01/summary.json Normal file
View file

@ -0,0 +1,369 @@
{
"preregistration_commit": "6acbe6d",
"masked_scoring_commit": "3690af1",
"independent_human_review": "pending",
"interpretation": "exploratory descriptive counts only",
"score_sha256": "d7d7fa0b688f5db93709821a076157acb2723791c99736fc296f08e1627b73b9",
"conditions": {
"patterns": {
"completed": 6,
"failed": 0,
"requirements_met": 21,
"requirements_possible": 24,
"criteria_met": {
"C1": 5,
"C2": 6,
"R": 5,
"Q": 5
},
"unsupported_claims": 2,
"calls": 6,
"prompt_tokens": 6297,
"completion_tokens": 1137,
"total_tokens": 7434,
"elapsed_seconds": 11.997,
"provider_reported_cost_usd": 0.0047316,
"over_requested_word_limit": 1,
"resolved_models": [
"google/gemini-2.5-flash"
],
"answers": [
{
"answer": "S01-U",
"run": "r01",
"met": 4,
"unsupported": 0,
"criteria": {
"C1": 1,
"C2": 1,
"R": 1,
"Q": 1
}
},
{
"answer": "S02-V",
"run": "r04",
"met": 4,
"unsupported": 0,
"criteria": {
"C1": 1,
"C2": 1,
"R": 1,
"Q": 1
}
},
{
"answer": "S03-V",
"run": "r05",
"met": 3,
"unsupported": 1,
"criteria": {
"C1": 1,
"C2": 1,
"R": 0,
"Q": 1
}
},
{
"answer": "S04-U",
"run": "r08",
"met": 4,
"unsupported": 0,
"criteria": {
"C1": 1,
"C2": 1,
"R": 1,
"Q": 1
}
},
{
"answer": "S05-V",
"run": "r09",
"met": 3,
"unsupported": 0,
"criteria": {
"C1": 0,
"C2": 1,
"R": 1,
"Q": 1
}
},
{
"answer": "S06-V",
"run": "r12",
"met": 3,
"unsupported": 1,
"criteria": {
"C1": 1,
"C2": 1,
"R": 1,
"Q": 0
}
}
],
"missed_requirements": 3
},
"prose": {
"completed": 6,
"failed": 0,
"requirements_met": 19,
"requirements_possible": 24,
"criteria_met": {
"C1": 5,
"C2": 6,
"R": 3,
"Q": 5
},
"unsupported_claims": 2,
"calls": 6,
"prompt_tokens": 6051,
"completion_tokens": 1192,
"total_tokens": 7243,
"elapsed_seconds": 11.317,
"provider_reported_cost_usd": 0.0047953,
"over_requested_word_limit": 0,
"resolved_models": [
"google/gemini-2.5-flash"
],
"answers": [
{
"answer": "S01-V",
"run": "r02",
"met": 3,
"unsupported": 1,
"criteria": {
"C1": 1,
"C2": 1,
"R": 0,
"Q": 1
}
},
{
"answer": "S02-U",
"run": "r03",
"met": 4,
"unsupported": 0,
"criteria": {
"C1": 1,
"C2": 1,
"R": 1,
"Q": 1
}
},
{
"answer": "S03-U",
"run": "r06",
"met": 3,
"unsupported": 1,
"criteria": {
"C1": 1,
"C2": 1,
"R": 0,
"Q": 1
}
},
{
"answer": "S04-V",
"run": "r07",
"met": 3,
"unsupported": 0,
"criteria": {
"C1": 1,
"C2": 1,
"R": 0,
"Q": 1
}
},
{
"answer": "S05-U",
"run": "r10",
"met": 3,
"unsupported": 0,
"criteria": {
"C1": 0,
"C2": 1,
"R": 1,
"Q": 1
}
},
{
"answer": "S06-U",
"run": "r11",
"met": 3,
"unsupported": 0,
"criteria": {
"C1": 1,
"C2": 1,
"R": 1,
"Q": 0
}
}
],
"missed_requirements": 5
}
},
"pairs": {
"S01": {
"patterns": {
"answer": "S01-U",
"run": "r01",
"met": 4,
"unsupported": 0,
"criteria": {
"C1": 1,
"C2": 1,
"R": 1,
"Q": 1
}
},
"prose": {
"answer": "S01-V",
"run": "r02",
"met": 3,
"unsupported": 1,
"criteria": {
"C1": 1,
"C2": 1,
"R": 0,
"Q": 1
}
},
"descriptive_advantage": "patterns"
},
"S02": {
"prose": {
"answer": "S02-U",
"run": "r03",
"met": 4,
"unsupported": 0,
"criteria": {
"C1": 1,
"C2": 1,
"R": 1,
"Q": 1
}
},
"patterns": {
"answer": "S02-V",
"run": "r04",
"met": 4,
"unsupported": 0,
"criteria": {
"C1": 1,
"C2": 1,
"R": 1,
"Q": 1
}
},
"descriptive_advantage": "tie"
},
"S03": {
"patterns": {
"answer": "S03-V",
"run": "r05",
"met": 3,
"unsupported": 1,
"criteria": {
"C1": 1,
"C2": 1,
"R": 0,
"Q": 1
}
},
"prose": {
"answer": "S03-U",
"run": "r06",
"met": 3,
"unsupported": 1,
"criteria": {
"C1": 1,
"C2": 1,
"R": 0,
"Q": 1
}
},
"descriptive_advantage": "tie"
},
"S04": {
"prose": {
"answer": "S04-V",
"run": "r07",
"met": 3,
"unsupported": 0,
"criteria": {
"C1": 1,
"C2": 1,
"R": 0,
"Q": 1
}
},
"patterns": {
"answer": "S04-U",
"run": "r08",
"met": 4,
"unsupported": 0,
"criteria": {
"C1": 1,
"C2": 1,
"R": 1,
"Q": 1
}
},
"descriptive_advantage": "patterns"
},
"S05": {
"patterns": {
"answer": "S05-V",
"run": "r09",
"met": 3,
"unsupported": 0,
"criteria": {
"C1": 0,
"C2": 1,
"R": 1,
"Q": 1
}
},
"prose": {
"answer": "S05-U",
"run": "r10",
"met": 3,
"unsupported": 0,
"criteria": {
"C1": 0,
"C2": 1,
"R": 1,
"Q": 1
}
},
"descriptive_advantage": "tie"
},
"S06": {
"prose": {
"answer": "S06-U",
"run": "r11",
"met": 3,
"unsupported": 0,
"criteria": {
"C1": 1,
"C2": 1,
"R": 1,
"Q": 0
}
},
"patterns": {
"answer": "S06-V",
"run": "r12",
"met": 3,
"unsupported": 1,
"criteria": {
"C1": 1,
"C2": 1,
"R": 1,
"Q": 0
}
},
"descriptive_advantage": "prose"
}
}
}