coulomb/llm-connect
4
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Restore qualified task identities and finish transport protocol types

Browse source 
Assistant: codex
Assistant-Model: gpt-5.6-luna
Assistant-Session: 01a07ff8-19d0-7820-b4d0-1353833cb7fc
This commit is contained in:
tegwick 2026-09-09 21:45:45 +02:00
parent 4055986c98
commit 68ffe9ef76
9 changed files with 899 additions and 157 deletions
Download patch file Download diff file Expand all files Collapse all files

697
docs/evidence/2026-09-09-legacy-task-qualification.json Normal file
View file

@ -0,0 +1,697 @@
[
{
"source_file": "workplans/llm-connect-WP-0001-foundation-gaaf-baseline.md",
"workplan_id": "LLM-WP-0001",
"workplan_uuid": "f7f08327-753f-4175-8591-ffa1c3188ebc",
"workplan_status": "completed",
"hub_status": "finished",
"tasks": [
{
"source_id": "T01",
"uuid": "c38c5a79-4ce5-4088-9a21-ac65e09b12ba",
"source_status": "done",
"hub_record_id": "LLM-WP-0001-T01",
"hub_status": "done",
"hub_parent": "f7f08327-753f-4175-8591-ffa1c3188ebc",
"source_id_original": "T01",
"source_id_qualified": "LLM-WP-0001-T01"
},
{
"source_id": "T02",
"uuid": "6a15c794-d0f7-4d9c-a3ac-850f8c5bd5e9",
"source_status": "done",
"hub_record_id": "LLM-WP-0001-T02",
"hub_status": "done",
"hub_parent": "f7f08327-753f-4175-8591-ffa1c3188ebc",
"source_id_original": "T02",
"source_id_qualified": "LLM-WP-0001-T02"
},
{
"source_id": "T03",
"uuid": "af1c63ac-e4be-495a-9fdb-68eddebfcb75",
"source_status": "done",
"hub_record_id": "LLM-WP-0001-T03",
"hub_status": "done",
"hub_parent": "f7f08327-753f-4175-8591-ffa1c3188ebc",
"source_id_original": "T03",
"source_id_qualified": "LLM-WP-0001-T03"
},
{
"source_id": "T04",
"uuid": "da5a7986-5c47-4c4c-a8f6-a58956127535",
"source_status": "done",
"hub_record_id": "LLM-WP-0001-T04",
"hub_status": "done",
"hub_parent": "f7f08327-753f-4175-8591-ffa1c3188ebc",
"source_id_original": "T04",
"source_id_qualified": "LLM-WP-0001-T04"
},
{
"source_id": "T05",
"uuid": "01237203-0582-4bc4-a308-075e991e8e99",
"source_status": "done",
"hub_record_id": "LLM-WP-0001-T05",
"hub_status": "done",
"hub_parent": "f7f08327-753f-4175-8591-ffa1c3188ebc",
"source_id_original": "T05",
"source_id_qualified": "LLM-WP-0001-T05"
},
{
"source_id": "T06",
"uuid": "2bee5174-d3d7-4267-9cee-6e0e9b5cc731",
"source_status": "done",
"hub_record_id": "LLM-WP-0001-T06",
"hub_status": "done",
"hub_parent": "f7f08327-753f-4175-8591-ffa1c3188ebc",
"source_id_original": "T06",
"source_id_qualified": "LLM-WP-0001-T06"
},
{
"source_id": "T07",
"uuid": "b6dccf3e-8742-486e-a6a7-82577866a3bc",
"source_status": "done",
"hub_record_id": "LLM-WP-0001-T07",
"hub_status": "done",
"hub_parent": "f7f08327-753f-4175-8591-ffa1c3188ebc",
"source_id_original": "T07",
"source_id_qualified": "LLM-WP-0001-T07"
},
{
"source_id": "T08",
"uuid": "cc05b67d-f956-458a-908f-2ff58b1d33d3",
"source_status": "done",
"hub_record_id": "LLM-WP-0001-T08",
"hub_status": "done",
"hub_parent": "f7f08327-753f-4175-8591-ffa1c3188ebc",
"source_id_original": "T08",
"source_id_qualified": "LLM-WP-0001-T08"
},
{
"source_id": "T09",
"uuid": "8f9ec054-79ab-411d-8204-9d764bbbed98",
"source_status": "done",
"hub_record_id": "LLM-WP-0001-T09",
"hub_status": "done",
"hub_parent": "f7f08327-753f-4175-8591-ffa1c3188ebc",
"source_id_original": "T09",
"source_id_qualified": "LLM-WP-0001-T09"
},
{
"source_id": "T10",
"uuid": "044ee879-6baa-42fd-a0a4-a43dac0eacbb",
"source_status": "done",
"hub_record_id": "LLM-WP-0001-T10",
"hub_status": "done",
"hub_parent": "f7f08327-753f-4175-8591-ffa1c3188ebc",
"source_id_original": "T10",
"source_id_qualified": "LLM-WP-0001-T10"
},
{
"source_id": "T11",
"uuid": "699eef00-e9df-4de0-b7e6-61cfaace9617",
"source_status": "done",
"hub_record_id": "LLM-WP-0001-T11",
"hub_status": "done",
"hub_parent": "f7f08327-753f-4175-8591-ffa1c3188ebc",
"source_id_original": "T11",
"source_id_qualified": "LLM-WP-0001-T11"
},
{
"source_id": "T12",
"uuid": "c0853a23-52ae-499e-9a49-e7b65749b508",
"source_status": "done",
"hub_record_id": "LLM-WP-0001-T12",
"hub_status": "done",
"hub_parent": "f7f08327-753f-4175-8591-ffa1c3188ebc",
"source_id_original": "T12",
"source_id_qualified": "LLM-WP-0001-T12"
}
]
},
{
"source_file": "workplans/llm-connect-WP-0002-core-extensions.md",
"workplan_id": "LLM-WP-0002",
"workplan_uuid": "448fa379-eb9e-4808-b3fa-0078f1e4eaba",
"workplan_status": "completed",
"hub_status": "finished",
"tasks": [
{
"source_id": "T01",
"uuid": "ae27c363-339a-4f78-9737-cf872698f6d8",
"source_status": "done",
"hub_record_id": "LLM-WP-0002-T01",
"hub_status": "done",
"hub_parent": "448fa379-eb9e-4808-b3fa-0078f1e4eaba",
"source_id_original": "T01",
"source_id_qualified": "LLM-WP-0002-T01"
},
{
"source_id": "T02",
"uuid": "ea6f6ef7-2cb2-48e2-b9c9-f2b84a1a242b",
"source_status": "done",
"hub_record_id": "LLM-WP-0002-T02",
"hub_status": "done",
"hub_parent": "448fa379-eb9e-4808-b3fa-0078f1e4eaba",
"source_id_original": "T02",
"source_id_qualified": "LLM-WP-0002-T02"
},
{
"source_id": "T03",
"uuid": "fe6dbb73-5d04-45e6-aa91-5eff79aae7ee",
"source_status": "done",
"hub_record_id": "LLM-WP-0002-T03",
"hub_status": "done",
"hub_parent": "448fa379-eb9e-4808-b3fa-0078f1e4eaba",
"source_id_original": "T03",
"source_id_qualified": "LLM-WP-0002-T03"
},
{
"source_id": "T04",
"uuid": "8fd21bc2-598e-4449-8c86-eacde760e23f",
"source_status": "done",
"hub_record_id": "LLM-WP-0002-T04",
"hub_status": "done",
"hub_parent": "448fa379-eb9e-4808-b3fa-0078f1e4eaba",
"source_id_original": "T04",
"source_id_qualified": "LLM-WP-0002-T04"
},
{
"source_id": "T05",
"uuid": "e15745f5-9bb7-45d6-a36b-3a345fb0e9f1",
"source_status": "done",
"hub_record_id": "LLM-WP-0002-T05",
"hub_status": "done",
"hub_parent": "448fa379-eb9e-4808-b3fa-0078f1e4eaba",
"source_id_original": "T05",
"source_id_qualified": "LLM-WP-0002-T05"
},
{
"source_id": "T06",
"uuid": "5af37ade-3dd0-4ce9-8ead-be9887913bab",
"source_status": "done",
"hub_record_id": "LLM-WP-0002-T06",
"hub_status": "done",
"hub_parent": "448fa379-eb9e-4808-b3fa-0078f1e4eaba",
"source_id_original": "T06",
"source_id_qualified": "LLM-WP-0002-T06"
},
{
"source_id": "T07",
"uuid": "e221e630-658f-4adb-9f00-7b7df7ab8cb4",
"source_status": "done",
"hub_record_id": "LLM-WP-0002-T07",
"hub_status": "done",
"hub_parent": "448fa379-eb9e-4808-b3fa-0078f1e4eaba",
"source_id_original": "T07",
"source_id_qualified": "LLM-WP-0002-T07"
},
{
"source_id": "T08",
"uuid": "a75c2b2a-e4ef-4cbd-9c5f-7e98c8d3d7e8",
"source_status": "done",
"hub_record_id": "LLM-WP-0002-T08",
"hub_status": "done",
"hub_parent": "448fa379-eb9e-4808-b3fa-0078f1e4eaba",
"source_id_original": "T08",
"source_id_qualified": "LLM-WP-0002-T08"
},
{
"source_id": "T09",
"uuid": "1c50889f-28ed-4c6e-a788-1fc7dcc5a2c3",
"source_status": "done",
"hub_record_id": "LLM-WP-0002-T09",
"hub_status": "done",
"hub_parent": "448fa379-eb9e-4808-b3fa-0078f1e4eaba",
"source_id_original": "T09",
"source_id_qualified": "LLM-WP-0002-T09"
},
{
"source_id": "T10",
"uuid": "fa4f9e80-ddee-4d05-a239-fe09e633b0cb",
"source_status": "done",
"hub_record_id": "LLM-WP-0002-T10",
"hub_status": "done",
"hub_parent": "448fa379-eb9e-4808-b3fa-0078f1e4eaba",
"source_id_original": "T10",
"source_id_qualified": "LLM-WP-0002-T10"
},
{
"source_id": "T11",
"uuid": "bca78609-7f7c-4548-8857-a72e4c760dc6",
"source_status": "done",
"hub_record_id": "LLM-WP-0002-T11",
"hub_status": "done",
"hub_parent": "448fa379-eb9e-4808-b3fa-0078f1e4eaba",
"source_id_original": "T11",
"source_id_qualified": "LLM-WP-0002-T11"
}
]
},
{
"source_file": "workplans/llm-connect-WP-0003-functional-extensions.md",
"workplan_id": "LLM-WP-0003",
"workplan_uuid": "7b463cdc-40a2-4cc5-8b55-b59cc5ae3443",
"workplan_status": "completed",
"hub_status": "finished",
"tasks": [
{
"source_id": "T01",
"uuid": "85cf92fd-cddd-4e19-8782-970f6480a37f",
"source_status": "done",
"hub_record_id": "LLM-WP-0003-T01",
"hub_status": "done",
"hub_parent": "7b463cdc-40a2-4cc5-8b55-b59cc5ae3443",
"source_id_original": "T01",
"source_id_qualified": "LLM-WP-0003-T01"
},
{
"source_id": "T02",
"uuid": "352701ce-4b21-4f5d-a22e-462136e58fd2",
"source_status": "done",
"hub_record_id": "LLM-WP-0003-T02",
"hub_status": "done",
"hub_parent": "7b463cdc-40a2-4cc5-8b55-b59cc5ae3443",
"source_id_original": "T02",
"source_id_qualified": "LLM-WP-0003-T02"
},
{
"source_id": "T03",
"uuid": "baeb9b39-7fee-4f2b-86cc-ce64ff9e9b95",
"source_status": "done",
"hub_record_id": "LLM-WP-0003-T03",
"hub_status": "done",
"hub_parent": "7b463cdc-40a2-4cc5-8b55-b59cc5ae3443",
"source_id_original": "T03",
"source_id_qualified": "LLM-WP-0003-T03"
},
{
"source_id": "T04",
"uuid": "aa4488c6-950e-4cea-99b1-89defa4677ce",
"source_status": "done",
"hub_record_id": "LLM-WP-0003-T04",
"hub_status": "done",
"hub_parent": "7b463cdc-40a2-4cc5-8b55-b59cc5ae3443",
"source_id_original": "T04",
"source_id_qualified": "LLM-WP-0003-T04"
},
{
"source_id": "T05",
"uuid": "a4ad9c9e-64a4-44f0-85f3-b9cfe9ef59f7",
"source_status": "done",
"hub_record_id": "LLM-WP-0003-T05",
"hub_status": "done",
"hub_parent": "7b463cdc-40a2-4cc5-8b55-b59cc5ae3443",
"source_id_original": "T05",
"source_id_qualified": "LLM-WP-0003-T05"
},
{
"source_id": "T06",
"uuid": "cf79bce2-8d1a-4708-90b2-5e6569908b14",
"source_status": "done",
"hub_record_id": "LLM-WP-0003-T06",
"hub_status": "done",
"hub_parent": "7b463cdc-40a2-4cc5-8b55-b59cc5ae3443",
"source_id_original": "T06",
"source_id_qualified": "LLM-WP-0003-T06"
},
{
"source_id": "T07",
"uuid": "c91964ab-7366-4b34-acd4-1ee12f96881e",
"source_status": "done",
"hub_record_id": "LLM-WP-0003-T07",
"hub_status": "done",
"hub_parent": "7b463cdc-40a2-4cc5-8b55-b59cc5ae3443",
"source_id_original": "T07",
"source_id_qualified": "LLM-WP-0003-T07"
},
{
"source_id": "T08",
"uuid": "e3115bb4-cf3b-4ca0-9992-136e317068ac",
"source_status": "done",
"hub_record_id": "LLM-WP-0003-T08",
"hub_status": "done",
"hub_parent": "7b463cdc-40a2-4cc5-8b55-b59cc5ae3443",
"source_id_original": "T08",
"source_id_qualified": "LLM-WP-0003-T08"
},
{
"source_id": "T09",
"uuid": "2caf5531-8e10-40e9-a595-8652882a10e0",
"source_status": "done",
"hub_record_id": "LLM-WP-0003-T09",
"hub_status": "done",
"hub_parent": "7b463cdc-40a2-4cc5-8b55-b59cc5ae3443",
"source_id_original": "T09",
"source_id_qualified": "LLM-WP-0003-T09"
},
{
"source_id": "T10",
"uuid": "dc3c81c2-698d-4fee-b1dd-1af156a4276f",
"source_status": "done",
"hub_record_id": "LLM-WP-0003-T10",
"hub_status": "done",
"hub_parent": "7b463cdc-40a2-4cc5-8b55-b59cc5ae3443",
"source_id_original": "T10",
"source_id_qualified": "LLM-WP-0003-T10"
},
{
"source_id": "T11",
"uuid": "848a1622-abdd-4938-8bb4-3da27f5f9867",
"source_status": "done",
"hub_record_id": "LLM-WP-0003-T11",
"hub_status": "done",
"hub_parent": "7b463cdc-40a2-4cc5-8b55-b59cc5ae3443",
"source_id_original": "T11",
"source_id_qualified": "LLM-WP-0003-T11"
}
]
},
{
"source_file": "workplans/llm-connect-WP-0004-adaptive-cost-quality-routing.md",
"workplan_id": "LLM-WP-0004",
"workplan_uuid": "e1807fab-e29e-4517-b362-95737a96582d",
"workplan_status": "completed",
"hub_status": "finished",
"tasks": [
{
"source_id": "T01",
"uuid": "1c285bec-c30b-45a8-a408-3f91d810a078",
"source_status": "done",
"hub_record_id": "LLM-WP-0004-T01",
"hub_status": "done",
"hub_parent": "e1807fab-e29e-4517-b362-95737a96582d",
"source_id_original": "T01",
"source_id_qualified": "LLM-WP-0004-T01"
},
{
"source_id": "T02",
"uuid": "5249f171-a047-499f-9ec4-cb50e1477765",
"source_status": "done",
"hub_record_id": "LLM-WP-0004-T02",
"hub_status": "done",
"hub_parent": "e1807fab-e29e-4517-b362-95737a96582d",
"source_id_original": "T02",
"source_id_qualified": "LLM-WP-0004-T02"
},
{
"source_id": "T03",
"uuid": "adb255cf-7e89-4fea-b822-6be437d99789",
"source_status": "done",
"hub_record_id": "LLM-WP-0004-T03",
"hub_status": "done",
"hub_parent": "e1807fab-e29e-4517-b362-95737a96582d",
"source_id_original": "T03",
"source_id_qualified": "LLM-WP-0004-T03"
},
{
"source_id": "T04",
"uuid": "51a33180-a99d-4aa4-96be-2fcee15bfbc3",
"source_status": "done",
"hub_record_id": "LLM-WP-0004-T04",
"hub_status": "done",
"hub_parent": "e1807fab-e29e-4517-b362-95737a96582d",
"source_id_original": "T04",
"source_id_qualified": "LLM-WP-0004-T04"
},
{
"source_id": "T05",
"uuid": "458610c5-c903-4b42-9602-cd511999c9ba",
"source_status": "done",
"hub_record_id": "LLM-WP-0004-T05",
"hub_status": "done",
"hub_parent": "e1807fab-e29e-4517-b362-95737a96582d",
"source_id_original": "T05",
"source_id_qualified": "LLM-WP-0004-T05"
},
{
"source_id": "T06",
"uuid": "c12a595b-90fc-4a80-8394-549edbda2031",
"source_status": "done",
"hub_record_id": "LLM-WP-0004-T06",
"hub_status": "done",
"hub_parent": "e1807fab-e29e-4517-b362-95737a96582d",
"source_id_original": "T06",
"source_id_qualified": "LLM-WP-0004-T06"
},
{
"source_id": "T07",
"uuid": "80b98e31-06fc-4462-b030-a12881095f93",
"source_status": "done",
"hub_record_id": "LLM-WP-0004-T07",
"hub_status": "done",
"hub_parent": "e1807fab-e29e-4517-b362-95737a96582d",
"source_id_original": "T07",
"source_id_qualified": "LLM-WP-0004-T07"
},
{
"source_id": "T08",
"uuid": "c2887fe3-bae6-4298-8c26-f9a519264dcf",
"source_status": "done",
"hub_record_id": "LLM-WP-0004-T08",
"hub_status": "done",
"hub_parent": "e1807fab-e29e-4517-b362-95737a96582d",
"source_id_original": "T08",
"source_id_qualified": "LLM-WP-0004-T08"
},
{
"source_id": "T09",
"uuid": "7a4fd87a-b0ba-41b0-8e1a-a60fdaded905",
"source_status": "done",
"hub_record_id": "LLM-WP-0004-T09",
"hub_status": "done",
"hub_parent": "e1807fab-e29e-4517-b362-95737a96582d",
"source_id_original": "T09",
"source_id_qualified": "LLM-WP-0004-T09"
},
{
"source_id": "T10",
"uuid": "8415a11d-d508-4d17-8082-10f93e9d16c5",
"source_status": "done",
"hub_record_id": "LLM-WP-0004-T10",
"hub_status": "done",
"hub_parent": "e1807fab-e29e-4517-b362-95737a96582d",
"source_id_original": "T10",
"source_id_qualified": "LLM-WP-0004-T10"
},
{
"source_id": "T11",
"uuid": "0e9f9f8e-5066-4257-913b-a19f5b3fc47d",
"source_status": "done",
"hub_record_id": "LLM-WP-0004-T11",
"hub_status": "done",
"hub_parent": "e1807fab-e29e-4517-b362-95737a96582d",
"source_id_original": "T11",
"source_id_qualified": "LLM-WP-0004-T11"
},
{
"source_id": "T12",
"uuid": "59d44712-1088-41ac-bad8-5d95db6f3a4f",
"source_status": "done",
"hub_record_id": "LLM-WP-0004-T12",
"hub_status": "done",
"hub_parent": "e1807fab-e29e-4517-b362-95737a96582d",
"source_id_original": "T12",
"source_id_qualified": "LLM-WP-0004-T12"
},
{
"source_id": "T13",
"uuid": "1927d369-f5f6-48d3-8f53-7e4f1cae370e",
"source_status": "done",
"hub_record_id": "LLM-WP-0004-T13",
"hub_status": "done",
"hub_parent": "e1807fab-e29e-4517-b362-95737a96582d",
"source_id_original": "T13",
"source_id_qualified": "LLM-WP-0004-T13"
},
{
"source_id": "T14",
"uuid": "4d4717c1-8849-4fed-8f8d-515901ecafe0",
"source_status": "done",
"hub_record_id": "LLM-WP-0004-T14",
"hub_status": "done",
"hub_parent": "e1807fab-e29e-4517-b362-95737a96582d",
"source_id_original": "T14",
"source_id_qualified": "LLM-WP-0004-T14"
},
{
"source_id": "T15",
"uuid": "304bd782-db15-4b7a-8d05-49e064a926c3",
"source_status": "done",
"hub_record_id": "LLM-WP-0004-T15",
"hub_status": "done",
"hub_parent": "e1807fab-e29e-4517-b362-95737a96582d",
"source_id_original": "T15",
"source_id_qualified": "LLM-WP-0004-T15"
},
{
"source_id": "T16",
"uuid": "62dd507f-536a-4623-8cbd-fa9f78e85ca6",
"source_status": "done",
"hub_record_id": "LLM-WP-0004-T16",
"hub_status": "done",
"hub_parent": "e1807fab-e29e-4517-b362-95737a96582d",
"source_id_original": "T16",
"source_id_qualified": "LLM-WP-0004-T16"
},
{
"source_id": "T17",
"uuid": "ccb73e92-1fca-42f9-8437-9b2b50e6424c",
"source_status": "done",
"hub_record_id": "LLM-WP-0004-T17",
"hub_status": "done",
"hub_parent": "e1807fab-e29e-4517-b362-95737a96582d",
"source_id_original": "T17",
"source_id_qualified": "LLM-WP-0004-T17"
},
{
"source_id": "T18",
"uuid": "b879d232-d6ce-4ff6-b534-616729ea5ad7",
"source_status": "done",
"hub_record_id": "LLM-WP-0004-T18",
"hub_status": "done",
"hub_parent": "e1807fab-e29e-4517-b362-95737a96582d",
"source_id_original": "T18",
"source_id_qualified": "LLM-WP-0004-T18"
},
{
"source_id": "T19",
"uuid": "99d2c1bc-f1d8-42b3-9e04-6eea49460943",
"source_status": "done",
"hub_record_id": "LLM-WP-0004-T19",
"hub_status": "done",
"hub_parent": "e1807fab-e29e-4517-b362-95737a96582d",
"source_id_original": "T19",
"source_id_qualified": "LLM-WP-0004-T19"
},
{
"source_id": "T20",
"uuid": "f533fbf4-484f-4408-8260-7e84e23bdc46",
"source_status": "done",
"hub_record_id": "LLM-WP-0004-T20",
"hub_status": "done",
"hub_parent": "e1807fab-e29e-4517-b362-95737a96582d",
"source_id_original": "T20",
"source_id_qualified": "LLM-WP-0004-T20"
},
{
"source_id": "T21",
"uuid": "7ef0c143-74b0-4740-81fa-819a826cf8f3",
"source_status": "done",
"hub_record_id": "LLM-WP-0004-T21",
"hub_status": "done",
"hub_parent": "e1807fab-e29e-4517-b362-95737a96582d",
"source_id_original": "T21",
"source_id_qualified": "LLM-WP-0004-T21"
},
{
"source_id": "T22",
"uuid": "c4c6743f-157b-4445-8576-9caa6421d463",
"source_status": "done",
"hub_record_id": "LLM-WP-0004-T22",
"hub_status": "done",
"hub_parent": "e1807fab-e29e-4517-b362-95737a96582d",
"source_id_original": "T22",
"source_id_qualified": "LLM-WP-0004-T22"
},
{
"source_id": "T23",
"uuid": "3a073ff7-0170-4a95-9c2a-a5daa84964e6",
"source_status": "done",
"hub_record_id": "LLM-WP-0004-T23",
"hub_status": "done",
"hub_parent": "e1807fab-e29e-4517-b362-95737a96582d",
"source_id_original": "T23",
"source_id_qualified": "LLM-WP-0004-T23"
}
]
},
{
"source_file": "workplans/llm-connect-WP-0005-cost-model-and-problem-class-estimators.md",
"workplan_id": "LLM-WP-0005",
"workplan_uuid": "869196c5-551b-4eef-b8d8-cca6f770a9b0",
"workplan_status": "finished",
"hub_status": "finished",
"tasks": [
{
"source_id": "T01",
"uuid": "535d3f12-911e-4b6a-87c3-b539c5986671",
"source_status": "done",
"hub_record_id": "LLM-WP-0005-T01",
"hub_status": "done",
"hub_parent": "869196c5-551b-4eef-b8d8-cca6f770a9b0",
"source_id_original": "T01",
"source_id_qualified": "LLM-WP-0005-T01"
},
{
"source_id": "T02",
"uuid": "691dd985-6a97-432d-8bf0-6cb99a9fbdcc",
"source_status": "done",
"hub_record_id": "LLM-WP-0005-T02",
"hub_status": "done",
"hub_parent": "869196c5-551b-4eef-b8d8-cca6f770a9b0",
"source_id_original": "T02",
"source_id_qualified": "LLM-WP-0005-T02"
},
{
"source_id": "T03",
"uuid": "ecf263d2-f40a-460e-9195-4e01135ef727",
"source_status": "done",
"hub_record_id": "LLM-WP-0005-T03",
"hub_status": "done",
"hub_parent": "869196c5-551b-4eef-b8d8-cca6f770a9b0",
"source_id_original": "T03",
"source_id_qualified": "LLM-WP-0005-T03"
},
{
"source_id": "T04",
"uuid": "f1860b10-7467-4ce3-9775-ab293cef3ed0",
"source_status": "done",
"hub_record_id": "LLM-WP-0005-T04",
"hub_status": "done",
"hub_parent": "869196c5-551b-4eef-b8d8-cca6f770a9b0",
"source_id_original": "T04",
"source_id_qualified": "LLM-WP-0005-T04"
},
{
"source_id": "T05",
"uuid": "950b74e9-ede8-477a-b6b7-c7af423d4ebb",
"source_status": "done",
"hub_record_id": "LLM-WP-0005-T05",
"hub_status": "done",
"hub_parent": "869196c5-551b-4eef-b8d8-cca6f770a9b0",
"source_id_original": "T05",
"source_id_qualified": "LLM-WP-0005-T05"
},
{
"source_id": "T06",
"uuid": "c47eca5f-4cb3-4f88-ac1b-38a9ae18e7e6",
"source_status": "done",
"hub_record_id": "LLM-WP-0005-T06",
"hub_status": "done",
"hub_parent": "869196c5-551b-4eef-b8d8-cca6f770a9b0",
"source_id_original": "T06",
"source_id_qualified": "LLM-WP-0005-T06"
},
{
"source_id": "T07",
"uuid": "c15fd1dc-48c3-40e9-abca-ba3ffe3684f9",
"source_status": "done",
"hub_record_id": "LLM-WP-0005-T07",
"hub_status": "done",
"hub_parent": "869196c5-551b-4eef-b8d8-cca6f770a9b0",
"source_id_original": "T07",
"source_id_qualified": "LLM-WP-0005-T07"
},
{
"source_id": "T08",
"uuid": "2993932a-334c-49f9-bb74-6ef4d3cbffcb",
"source_status": "done",
"hub_record_id": "LLM-WP-0005-T08",
"hub_status": "done",
"hub_parent": "869196c5-551b-4eef-b8d8-cca6f770a9b0",
"source_id_original": "T08",
"source_id_qualified": "LLM-WP-0005-T08"
}
]
}
]

11
docs/evidence/2026-09-09-request-admission-quality.json Normal file
View file

@ -0,0 +1,11 @@
{
"baseline": {
"ruff_errors": 177,
"mypy_errors": 36
},
"current": {
"ruff_errors": 177,
"mypy_errors": 36
},
"unchanged_baseline": true
}

60
llm_connect/messages_gate.py
View file

@ -15,7 +15,7 @@ import threading
import time import time
from dataclasses import asdict, dataclass from dataclasses import asdict, dataclass
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from typing import Protocol from typing import Any, NoReturn, Protocol, TypeGuard, cast
from urllib.parse import urlsplit from urllib.parse import urlsplit
@ -23,11 +23,11 @@ class RequestRefused(RuntimeError):
"""Bounded refusal; never carries request content or credentials.""" """Bounded refusal; never carries request content or credentials."""
def _integer(value: object, upper: int) -> bool: def _integer(value: object, upper: int) -> TypeGuard[int]:
return type(value) is int and 0 < value <= upper return type(value) is int and 0 < value <= upper
def _unique(pairs): def _unique(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
result = {} result = {}
for key, value in pairs: for key, value in pairs:
if key in result: if key in result:
@ -36,8 +36,8 @@ def _unique(pairs):
return result return result
def _json(raw: bytes): def _json(raw: bytes) -> Any:
def invalid(_): def invalid(_: str) -> NoReturn:
raise ValueError("nonfinite number") raise ValueError("nonfinite number")
return json.loads(raw, object_pairs_hook=_unique, parse_constant=invalid) return json.loads(raw, object_pairs_hook=_unique, parse_constant=invalid)
@ -62,13 +62,13 @@ class MessagesPolicy:
max_body_bytes: int = 2_000_000 max_body_bytes: int = 2_000_000
timeout_seconds: int = 120 timeout_seconds: int = 120
def __post_init__(self): def __post_init__(self) -> None:
for value in (self.tariff_ref, self.model): for value in (self.tariff_ref, self.model):
if not isinstance(value, str) or not re.fullmatch( if not isinstance(value, str) or not re.fullmatch(
r"[A-Za-z0-9][A-Za-z0-9._:/@+-]{0,199}", value r"[A-Za-z0-9][A-Za-z0-9._:/@+-]{0,199}", value
): ):
raise RequestRefused("invalid request policy identity") raise RequestRefused("invalid request policy identity")
for value, upper in ( for bound, upper in (
(self.context_tokens, 10_000_000), (self.context_tokens, 10_000_000),
(self.max_output_tokens, 1_000_000), (self.max_output_tokens, 1_000_000),
(self.input_microusd_per_token, 1_000_000), (self.input_microusd_per_token, 1_000_000),
@ -76,7 +76,7 @@ class MessagesPolicy:
(self.max_body_bytes, 2_000_000), (self.max_body_bytes, 2_000_000),
(self.timeout_seconds, 900), (self.timeout_seconds, 900),
): ):
if not _integer(value, upper): if not _integer(bound, upper):
raise RequestRefused("invalid request policy bound") raise RequestRefused("invalid request policy bound")
if not isinstance(self.allowed_betas, tuple) or len(set(self.allowed_betas)) != len( if not isinstance(self.allowed_betas, tuple) or len(set(self.allowed_betas)) != len(
self.allowed_betas self.allowed_betas
@ -128,7 +128,7 @@ class MessagesPolicy:
}: }:
raise RequestRefused("only keep-all thinking context admitted") raise RequestRefused("only keep-all thinking context admitted")
def cache(value): def cache(value: Any) -> None:
if ( if (
not isinstance(value, dict) not isinstance(value, dict)
or set(value) - {"type", "ttl"} or set(value) - {"type", "ttl"}
@ -137,7 +137,7 @@ class MessagesPolicy:
): ):
raise RequestRefused("cache mode not admitted") raise RequestRefused("cache mode not admitted")
def blocks(value, *, system=False, nested=False): def blocks(value: Any, *, system: bool = False, nested: bool = False) -> None:
if isinstance(value, str): if isinstance(value, str):
return return
if not isinstance(value, list) or len(value) > 10000: if not isinstance(value, list) or len(value) > 10000:
@ -259,12 +259,12 @@ class RequestMeter(Protocol):
class _Stream: class _Stream:
"""Observe terminal usage without persisting provider content.""" """Observe terminal usage without persisting provider content."""
def __init__(self, policy: MessagesPolicy): def __init__(self, policy: MessagesPolicy) -> None:
self.policy = policy self.policy = policy
self.started = self.stopped = self.delta = False self.started = self.stopped = self.delta = False
self.usage = {} self.usage: dict[str, Any] = {}
def event(self, raw: bytes): def event(self, raw: bytes) -> None:
data = b"\n".join( data = b"\n".join(
line[5:].lstrip() for line in raw.splitlines() if line.startswith(b"data:") line[5:].lstrip() for line in raw.splitlines() if line.startswith(b"data:")
) )
@ -304,7 +304,7 @@ class _Stream:
): ):
raise RequestRefused("provider content feature not admitted") raise RequestRefused("provider content feature not admitted")
def _usage(self, value): def _usage(self, value: Any) -> None:
if not isinstance(value, dict): if not isinstance(value, dict):
raise RequestRefused("provider usage incomplete") raise RequestRefused("provider usage incomplete")
for key, count in value.items(): for key, count in value.items():
@ -323,7 +323,7 @@ class _Stream:
def cost(self) -> int: def cost(self) -> int:
if not self.stopped: if not self.stopped:
raise RequestRefused("provider stream incomplete") raise RequestRefused("provider stream incomplete")
counts = {} counts: dict[str, int] = {}
for key in ( for key in (
"input_tokens", "input_tokens",
"output_tokens", "output_tokens",
@ -341,11 +341,15 @@ class _Stream:
) )
class _OwnerHTTPServer(ThreadingHTTPServer):
owner: MessagesServer
class _Handler(BaseHTTPRequestHandler): class _Handler(BaseHTTPRequestHandler):
def log_message(self, *args): def log_message(self, format: str, *args: Any) -> None:
pass pass
def _error(self, status, code): def _error(self, status: int, code: str) -> None:
raw = json.dumps( raw = json.dumps(
{"type": "error", "error": {"type": "invalid_request_error", "message": code}} {"type": "error", "error": {"type": "invalid_request_error", "message": code}}
).encode() ).encode()
@ -355,8 +359,8 @@ class _Handler(BaseHTTPRequestHandler):
self.end_headers() self.end_headers()
self.wfile.write(raw) self.wfile.write(raw)
def do_POST(self): def do_POST(self) -> None:
owner = self.server.owner owner = cast(_OwnerHTTPServer, self.server).owner
upstream = None upstream = None
sent = False sent = False
try: try:
@ -402,7 +406,9 @@ class _Handler(BaseHTTPRequestHandler):
else http.client.HTTPConnection else http.client.HTTPConnection
) )
upstream = kind( upstream = kind(
owner.endpoint.hostname, owner.endpoint.port, timeout=owner.policy.timeout_seconds cast(str, owner.endpoint.hostname),
owner.endpoint.port,
timeout=owner.policy.timeout_seconds,
) )
headers = { headers = {
"Content-Type": "application/json", "Content-Type": "application/json",
@ -490,7 +496,7 @@ class MessagesServer:
host: str = "127.0.0.1", host: str = "127.0.0.1",
port: int = 0, port: int = 0,
allow_test_http: bool = False, allow_test_http: bool = False,
): ) -> None:
endpoint = urlsplit(upstream_url) endpoint = urlsplit(upstream_url)
if endpoint.scheme != "https" and not ( if endpoint.scheme != "https" and not (
allow_test_http allow_test_http
@ -515,19 +521,19 @@ class MessagesServer:
raise RequestRefused("explicit provider credential required") raise RequestRefused("explicit provider credential required")
self.policy, self.meter, self.endpoint = policy, meter, endpoint self.policy, self.meter, self.endpoint = policy, meter, endpoint
self._provider_key = provider_key self._provider_key = provider_key
self._httpd = ThreadingHTTPServer((host, port), _Handler) self._httpd = _OwnerHTTPServer((host, port), _Handler)
self._httpd.owner = self self._httpd.owner = self
self._thread = None self._thread: threading.Thread | None = None
@property @property
def port(self): def port(self) -> int:
return self._httpd.server_address[1] return int(self._httpd.server_address[1])
def start(self): def start(self) -> None:
self._thread = threading.Thread(target=self._httpd.serve_forever, daemon=True) self._thread = threading.Thread(target=self._httpd.serve_forever, daemon=True)
self._thread.start() self._thread.start()
def stop(self): def stop(self) -> None:
if self._thread is not None: if self._thread is not None:
self._httpd.shutdown() self._httpd.shutdown()
self._thread.join() self._thread.join()

28
workplans/LLM-WP-0009-owner-metered-messages-transport.md
View file

@ -71,3 +71,31 @@ review compatibility of the exact CLI/beta combination against the actual
provider before accepting a live profile. Local fake-provider evidence cannot provider before accepting a live profile. Local fake-provider evidence cannot
close this task or establish a hard live EUR ceiling. Reuse GLAS-WP-0015 identity close this task or establish a hard live EUR ceiling. Reuse GLAS-WP-0015 identity
and native-delivery owner work; completed verifier CCRs are not reopened. and native-delivery owner work; completed verifier CCRs are not reopened.
Pre-release quality return: configured repository-wide checks expose 177 Ruff
diagnostics and 36 mypy errors, reproduced identically at the original 00560945
source baseline. The new transport adds none after its protocol types were
completed. The 263 passing tests are not a claim of green full-repository CI.
Resolve or explicitly disposition those existing checks before an owner accepts
the protected artifact/release. Evidence:
`docs/evidence/2026-09-09-request-admission-quality.json`.
## Repair historical source identities blocking primary synchronization
```task
id: LLM-WP-0009-T04
status: done
priority: medium
```
HFACT-WP-0001-T02 side quest: qualified 65 historical parent-local task IDs to
match their already-existing Hub record IDs across five finished workplans.
Preserved all workplan/task UUIDs, parent links, statuses and task content;
qualified local references and retained an exact before/after mapping in
`docs/evidence/2026-09-09-legacy-task-qualification.json`. This is source identity
repair, not a UUID migration, record recreation or retirement. Six missing
historical ad-hoc source bindings also resolve to existing matching Hub UUIDs;
only Repo Manager's managed-field write may restore those pointers. The
AGENTS.md versus registry prefix disagreement remains a future-instruction
issue; published workplan IDs are unchanged.

48
workplans/llm-connect-WP-0001-foundation-gaaf-baseline.md
View file

@ -28,7 +28,7 @@ and state-hub housekeeping.
## Tasks ## Tasks
```task ```task
id: T01 id: LLM-WP-0001-T01
title: 'Create SCOPE.md' title: 'Create SCOPE.md'
priority: high priority: high
status: done status: done
@ -36,7 +36,7 @@ state_hub_task_id: "c38c5a79-4ce5-4088-9a21-ac65e09b12ba"
``` ```
```task ```task
id: T02 id: LLM-WP-0001-T02
title: 'Fill .claude/rules/ stubs: architecture.md, stack-and-commands.md, repo-boundary.md' title: 'Fill .claude/rules/ stubs: architecture.md, stack-and-commands.md, repo-boundary.md'
priority: high priority: high
status: done status: done
@ -44,7 +44,7 @@ state_hub_task_id: "6a15c794-d0f7-4d9c-a3ac-850f8c5bd5e9"
``` ```
```task ```task
id: T03 id: LLM-WP-0001-T03
title: 'Create ARCHITECTURE-LAYERS.md with layer map, scorecard stub, next-review date' title: 'Create ARCHITECTURE-LAYERS.md with layer map, scorecard stub, next-review date'
priority: high priority: high
status: done status: done
@ -52,7 +52,7 @@ state_hub_task_id: "af1c63ac-e4be-495a-9fdb-68eddebfcb75"
``` ```
```task ```task
id: T04 id: LLM-WP-0001-T04
title: 'Create /contracts/ tree (core/, functional/, config/)' title: 'Create /contracts/ tree (core/, functional/, config/)'
priority: high priority: high
status: done status: done
@ -60,7 +60,7 @@ state_hub_task_id: "da5a7986-5c47-4c4c-a8f6-a58956127535"
``` ```
```task ```task
id: T05 id: LLM-WP-0001-T05
title: 'Core contract doc: LLMAdapter interface invariants, RunConfig/LLMResponse field contracts' title: 'Core contract doc: LLMAdapter interface invariants, RunConfig/LLMResponse field contracts'
priority: high priority: high
status: done status: done
@ -68,7 +68,7 @@ state_hub_task_id: "01237203-0582-4bc4-a308-075e991e8e99"
``` ```
```task ```task
id: T06 id: LLM-WP-0001-T06
title: 'Functional contract stubs for all 4 adapters + embedding adapters (maturity: Beta)' title: 'Functional contract stubs for all 4 adapters + embedding adapters (maturity: Beta)'
priority: medium priority: medium
status: done status: done
@ -76,7 +76,7 @@ state_hub_task_id: "2bee5174-d3d7-4267-9cee-6e0e9b5cc731"
``` ```
```task ```task
id: T07 id: LLM-WP-0001-T07
title: 'Create tests/ with conftest.py, wire pytest in pyproject.toml' title: 'Create tests/ with conftest.py, wire pytest in pyproject.toml'
priority: high priority: high
status: done status: done
@ -84,7 +84,7 @@ state_hub_task_id: "b6dccf3e-8742-486e-a6a7-82577866a3bc"
``` ```
```task ```task
id: T08 id: LLM-WP-0001-T08
title: 'Unit tests: RunConfig, LLMResponse, MockLLMAdapter, full exception hierarchy' title: 'Unit tests: RunConfig, LLMResponse, MockLLMAdapter, full exception hierarchy'
priority: high priority: high
status: done status: done
@ -92,7 +92,7 @@ state_hub_task_id: "cc05b67d-f956-458a-908f-2ff58b1d33d3"
``` ```
```task ```task
id: T09 id: LLM-WP-0001-T09
title: 'Unit tests: create_adapter (all providers + unknown provider error), create_embedding_adapter' title: 'Unit tests: create_adapter (all providers + unknown provider error), create_embedding_adapter'
priority: high priority: high
status: done status: done
@ -100,7 +100,7 @@ state_hub_task_id: "8f9ec054-79ab-411d-8204-9d764bbbed98"
``` ```
```task ```task
id: T10 id: LLM-WP-0001-T10
title: 'Add ruff, mypy to dev deps in pyproject.toml' title: 'Add ruff, mypy to dev deps in pyproject.toml'
priority: medium priority: medium
status: done status: done
@ -108,7 +108,7 @@ state_hub_task_id: "044ee879-6baa-42fd-a0a4-a43dac0eacbb"
``` ```
```task ```task
id: T11 id: LLM-WP-0001-T11
title: 'CI workflow: pytest + ruff + mypy' title: 'CI workflow: pytest + ruff + mypy'
priority: medium priority: medium
status: done status: done
@ -116,7 +116,7 @@ state_hub_task_id: "699eef00-e9df-4de0-b7e6-61cfaace9617"
``` ```
```task ```task
id: T12 id: LLM-WP-0001-T12
title: 'State hub: register this host path, SBOM refresh' title: 'State hub: register this host path, SBOM refresh'
priority: low priority: low
status: done status: done
@ -125,18 +125,18 @@ state_hub_task_id: "c0853a23-52ae-499e-9a49-e7b65749b508"
| ID | Title | Priority | Status | | ID | Title | Priority | Status |
|-----|-------|----------|--------| |-----|-------|----------|--------|
| T01 | Create `SCOPE.md` | high | done | | LLM-WP-0001-T01 | Create `SCOPE.md` | high | done |
| T02 | Fill `.claude/rules/` stubs: `architecture.md`, `stack-and-commands.md`, `repo-boundary.md` | high | done | | LLM-WP-0001-T02 | Fill `.claude/rules/` stubs: `architecture.md`, `stack-and-commands.md`, `repo-boundary.md` | high | done |
| T03 | Create `ARCHITECTURE-LAYERS.md` with layer map, scorecard stub, next-review date | high | done | | LLM-WP-0001-T03 | Create `ARCHITECTURE-LAYERS.md` with layer map, scorecard stub, next-review date | high | done |
| T04 | Create `/contracts/` tree (`core/`, `functional/`, `config/`) | high | done | | LLM-WP-0001-T04 | Create `/contracts/` tree (`core/`, `functional/`, `config/`) | high | done |
| T05 | Core contract doc: `LLMAdapter` interface invariants, `RunConfig`/`LLMResponse` field contracts | high | done | | LLM-WP-0001-T05 | Core contract doc: `LLMAdapter` interface invariants, `RunConfig`/`LLMResponse` field contracts | high | done |
| T06 | Functional contract stubs for all 4 adapters + embedding adapters (maturity: Beta) | medium | done | | LLM-WP-0001-T06 | Functional contract stubs for all 4 adapters + embedding adapters (maturity: Beta) | medium | done |
| T07 | Create `tests/` with `conftest.py`, wire pytest in `pyproject.toml` | high | done | | LLM-WP-0001-T07 | Create `tests/` with `conftest.py`, wire pytest in `pyproject.toml` | high | done |
| T08 | Unit tests: `RunConfig`, `LLMResponse`, `MockLLMAdapter`, full exception hierarchy | high | done | | LLM-WP-0001-T08 | Unit tests: `RunConfig`, `LLMResponse`, `MockLLMAdapter`, full exception hierarchy | high | done |
| T09 | Unit tests: `create_adapter` (all providers + unknown provider error), `create_embedding_adapter` | high | done | | LLM-WP-0001-T09 | Unit tests: `create_adapter` (all providers + unknown provider error), `create_embedding_adapter` | high | done |
| T10 | Add `ruff`, `mypy` to dev deps in `pyproject.toml` | medium | done | | LLM-WP-0001-T10 | Add `ruff`, `mypy` to dev deps in `pyproject.toml` | medium | done |
| T11 | CI workflow: pytest + ruff + mypy | medium | done | | LLM-WP-0001-T11 | CI workflow: pytest + ruff + mypy | medium | done |
| T12 | State hub: register this host path, SBOM refresh | low | done | | LLM-WP-0001-T12 | State hub: register this host path, SBOM refresh | low | done |
## Exit criteria ## Exit criteria

46
workplans/llm-connect-WP-0002-core-extensions.md
View file

@ -38,12 +38,12 @@ Both changes are Core-layer modifications under GAAF-2026:
`asyncio.get_event_loop().run_in_executor(None, ...)` fallback so existing `asyncio.get_event_loop().run_in_executor(None, ...)` fallback so existing
adapters remain valid; native async overrides are provided per adapter. adapters remain valid; native async overrides are provided per adapter.
Core contract doc (from WP-0001 T05) must be updated after each change. Core contract doc (from LLM-WP-0001-T05) must be updated after each change.
## Tasks ## Tasks
```task ```task
id: T01 id: LLM-WP-0002-T01
title: 'BudgetTracker dataclass: total, spent, remaining(), thread-safe increment' title: 'BudgetTracker dataclass: total, spent, remaining(), thread-safe increment'
priority: high priority: high
status: done status: done
@ -51,7 +51,7 @@ state_hub_task_id: "ae27c363-339a-4f78-9737-cf872698f6d8"
``` ```
```task ```task
id: T02 id: LLM-WP-0002-T02
title: 'LLMBudgetExceededError(LLMError) in exceptions.py' title: 'LLMBudgetExceededError(LLMError) in exceptions.py'
priority: high priority: high
status: done status: done
@ -59,7 +59,7 @@ state_hub_task_id: "ea6f6ef7-2cb2-48e2-b9c9-f2b84a1a242b"
``` ```
```task ```task
id: T03 id: LLM-WP-0002-T03
title: 'Optional budget_tracker field on RunConfig' title: 'Optional budget_tracker field on RunConfig'
priority: high priority: high
status: done status: done
@ -67,7 +67,7 @@ state_hub_task_id: "fe6dbb73-5d04-45e6-aa91-5eff79aae7ee"
``` ```
```task ```task
id: T04 id: LLM-WP-0002-T04
title: 'Enforcement: adapters check/update tracker, raise LLMBudgetExceededError when exceeded' title: 'Enforcement: adapters check/update tracker, raise LLMBudgetExceededError when exceeded'
priority: high priority: high
status: done status: done
@ -75,7 +75,7 @@ state_hub_task_id: "8fd21bc2-598e-4449-8c86-eacde760e23f"
``` ```
```task ```task
id: T05 id: LLM-WP-0002-T05
title: 'Update Core contract doc for BudgetTracker and RunConfig changes' title: 'Update Core contract doc for BudgetTracker and RunConfig changes'
priority: medium priority: medium
status: done status: done
@ -83,7 +83,7 @@ state_hub_task_id: "e15745f5-9bb7-45d6-a36b-3a345fb0e9f1"
``` ```
```task ```task
id: T06 id: LLM-WP-0002-T06
title: 'Tests: single call, delegation chain, exceeded error, multi-adapter shared tracker' title: 'Tests: single call, delegation chain, exceeded error, multi-adapter shared tracker'
priority: high priority: high
status: done status: done
@ -91,7 +91,7 @@ state_hub_task_id: "5af37ade-3dd0-4ce9-8ead-be9887913bab"
``` ```
```task ```task
id: T07 id: LLM-WP-0002-T07
title: 'Add async_execute_prompt to LLMAdapter ABC with default executor fallback' title: 'Add async_execute_prompt to LLMAdapter ABC with default executor fallback'
priority: high priority: high
status: done status: done
@ -99,7 +99,7 @@ state_hub_task_id: "e221e630-658f-4adb-9f00-7b7df7ab8cb4"
``` ```
```task ```task
id: T08 id: LLM-WP-0002-T08
title: 'Native async override in OpenAIAdapter, GeminiAdapter, OpenRouterAdapter' title: 'Native async override in OpenAIAdapter, GeminiAdapter, OpenRouterAdapter'
priority: high priority: high
status: done status: done
@ -107,7 +107,7 @@ state_hub_task_id: "a75c2b2a-e4ef-4cbd-9c5f-7e98c8d3d7e8"
``` ```
```task ```task
id: T09 id: LLM-WP-0002-T09
title: 'Native async for ClaudeCodeAdapter via asyncio.create_subprocess_exec' title: 'Native async for ClaudeCodeAdapter via asyncio.create_subprocess_exec'
priority: high priority: high
status: done status: done
@ -115,7 +115,7 @@ state_hub_task_id: "1c50889f-28ed-4c6e-a788-1fc7dcc5a2c3"
``` ```
```task ```task
id: T10 id: LLM-WP-0002-T10
title: 'Update Core contract doc for async_execute_prompt' title: 'Update Core contract doc for async_execute_prompt'
priority: medium priority: medium
status: done status: done
@ -123,7 +123,7 @@ state_hub_task_id: "fa4f9e80-ddee-4d05-a239-fe09e633b0cb"
``` ```
```task ```task
id: T11 id: LLM-WP-0002-T11
title: 'Tests: asyncio.gather over N adapters, timeout propagation, budget interaction' title: 'Tests: asyncio.gather over N adapters, timeout propagation, budget interaction'
priority: high priority: high
status: done status: done
@ -134,22 +134,22 @@ state_hub_task_id: "bca78609-7f7c-4548-8857-a72e4c760dc6"
| ID | Title | Priority | Status | | ID | Title | Priority | Status |
|-----|-------|----------|--------| |-----|-------|----------|--------|
| T01 | `BudgetTracker` dataclass: `total`, `spent`, `remaining()`, thread-safe increment | high | done | | LLM-WP-0002-T01 | `BudgetTracker` dataclass: `total`, `spent`, `remaining()`, thread-safe increment | high | done |
| T02 | `LLMBudgetExceededError(LLMError)` in `exceptions.py` | high | done | | LLM-WP-0002-T02 | `LLMBudgetExceededError(LLMError)` in `exceptions.py` | high | done |
| T03 | Optional `budget_tracker: BudgetTracker \| None` field on `RunConfig` | high | done | | LLM-WP-0002-T03 | Optional `budget_tracker: BudgetTracker \| None` field on `RunConfig` | high | done |
| T04 | Enforcement: each adapter checks/updates tracker around call; raises on exceeded | high | done | | LLM-WP-0002-T04 | Enforcement: each adapter checks/updates tracker around call; raises on exceeded | high | done |
| T05 | Update Core contract doc | medium | done | | LLM-WP-0002-T05 | Update Core contract doc | medium | done |
| T06 | Tests: single call, delegation chain (A→B→C shared tracker), exceeded error, multi-adapter | high | done | | LLM-WP-0002-T06 | Tests: single call, delegation chain (A→B→C shared tracker), exceeded error, multi-adapter | high | done |
### FR-3 — async_execute_prompt ### FR-3 — async_execute_prompt
| ID | Title | Priority | Status | | ID | Title | Priority | Status |
|-----|-------|----------|--------| |-----|-------|----------|--------|
| T07 | Add `async_execute_prompt` to `LLMAdapter` ABC with default executor fallback | high | done | | LLM-WP-0002-T07 | Add `async_execute_prompt` to `LLMAdapter` ABC with default executor fallback | high | done |
| T08 | Native async override in `OpenAIAdapter`, `GeminiAdapter`, `OpenRouterAdapter` | high | done | | LLM-WP-0002-T08 | Native async override in `OpenAIAdapter`, `GeminiAdapter`, `OpenRouterAdapter` | high | done |
| T09 | Native async for `ClaudeCodeAdapter` via `asyncio.create_subprocess_exec` | high | done | | LLM-WP-0002-T09 | Native async for `ClaudeCodeAdapter` via `asyncio.create_subprocess_exec` | high | done |
| T10 | Update Core contract doc | medium | done | | LLM-WP-0002-T10 | Update Core contract doc | medium | done |
| T11 | Tests: `asyncio.gather` over N adapters, timeout propagation, budget interaction | high | done | | LLM-WP-0002-T11 | Tests: `asyncio.gather` over N adapters, timeout propagation, budget interaction | high | done |
## Exit criteria ## Exit criteria

44
workplans/llm-connect-WP-0003-functional-extensions.md
View file

@ -37,7 +37,7 @@ Both additions are Functional-layer under GAAF-2026:
## Tasks ## Tasks
```task ```task
id: T01 id: LLM-WP-0003-T01
title: 'RoutingPolicy data model: rules list with task_type, prefer, max_cost_per_1k, fallback' title: 'RoutingPolicy data model: rules list with task_type, prefer, max_cost_per_1k, fallback'
priority: high priority: high
status: done status: done
@ -45,7 +45,7 @@ state_hub_task_id: "85cf92fd-cddd-4e19-8782-970f6480a37f"
``` ```
```task ```task
id: T02 id: LLM-WP-0003-T02
title: 'policy.resolve(task_type) returns configured LLMAdapter' title: 'policy.resolve(task_type) returns configured LLMAdapter'
priority: high priority: high
status: done status: done
@ -53,7 +53,7 @@ state_hub_task_id: "352701ce-4b21-4f5d-a22e-462136e58fd2"
``` ```
```task ```task
id: T03 id: LLM-WP-0003-T03
title: 'Export RoutingPolicy from llm_connect.__init__ and update __all__' title: 'Export RoutingPolicy from llm_connect.__init__ and update __all__'
priority: medium priority: medium
status: done status: done
@ -61,7 +61,7 @@ state_hub_task_id: "baeb9b39-7fee-4f2b-86cc-ce64ff9e9b95"
``` ```
```task ```task
id: T04 id: LLM-WP-0003-T04
title: 'Functional contract doc for RoutingPolicy' title: 'Functional contract doc for RoutingPolicy'
priority: medium priority: medium
status: done status: done
@ -69,7 +69,7 @@ state_hub_task_id: "aa4488c6-950e-4cea-99b1-89defa4677ce"
``` ```
```task ```task
id: T05 id: LLM-WP-0003-T05
title: 'Tests: rule match, cost-cap fallback, unknown task_type fallback, no-match default' title: 'Tests: rule match, cost-cap fallback, unknown task_type fallback, no-match default'
priority: high priority: high
status: done status: done
@ -77,7 +77,7 @@ state_hub_task_id: "a4ad9c9e-64a4-44f0-85f3-b9cfe9ef59f7"
``` ```
```task ```task
id: T06 id: LLM-WP-0003-T06
title: 'Design /execute JSON schema (request: provider, model, prompt, config; response: LLMResponse)' title: 'Design /execute JSON schema (request: provider, model, prompt, config; response: LLMResponse)'
priority: high priority: high
status: done status: done
@ -85,7 +85,7 @@ state_hub_task_id: "cf79bce2-8d1a-4708-90b2-5e6569908b14"
``` ```
```task ```task
id: T07 id: LLM-WP-0003-T07
title: 'Implement llm_connect/server.py: POST /execute, GET /health' title: 'Implement llm_connect/server.py: POST /execute, GET /health'
priority: high priority: high
status: done status: done
@ -93,7 +93,7 @@ state_hub_task_id: "c91964ab-7366-4b34-acd4-1ee12f96881e"
``` ```
```task ```task
id: T08 id: LLM-WP-0003-T08
title: 'python -m llm_connect.server --port N --provider X --model Y CLI entry point' title: 'python -m llm_connect.server --port N --provider X --model Y CLI entry point'
priority: high priority: high
status: done status: done
@ -101,7 +101,7 @@ state_hub_task_id: "e3115bb4-cf3b-4ca0-9992-136e317068ac"
``` ```
```task ```task
id: T09 id: LLM-WP-0003-T09
title: 'Add server optional dep (httpx or aiohttp) to pyproject.toml' title: 'Add server optional dep (httpx or aiohttp) to pyproject.toml'
priority: medium priority: medium
status: done status: done
@ -109,7 +109,7 @@ state_hub_task_id: "2caf5531-8e10-40e9-a595-8652882a10e0"
``` ```
```task ```task
id: T10 id: LLM-WP-0003-T10
title: 'Functional contract doc: HTTP API schema (request/response shapes, error codes)' title: 'Functional contract doc: HTTP API schema (request/response shapes, error codes)'
priority: medium priority: medium
status: done status: done
@ -117,7 +117,7 @@ state_hub_task_id: "dc3c81c2-698d-4fee-b1dd-1af156a4276f"
``` ```
```task ```task
id: T11 id: LLM-WP-0003-T11
title: 'Tests: server POST round-trip (MockAdapter), GET /health, error responses' title: 'Tests: server POST round-trip (MockAdapter), GET /health, error responses'
priority: high priority: high
status: done status: done
@ -128,22 +128,22 @@ state_hub_task_id: "848a1622-abdd-4938-8bb4-3da27f5f9867"
| ID | Title | Priority | Status | | ID | Title | Priority | Status |
|-----|-------|----------|--------| |-----|-------|----------|--------|
| T01 | `RoutingPolicy` data model: `rules` list with `task_type`, `prefer`, `max_cost_per_1k`, `fallback` | high | done | | LLM-WP-0003-T01 | `RoutingPolicy` data model: `rules` list with `task_type`, `prefer`, `max_cost_per_1k`, `fallback` | high | done |
| T02 | `policy.resolve(task_type)` → returns configured `LLMAdapter` | high | done | | LLM-WP-0003-T02 | `policy.resolve(task_type)` → returns configured `LLMAdapter` | high | done |
| T03 | Export from `llm_connect.__init__` and update `__all__` | medium | done | | LLM-WP-0003-T03 | Export from `llm_connect.__init__` and update `__all__` | medium | done |
| T04 | Functional contract doc for `RoutingPolicy` | medium | done | | LLM-WP-0003-T04 | Functional contract doc for `RoutingPolicy` | medium | done |
| T05 | Tests: rule match, cost-cap fallback, unknown task_type fallback, no-match default | high | done | | LLM-WP-0003-T05 | Tests: rule match, cost-cap fallback, unknown task_type fallback, no-match default | high | done |
### FR-1 — HTTP serve mode ### FR-1 — HTTP serve mode
| ID | Title | Priority | Status | | ID | Title | Priority | Status |
|-----|-------|----------|--------| |-----|-------|----------|--------|
| T06 | Design `/execute` JSON schema (request: provider, model, prompt, config; response: LLMResponse fields) | high | done | | LLM-WP-0003-T06 | Design `/execute` JSON schema (request: provider, model, prompt, config; response: LLMResponse fields) | high | done |
| T07 | Implement `llm_connect/server.py` — minimal HTTP server, `POST /execute`, `GET /health` | high | done | | LLM-WP-0003-T07 | Implement `llm_connect/server.py` — minimal HTTP server, `POST /execute`, `GET /health` | high | done |
| T08 | `python -m llm_connect.server --port N --provider X --model Y` CLI entry point | high | done | | LLM-WP-0003-T08 | `python -m llm_connect.server --port N --provider X --model Y` CLI entry point | high | done |
| T09 | Add `httpx` or `aiohttp` server dep under `[project.optional-dependencies] server` | medium | done | | LLM-WP-0003-T09 | Add `httpx` or `aiohttp` server dep under `[project.optional-dependencies] server` | medium | done |
| T10 | Functional contract doc (API schema — request/response shapes, error codes) | medium | done | | LLM-WP-0003-T10 | Functional contract doc (API schema — request/response shapes, error codes) | medium | done |
| T11 | Tests: spin up server in subprocess or via `TestClient`, POST round-trip (MockAdapter), error responses | high | done | | LLM-WP-0003-T11 | Tests: spin up server in subprocess or via `TestClient`, POST round-trip (MockAdapter), error responses | high | done |
## Exit criteria ## Exit criteria

104
workplans/llm-connect-WP-0004-adaptive-cost-quality-routing.md
View file

@ -78,7 +78,7 @@ The fenced `task` blocks below are the State Hub registration index. Keep them
in sync with the detailed task tables that follow. in sync with the detailed task tables that follow.
```task ```task
id: T01 id: LLM-WP-0004-T01
title: 'QualityObservation dataclass: task_type, adapter_id, model_id, cost_usd, quality_score (0..1), latency_ms, tokens_in, tokens_out, baseline_adapter_id, recorded_at, tags' title: 'QualityObservation dataclass: task_type, adapter_id, model_id, cost_usd, quality_score (0..1), latency_ms, tokens_in, tokens_out, baseline_adapter_id, recorded_at, tags'
priority: high priority: high
status: done status: done
@ -86,7 +86,7 @@ state_hub_task_id: "1c285bec-c30b-45a8-a408-3f91d810a078"
``` ```
```task ```task
id: T02 id: LLM-WP-0004-T02
title: 'QualityLedger append-only JSONL store with file-locked writes, configurable path, simple query helpers (by_task_type, recent, mean_quality)' title: 'QualityLedger append-only JSONL store with file-locked writes, configurable path, simple query helpers (by_task_type, recent, mean_quality)'
priority: high priority: high
status: done status: done
@ -94,7 +94,7 @@ state_hub_task_id: "5249f171-a047-499f-9ec4-cb50e1477765"
``` ```
```task ```task
id: T03 id: LLM-WP-0004-T03
title: 'TTL helpers: prune_before(timestamp) and is_stale(observation, max_age)' title: 'TTL helpers: prune_before(timestamp) and is_stale(observation, max_age)'
priority: medium priority: medium
status: done status: done
@ -102,7 +102,7 @@ state_hub_task_id: "adb255cf-7e89-4fea-b822-6be437d99789"
``` ```
```task ```task
id: T04 id: LLM-WP-0004-T04
title: 'Functional contract doc for the ledger schema and quality_score semantics' title: 'Functional contract doc for the ledger schema and quality_score semantics'
priority: medium priority: medium
status: done status: done
@ -110,7 +110,7 @@ state_hub_task_id: "51a33180-a99d-4aa4-96be-2fcee15bfbc3"
``` ```
```task ```task
id: T05 id: LLM-WP-0004-T05
title: 'Tests: ledger round-trip, concurrent writes, query helpers, TTL, malformed-line resilience' title: 'Tests: ledger round-trip, concurrent writes, query helpers, TTL, malformed-line resilience'
priority: high priority: high
status: done status: done
@ -118,7 +118,7 @@ state_hub_task_id: "458610c5-c903-4b42-9602-cd511999c9ba"
``` ```
```task ```task
id: T06 id: LLM-WP-0004-T06
title: 'GradingResult dataclass: quality_score, notes, grader_id, baseline_response, candidate_response' title: 'GradingResult dataclass: quality_score, notes, grader_id, baseline_response, candidate_response'
priority: high priority: high
status: done status: done
@ -126,7 +126,7 @@ state_hub_task_id: "c12a595b-90fc-4a80-8394-549edbda2031"
``` ```
```task ```task
id: T07 id: LLM-WP-0004-T07
title: 'BaselineGrader protocol plus PairedGrader that runs baseline and candidate calls and delegates to a Judge' title: 'BaselineGrader protocol plus PairedGrader that runs baseline and candidate calls and delegates to a Judge'
priority: high priority: high
status: done status: done
@ -134,7 +134,7 @@ state_hub_task_id: "80b98e31-06fc-4462-b030-a12881095f93"
``` ```
```task ```task
id: T08 id: LLM-WP-0004-T08
title: 'Judge protocol and built-ins: ExactMatchJudge, EmbeddingSimilarityJudge, LLMJudge' title: 'Judge protocol and built-ins: ExactMatchJudge, EmbeddingSimilarityJudge, LLMJudge'
priority: high priority: high
status: done status: done
@ -142,7 +142,7 @@ state_hub_task_id: "c2887fe3-bae6-4298-8c26-f9a519264dcf"
``` ```
```task ```task
id: T09 id: LLM-WP-0004-T09
title: 'Functional contract doc covering judge bias caveats' title: 'Functional contract doc covering judge bias caveats'
priority: medium priority: medium
status: done status: done
@ -150,7 +150,7 @@ state_hub_task_id: "7a4fd87a-b0ba-41b0-8e1a-a60fdaded905"
``` ```
```task ```task
id: T10 id: LLM-WP-0004-T10
title: 'Tests: judges with canned inputs, stable grader result, deterministic LLMJudge rubric seed' title: 'Tests: judges with canned inputs, stable grader result, deterministic LLMJudge rubric seed'
priority: high priority: high
status: done status: done
@ -158,7 +158,7 @@ state_hub_task_id: "8415a11d-d508-4d17-8082-10f93e9d16c5"
``` ```
```task ```task
id: T11 id: LLM-WP-0004-T11
title: 'AdaptiveRoutingPolicy extends RoutingPolicy and selects the cheapest adapter whose observed mean quality clears the floor' title: 'AdaptiveRoutingPolicy extends RoutingPolicy and selects the cheapest adapter whose observed mean quality clears the floor'
priority: high priority: high
status: done status: done
@ -166,7 +166,7 @@ state_hub_task_id: "0e9f9f8e-5066-4257-913b-a19f5b3fc47d"
``` ```
```task ```task
id: T12 id: LLM-WP-0004-T12
title: 'Tie-breaking: prefer lower observed cost, then explicit preferred adapter from static rules' title: 'Tie-breaking: prefer lower observed cost, then explicit preferred adapter from static rules'
priority: medium priority: medium
status: done status: done
@ -174,7 +174,7 @@ state_hub_task_id: "59d44712-1088-41ac-bad8-5d95db6f3a4f"
``` ```
```task ```task
id: T13 id: LLM-WP-0004-T13
title: 'Cold-start behaviour falls through to static RoutingPolicy.resolve when observations are missing' title: 'Cold-start behaviour falls through to static RoutingPolicy.resolve when observations are missing'
priority: high priority: high
status: done status: done
@ -182,7 +182,7 @@ state_hub_task_id: "1927d369-f5f6-48d3-8f53-7e4f1cae370e"
``` ```
```task ```task
id: T14 id: LLM-WP-0004-T14
title: 'Functional contract doc for adaptive policy and sample-size/freshness trade-off' title: 'Functional contract doc for adaptive policy and sample-size/freshness trade-off'
priority: medium priority: medium
status: done status: done
@ -190,7 +190,7 @@ state_hub_task_id: "4d4717c1-8849-4fed-8f8d-515901ecafe0"
``` ```
```task ```task
id: T15 id: LLM-WP-0004-T15
title: 'Tests: floor enforcement, tie-break, cold-start, window-size effect, fallback chain' title: 'Tests: floor enforcement, tie-break, cold-start, window-size effect, fallback chain'
priority: high priority: high
status: done status: done
@ -198,7 +198,7 @@ state_hub_task_id: "304bd782-db15-4b7a-8d05-49e064a926c3"
``` ```
```task ```task
id: T16 id: LLM-WP-0004-T16
title: 'ShadowingAdapter wraps a candidate adapter, also invokes the baseline adapter, grades, and appends to QualityLedger' title: 'ShadowingAdapter wraps a candidate adapter, also invokes the baseline adapter, grades, and appends to QualityLedger'
priority: medium priority: medium
status: done status: done
@ -206,7 +206,7 @@ state_hub_task_id: "62dd507f-536a-4623-8cbd-fa9f78e85ca6"
``` ```
```task ```task
id: T17 id: LLM-WP-0004-T17
title: 'Sampling: caller-configurable shadow_rate so production load is not doubled' title: 'Sampling: caller-configurable shadow_rate so production load is not doubled'
priority: medium priority: medium
status: done status: done
@ -214,7 +214,7 @@ state_hub_task_id: "ccb73e92-1fca-42f9-8437-9b2b50e6424c"
``` ```
```task ```task
id: T18 id: LLM-WP-0004-T18
title: 'Failure isolation: shadow errors never affect the candidate response returned to the caller' title: 'Failure isolation: shadow errors never affect the candidate response returned to the caller'
priority: high priority: high
status: done status: done
@ -222,7 +222,7 @@ state_hub_task_id: "b879d232-d6ce-4ff6-b534-616729ea5ad7"
``` ```
```task ```task
id: T19 id: LLM-WP-0004-T19
title: 'Functional contract doc for ShadowingAdapter' title: 'Functional contract doc for ShadowingAdapter'
priority: low priority: low
status: done status: done
@ -230,7 +230,7 @@ state_hub_task_id: "99d2c1bc-f1d8-42b3-9e04-6eea49460943"
``` ```
```task ```task
id: T20 id: LLM-WP-0004-T20
title: 'Tests: candidate response survives baseline failure, ledger sampling rate, sync vs async modes' title: 'Tests: candidate response survives baseline failure, ledger sampling rate, sync vs async modes'
priority: high priority: high
status: done status: done
@ -238,7 +238,7 @@ state_hub_task_id: "f533fbf4-484f-4408-8260-7e84e23bdc46"
``` ```
```task ```task
id: T21 id: LLM-WP-0004-T21
title: 'Example script: route fixture batch through three candidate adapters and populate the ledger' title: 'Example script: route fixture batch through three candidate adapters and populate the ledger'
priority: medium priority: medium
status: done status: done
@ -246,7 +246,7 @@ state_hub_task_id: "7ef0c143-74b0-4740-81fa-819a826cf8f3"
``` ```
```task ```task
id: T22 id: LLM-WP-0004-T22
title: 'Integration test: cold-start, static fallback, first observations, convergence to cheapest qualifying adapter' title: 'Integration test: cold-start, static fallback, first observations, convergence to cheapest qualifying adapter'
priority: high priority: high
status: done status: done
@ -254,61 +254,61 @@ state_hub_task_id: "c4c6743f-157b-4445-8576-9caa6421d463"
``` ```
```task ```task
id: T23 id: LLM-WP-0004-T23
title: 'Consumer integration guide showing how infospace-bench wires task types into adaptive policy' title: 'Consumer integration guide showing how infospace-bench wires task types into adaptive policy'
priority: medium priority: medium
status: done status: done
state_hub_task_id: "3a073ff7-0170-4a95-9c2a-a5daa84964e6" state_hub_task_id: "3a073ff7-0170-4a95-9c2a-a5daa84964e6"
``` ```
### T01 — Quality observation data model + ledger ### LLM-WP-0004-T01 — Quality observation data model + ledger
| ID | Title | Priority | Status | | ID | Title | Priority | Status |
|-----|------------------------------------ |-----|------------------------------------
---------------------------------------------------------------------------------------------------------------------------------------------------------|----------|--------| ---------------------------------------------------------------------------------------------------------------------------------------------------------|----------|--------|
| T01 | `QualityObservation` dataclass: `task_type`, `adapter_id`, `model_id`, `cost_usd`, `quality_score` (0..1), `latency_ms`, `tokens_in`, `tokens_out`, `baseline_adapter_id`, `recorded_at`, `tags` | high | done | | LLM-WP-0004-T01 | `QualityObservation` dataclass: `task_type`, `adapter_id`, `model_id`, `cost_usd`, `quality_score` (0..1), `latency_ms`, `tokens_in`, `tokens_out`, `baseline_adapter_id`, `recorded_at`, `tags` | high | done |
| T02 | `QualityLedger` append-only JSONL store with file-locked writes, configurable path, simple query helpers (`by_task_type`, `recent`, `mean_quality`) | high | done | | LLM-WP-0004-T02 | `QualityLedger` append-only JSONL store with file-locked writes, configurable path, simple query helpers (`by_task_type`, `recent`, `mean_quality`) | high | done |
| T03 | TTL helpers: `prune_before(timestamp)` and `is_stale(observation, max_age)` so callers can refresh observations without re-reading the whole ledger | medium | done | | LLM-WP-0004-T03 | TTL helpers: `prune_before(timestamp)` and `is_stale(observation, max_age)` so callers can refresh observations without re-reading the whole ledger | medium | done |
| T04 | Functional contract doc for the ledger schema and the field semantics of `quality_score` | medium | done | | LLM-WP-0004-T04 | Functional contract doc for the ledger schema and the field semantics of `quality_score` | medium | done |
| T05 | Tests: round-trip, concurrent writes, query helpers, TTL, malformed-line resilience | high | done | | LLM-WP-0004-T05 | Tests: round-trip, concurrent writes, query helpers, TTL, malformed-line resilience | high | done |
### T02 — Baseline grader ### LLM-WP-0004-T02 — Baseline grader
| ID | Title | Priority | Status | | ID | Title | Priority | Status |
|-----|----------------------------------------------------------------------------------------------------------------------------------------------------|----------|--------| |-----|----------------------------------------------------------------------------------------------------------------------------------------------------|----------|--------|
| T06 | `GradingResult` dataclass: `quality_score`, `notes`, `grader_id`, `baseline_response`, `candidate_response` | high | done | | LLM-WP-0004-T06 | `GradingResult` dataclass: `quality_score`, `notes`, `grader_id`, `baseline_response`, `candidate_response` | high | done |
| T07 | `BaselineGrader` protocol: `.grade(baseline_adapter, candidate_adapter, prompt, run_config)` → `GradingResult`; built-in concrete `PairedGrader` runs both calls and delegates to a `Judge` | high | done | | LLM-WP-0004-T07 | `BaselineGrader` protocol: `.grade(baseline_adapter, candidate_adapter, prompt, run_config)` → `GradingResult`; built-in concrete `PairedGrader` runs both calls and delegates to a `Judge` | high | done |
| T08 | `Judge` protocol + three built-ins: `ExactMatchJudge`, `EmbeddingSimilarityJudge` (uses an embedding adapter), `LLMJudge` (uses a third adapter with a fixed rubric prompt) | high | done | | LLM-WP-0004-T08 | `Judge` protocol + three built-ins: `ExactMatchJudge`, `EmbeddingSimilarityJudge` (uses an embedding adapter), `LLMJudge` (uses a third adapter with a fixed rubric prompt) | high | done |
| T09 | Functional contract doc covering judge bias caveats (length bias, format bias, position bias for `LLMJudge`) | medium | done | | LLM-WP-0004-T09 | Functional contract doc covering judge bias caveats (length bias, format bias, position bias for `LLMJudge`) | medium | done |
| T10 | Tests: each judge against canned inputs, grader emits stable result with both responses preserved, deterministic seed for `LLMJudge` rubric | high | done | | LLM-WP-0004-T10 | Tests: each judge against canned inputs, grader emits stable result with both responses preserved, deterministic seed for `LLMJudge` rubric | high | done |
### T03 — Adaptive routing policy ### LLM-WP-0004-T03 — Adaptive routing policy
| ID | Title | Priority | Status | | ID | Title | Priority | Status |
|-----|--------------------------------------------------------------------------------------------------------------------------------------------------|----------|--------| |-----|--------------------------------------------------------------------------------------------------------------------------------------------------|----------|--------|
| T11 | `AdaptiveRoutingPolicy` extends `RoutingPolicy`: given `task_type` + `quality_floor` + `ledger`, returns the cheapest adapter whose observed mean quality clears the floor over a configurable window | high | done | | LLM-WP-0004-T11 | `AdaptiveRoutingPolicy` extends `RoutingPolicy`: given `task_type` + `quality_floor` + `ledger`, returns the cheapest adapter whose observed mean quality clears the floor over a configurable window | high | done |
| T12 | Tie-breaking: when two adapters meet the floor, prefer lower observed cost; if still tied, prefer the explicitly-preferred adapter from the underlying static rules | medium | done | | LLM-WP-0004-T12 | Tie-breaking: when two adapters meet the floor, prefer lower observed cost; if still tied, prefer the explicitly-preferred adapter from the underlying static rules | medium | done |
| T13 | Cold-start behaviour: when no observations exist for a `(task_type, adapter)` pair, fall through to the static `RoutingPolicy.resolve` result so the system stays usable on day zero | high | done | | LLM-WP-0004-T13 | Cold-start behaviour: when no observations exist for a `(task_type, adapter)` pair, fall through to the static `RoutingPolicy.resolve` result so the system stays usable on day zero | high | done |
| T14 | Functional contract doc; document the trade-off between sample size and freshness | medium | done | | LLM-WP-0004-T14 | Functional contract doc; document the trade-off between sample size and freshness | medium | done |
| T15 | Tests: floor enforcement, tie-break, cold-start, window-size effect, fallback chain | high | done | | LLM-WP-0004-T15 | Tests: floor enforcement, tie-break, cold-start, window-size effect, fallback chain | high | done |
### T04 — Shadow-mode observation wrapper ### LLM-WP-0004-T04 — Shadow-mode observation wrapper
| ID | Title | Priority | Status | | ID | Title | Priority | Status |
|-----|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|--------| |-----|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|--------|
| T16 | `ShadowingAdapter` wraps a candidate adapter; on each call, also invokes the baseline adapter (sync or via a thread pool), grades, and appends to a `QualityLedger` | medium | done | | LLM-WP-0004-T16 | `ShadowingAdapter` wraps a candidate adapter; on each call, also invokes the baseline adapter (sync or via a thread pool), grades, and appends to a `QualityLedger` | medium | done |
| T17 | Sampling: caller-configurable fraction (`shadow_rate=0.1` means grade one call in ten) so production load is not doubled | medium | done | | LLM-WP-0004-T17 | Sampling: caller-configurable fraction (`shadow_rate=0.1` means grade one call in ten) so production load is not doubled | medium | done |
| T18 | Failure isolation: shadow errors never affect the candidate response returned to the caller; failures are logged but not raised | high | done | | LLM-WP-0004-T18 | Failure isolation: shadow errors never affect the candidate response returned to the caller; failures are logged but not raised | high | done |
| T19 | Functional contract doc | low | done | | LLM-WP-0004-T19 | Functional contract doc | low | done |
| T20 | Tests: candidate response always returned even when baseline raises, ledger gets exactly `shadow_rate × calls` entries (within tolerance), sync vs async modes | high | done | | LLM-WP-0004-T20 | Tests: candidate response always returned even when baseline raises, ledger gets exactly `shadow_rate × calls` entries (within tolerance), sync vs async modes | high | done |
### T05 — End-to-end example + integration test ### LLM-WP-0004-T05 — End-to-end example + integration test
| ID | Title | Priority | Status | | ID | Title | Priority | Status |
|-----|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|--------| |-----|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|--------|
| T21 | Example script: route a small fixture batch through three candidate adapters (one OpenRouter cheap, one OpenRouter mid, `ClaudeCodeAdapter` as baseline), grade each, populate ledger | medium | done | | LLM-WP-0004-T21 | Example script: route a small fixture batch through three candidate adapters (one OpenRouter cheap, one OpenRouter mid, `ClaudeCodeAdapter` as baseline), grade each, populate ledger | medium | done |
| T22 | Integration test with mocked adapters covering: cold-start → static fallback → first observations → adaptive selection converges to the cheapest qualifying adapter | high | done | | LLM-WP-0004-T22 | Integration test with mocked adapters covering: cold-start → static fallback → first observations → adaptive selection converges to the cheapest qualifying adapter | high | done |
| T23 | Brief consumer-integration guide in `docs/` showing how `infospace-bench` (or any caller) wires task-type-per-stage into the adaptive policy | medium | done | | LLM-WP-0004-T23 | Brief consumer-integration guide in `docs/` showing how `infospace-bench` (or any caller) wires task-type-per-stage into the adaptive policy | medium | done |
## Risks and open questions ## Risks and open questions
@ -364,5 +364,5 @@ state_hub_task_id: "3a073ff7-0170-4a95-9c2a-a5daa84964e6"
- Wire the shadow-mode wrapper for the first multi-chapter run so the - Wire the shadow-mode wrapper for the first multi-chapter run so the
ledger fills up while real generation proceeds ledger fills up while real generation proceeds
That workplan should be drafted after T01–T03 of this workplan land, so That workplan should be drafted after LLM-WP-0004-T01–LLM-WP-0004-T03 of this workplan land, so
that the consumer-side wiring is anchored in a stable llm-connect API. that the consumer-side wiring is anchored in a stable llm-connect API.

18
workplans/llm-connect-WP-0005-cost-model-and-problem-class-estimators.md
View file

@ -181,7 +181,7 @@ keeps working (LLM-WP-0004 owns the ledger).
## Tasks ## Tasks
```task ```task
id: T01 id: LLM-WP-0005-T01
title: 'ModelRate + ModelRateRegistry data model, YAML loader, default-registry seed of nine OpenRouter models' title: 'ModelRate + ModelRateRegistry data model, YAML loader, default-registry seed of nine OpenRouter models'
priority: high priority: high
status: done status: done
@ -189,7 +189,7 @@ state_hub_task_id: "535d3f12-911e-4b6a-87c3-b539c5986671"
``` ```
```task ```task
id: T02 id: LLM-WP-0005-T02
title: 'CostModel.estimate_cost() pure function; tests for known model, unknown model, registry override, zero-token edge' title: 'CostModel.estimate_cost() pure function; tests for known model, unknown model, registry override, zero-token edge'
priority: high priority: high
status: done status: done
@ -197,7 +197,7 @@ state_hub_task_id: "691dd985-6a97-432d-8bf0-6cb99a9fbdcc"
``` ```
```task ```task
id: T03 id: LLM-WP-0005-T03
title: 'ProblemClass protocol + TokenEstimate + ProblemClassRegistry' title: 'ProblemClass protocol + TokenEstimate + ProblemClassRegistry'
priority: high priority: high
status: done status: done
@ -205,7 +205,7 @@ state_hub_task_id: "ecf263d2-f40a-460e-9195-4e01135ef727"
``` ```
```task ```task
id: T04 id: LLM-WP-0005-T04
title: 'Built-in classes: chunk-summarization, entity-extraction, relation-extraction, judge-eval, report-synthesis' title: 'Built-in classes: chunk-summarization, entity-extraction, relation-extraction, judge-eval, report-synthesis'
priority: high priority: high
status: done status: done
@ -213,7 +213,7 @@ state_hub_task_id: "f1860b10-7467-4ce3-9775-ab293cef3ed0"
``` ```
```task ```task
id: T05 id: LLM-WP-0005-T05
title: 'ProblemClass.fit() adapts tunable params from QualityLedger observations' title: 'ProblemClass.fit() adapts tunable params from QualityLedger observations'
priority: medium priority: medium
status: done status: done
@ -221,7 +221,7 @@ state_hub_task_id: "950b74e9-ede8-477a-b6b7-c7af423d4ebb"
``` ```
```task ```task
id: T06 id: LLM-WP-0005-T06
title: 'CLI helpers: llm-connect rates show, llm-connect classes show, llm-connect classes fit <ledger>' title: 'CLI helpers: llm-connect rates show, llm-connect classes show, llm-connect classes fit <ledger>'
priority: medium priority: medium
status: done status: done
@ -229,7 +229,7 @@ state_hub_task_id: "c47eca5f-4cb3-4f88-ac1b-38a9ae18e7e6"
``` ```
```task ```task
id: T07 id: LLM-WP-0005-T07
title: 'Functional contract docs under contracts/functional/ for rates, costs, and problem classes' title: 'Functional contract docs under contracts/functional/ for rates, costs, and problem classes'
priority: medium priority: medium
status: done status: done
@ -237,7 +237,7 @@ state_hub_task_id: "c15fd1dc-48c3-40e9-abca-ba3ffe3684f9"
``` ```
```task ```task
id: T08 id: LLM-WP-0005-T08
title: 'Consumer migration note for infospace-bench: replace plan_generation_summary cost+token math with llm-connect calls' title: 'Consumer migration note for infospace-bench: replace plan_generation_summary cost+token math with llm-connect calls'
priority: medium priority: medium
status: done status: done
@ -316,7 +316,7 @@ Out of scope:
## Consumer-side follow-up ## Consumer-side follow-up
Once T01-T04 land in llm-connect, `infospace-bench` opens a thin Once `LLM-WP-0005-T01` through `LLM-WP-0005-T04` land in llm-connect, `infospace-bench` opens a thin
companion workplan to: companion workplan to:
- Replace `_CALLS_PER_CHUNK_BY_WORKFLOW` + `_profile_template_words` + - Replace `_CALLS_PER_CHUNK_BY_WORKFLOW` + `_profile_template_words` +