From 33da93cd6689ab8ddd524e63540c5fc0927ddf42 Mon Sep 17 00:00:00 2001 From: tegwick Date: Mon, 14 Sep 2026 02:38:37 +0200 Subject: [PATCH] Prepare no-network trial plans for native recipient review Assistant: codex Assistant-Model: gpt-6-astra Assistant-Session: 01a09cbc-910e-77a0-b336-3130194350b3 --- .custodian-brief.md | 5 + docs/data-contracts.md | 16 + docs/evidence/2026-09-14-trial-full-plan.json | 499 ++++++++++++++++++ docs/evidence/2026-09-14-trial-inputs.json | 27 + .../evidence/2026-09-14-trial-pilot-plan.json | 67 +++ intelligence_radar/__main__.py | 6 +- intelligence_radar/trials.py | 50 +- tests/test_trials.py | 32 +- ...evidence-coverage-and-field-calibration.md | 11 + 9 files changed, 709 insertions(+), 4 deletions(-) create mode 100644 docs/evidence/2026-09-14-trial-full-plan.json create mode 100644 docs/evidence/2026-09-14-trial-inputs.json create mode 100644 docs/evidence/2026-09-14-trial-pilot-plan.json diff --git a/.custodian-brief.md b/.custodian-brief.md index ed5aa8f..e85aa6a 100644 --- a/.custodian-brief.md +++ b/.custodian-brief.md @@ -28,3 +28,8 @@ reserved pending reconciliation. Credential delivery blocks T02; T01/T03 continu 2026-09-14: Existing Bao key confirmed by user. Access is blocked by the known WARDEN-WP-0039-T03 caller/owner contract, not a request for a replacement key. Native repair packet and dependencies: docs/evidence/2026-09-14-openrouter-access.md. + +Latest: FLEX-WP-0026 supplies the native contract; SECRETS-WP-0010-T03 owns +remaining real approval/delivery. Radar `trials --plan` prepares exact pilot/full +review inputs without credentials or spending. 35 tests pass. See +docs/evidence/2026-09-14-trial-inputs.json and the adjacent trial plans. diff --git a/docs/data-contracts.md b/docs/data-contracts.md index 52475c2..3038473 100644 --- a/docs/data-contracts.md +++ b/docs/data-contracts.md @@ -136,3 +136,19 @@ Reconciliation is currently manual and must preserve old events. Never delete reservations or assume a timeout was free. Controlled work samples and close variants are not representative production outcomes; they are not automatically added to advisor calibration. + +### Review trials before admission + +Add `--plan` to the trial command to render config/suite identities, exact run +IDs, request hashes and campaign accounting. It requires no key and makes no +network calls or ledger writes. Settled runs contribute actual recorded cost; +pending reservations are counted once; only unseen runs add new reservations. +Budget blockers are explicit. The plan always states `execution_authorized: false`: +it neither proves current provider prices/access nor grants native delivery. + +`docs/evidence/2026-09-14-trial-{pilot,full}-plan.json` and +`2026-09-14-trial-inputs.json` are review snapshots. The latter hashes raw source +and input bytes; plan config/suite hashes use canonical JSON. The owner must +install and admit a fixed recipient separately, then rerender against current +billing before execution. Mutable ledger state and provider availability may +change after planning. diff --git a/docs/evidence/2026-09-14-trial-full-plan.json b/docs/evidence/2026-09-14-trial-full-plan.json new file mode 100644 index 0000000..230ca45 --- /dev/null +++ b/docs/evidence/2026-09-14-trial-full-plan.json @@ -0,0 +1,499 @@ +{ + "schema_version": 1, + "campaign": "ir-openrouter-pilot-20260913", + "pilot": false, + "execution_authorized": false, + "scope": "review inputs only; requires admitted recipient, credential delivery and billing reconciliation", + "config_sha256": "6cd62c6f05f08e2be09af34b033c1edee1a24f94c4f81b524921eb12d2276582", + "suite_sha256": "addd8a8d76b3a0f31fc1e53a1e49d38fcea9a0e5498de802df0b88b96ea26174", + "budget_usd": "10.00", + "spent_usd": "0", + "pending_usd": "0.023712", + "new_reservations_usd": "0.6961580", + "planned_committed_usd": "0.7198700", + "budget_blockers": [ + "unresolved billing reservations" + ], + "trials": [ + { + "case_id": "provenance-1", + "model": "anthropic/claude-sonnet-5", + "run_id": "29a46a1d6c2d9ef6cfce8de6efef75cddcfecee346c1b4babf57119f1a241d71", + "status": "reserved", + "reserved_usd": "0.023712", + "request_sha256": "b35286b8a60cf274694cfa3eb8d48a5ba095b19e89d7283d90b91fd1edb0e734" + }, + { + "case_id": "provenance-1", + "model": "anthropic/claude-haiku-4.5", + "run_id": "38696b9176e0e9c1350cfcf5230d0ba9585d5b281c8ea3f37b19c711872a3c01", + "status": "new", + "reserved_usd": "0.011856", + "request_sha256": "d61b363473018a88baea632c3ea8fd4780800e818fa095ffb711295cd0ffa465" + }, + { + "case_id": "provenance-1", + "model": "deepseek/deepseek-v4-flash", + "run_id": "13d2f7aa2eccbd6396e3b210471676e2bac4535972759a8de516d8537c67e685", + "status": "new", + "reserved_usd": "0.0005712", + "request_sha256": "43a301e8c02e0c4dc253ea3cc307f290f8dfa7b482ef75220e786144b987561d" + }, + { + "case_id": "provenance-2", + "model": "anthropic/claude-sonnet-5", + "run_id": "fe2d8ff3793319fdf4e3e68a8326db059d520357f89592b3a854cca89aa18051", + "status": "new", + "reserved_usd": "0.023712", + "request_sha256": "b8509d80ea547058fe0c1c053dc63ea17eedb753be661b4456df3b8377719d63" + }, + { + "case_id": "provenance-2", + "model": "anthropic/claude-haiku-4.5", + "run_id": "81b4e62b98aaa8aac4d924044bdb20c37cf66367ad24d20274f543b2fdd2cdf0", + "status": "new", + "reserved_usd": "0.011856", + "request_sha256": "0faeba8bbba3ac683e5e3dd61711bcf27928a18ba928228c686b286e8b50eda6" + }, + { + "case_id": "provenance-2", + "model": "deepseek/deepseek-v4-flash", + "run_id": "0f0dbc7620cc6e8e24cc7a13edb541cc944d1ef0b651d8448a63d19af668e099", + "status": "new", + "reserved_usd": "0.0005712", + "request_sha256": "3cd58175178c33d4f52e88f8d710bc2361ed2630db4eb58818fe0776cb3f43da" + }, + { + "case_id": "provenance-3", + "model": "anthropic/claude-sonnet-5", + "run_id": "58a1afd9e00854b5fd62ebdd8c35e5878f4234791e685f1c7a7e22bda676f915", + "status": "new", + "reserved_usd": "0.023712", + "request_sha256": "fc2381e858842f5c15d639e8d8b7e3d4937002a6c02dc77a6c0e3a3bc166770c" + }, + { + "case_id": "provenance-3", + "model": "anthropic/claude-haiku-4.5", + "run_id": "1de340ef91e3a0b1c27b4f968996734403e0eb88634825e759cb177c0765efe6", + "status": "new", + "reserved_usd": "0.011856", + "request_sha256": "733d4c4efb527f068708742f75730f6b63cbe6518767effe0391579f93dc248d" + }, + { + "case_id": "provenance-3", + "model": "deepseek/deepseek-v4-flash", + "run_id": "4ecc90936455a07d5dc2e3c94449bbee8d4cfb3142fa2037aed08d678a116583", + "status": "new", + "reserved_usd": "0.0005712", + "request_sha256": "2d0f3a0ab9dd902cd192046a7b1dbbe0e3d23e3ccec6f8281ef3ed2e226caf28" + }, + { + "case_id": "provenance-4", + "model": "anthropic/claude-sonnet-5", + "run_id": "e0c64ca21121f96c52422ec3870e42f668a6acc1b6053bade415be977086a68e", + "status": "new", + "reserved_usd": "0.023712", + "request_sha256": "467a224d79ad6a24975aae1d617f8e19c537d4cd4d7b57a553de0b1ab26440dc" + }, + { + "case_id": "provenance-4", + "model": "anthropic/claude-haiku-4.5", + "run_id": "fb4997541e890c135d4700e713867faac5b217a0d55aa8072734047fbebb8882", + "status": "new", + "reserved_usd": "0.011856", + "request_sha256": "8d6370ae9995457a7c5478e96a92412ec92fe57f443fc81ab4bab17d9e9fdb58" + }, + { + "case_id": "provenance-4", + "model": "deepseek/deepseek-v4-flash", + "run_id": "cbe6002b877624c6bba198bdf6bdb7134139d7d0c94b8f1dda5240f905b3eb20", + "status": "new", + "reserved_usd": "0.0005712", + "request_sha256": "76c18a7050625a702f9e62cf52cc41a0300ad4f5c7732cbc3cd484179ec08946" + }, + { + "case_id": "cost-1", + "model": "anthropic/claude-sonnet-5", + "run_id": "846bfcc1b25604707ac491a3080342b10818e2daf1ff80913dee2680b0342ca8", + "status": "new", + "reserved_usd": "0.023368", + "request_sha256": "d9af7430636fcb84665f3e1990e69d3ed3b4d35cf502c761f8229446be960e30" + }, + { + "case_id": "cost-1", + "model": "anthropic/claude-haiku-4.5", + "run_id": "261816dca40d7432b27cd1583522153f5cfd75c7d44b916c3388465f4c9c9dcd", + "status": "new", + "reserved_usd": "0.011684", + "request_sha256": "94af714b2eb37090877ba813f489bb154bb54e84c015175f4878ac40f2a28d32" + }, + { + "case_id": "cost-1", + "model": "deepseek/deepseek-v4-flash", + "run_id": "9f798c75ca4252b5d05b36be89d7c333efa40a55a49a0c4b47e08f4893a52db0", + "status": "new", + "reserved_usd": "0.000554", + "request_sha256": "760adc1737ae17f01b7db340535a38b7e1af1fa335aa3a96e9aefab40500918e" + }, + { + "case_id": "cost-2", + "model": "anthropic/claude-sonnet-5", + "run_id": "2aef4c907e19a43e977fdc1b25d5ce92257093b16f992dbb67ccbadd698351cc", + "status": "new", + "reserved_usd": "0.023368", + "request_sha256": "04f2af26ea168333457eee66f22bcccce462b5281e9f656bd1bf6db8cba310bb" + }, + { + "case_id": "cost-2", + "model": "anthropic/claude-haiku-4.5", + "run_id": "e4e4e060b1153a3458051b1f98e55e246c0d7143d4a6818943f644fa2f130a86", + "status": "new", + "reserved_usd": "0.011684", + "request_sha256": "8e1249c510efe49d30723f9c66f1971e4923718f37cf4c5a0e2c79206026b493" + }, + { + "case_id": "cost-2", + "model": "deepseek/deepseek-v4-flash", + "run_id": "7e403094285cb8b5904932e19c8b204780370f57d160ddf79a2293d2a920e364", + "status": "new", + "reserved_usd": "0.000554", + "request_sha256": "fdb9e8f0be7e2697e87e938ee5cddb162da989e7e6a1d7e88ce7ab5ff34813c5" + }, + { + "case_id": "cost-3", + "model": "anthropic/claude-sonnet-5", + "run_id": "03c75143d3ffaad2c2fc3ea7dd605c335c9b1364c26cc0ded1dca1bab6478c82", + "status": "new", + "reserved_usd": "0.023368", + "request_sha256": "029f4049a7e81e5247bf9a84a094db9b4c99ce189905a0c41d1bff529e7e5d29" + }, + { + "case_id": "cost-3", + "model": "anthropic/claude-haiku-4.5", + "run_id": "2ed8938a1348613fb194a7bf932b55ecc917bf01c66a0eba6916dcb40b65fa1c", + "status": "new", + "reserved_usd": "0.011684", + "request_sha256": "2fd0cad1100ee5541cde0f1b73d30444b03e2df952a21d2cbb0000dd1da35d77" + }, + { + "case_id": "cost-3", + "model": "deepseek/deepseek-v4-flash", + "run_id": "b8953942256f8f6990321f5a39e38ee2ab1e33b89414717a021052e7db00c6aa", + "status": "new", + "reserved_usd": "0.000554", + "request_sha256": "44ef690e9b2df10f15cb22b4d919605743b73a302e22697e69f80a4c17ea071f" + }, + { + "case_id": "cost-4", + "model": "anthropic/claude-sonnet-5", + "run_id": "bb9692c71a153160053a23fda70017a478dae6e8ee0c51f49337e11557165d3b", + "status": "new", + "reserved_usd": "0.023368", + "request_sha256": "4339370950f56bb473a1ec0431a8592b879315eed6c5cbe512071223c2de409f" + }, + { + "case_id": "cost-4", + "model": "anthropic/claude-haiku-4.5", + "run_id": "01e94c7e8aed5d56bf28653e3c5a7bf5cfa44a68e871446eef6c85fec8f126e9", + "status": "new", + "reserved_usd": "0.011684", + "request_sha256": "650c98cc749bbd88ba3a5bc096514957ddc8e0fc6fdb429f19751acd7567c7bc" + }, + { + "case_id": "cost-4", + "model": "deepseek/deepseek-v4-flash", + "run_id": "261090095007f0dd20892f583a814e19a5b0c55f0d8b3f38ff82a9d907d7910c", + "status": "new", + "reserved_usd": "0.000554", + "request_sha256": "e1428f17d12e3348404aa21ef0159d5c53eac949c620aaa8b6530c361f6dd14b" + }, + { + "case_id": "eligibility-1", + "model": "anthropic/claude-sonnet-5", + "run_id": "a21f57ee91ad4fd87a206a1dc1461920ed3627c42ee162ee1464e1d5179f85d7", + "status": "new", + "reserved_usd": "0.023692", + "request_sha256": "0dd1352a5aa6388e74588d10b76411167332b3988606d1aa17cde6c281b1782b" + }, + { + "case_id": "eligibility-1", + "model": "anthropic/claude-haiku-4.5", + "run_id": "f2f96773fd8d20ba245161cc95daa3eef59e31813ec4d398a852970b59c2b615", + "status": "new", + "reserved_usd": "0.011846", + "request_sha256": "2d36ee7067ff4035469c39970e907e7d3f4ce13d3e7a4cdcf795630e7f900e23" + }, + { + "case_id": "eligibility-1", + "model": "deepseek/deepseek-v4-flash", + "run_id": "ed59296ebff47f502905c25cd618c77498e3ce888c4d2d83d6b4d7d2be28de67", + "status": "new", + "reserved_usd": "0.0005702", + "request_sha256": "f083282efed68529ae57ad803d352f4a83dd4110e0f3aa70856da08da93d0231" + }, + { + "case_id": "eligibility-2", + "model": "anthropic/claude-sonnet-5", + "run_id": "79138be1d6febe7b5b51a19ce72189912fee1c3455fe3aa14b004b1abb3055b8", + "status": "new", + "reserved_usd": "0.023692", + "request_sha256": "ce38af84406984763279d9c0a51f41596fc3b544f2164d025c4dec70477bc25a" + }, + { + "case_id": "eligibility-2", + "model": "anthropic/claude-haiku-4.5", + "run_id": "a800b8d5fda5868cd8282c2b8855c3632e45815916e83f07468d287ee30c75d7", + "status": "new", + "reserved_usd": "0.011846", + "request_sha256": "dd6ac1ae4112a5841cf887ff372a81ef56aebe940f10a1fd628be9c9b48fa4fa" + }, + { + "case_id": "eligibility-2", + "model": "deepseek/deepseek-v4-flash", + "run_id": "2a94122cb6c69d829f0fe56f6f0980f18b4af975fbc700aac07e97d958fc7588", + "status": "new", + "reserved_usd": "0.0005702", + "request_sha256": "4e0d9171a3eab7c7c008ceb257c82ac29ec29be83cd9945d2856e55ebdfdb3cf" + }, + { + "case_id": "eligibility-3", + "model": "anthropic/claude-sonnet-5", + "run_id": "67de1a770fb5c3e69637cbb7b0e51e0a1cd223963b5e912a1a6cd9b43242eb05", + "status": "new", + "reserved_usd": "0.023692", + "request_sha256": "edfbc0e96c430abc09fada2f6a7610b37cd03ba499202b9a891a6465088471cd" + }, + { + "case_id": "eligibility-3", + "model": "anthropic/claude-haiku-4.5", + "run_id": "8d5eecd0f948912f515c54fb672ffd88cf5907194e23d4292d2a51231182a4bf", + "status": "new", + "reserved_usd": "0.011846", + "request_sha256": "5689ed57728a65c51570d363de9545c8ecf2cded832bf0a681b5106f61819ee7" + }, + { + "case_id": "eligibility-3", + "model": "deepseek/deepseek-v4-flash", + "run_id": "27186c09f8bd32be57c28e5003356070666499b6adac7b11b7c147d53f5fa4ba", + "status": "new", + "reserved_usd": "0.0005702", + "request_sha256": "92fac158ad4182e4f201783a8a3558c55657dd2d19f4cd47baaad19327b0ba77" + }, + { + "case_id": "eligibility-4", + "model": "anthropic/claude-sonnet-5", + "run_id": "cb44444bf90f2c72374fab34e115697c2244176ff8e06d1534f3da98837559c0", + "status": "new", + "reserved_usd": "0.023692", + "request_sha256": "3d63811d7c72806d0bb206e9166c8318b8f162189617bcfcd9c0e9216e1b6c06" + }, + { + "case_id": "eligibility-4", + "model": "anthropic/claude-haiku-4.5", + "run_id": "29524213eac29748f16dc03948b30925057789a0d7af120d3d4eea09a8b24a64", + "status": "new", + "reserved_usd": "0.011846", + "request_sha256": "63ee1db0bb4448c490897b5bcc462d11f8ff9be2174e0fcdf68189a9721f8058" + }, + { + "case_id": "eligibility-4", + "model": "deepseek/deepseek-v4-flash", + "run_id": "1c25053df035bc63b04033e77d61f8da45711142861debb0ca06bf85bb432f9f", + "status": "new", + "reserved_usd": "0.0005702", + "request_sha256": "3c5f5ec0d2e5bed578f9b3b8da3fa48334077c1bbacc9b7f0c6b35c1eddcfa5f" + }, + { + "case_id": "comparability-1", + "model": "anthropic/claude-sonnet-5", + "run_id": "64b6bc5641a7157e2533ff72c2b9750c37d8d12ade876607b5cbede931d953ce", + "status": "new", + "reserved_usd": "0.02406", + "request_sha256": "c12e7b14a4725082d94f3c304b1b374b078dff96e31b38d8a32d53057a9e6bfa" + }, + { + "case_id": "comparability-1", + "model": "anthropic/claude-haiku-4.5", + "run_id": "cff1c435afc69df6649c0de938c96648d1c81c08c667b77be484d579536d7d3a", + "status": "new", + "reserved_usd": "0.01203", + "request_sha256": "eefd247a5bab6793009b78e71a6c39f30ac99c36390ca09c73bc40f07f2e7a62" + }, + { + "case_id": "comparability-1", + "model": "deepseek/deepseek-v4-flash", + "run_id": "9f3d77846fee2afb0bc102efbd8050b7b6386e24e01699ddce3e5faaa7fc7d92", + "status": "new", + "reserved_usd": "0.0005886", + "request_sha256": "68724468ec3e2254964c4c274a9cbca9fc80767c243381ba276192c248acb9f4" + }, + { + "case_id": "comparability-2", + "model": "anthropic/claude-sonnet-5", + "run_id": "cbf5d1305c46c7092b5dc62430723c03db161accd82fd72d22d95c8e04534ef8", + "status": "new", + "reserved_usd": "0.02406", + "request_sha256": "3360d29f253bacd7cc4acf6c630e5796595edbbcb6b81fda44891736d12ad161" + }, + { + "case_id": "comparability-2", + "model": "anthropic/claude-haiku-4.5", + "run_id": "445b1abdf1dd6c2c8f04c54c9dc760163fe39d9c7b9255cb85d81dd3c8700cce", + "status": "new", + "reserved_usd": "0.01203", + "request_sha256": "96570e480894d8c436fa78c313f1b2a29fb13b44040192e5c9b28f5bb9ff5d2d" + }, + { + "case_id": "comparability-2", + "model": "deepseek/deepseek-v4-flash", + "run_id": "fc665542c2305a3c6b59d24f61648b2752855b0e177b498d3f776590fb6f339f", + "status": "new", + "reserved_usd": "0.0005886", + "request_sha256": "dede416a3316ba6ee1d05d7901514d88ced382af89e957fd322decc4237f3996" + }, + { + "case_id": "comparability-3", + "model": "anthropic/claude-sonnet-5", + "run_id": "cccb9b82c6ec1df25ee1d07e4907b20f2ac0cddc73d06d10a75eb057d8156cea", + "status": "new", + "reserved_usd": "0.02406", + "request_sha256": "d829875846a6c484096792e77a13b99be722349bac088b5e43fa6294f950e93b" + }, + { + "case_id": "comparability-3", + "model": "anthropic/claude-haiku-4.5", + "run_id": "d300af434588838acbe495db2023cf98b91190204b0da1984338db9ab353a9b5", + "status": "new", + "reserved_usd": "0.01203", + "request_sha256": "01ad93d173e19ed04c7192b3ab59a90e97d4e990c8d498e1e55f4e63c791aaad" + }, + { + "case_id": "comparability-3", + "model": "deepseek/deepseek-v4-flash", + "run_id": "c708e5dc1bd60b3a13f2aa0687ae4d7f5014add874ee189bdcf9ae8f61a2ec9e", + "status": "new", + "reserved_usd": "0.0005886", + "request_sha256": "2bb12d3ec9fbb287d424795830753b897e0fcf9bbaac87efb6044883c3d866b4" + }, + { + "case_id": "comparability-4", + "model": "anthropic/claude-sonnet-5", + "run_id": "75625c4e7fff465c8914461407672503cccbd5d03b47d799a31dd83dfa9051cd", + "status": "new", + "reserved_usd": "0.02406", + "request_sha256": "7723903ab2361fe9dab2d50d3eb86c091dd4f663a8053ba8da5a9e2ebb8fc32c" + }, + { + "case_id": "comparability-4", + "model": "anthropic/claude-haiku-4.5", + "run_id": "eafafec40c8458eaf8776f7040297d6eba3823e6360f39fbe497e1d18e4b4c71", + "status": "new", + "reserved_usd": "0.01203", + "request_sha256": "f28dae05e06472085223522955755e967df1729b871f3fc0be8ef3b00f5808dc" + }, + { + "case_id": "comparability-4", + "model": "deepseek/deepseek-v4-flash", + "run_id": "9416532af6fdb1061bc5b676d3ac6cbc33ddbf1eb57c5f66c90a3884f99842db", + "status": "new", + "reserved_usd": "0.0005886", + "request_sha256": "d0544462f04dacef1d623c01bc378736f27b43577c67dcebfb48168e32659fbc" + }, + { + "case_id": "contradiction-1", + "model": "anthropic/claude-sonnet-5", + "run_id": "05ae10aa173fba2a78b714aaf21e7224e495b6e130616d87845e2811a3b85515", + "status": "new", + "reserved_usd": "0.023258", + "request_sha256": "75f4b6e2e3e5122e0d106d58d59771788079996bc997d62bc250fe0722723057" + }, + { + "case_id": "contradiction-1", + "model": "anthropic/claude-haiku-4.5", + "run_id": "cab8495ad913f7da5adf3fb00bf1cd379a8149b233e3a1357a63ba94f7e22443", + "status": "new", + "reserved_usd": "0.011629", + "request_sha256": "1436aa4d7b2b4665d8e38ab082fd3c558b4f5997b29dd6c49237d10302dd08e6" + }, + { + "case_id": "contradiction-1", + "model": "deepseek/deepseek-v4-flash", + "run_id": "19b3f794d4d64edc2357ce163ee939fe95239df8345570707bfc1ad06ff3d44e", + "status": "new", + "reserved_usd": "0.0005485", + "request_sha256": "70c6ea5265db3f679ea8a3e5d3ca51bb16bf2d46b36044e7206aee128204b20e" + }, + { + "case_id": "contradiction-2", + "model": "anthropic/claude-sonnet-5", + "run_id": "e696043142012027396636ac5ad66a63af7b63a5b3606fd6bff49e37c26f8646", + "status": "new", + "reserved_usd": "0.023258", + "request_sha256": "9d30d51891817f28899d1d83a92297dbd1a8e0211128db490e025c82d4733b50" + }, + { + "case_id": "contradiction-2", + "model": "anthropic/claude-haiku-4.5", + "run_id": "746338999121ea05fd7e8aa7f1822c0709ed4af8eb75e7ee65227aaf5b1933f0", + "status": "new", + "reserved_usd": "0.011629", + "request_sha256": "673430ab6cf1faee33fe97d9025467497fdf7f40bba3f546f3785c7603c279c2" + }, + { + "case_id": "contradiction-2", + "model": "deepseek/deepseek-v4-flash", + "run_id": "7ab82f8989b7ccf4e36a054ebc9307d53170bffcc54475ada0618a8b131800d5", + "status": "new", + "reserved_usd": "0.0005485", + "request_sha256": "bbffdd81469f8625b88a56cf9fa82417d72a2ea279a1ca67945afbb34fd36e47" + }, + { + "case_id": "contradiction-3", + "model": "anthropic/claude-sonnet-5", + "run_id": "2af0946e4c4dd6671a6bcb82435dd0fe6c5b8c683ea84d876aca614ff8dea52c", + "status": "new", + "reserved_usd": "0.023258", + "request_sha256": "efd1224ac30dae379f655e9c0390329620f696bfb703fac6850a077b5810cad6" + }, + { + "case_id": "contradiction-3", + "model": "anthropic/claude-haiku-4.5", + "run_id": "39f2551334276ac65dce1f79d27380a0b7d2781a049e53ba68fc07e3e391e0b7", + "status": "new", + "reserved_usd": "0.011629", + "request_sha256": "322e9ec4d3afddeb1a35d47900f4c26da9b6efbc1b538580c50a7e2a493f97af" + }, + { + "case_id": "contradiction-3", + "model": "deepseek/deepseek-v4-flash", + "run_id": "abe63efa9747fa9aba8d4658cd0ccd4edd1076f69d9c697101afcd14ccba89a3", + "status": "new", + "reserved_usd": "0.0005485", + "request_sha256": "ab428d0a9cf0c0aa49882c6e479425d505eee1339cc26f16838ac2b278ae6786" + }, + { + "case_id": "contradiction-4", + "model": "anthropic/claude-sonnet-5", + "run_id": "7878893e437a7bf66d46e40cf2bdeae0c0fffcbc947a656e35bccbb3274cef29", + "status": "new", + "reserved_usd": "0.023258", + "request_sha256": "1ddece5f21aed43e46d25d37df5f49356b45ff5fc6bb4ca7e4cea23062e93899" + }, + { + "case_id": "contradiction-4", + "model": "anthropic/claude-haiku-4.5", + "run_id": "7e2a32653c6b13c6cdb40058a61196681ccfb66704a7b4f53d85072d787cc267", + "status": "new", + "reserved_usd": "0.011629", + "request_sha256": "3ea826a54363ebf96e3d6b285a04c7cf86e9a83b93c309eb80da48ae57f74767" + }, + { + "case_id": "contradiction-4", + "model": "deepseek/deepseek-v4-flash", + "run_id": "b71a830ec8c59a1bdbdb4d784d4f429b15bff304737c4611c099ed65458f7bf5", + "status": "new", + "reserved_usd": "0.0005485", + "request_sha256": "cb049de92154ddefe6452bfa72ede7cb236155b73834f0947cb04a1686dd1f36" + } + ] +} diff --git a/docs/evidence/2026-09-14-trial-inputs.json b/docs/evidence/2026-09-14-trial-inputs.json new file mode 100644 index 0000000..b0497c9 --- /dev/null +++ b/docs/evidence/2026-09-14-trial-inputs.json @@ -0,0 +1,27 @@ +{ + "status": "review input; not installed recipient admission", + "files": { + "intelligence_radar/__init__.py": "8075fe2809bbce1921ca24d79b7680e5d499f926b1f68f77d1bdf345932c022d", + "intelligence_radar/__main__.py": "9b9e3f9ab99cd21925231b742b04cc3c62781b8edb70cee470c2ed32ea720d0a", + "intelligence_radar/advisor.py": "e853ba0c4859d7927549083be4508648230e78fd190ca2595ca865e72810918b", + "intelligence_radar/bundles.py": "fd651b568410e36c875511ee7e75edbfc9ff36b29991cfe57138232c3f692e39", + "intelligence_radar/ingest.py": "a4e361286cbf93e2268b6506dab16dc092baa8025e2d82f5578250693f98ba78", + "intelligence_radar/scheduler.py": "95413d5c7b8bbe9e4288fac18eb7ed1f80e3d855bba0c3ca33dbc6934172f7c4", + "intelligence_radar/store.py": "87746bc130de7ed2b81b091793ca292a1d6c2a65f7cef68849d0ac9152e8d382", + "intelligence_radar/trials.py": "f19f63f34963eae54ec3f42f86ed07c1dcb4e9ef28259df9d6030d6c8dad929f", + "ops/openrouter-pilot.json": "c7690151ec95c0f1a448a0e7db43ee7db3175511bd5d0803110e440fc4a80a6d", + "examples/work-sample-suite.json": "d96a445d3485d558af49c639cbc1811bbfb2955e0074f4747d74f79dac0d8b66" + }, + "proposed_pilot_argv": [ + "python3", + "-m", + "intelligence_radar", + "trials", + "--config", + "ops/openrouter-pilot.json", + "--suite", + "examples/work-sample-suite.json", + "--pilot" + ], + "installation_requirements": "Owner must install and pin trusted interpreter, all imported code, config/suite, exact argv, private cwd and fixed environment. Rolling checkout paths are not an admitted recipient. Revalidate model availability and price caps before paid execution." +} diff --git a/docs/evidence/2026-09-14-trial-pilot-plan.json b/docs/evidence/2026-09-14-trial-pilot-plan.json new file mode 100644 index 0000000..fa69a77 --- /dev/null +++ b/docs/evidence/2026-09-14-trial-pilot-plan.json @@ -0,0 +1,67 @@ +{ + "schema_version": 1, + "campaign": "ir-openrouter-pilot-20260913", + "pilot": true, + "execution_authorized": false, + "scope": "review inputs only; requires admitted recipient, credential delivery and billing reconciliation", + "config_sha256": "6cd62c6f05f08e2be09af34b033c1edee1a24f94c4f81b524921eb12d2276582", + "suite_sha256": "addd8a8d76b3a0f31fc1e53a1e49d38fcea9a0e5498de802df0b88b96ea26174", + "budget_usd": "10.00", + "spent_usd": "0", + "pending_usd": "0.023712", + "new_reservations_usd": "0.0485664", + "planned_committed_usd": "0.0722784", + "budget_blockers": [ + "unresolved billing reservations" + ], + "trials": [ + { + "case_id": "provenance-1", + "model": "anthropic/claude-sonnet-5", + "run_id": "29a46a1d6c2d9ef6cfce8de6efef75cddcfecee346c1b4babf57119f1a241d71", + "status": "reserved", + "reserved_usd": "0.023712", + "request_sha256": "b35286b8a60cf274694cfa3eb8d48a5ba095b19e89d7283d90b91fd1edb0e734" + }, + { + "case_id": "provenance-1", + "model": "anthropic/claude-haiku-4.5", + "run_id": "38696b9176e0e9c1350cfcf5230d0ba9585d5b281c8ea3f37b19c711872a3c01", + "status": "new", + "reserved_usd": "0.011856", + "request_sha256": "d61b363473018a88baea632c3ea8fd4780800e818fa095ffb711295cd0ffa465" + }, + { + "case_id": "provenance-1", + "model": "deepseek/deepseek-v4-flash", + "run_id": "13d2f7aa2eccbd6396e3b210471676e2bac4535972759a8de516d8537c67e685", + "status": "new", + "reserved_usd": "0.0005712", + "request_sha256": "43a301e8c02e0c4dc253ea3cc307f290f8dfa7b482ef75220e786144b987561d" + }, + { + "case_id": "provenance-2", + "model": "anthropic/claude-sonnet-5", + "run_id": "fe2d8ff3793319fdf4e3e68a8326db059d520357f89592b3a854cca89aa18051", + "status": "new", + "reserved_usd": "0.023712", + "request_sha256": "b8509d80ea547058fe0c1c053dc63ea17eedb753be661b4456df3b8377719d63" + }, + { + "case_id": "provenance-2", + "model": "anthropic/claude-haiku-4.5", + "run_id": "81b4e62b98aaa8aac4d924044bdb20c37cf66367ad24d20274f543b2fdd2cdf0", + "status": "new", + "reserved_usd": "0.011856", + "request_sha256": "0faeba8bbba3ac683e5e3dd61711bcf27928a18ba928228c686b286e8b50eda6" + }, + { + "case_id": "provenance-2", + "model": "deepseek/deepseek-v4-flash", + "run_id": "0f0dbc7620cc6e8e24cc7a13edb541cc944d1ef0b651d8448a63d19af668e099", + "status": "new", + "reserved_usd": "0.0005712", + "request_sha256": "3cd58175178c33d4f52e88f8d710bc2361ed2630db4eb58818fe0776cb3f43da" + } + ] +} diff --git a/intelligence_radar/__main__.py b/intelligence_radar/__main__.py index 2f3ba4a..8daf235 100644 --- a/intelligence_radar/__main__.py +++ b/intelligence_radar/__main__.py @@ -184,11 +184,13 @@ def main(argv=None): trials_command.add_argument('--config', type=Path, required=True) trials_command.add_argument('--suite', type=Path, required=True) trials_command.add_argument('--pilot', action='store_true') + trials_command.add_argument('--plan', action='store_true', help='review request identities and budget without calls or writes') args = parser.parse_args(argv) try: if args.command == 'trials': - from .trials import run - print(json.dumps(run(args.root, json.loads(args.config.read_text()), + from .trials import plan, run + action = plan if args.plan else run + print(json.dumps(action(args.root, json.loads(args.config.read_text()), json.loads(args.suite.read_text()), args.pilot), indent=2)) elif args.command == 'export': from .bundles import export_bundle diff --git a/intelligence_radar/trials.py b/intelligence_radar/trials.py index ddef84b..912664f 100644 --- a/intelligence_radar/trials.py +++ b/intelligence_radar/trials.py @@ -54,6 +54,54 @@ def plan_trial(case, model): return payload,reserved +def trial_id(campaign, case, payload): + return hashlib.sha256(canonical({'campaign':campaign,'case':case,'payload':payload})).hexdigest() + + +def plan(root, config, suite, pilot=False): + """Render review inputs without credentials, network calls or ledger writes.""" + campaign = config['campaign'] + limit = money(config['budget_usd']) + for label, items in [('case', suite['cases']), ('model', config['models'])]: + ids = [item['id'] for item in items] + if not ids or len(ids) != len(set(ids)): + raise ValueError(f'{label} IDs must be nonempty and unique') + events = [(i, e) for i, e in all_records(root, 'trial-events') if e['campaign'] == campaign] + held = {i: e for i, e in events if e['kind'] == 'reserved'} + settled = {e['reservation_id']: e for _, e in events if e['kind'] == 'settled'} + spent = sum((money(e['actual_usd']) for e in settled.values()), Decimal(0)) + pending = sum((money(e['reserved_usd']) for i, e in held.items() if i not in settled), Decimal(0)) + existing = {e['run_id']: 'settled' if i in settled else 'reserved' for i, e in held.items()} + rows = [] + new_total = Decimal(0) + for case in suite['cases'][:2] if pilot else suite['cases']: + for model in config['models']: + payload, reserved = plan_trial(case, model) + ident = trial_id(campaign, case, payload) + status = existing.get(ident, 'new') + if status == 'new': + new_total += reserved + rows.append({'case_id': case['id'], 'model': model['id'], 'run_id': ident, + 'status': status, 'reserved_usd': str(reserved), + 'request_sha256': hashlib.sha256(canonical(payload)).hexdigest()}) + blockers = [] + if pending: + blockers.append('unresolved billing reservations') + if any(money(e['actual_usd']) > money(held[i]['reserved_usd']) for i, e in settled.items()): + blockers.append('campaign halted after provider cost exceeded reservation') + total = spent + pending + new_total + if total > limit: + blockers.append('planned reservations exceed campaign budget') + return {'schema_version': 1, 'campaign': campaign, 'pilot': pilot, + 'execution_authorized': False, + 'scope': 'review inputs only; requires admitted recipient, credential delivery and billing reconciliation', + 'config_sha256': hashlib.sha256(canonical(config)).hexdigest(), + 'suite_sha256': hashlib.sha256(canonical(suite)).hexdigest(), + 'budget_usd': str(limit), 'spent_usd': str(spent), 'pending_usd': str(pending), + 'new_reservations_usd': str(new_total), 'planned_committed_usd': str(total), + 'budget_blockers': blockers, 'trials': rows} + + def execute(root, config, case, model, opener=urlopen): root=Path(root) if not os.environ.get('OPENROUTER_API_KEY'): @@ -61,7 +109,7 @@ def execute(root, config, case, model, opener=urlopen): campaign=config['campaign'] limit=money(config['budget_usd']) payload,reserved=plan_trial(case,model) - run_id=hashlib.sha256(canonical({'campaign':campaign,'case':case,'payload':payload})).hexdigest() + run_id=trial_id(campaign, case, payload) lockpath=root/'.runtime/trials.lock';lockpath.parent.mkdir(parents=True,exist_ok=True) with lockpath.open('a') as lock: fcntl.flock(lock,fcntl.LOCK_EX) diff --git a/tests/test_trials.py b/tests/test_trials.py index 035b76b..fad3944 100644 --- a/tests/test_trials.py +++ b/tests/test_trials.py @@ -6,7 +6,7 @@ import tempfile import unittest from unittest.mock import patch -from intelligence_radar.trials import budget, execute, verify +from intelligence_radar.trials import budget, execute, plan, verify class TrialTests(unittest.TestCase): @@ -68,3 +68,33 @@ class TrialTests(unittest.TestCase): with self.assertRaisesRegex(ValueError, 'campaign halted'): execute(self.root, self.config, case, self.model, opener=lambda *a, **k: self.fail('halted campaign made a call')) + + def test_plan_needs_no_key_and_writes_nothing(self): + config = dict(self.config, models=[self.model]) + with patch.dict(os.environ, {}, clear=True): + result = plan(self.root, config, {'cases': [self.case]}) + self.assertFalse(result['execution_authorized']) + self.assertEqual(result['budget_blockers'], []) + self.assertEqual(list(self.root.iterdir()), []) + executed = execute(self.root, config, self.case, self.model, opener=self.response) + self.assertEqual(result['trials'][0]['run_id'], executed['run_id']) + after = plan(self.root, config, {'cases': [self.case]}) + self.assertEqual(after['new_reservations_usd'], '0') + self.assertEqual(after['planned_committed_usd'], '0.001') + + def test_plan_counts_pending_once_and_exposes_blocker(self): + def fail(*args, **kwargs): + raise TimeoutError('timeout') + with self.assertRaises(TimeoutError): + execute(self.root, self.config, self.case, self.model, opener=fail) + result = plan(self.root, dict(self.config, models=[self.model]), {'cases': [self.case]}) + self.assertEqual(result['budget_blockers'], ['unresolved billing reservations']) + self.assertEqual(result['new_reservations_usd'], '0') + self.assertEqual(result['planned_committed_usd'], budget(self.root, 'test')['pending_usd']) + + def test_plan_refuses_duplicates_and_marks_over_budget(self): + config = dict(self.config, models=[self.model], budget_usd='0') + with self.assertRaisesRegex(ValueError, 'unique'): + plan(self.root, config, {'cases': [self.case, self.case]}) + result = plan(self.root, config, {'cases': [self.case]}) + self.assertIn('planned reservations exceed campaign budget', result['budget_blockers']) diff --git a/workplans/IR-WP-0004-evidence-coverage-and-field-calibration.md b/workplans/IR-WP-0004-evidence-coverage-and-field-calibration.md index d7427e3..ba5f76a 100644 --- a/workplans/IR-WP-0004-evidence-coverage-and-field-calibration.md +++ b/workplans/IR-WP-0004-evidence-coverage-and-field-calibration.md @@ -85,3 +85,14 @@ Keep draft creation separate from reviewed publication and any notifications. path, dependency and replay checks before publication. Default `make` lists targets. All 32 tests and repository/archive integrity checks pass. Curated publication, hosted workers and decomposition comparisons remain open. + +### Trial admission preparation — 2026-09-14 + +Owner work now lives in FLEX-WP-0026 and SECRETS-WP-0010-T03. The native +contract, current PDP, Approval Engine deployment and scoped reader proof have +progressed; real human approval and native OpenRouter delivery remain open. +Sent readiness follow-up to secrets-engine. Added a no-network `trials --plan` +with exact request identities, input hashes and campaign reservations; retained +pilot/full review snapshots and source/input hash manifest. These are review +inputs, not installation pins or delivery authorization. All 35 tests and +repository integrity checks pass. T02 remains wait on native delivery and billing.