Reserve worker spend durably before governed dispatch
Some checks failed
Governed runtime contract / contract (push) Has been cancelled
Some checks failed
Governed runtime contract / contract (push) Has been cancelled
Assistant: codex Assistant-Model: gpt-5.6-luna Assistant-Session: 01a07ff8-19d0-7820-b4d0-1353833cb7fc
This commit is contained in:
parent
4ffb8acb19
commit
38b25fbc26
14 changed files with 1378 additions and 9 deletions
100
docs/evidence/2026-09-09-spend-admission.json
Normal file
100
docs/evidence/2026-09-09-spend-admission.json
Normal file
|
|
@ -0,0 +1,100 @@
|
|||
{
|
||||
"schema": "rein-spend-admission-evidence/v1",
|
||||
"observed_at": "2026-09-09T14:37:52.167561+00:00",
|
||||
"workplan": "REINAH-WP-0003",
|
||||
"task": "REINAH-WP-0003-T05",
|
||||
"source_base": "4ffb8acb190df235fc7d1ba20e2643e33f596286",
|
||||
"source_files_sha256": {
|
||||
"rein_aharness/spend_admission.py": "334a716da6a0900bca4fdb1fd0833c2446efe08ffb59ae9ee17ce85558c67b81",
|
||||
"rein_aharness/claim_loop.py": "9fc1796d9dc822f2dd93f7c3edf4fc163bc2b0e502d4c95664db741bc9408180",
|
||||
"rein_aharness/glas_execution.py": "4dc90e919de113f2f7469a747062a9ba58d50be33611c4ba2bad08da8fb894d8",
|
||||
"rein_aharness/ops_run_client.py": "ac9263fcac5fa7a007819987b2af11338409811a3dba42a263e81c7efc7ac282",
|
||||
"rein_aharness/cli.py": "032cc133e1a7edfe4f5353cee6fe1aa55499ff526f47fc8c231e711fc39da597",
|
||||
"tests/test_spend_admission.py": "6b11b3ca782789f4f91d238b6646cde755117bed50713d63e9135a3be787e3e4",
|
||||
"tests/test_repository_artifact_bwrap.py": "39cc3e19f46c07e4178387cdcf70c5b32c4a75e1923da75293d8028747d48453",
|
||||
"scripts/verify-runtime-contracts.sh": "5617882d62e749a8ef698f11091fd2513f87553e739f6033b1c76f43e16bb492",
|
||||
"scripts/verify-recovery-contracts.sh": "b1bf99e23f33a99b03b383b70c1312bb8b49f10b583f528b4b43a3347431e794",
|
||||
"docs/spend-admission.md": "d4814f79b1cb05446fd289a1f6a1237e66321d920c86204abf3ebc908d9fc36e",
|
||||
"docs/sandbox-artifact-return.md": "788c0d7b04f749a9aaa3f59db04887d82f5c40082b2c4e3d94ce8a1a7a8b0673",
|
||||
"deploy/systemd/claim-loop.env.example": "6cd1e8d62c59735f6f53a126af4326dc1ec15cb3409dfbacb2fe4cd76dbb4b26",
|
||||
"workplans/REINAH-WP-0003-governed-runtime-integrity.md": "6c8668866a7336ee4b698f15c7bf926d8c30eef51d689d496943756bb37564c8"
|
||||
},
|
||||
"validation": {
|
||||
"full_suite": {
|
||||
"passed": 330,
|
||||
"skipped": 0,
|
||||
"real_bwrap_enabled": true
|
||||
},
|
||||
"spend_and_actual_bwrap": {
|
||||
"passed": 41
|
||||
},
|
||||
"runtime_contract_gate": {
|
||||
"passed": 90
|
||||
},
|
||||
"recovery_contract_gate": {
|
||||
"passed": 52
|
||||
},
|
||||
"gate_warning": "pytest cache could not write in restricted owner checkout; test assertions passed",
|
||||
"overlap_note": "Targeted suites and gates overlap the full suite; counts are not additional independent executions."
|
||||
},
|
||||
"proof_scope": {
|
||||
"actual_bwrap_owner_glas_worker_artifact_import": true,
|
||||
"queue": "fixture",
|
||||
"authoring": "deterministic fixture",
|
||||
"accounting": "fixture",
|
||||
"model_requests": 0,
|
||||
"natural_factory_claims": 0,
|
||||
"deployment_changed": false,
|
||||
"factory_policy_provisioned": false,
|
||||
"dispatch_enabled": false
|
||||
},
|
||||
"reuse_discovery": {
|
||||
"url": "https://reuse.coulomb.social/v1/federated",
|
||||
"composed_at": "2026-09-09T12:18:07+00:00",
|
||||
"stale": false,
|
||||
"capabilities": 65,
|
||||
"sources": 61,
|
||||
"snapshot_sha256": "340c5b89a449bb334e6a4922710046011aaa9e302a78297bd2d61f9b8db345b9",
|
||||
"result": "No reservation capability found in hosted index; inspected llm-connect reporting/token control, Railiance Fabric models and agentic-resources session memory do not provide admission."
|
||||
},
|
||||
"implementation": "Private immutable-policy SQLite ledger before dispatch, conservative daily/total micro-euro reservations, full-charge success, durable unknown holds, replay denial, explicit receipt-backed reconciliation and sticky overrun freeze; existing Glas cached catalog, grants, repository transaction and close outbox reused.",
|
||||
"remaining": [
|
||||
"HFACT-WP-0001-T01: demonstrated provider maximum liability and conservative FX, finalized operating acceptance",
|
||||
"REINAH-WP-0003-T05/T06 and HFACT-WP-0001-T03/T04/T05: protected installation, exact identity/credential/egress/placement and natural queue/model proof"
|
||||
],
|
||||
"owner_snapshot": {
|
||||
"observed_at": "2026-09-09T13:13:20.768438+00:00",
|
||||
"tasks": {
|
||||
"secrets-replay": {
|
||||
"id": "3eb9cff8-1441-5437-9e92-a2b655c82d04",
|
||||
"status": "wait",
|
||||
"needs_human": false,
|
||||
"updated_at": "2026-09-09T07:31:58.416393Z"
|
||||
},
|
||||
"native-delivery": {
|
||||
"id": "f8069c8a-ad6b-5d0b-9a36-c2326699437d",
|
||||
"status": "wait",
|
||||
"needs_human": false,
|
||||
"updated_at": "2026-09-09T07:31:58.646322Z"
|
||||
},
|
||||
"client-side": {
|
||||
"id": "68bff751-e48b-548b-8fb4-dfb3b16210c2",
|
||||
"status": "wait",
|
||||
"needs_human": false,
|
||||
"updated_at": "2026-09-09T12:41:29.448507Z"
|
||||
},
|
||||
"audit-custody": {
|
||||
"id": "fd4a4ac3-e525-57c1-9179-e0fcd0226913",
|
||||
"status": "progress",
|
||||
"needs_human": false,
|
||||
"updated_at": "2026-09-08T18:38:29.290026Z"
|
||||
},
|
||||
"approval-deployment": {
|
||||
"id": "f0aa2e6d-19e6-5b43-886c-efa4e3de5f22",
|
||||
"status": "wait",
|
||||
"needs_human": false,
|
||||
"updated_at": "2026-09-09T12:38:47.065340Z"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -52,3 +52,7 @@ including response-lost close replay. Its queue and task authoring are fixtures;
|
|||
it proves neither a natural Activity Core claim nor model/provider admission.
|
||||
`tests/test_native_limits.py` exercises control propagation, terminal accounting,
|
||||
invalid/exhausted results and refusal of older CLI versions without inference.
|
||||
|
||||
The subsequent [durable spend admission](spend-admission.md) return implements
|
||||
private daily/total reservation and unknown-outcome recovery in the worker.
|
||||
Provider liability/FX proof and final operating admission remain open.
|
||||
|
|
|
|||
133
docs/spend-admission.md
Normal file
133
docs/spend-admission.md
Normal file
|
|
@ -0,0 +1,133 @@
|
|||
# Durable worker spend admission
|
||||
|
||||
The claim worker can reserve a declared maximum liability before invoking Glas.
|
||||
This is an opt-in control for a single host and one immutable operating envelope.
|
||||
It adds no service, provider calls or credentials. llm-connect remains the usage
|
||||
reporting owner; its reporting ledger and bundled FX snapshot are not admission
|
||||
inputs. Reuse discovery on 2026-09-09 found no reservation capability in the fresh
|
||||
hosted reuse-surface index (65 capabilities, 61 sources), or in the inspected
|
||||
llm-connect, Railiance Fabric and agentic-resources implementations. This is a
|
||||
bounded discovery result, not proof that every fleet capability is registered.
|
||||
|
||||
No factory policy is provisioned by this change. HFACT-WP-0001-T01 still owns
|
||||
provider enforcement/overrun proof, accepted conservative FX, final operating
|
||||
admission and the deployment return under REINAH-WP-0003-T05/T06.
|
||||
|
||||
## Admission and accounting
|
||||
|
||||
The worker replays pending terminal closes first, then checks spend capacity
|
||||
before claiming. A configured worker refuses profile-absent work. Immediately
|
||||
before invoking Glas it checks the queue owner, ActivityDefinition, resolved
|
||||
repository, project, actor, repository grant, exact profile and descriptor digests,
|
||||
and the native USD/turn limits. The same cached catalog supplies the checked
|
||||
profile to Glas. Policy, ledger and project come from trusted worker configuration;
|
||||
queue execution references and prompt text cannot supply them.
|
||||
|
||||
An SQLite `BEGIN IMMEDIATE` transaction inserts the reservation before dispatch.
|
||||
The ledger opens in existing-file mode, with synchronous FULL transactions. A
|
||||
missing/corrupt ledger, changed policy, invalid FX/amount, expired policy, backward
|
||||
clock, exhausted daily/total capacity or unresolved reservation refuses execution.
|
||||
Concurrent processes sharing the file cannot admit two active runs. A run ID or
|
||||
ActivityDefinition/idempotency-key pair cannot be spent again, even on a new
|
||||
claim attempt, after reconciliation or under a replacement queue row ID.
|
||||
|
||||
The reservation is `ceil(max_liability_usd * eur_per_usd * 1,000,000)` integer
|
||||
micro-euros; envelope ceilings round down. Monetary arithmetic uses decimal
|
||||
values without reporting-FX defaults. `max_budget_usd` is the native CLI stop
|
||||
threshold and must fit inside the declared maximum liability. These two values
|
||||
are separate because admission must account for the provider's demonstrated
|
||||
maximum exposure, including any bounded in-flight overshoot.
|
||||
|
||||
On complete success with finite accounting and confirmed session cleanup and
|
||||
sandbox destruction, the ledger charges the **full reserved amount**. It never
|
||||
refunds capacity based on the sandbox's self-reported cost. Missing accounting,
|
||||
execution failure, cancellation, lost lease or crash leaves the reservation held
|
||||
and blocks further admission, including after midnight or restart. An observed
|
||||
cost above declared liability is recorded conservatively and permanently marks
|
||||
the envelope breached. Reconciliation does not clear that breach.
|
||||
|
||||
A completed or reconciled reservation counts in total once and, conservatively,
|
||||
against every local calendar day from admission through completion/reconciliation.
|
||||
A run crossing midnight therefore uses its full reservation in both daily totals.
|
||||
Unresolved work blocks new admission globally; no timer silently releases it.
|
||||
This intentionally sacrifices some utilization for the first bounded pilot.
|
||||
More exact billing reconciliation can follow measured use without weakening the
|
||||
unknown-outcome boundary.
|
||||
|
||||
## Operator configuration and recovery
|
||||
|
||||
Use a private directory on durable local storage outside every sandbox mount and
|
||||
target checkout. The directory must be worker-owned mode 0700; policy and ledger
|
||||
must be worker-owned regular files mode 0600, without hardlinks or symlinks.
|
||||
All admitted processes must use this one ledger. Separate copies, network-file
|
||||
locking, multi-host admission, rollback to stale backups and hostile host owners
|
||||
are outside v1. The protected deployment must keep both files and the launch
|
||||
configuration inaccessible to the workload and preserve them across restarts.
|
||||
|
||||
Configure the following only after the operating envelope is accepted:
|
||||
|
||||
```text
|
||||
AGENT_HARNESS_REQUIRE_SPEND_ADMISSION=1
|
||||
AGENT_HARNESS_SPEND_POLICY=/absolute/private/spend-policy.json
|
||||
AGENT_HARNESS_SPEND_LEDGER=/absolute/private/spend.sqlite3
|
||||
AGENT_HARNESS_EXECUTION_PROJECT=<accepted-project>
|
||||
```
|
||||
|
||||
Any nonempty REQUIRE setting requires admission; missing policy/ledger then
|
||||
refuses before claim. With all spend settings absent, the existing worker
|
||||
compatibility behavior remains. The factory deployment must set REQUIRE and
|
||||
verify missing-configuration denial as part of its owner acceptance.
|
||||
|
||||
The strict JSON `SpendPolicy` fields are:
|
||||
|
||||
| Fields | Contract |
|
||||
| --- | --- |
|
||||
| `version`, `envelope_id`, `authority_ref` | Version `1`, unique envelope identity, reference to the accepted owner record. Local configuration is trusted; this module does not verify signatures or grant authority. |
|
||||
| `valid_from`, `expires_at`, `timezone` | Explicit timezone-aware validity interval and budget calendar, for example Europe/Berlin. |
|
||||
| `worker_id`, `activity_definition_id`, `target_repo`, `project` | Exact admitted scope; target is an absolute path. Actor is `agt`. |
|
||||
| `profile_ref`, `profile_sha256`, `descriptor_sha256`, `repository_grant_id` | Versioned profile plus SHA-256 of canonical `model_dump(mode="json")` for the resolved profile and descriptor; `spend_admission.digest` supplies the canonical serializer. Grant identity is the existing `RepositoryGrant.grant_id`. |
|
||||
| `max_budget_usd`, `max_liability_usd`, `eur_per_usd`, `per_run_eur`, `daily_eur`, `total_eur` | Positive decimal **strings**. Explicit liability/FX must fit per-run, daily and total ceilings. No implicit exchange rate or provider price. |
|
||||
| `max_turns` | Positive integer matching the resolved native profile. |
|
||||
|
||||
Provision the ledger once using `rein-aharness spend init --policy <path>
|
||||
--ledger <path>`. Initialization refuses an existing file. Worker execution never
|
||||
creates an empty replacement. Keep the accepted policy immutable; replacing its
|
||||
contents under the same ledger refuses, rather than resetting spent capacity.
|
||||
|
||||
Inspect with `rein-aharness spend status --policy <path> --ledger <path>`.
|
||||
If a reservation is held, the operator must first confirm provider execution has
|
||||
stopped and obtain final accounting. Record the evidence in the owning work record,
|
||||
then run:
|
||||
|
||||
```text
|
||||
rein-aharness spend reconcile --policy <path> --ledger <path> \
|
||||
--run-id <id> --cost-usd <final-usd> --receipt <bounded-evidence-reference> \
|
||||
--provider-stopped
|
||||
```
|
||||
|
||||
The flag is an operator attestation, not an automatic provider query. The local
|
||||
receipt must refer to reviewed termination and final accounting; it must contain
|
||||
no secrets. Reconciliation charges at least the full reservation and any higher
|
||||
observed liability, cannot discard a known higher cost, is idempotent for the same
|
||||
receipt/cost, and never authorizes rerunning that demand. There is no refund,
|
||||
reset, policy migration or breach-clear command. A breached envelope requires a
|
||||
new owner decision that accounts for existing spend before any further admission.
|
||||
|
||||
## Proof and remaining boundary
|
||||
|
||||
`tests/test_spend_admission.py` covers pre-dispatch denial, concurrent processes,
|
||||
crash/reopen, identity/profile/grant changes, replay, unknown and overrun outcomes,
|
||||
midnight, total exhaustion, decimal rounding and operator reconciliation. It is
|
||||
included in both runtime-contract and recovery gates.
|
||||
|
||||
The opt-in `REIN_REAL_BWRAP=1` test also runs actual bwrap, sandbox owner transport,
|
||||
Glas, repository import and durable-close replay with admission enabled. Queue
|
||||
responses, accounting and authoring are deterministic fixtures; no model is
|
||||
called. The fixture's policy and FX have no production authority.
|
||||
|
||||
This ledger enforces allocation against **declared** maximum liability. It cannot
|
||||
make an opaque provider/tool loop honor that maximum, establish a hard EUR limit,
|
||||
or verify the operator's FX assumption. G0 remains blocked until those semantics
|
||||
are proven for the pinned provider/CLI, including retries, cache, subagents and
|
||||
in-flight work. Exact credential/identity/egress/placement and protected-runtime
|
||||
installation are also still required before natural factory execution.
|
||||
Loading…
Add table
Add a link
Reference in a new issue