From c30806b968e408431a5afd3c588bbf4ff6fdfb45 Mon Sep 17 00:00:00 2001 From: tegwick Date: Wed, 9 Sep 2026 21:50:12 +0200 Subject: [PATCH] Reserve provider requests inside the durable worker envelope Assistant: codex Assistant-Model: gpt-5.6-luna Assistant-Session: 01a07ff8-19d0-7820-b4d0-1353833cb7fc --- deploy/runtime-contract-lock.json | 2 +- docs/evidence/2026-09-09-guarded-overrun.json | 105 ++++++ docs/evidence/2026-09-09-guarded-tools.json | 204 ++++++++++++ docs/request-admission.md | 43 +++ docs/spend-admission.md | 9 +- rein_aharness/request_admission.py | 239 +++++++++++++ rein_aharness/spend_admission.py | 27 ++ scripts/verify-recovery-contracts.sh | 1 + scripts/verify-runtime-contracts.sh | 1 + tests/native_cli_fixture.py | 113 ++++++- tests/test_native_cli_boundary.py | 51 ++- tests/test_request_admission.py | 313 ++++++++++++++++++ ...INAH-WP-0003-governed-runtime-integrity.md | 18 + 13 files changed, 1114 insertions(+), 12 deletions(-) create mode 100644 docs/evidence/2026-09-09-guarded-overrun.json create mode 100644 docs/evidence/2026-09-09-guarded-tools.json create mode 100644 docs/request-admission.md create mode 100644 rein_aharness/request_admission.py create mode 100644 tests/test_request_admission.py diff --git a/deploy/runtime-contract-lock.json b/deploy/runtime-contract-lock.json index 4d69798..17677d0 100644 --- a/deploy/runtime-contract-lock.json +++ b/deploy/runtime-contract-lock.json @@ -7,7 +7,7 @@ }, "dependencies": [ { - "commit": "00560945f81ba6ff1f5cacd9fe99c7fe756cc4b1", + "commit": "dc77f434a8b279436efa64e279af58372d275d8a", "distribution": "llm-connect", "source": "../llm-connect", "version": "0.1.0" diff --git a/docs/evidence/2026-09-09-guarded-overrun.json b/docs/evidence/2026-09-09-guarded-overrun.json new file mode 100644 index 0000000..e035277 --- /dev/null +++ b/docs/evidence/2026-09-09-guarded-overrun.json @@ -0,0 +1,105 @@ +{ + "case": "guarded-overrun", + "binary_version": "2.1.266 (Claude Code)", + "binary_sha256": "19842705e989393fce936804df6d2ab034860e24b8f8880357981d87ffd83fac", + "argv": [ + "/opt/claude", + "--print", + "--permission-mode", + "dontAsk", + "--allowedTools", + "Read,Write,Edit,Glob,Grep,Bash(git add:*),Bash(git commit:*),Bash(git status),Bash(git log:*),Bash(git diff:*),Bash(date:*),Bash(ls:*)", + "--output-format", + "json", + "--max-budget-usd", + "0.01", + "--max-turns", + "4", + "--bare", + "--setting-sources", + "", + "--strict-mcp-config", + "--mcp-config", + "{\"mcpServers\":{}}", + "--disallowedTools", + "mcp__*", + "--tools", + "Read,Write,Edit,Glob,Grep,Bash", + "--no-session-persistence", + "--model", + "claude-sonnet-4-6" + ], + "scope": "synthetic API in isolated network namespace; no real credential or provider request", + "returncode": 1, + "terminal": { + "type": "result", + "subtype": "success", + "is_error": true, + "total_cost_usd": 0, + "num_turns": 1, + "usage": { + "output_tokens_details": { + "thinking_tokens": 0 + }, + "input_tokens": 0, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "output_tokens": 0, + "server_tool_use": { + "web_search_requests": 0, + "web_fetch_requests": 0 + }, + "service_tier": "standard", + "cache_creation": { + "ephemeral_1h_input_tokens": 0, + "ephemeral_5m_input_tokens": 0 + }, + "inference_geo": "", + "iterations": [], + "speed": "standard" + }, + "permission_denials": [] + }, + "requests": [], + "markers": { + "hook-ran": false, + "mcp-ran": false, + "denied-ran": false + }, + "created_file": null, + "network_namespace": "net:[4026532319]", + "guard_attempts": [ + { + "fields": [ + "context_management", + "max_tokens", + "messages", + "metadata", + "model", + "output_config", + "stream", + "system", + "thinking", + "tools" + ], + "betas": "claude-code-20250219,interleaved-thinking-2025-05-14,thinking-token-count-2026-05-13,context-management-2025-06-27,prompt-caching-scope-2026-01-05,effort-2025-11-24", + "context_management": { + "edits": [ + { + "type": "clear_thinking_20251015", + "keep": "all" + } + ] + }, + "thinking": { + "type": "adaptive", + "display": "omitted" + }, + "output_config": { + "effort": "high" + }, + "liability_microusd": 1080000 + } + ], + "request_reservations": [] +} diff --git a/docs/evidence/2026-09-09-guarded-tools.json b/docs/evidence/2026-09-09-guarded-tools.json new file mode 100644 index 0000000..00c0239 --- /dev/null +++ b/docs/evidence/2026-09-09-guarded-tools.json @@ -0,0 +1,204 @@ +{ + "case": "guarded-tools", + "binary_version": "2.1.266 (Claude Code)", + "binary_sha256": "19842705e989393fce936804df6d2ab034860e24b8f8880357981d87ffd83fac", + "argv": [ + "/opt/claude", + "--print", + "--permission-mode", + "dontAsk", + "--allowedTools", + "Read,Write,Edit,Glob,Grep,Bash(git add:*),Bash(git commit:*),Bash(git status),Bash(git log:*),Bash(git diff:*),Bash(date:*),Bash(ls:*)", + "--output-format", + "json", + "--max-budget-usd", + "1", + "--max-turns", + "4", + "--bare", + "--setting-sources", + "", + "--strict-mcp-config", + "--mcp-config", + "{\"mcpServers\":{}}", + "--disallowedTools", + "mcp__*", + "--tools", + "Read,Write,Edit,Glob,Grep,Bash", + "--no-session-persistence", + "--model", + "claude-sonnet-4-6" + ], + "scope": "synthetic API in isolated network namespace; no real credential or provider request", + "returncode": 0, + "terminal": { + "type": "result", + "subtype": "success", + "is_error": false, + "total_cost_usd": 0.0009000000000000001, + "num_turns": 4, + "usage": { + "input_tokens": 200, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "output_tokens": 20, + "output_tokens_details": { + "thinking_tokens": 0 + }, + "server_tool_use": { + "web_search_requests": 0, + "web_fetch_requests": 0 + }, + "service_tier": "standard", + "cache_creation": { + "ephemeral_1h_input_tokens": 0, + "ephemeral_5m_input_tokens": 0 + }, + "inference_geo": "", + "iterations": [], + "speed": "standard" + }, + "permission_denials": [ + { + "tool_name": "Bash", + "tool_use_id": "tool_denied", + "tool_input": { + "command": "printf bypass > /work/denied-ran", + "description": "Fixture forbidden operation" + } + } + ] + }, + "requests": [ + { + "path": "/v1/messages", + "model": "claude-sonnet-4-6", + "max_tokens": 32000, + "tools": [ + "Bash", + "Edit", + "Read" + ], + "tool_results": [], + "ambient_context_loaded": false + }, + { + "path": "/v1/messages", + "model": "claude-sonnet-4-6", + "max_tokens": 32000, + "tools": [ + "Bash", + "Edit", + "Read" + ], + "tool_results": [ + { + "id": "tool_allowed", + "is_error": false + }, + { + "id": "tool_create", + "is_error": false + }, + { + "id": "tool_denied", + "is_error": true + } + ], + "ambient_context_loaded": false + } + ], + "markers": { + "hook-ran": false, + "mcp-ran": false, + "denied-ran": false + }, + "created_file": "fixture-created\n", + "network_namespace": "net:[4026532319]", + "guard_attempts": [ + { + "fields": [ + "context_management", + "max_tokens", + "messages", + "metadata", + "model", + "output_config", + "stream", + "system", + "thinking", + "tools" + ], + "betas": "claude-code-20250219,interleaved-thinking-2025-05-14,thinking-token-count-2026-05-13,context-management-2025-06-27,prompt-caching-scope-2026-01-05,effort-2025-11-24", + "context_management": { + "edits": [ + { + "type": "clear_thinking_20251015", + "keep": "all" + } + ] + }, + "thinking": { + "type": "adaptive", + "display": "omitted" + }, + "output_config": { + "effort": "high" + }, + "liability_microusd": 1080000 + }, + { + "fields": [ + "context_management", + "max_tokens", + "messages", + "metadata", + "model", + "output_config", + "stream", + "system", + "thinking", + "tools" + ], + "betas": "claude-code-20250219,interleaved-thinking-2025-05-14,thinking-token-count-2026-05-13,context-management-2025-06-27,prompt-caching-scope-2026-01-05,effort-2025-11-24", + "context_management": { + "edits": [ + { + "type": "clear_thinking_20251015", + "keep": "all" + } + ] + }, + "thinking": { + "type": "adaptive", + "display": "omitted" + }, + "output_config": { + "effort": "high" + }, + "liability_microusd": 1080000 + } + ], + "request_reservations": [ + { + "receipt": "183fdd9d-acb4-44ec-8393-9361d978ab70", + "run_id": "fixture-run", + "policy_sha256": "37b66853a37b5e7e772f128687cc96085fa7fb7c4a5687beb8ed53aab9d9daf3", + "lease_id": "fixture-lease", + "liability_microusd": 1080000, + "state": "charged", + "observed_microusd": 450, + "created_at": "2026-09-09T19:41:28.936634+00:00" + }, + { + "receipt": "23f1e246-7011-437b-9db3-a1c7db1dacf5", + "run_id": "fixture-run", + "policy_sha256": "37b66853a37b5e7e772f128687cc96085fa7fb7c4a5687beb8ed53aab9d9daf3", + "lease_id": "fixture-lease", + "liability_microusd": 1080000, + "state": "charged", + "observed_microusd": 450, + "created_at": "2026-09-09T19:41:29.290569+00:00" + } + ] +} diff --git a/docs/request-admission.md b/docs/request-admission.md new file mode 100644 index 0000000..c30f9b3 --- /dev/null +++ b/docs/request-admission.md @@ -0,0 +1,43 @@ +# Child request reservations in the worker envelope + +`rein_aharness.request_admission.RequestLedger` implements llm-connect's +owner-hosted Messages meter in the existing private `SpendLedger` database. +It introduces no second daily/total allowance and does not install a service. + +The trusted provisioning path calls `RequestLedger.initialize()` before any run +reservation. Missing tables or state refuse; initialization is additive and +cannot reset existing request tables. After the normal validated parent dispatch +reservation, the owner calls `bind_route(run_id, policy_sha256, lease_id=..., +expires_at=...)` with the real accepted lease expiry. This returns one random +token whose hash is stored with run, lease, exact request policy and expiry. +The owner supplies that token and the gateway URL to the workload; no provider +key or ledger path belongs in the sandbox. There is no replacement/renewal path +for a run; expiry or revocation requires owner handling, not workload replay. + +Each forward first commits a child hold in micro-USD. Requests serialize with +SQLite `BEGIN IMMEDIATE`; a pending or unknown request blocks additional calls. +Sum of full child liabilities cannot exceed the parent's admitted USD maximum. +Trusted terminal usage charges the full child reservation without refunds. +Known overrun records the larger amount and freezes the whole envelope. Request +receipts contain only bounded IDs/digests, amounts, times and disposition. + +Parent observation revokes the route, including unknown accounting results. +Even an apparent successful CLI result cannot settle a parent with pending child +requests. Existing operator reconciliation still requires attested provider +termination and final accounting; it settles child holds conservatively and +never reopens a route or grants another demand attempt. `revoke_route` is the +trusted callback for real lease loss; an expiry also denies new admissions. +In-flight cancellation retains liability until outcome is known. + +The downstream protocol is documented in +`llm-connect/contracts/functional/messages-admission.md` (LLM-WP-0009). +The default worker/serve configurations do not yet host or inject this route. +REINAH-WP-0003-T05/T06 and LLM-WP-0009-T03 retain protected owner hosting, real +lease/token wiring, provider custody, enforced egress and bypass proof, accepted +tariffs/FX, runtime installation and admitted real-profile/queue execution. + +The installed-CLI fake-provider proof uses synthetic bounds and dummy keys. +With a USD 0.01 parent the same native request is refused with zero upstream +calls; with adequate fixture capacity a two-request tool session succeeds. +That verifies request admission and accounting, not a live EUR ceiling or +owner/workload secret and network separation. diff --git a/docs/spend-admission.md b/docs/spend-admission.md index cdfc91e..a490f06 100644 --- a/docs/spend-admission.md +++ b/docs/spend-admission.md @@ -134,6 +134,9 @@ installation are also still required before natural factory execution. The [native CLI proof](native-cli-boundary-proof.md) now demonstrates the missing pre-request bound: one fixture response overshot a USD 0.01 threshold to an -estimated USD 0.18015. HFACT T01 must implement and prove a metered request path -outside the workload, including direct-provider bypass denial. The full-charge -reservation/reconciliation implementation remains unchanged. +estimated USD 0.18015. The [request extension](request-admission.md) now provides +child holds and the llm-connect transport proves reserve-before-forward with the +installed CLI. HFACT T01 retains owner hosting, actual lease/token integration, +accepted tariffs/FX and direct-provider bypass denial. Parent observation also +revokes request routes and retains unresolved children; operator reconciliation +keeps full liability while closing the route. diff --git a/rein_aharness/request_admission.py b/rein_aharness/request_admission.py new file mode 100644 index 0000000..7a9b0c4 --- /dev/null +++ b/rein_aharness/request_admission.py @@ -0,0 +1,239 @@ +"""Request holds inside the worker's existing private spend ledger. + +Provision explicitly before use. Only a trusted owner may bind/revoke a route; +the workload gets an opaque token, never the parent policy, ledger or provider +credential. Lease expiry must come from the accepted queue lease, not a prompt. +""" + +from __future__ import annotations + +import hashlib +import re +import secrets +import uuid +from datetime import UTC, datetime + +from rein_aharness.spend_admission import ( + SpendAdmissionError, + SpendLedger, + cap_micros, + converted_micros, + timestamp, +) + + +def _token_hash(token: str) -> str: + if not isinstance(token, str) or not re.fullmatch(r"[A-Za-z0-9_-]{43,100}", token): + raise SpendAdmissionError("invalid request route") + return hashlib.sha256(token.encode()).hexdigest() + + +def _identity(value: str) -> None: + if not isinstance(value, str) or not re.fullmatch( + r"[A-Za-z0-9][A-Za-z0-9._:/@+-]{0,199}", value + ): + raise SpendAdmissionError("invalid request identity") + + +class RequestLedger: + def __init__(self, parent: SpendLedger): + self.parent = parent + + def initialize(self) -> None: + """Explicit, additive schema provisioning; existing request tables refuse.""" + with self.parent._db() as db: + if db.execute("SELECT 1 FROM reservations WHERE state='held'").fetchone(): + raise SpendAdmissionError("provision request tables before dispatch") + db.execute("""CREATE TABLE request_routes ( + run_id TEXT PRIMARY KEY, token_sha256 TEXT UNIQUE NOT NULL, + policy_sha256 TEXT NOT NULL, lease_id TEXT NOT NULL, + expires_at TEXT NOT NULL, revoked INTEGER NOT NULL CHECK(revoked IN (0,1)))""") + db.execute("""CREATE TABLE request_reservations ( + receipt TEXT PRIMARY KEY, run_id TEXT NOT NULL, + policy_sha256 TEXT NOT NULL, lease_id TEXT NOT NULL, + liability_microusd INTEGER NOT NULL CHECK(liability_microusd>0), + state TEXT NOT NULL CHECK(state IN ('held','charged')), + observed_microusd INTEGER, created_at TEXT NOT NULL)""") + + def bind_route( + self, + run_id: str, + policy_sha256: str, + *, + lease_id: str, + expires_at: str, + now: datetime | None = None, + ) -> str: + """Trusted owner only; a parent run can never get a replacement route.""" + _identity(run_id) + _identity(lease_id) + if not isinstance(policy_sha256, str) or not re.fullmatch( + r"[0-9a-f]{64}", policy_sha256 + ): + raise SpendAdmissionError("invalid request policy digest") + token = secrets.token_urlsafe(32) + with self.parent._db() as db: + clock = now or datetime.now(UTC) + self.parent._clock(db, clock, admission=True) + if ( + not clock + < timestamp(expires_at) + <= timestamp(self.parent.policy.expires_at) + ): + raise SpendAdmissionError("route must expire with the admitted lease") + row = db.execute( + "SELECT * FROM reservations WHERE run_id=?", (run_id,) + ).fetchone() + if ( + row is None + or row["state"] != "held" + or row["definition_id"] != self.parent.policy.activity_definition_id + ): + raise SpendAdmissionError("active admitted parent required") + if db.execute("SELECT breached FROM envelope").fetchone()[0]: + raise SpendAdmissionError("spend envelope breached") + db.execute( + "INSERT INTO request_routes VALUES (?, ?, ?, ?, ?, 0)", + (run_id, _token_hash(token), policy_sha256, lease_id, expires_at), + ) + return token + + def revoke_route(self, run_id: str) -> None: + with self.parent._db() as db: + changed = db.execute( + "UPDATE request_routes SET revoked=1 WHERE run_id=?", (run_id,) + ) + if changed.rowcount != 1: + raise SpendAdmissionError("unknown request route") + + def _active(self, db, row, now): + self.parent._clock(db, now, admission=True) + if row is None or row["revoked"] or now >= timestamp(row["expires_at"]): + raise SpendAdmissionError("request route expired or revoked") + parent = db.execute( + "SELECT state FROM reservations WHERE run_id=?", (row["run_id"],) + ).fetchone() + if ( + parent is None + or parent["state"] != "held" + or db.execute("SELECT breached FROM envelope").fetchone()[0] + ): + raise SpendAdmissionError("request parent not active") + + def reserve_request( + self, + token: str, + policy_sha256: str, + liability_microusd: int, + *, + now: datetime | None = None, + ) -> str: + token_hash = _token_hash(token) + if ( + type(liability_microusd) is not int + or not 0 < liability_microusd <= 1_000_000_000_000 + ): + raise SpendAdmissionError("invalid request liability") + receipt = str(uuid.uuid4()) + with self.parent._db() as db: + clock = now or datetime.now(UTC) + route = db.execute( + "SELECT * FROM request_routes WHERE token_sha256=?", (token_hash,) + ).fetchone() + self._active(db, route, clock) + if route["policy_sha256"] != policy_sha256: + raise SpendAdmissionError("request policy not admitted") + if db.execute( + "SELECT 1 FROM request_reservations WHERE state='held'" + ).fetchone(): + raise SpendAdmissionError("request outcome unresolved") + charged = db.execute( + "SELECT COALESCE(SUM(liability_microusd),0) FROM request_reservations WHERE run_id=?", + (route["run_id"],), + ).fetchone()[0] + if charged + liability_microusd > cap_micros( + self.parent.policy.max_liability_usd + ): + raise SpendAdmissionError("request exceeds remaining parent capacity") + db.execute( + "INSERT INTO request_reservations VALUES (?, ?, ?, ?, ?, 'held', NULL, ?)", + ( + receipt, + route["run_id"], + policy_sha256, + route["lease_id"], + liability_microusd, + clock.isoformat(), + ), + ) + return receipt + + def request_active(self, receipt: str, *, now: datetime | None = None) -> bool: + with self.parent._db() as db: + request = db.execute( + "SELECT * FROM request_reservations WHERE receipt=?", (receipt,) + ).fetchone() + if request is None or request["state"] != "held": + return False + route = db.execute( + "SELECT * FROM request_routes WHERE run_id=?", (request["run_id"],) + ).fetchone() + try: + self._active(db, route, now or datetime.now(UTC)) + except SpendAdmissionError: + return False + return True + + def complete_request( + self, receipt: str, observed_microusd: int, *, now: datetime | None = None + ) -> None: + if ( + type(observed_microusd) is not int + or not 0 <= observed_microusd <= 100_000_000_000_000 + ): + raise SpendAdmissionError("request accounting incomplete") + with self.parent._db() as db: + self.parent._clock(db, now or datetime.now(UTC), admission=False) + row = db.execute( + "SELECT * FROM request_reservations WHERE receipt=?", (receipt,) + ).fetchone() + if row is None or row["state"] != "held": + raise SpendAdmissionError("request completion replay refused") + breached = observed_microusd > row["liability_microusd"] + db.execute( + "UPDATE request_reservations SET state=?, liability_microusd=?, observed_microusd=? WHERE receipt=?", + ( + "held" if breached else "charged", + max(row["liability_microusd"], observed_microusd), + observed_microusd, + receipt, + ), + ) + if breached: + db.execute("UPDATE envelope SET breached=1") + db.execute( + "UPDATE request_routes SET revoked=1 WHERE run_id=?", + (row["run_id"],), + ) + total = db.execute( + "SELECT SUM(liability_microusd) FROM request_reservations WHERE run_id=?", + (row["run_id"],), + ).fetchone()[0] + # Format integer micros without float rounding. + usd = f"{total // 1_000_000}.{total % 1_000_000:06d}" + liability = converted_micros(usd, self.parent.policy.eur_per_usd) + db.execute( + "UPDATE reservations SET liability=MAX(liability, ?) WHERE run_id=?", + (liability, row["run_id"]), + ) + if breached: + raise SpendAdmissionError("request liability breached; envelope frozen") + + def status(self): + with self.parent._db() as db: + return [ + dict(row) + for row in db.execute( + "SELECT * FROM request_reservations ORDER BY created_at, receipt" + ) + ] diff --git a/rein_aharness/spend_admission.py b/rein_aharness/spend_admission.py index ee7b2d2..927374d 100644 --- a/rein_aharness/spend_admission.py +++ b/rein_aharness/spend_admission.py @@ -385,9 +385,12 @@ class SpendLedger: try: cost = amount(evidence.get("cost_usd"), positive=False) except SpendAdmissionError: + with self._db() as db: + self._close_requests(db, run_id) return False # Unknown remains held, even for apparent success. with self._db() as db: day = self._clock(db, now or datetime.now(UTC), admission=False) + requests_resolved = self._close_requests(db, run_id) row = db.execute( "SELECT * FROM reservations WHERE run_id=?", (run_id,) ).fetchone() @@ -405,6 +408,7 @@ class SpendLedger: and evidence.get("session_cleanup") == "succeeded" and evidence.get("sandbox_destroy") == "succeeded" and not breached + and requests_resolved ) db.execute( "UPDATE reservations SET liability=?, observed_usd=?, state=?, end_day=? WHERE run_id=?", @@ -420,6 +424,28 @@ class SpendLedger: db.execute("UPDATE envelope SET breached=1") return complete + @staticmethod + def _close_requests( + db: sqlite3.Connection, run_id: str, *, reconcile: bool = False + ) -> bool: + # Existing ledgers without the opt-in request extension remain compatible. + if not db.execute( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name='request_routes'" + ).fetchone(): + return True + db.execute("UPDATE request_routes SET revoked=1 WHERE run_id=?", (run_id,)) + if reconcile: + # The existing operator reconciliation attests provider termination + # and final accounting. Full child liabilities are retained. + db.execute( + "UPDATE request_reservations SET state='charged' WHERE run_id=?", + (run_id,), + ) + return not db.execute( + "SELECT 1 FROM request_reservations WHERE run_id=? AND state='held'", + (run_id,), + ).fetchone() + def reconcile( self, run_id: str, *, cost_usd: str, receipt: str, now: datetime | None = None ) -> None: @@ -446,6 +472,7 @@ class SpendLedger: raise SpendAdmissionError( "reconciliation cannot discard observed liability" ) + self._close_requests(db, run_id, reconcile=True) liability = max( row["liability"], converted_micros(str(cost), self.policy.eur_per_usd) ) diff --git a/scripts/verify-recovery-contracts.sh b/scripts/verify-recovery-contracts.sh index bd89084..242f0a7 100755 --- a/scripts/verify-recovery-contracts.sh +++ b/scripts/verify-recovery-contracts.sh @@ -11,6 +11,7 @@ PYTHONPATH="${REPO_ROOT}:${REPO_ROOT}/../llm-connect" \ "${PYTHON_BIN}" -m pytest \ tests/test_native_limits.py \ tests/test_spend_admission.py \ + tests/test_request_admission.py \ tests/test_claim_loop.py::test_process_one_initial_heartbeat_rejection_refuses_dispatch \ tests/test_claim_loop.py::test_process_one_lease_loss_cancels_registered_adapter_process \ tests/test_claim_loop.py::test_profiled_close_failure_happens_after_repository_lock_release \ diff --git a/scripts/verify-runtime-contracts.sh b/scripts/verify-runtime-contracts.sh index bfb44fa..7560838 100755 --- a/scripts/verify-runtime-contracts.sh +++ b/scripts/verify-runtime-contracts.sh @@ -12,6 +12,7 @@ PYTHONPATH="${REPO_ROOT}:${REPO_ROOT}/../llm-connect" \ "${PYTHON_BIN}" -m pytest \ tests/test_native_limits.py \ tests/test_spend_admission.py \ + tests/test_request_admission.py \ tests/test_glas_execution.py \ tests/test_ops_run_client.py \ tests/test_claim_loop.py \ diff --git a/tests/native_cli_fixture.py b/tests/native_cli_fixture.py index e9d98d8..0c13b20 100644 --- a/tests/native_cli_fixture.py +++ b/tests/native_cli_fixture.py @@ -10,6 +10,7 @@ from pathlib import Path root = Path("/work") settings = json.loads((root / "case.json").read_text()) seen = [] +guard_attempts = [] class Handler(http.server.BaseHTTPRequestHandler): @@ -47,7 +48,7 @@ class Handler(http.server.BaseHTTPRequestHandler): self.wfile.write(b'{"input_tokens":100}') return message_number = sum("count_tokens" not in request["path"] for request in seen) - if settings["case"] == "tools" and message_number == 1: + if settings["case"] in ("tools", "guarded-tools") and message_number == 1: blocks = [ { "type": "tool_use", @@ -83,7 +84,9 @@ class Handler(http.server.BaseHTTPRequestHandler): blocks = [{"type": "text", "text": "fixture complete"}] stop = "end_turn" usage = { - "input_tokens": 60000 if settings["case"] == "overrun" else 100, + "input_tokens": 60000 + if settings["case"] in ("overrun", "guarded-overrun") + else 100, "output_tokens": 0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0, @@ -155,6 +158,104 @@ class Handler(http.server.BaseHTTPRequestHandler): server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler) threading.Thread(target=server.serve_forever, daemon=True).start() +gate = meter = None +route_token = "fixture-no-provider-secret" +base_url = "http://127.0.0.1:" + str(server.server_port) +if settings["case"].startswith("guarded-"): + from datetime import UTC, datetime, timedelta + from types import SimpleNamespace + + from llm_connect.messages_gate import MessagesPolicy, MessagesServer, RequestRefused + from rein_aharness.request_admission import RequestLedger + from rein_aharness.spend_admission import SpendLedger, SpendPolicy + + class ObservedPolicy(MessagesPolicy): + def validate(self, data, betas): + row = { + "fields": sorted(data), + "betas": betas, + "context_management": data.get("context_management"), + "thinking": data.get("thinking"), + "output_config": data.get("output_config"), + } + guard_attempts.append(row) + try: + value = super().validate(data, betas) + row["liability_microusd"] = value + return value + except RequestRefused as exc: + row["refusal"] = str(exc) + raise + + # Synthetic accepted bounds ONLY, never a live price/FX/authority packet. + policy = ObservedPolicy( + "fixture:no-live-tariff", + "claude-sonnet-4-6", + 200000, + 32000, + 3, + 15, + allowed_betas=( + "claude-code-20250219", + "interleaved-thinking-2025-05-14", + "thinking-token-count-2026-05-13", + "context-management-2025-06-27", + "prompt-caching-scope-2026-01-05", + "effort-2025-11-24", + ), + ) + cap = "0.01" if settings["case"] == "guarded-overrun" else "5" + private = root / "private" + private.mkdir(mode=0o700) + now = datetime.now(UTC) + expiry = (now + timedelta(minutes=5)).isoformat() + parent_policy = SpendPolicy( + "1", + "fixture", + "fixture:no-live-authority", + (now - timedelta(minutes=1)).isoformat(), + expiry, + "UTC", + "fixture-worker", + "fixture-definition", + "/work/target", + "fixture", + "harness.fixture@1.0.0", + "a" * 64, + "b" * 64, + "fixture-grant", + cap, + cap, + 4, + "1", + cap, + cap, + cap, + ) + parent = SpendLedger(private / "spend.sqlite3", parent_policy) + parent.initialize() + meter = RequestLedger(parent) + meter.initialize() + parent.reserve( + SimpleNamespace( + id="fixture-run", + activity_definition_id="fixture-definition", + idempotency_key="fixture", + attempt=1, + ) + ) + route_token = meter.bind_route( + "fixture-run", policy.sha256, lease_id="fixture-lease", expires_at=expiry + ) + gate = MessagesServer( + policy, + meter, + provider_key="fixture-owner-only-key", + upstream_url=base_url, + allow_test_http=True, + ) + gate.start() + base_url = "http://127.0.0.1:" + str(gate.port) (root / "target" / ".claude").mkdir(parents=True) (root / "target" / ".claude" / "settings.json").write_text( json.dumps( @@ -181,8 +282,8 @@ threading.Thread(target=server.serve_forever, daemon=True).start() subprocess.run(["git", "init", "-q", str(root / "target")], check=True) env = dict( os.environ, - ANTHROPIC_BASE_URL="http://127.0.0.1:" + str(server.server_port), - ANTHROPIC_API_KEY="fixture-no-provider-secret", + ANTHROPIC_BASE_URL=base_url, + ANTHROPIC_API_KEY=route_token, CLAUDE_CONFIG_DIR="/work/config", CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC="1", DISABLE_AUTOUPDATER="1", @@ -222,7 +323,11 @@ try: if (root / "target" / "result.txt").exists() else None, "network_namespace": os.readlink("/proc/self/ns/net"), + "guard_attempts": guard_attempts, + "request_reservations": meter.status() if meter else [], } print(json.dumps(output)) finally: + if gate: + gate.stop() server.shutdown() diff --git a/tests/test_native_cli_boundary.py b/tests/test_native_cli_boundary.py index 70bb918..1605084 100644 --- a/tests/test_native_cli_boundary.py +++ b/tests/test_native_cli_boundary.py @@ -21,7 +21,9 @@ pytestmark = pytest.mark.skipif( ) -@pytest.mark.parametrize("case", ["overrun", "tools"]) +@pytest.mark.parametrize( + "case", ["overrun", "tools", "guarded-overrun", "guarded-tools"] +) def test_installed_cli_boundary(tmp_path, case): binary = Path( os.environ.get("REIN_CLAUDE_PROOF_BINARY") or shutil.which("claude") or "" @@ -31,7 +33,7 @@ def test_installed_cli_boundary(tmp_path, case): "revalidate the fixture for this binary version before claiming the pinned proof" ) adapter = AgenticClaudeCodeAdapter(workdir=tmp_path, model="claude-sonnet-4-6") - cap = 0.01 if case == "overrun" else 1 + cap = 0.01 if case in ("overrun", "guarded-overrun") else 1 argv = adapter._build_command( RunConfig(model_params={"max_budget_usd": cap, "max_turns": 4}) ) @@ -45,6 +47,23 @@ def test_installed_cli_boundary(tmp_path, case): shutil.copyfile( Path(__file__).with_name("native_cli_fixture.py"), tmp_path / "fixture.py" ) + if case.startswith("guarded-"): + import llm_connect.messages_gate + import rein_aharness.request_admission + import rein_aharness.spend_admission + + for package, modules in { + "llm_connect": [llm_connect.messages_gate], + "rein_aharness": [ + rein_aharness.request_admission, + rein_aharness.spend_admission, + ], + }.items(): + target = tmp_path / package + target.mkdir() + (target / "__init__.py").write_text("") + for module in modules: + shutil.copyfile(module.__file__, target / Path(module.__file__).name) command = [ "bwrap", "--unshare-all", @@ -93,7 +112,9 @@ def test_installed_cli_boundary(tmp_path, case): "/usr/bin/python3", "/work/fixture.py", ] - proc = subprocess.run(command, capture_output=True, text=True, timeout=55, check=False) + proc = subprocess.run( + command, capture_output=True, text=True, timeout=55, check=False + ) assert proc.returncode == 0, proc.stderr[-1000:] result = json.loads(proc.stdout) assert result["network_namespace"] != os.readlink("/proc/self/ns/net") @@ -111,7 +132,20 @@ def test_installed_cli_boundary(tmp_path, case): assert request["model"] == "claude-sonnet-4-6" assert not request["ambient_context_loaded"] terminal = result["terminal"] - if case == "overrun": + if case.startswith("guarded-"): + assert all("refusal" not in row for row in result["guard_attempts"]), ( + json.dumps(result["guard_attempts"]) + ) + if case == "guarded-overrun": + assert result["guard_attempts"], result + assert all( + row.get("liability_microusd") == 1_080_000 + for row in result["guard_attempts"] + ), result + assert not requests and not result["request_reservations"] + assert result["returncode"] != 0 and terminal["is_error"] + assert terminal["total_cost_usd"] == 0 + elif case == "overrun": assert len(requests) == 1 assert proc.returncode == 0 and result["returncode"] == 1 assert terminal["subtype"] == "error_max_budget_usd" @@ -129,6 +163,15 @@ def test_installed_cli_boundary(tmp_path, case): assert result["created_file"] == "fixture-created\n" usage, cost = terminal_accounting(terminal, max_budget_usd=cap, max_turns=4) assert usage["total_tokens"] > 0 and cost > 0 + if case == "guarded-tools": + assert len(result["request_reservations"]) == 2 + assert all( + row["state"] == "charged" for row in result["request_reservations"] + ) + assert ( + sum(row["liability_microusd"] for row in result["request_reservations"]) + == 2_160_000 + ) if output_dir := os.environ.get("REIN_CLAUDE_PROOF_OUTPUT"): target = Path(output_dir) target.mkdir(parents=True, exist_ok=True) diff --git a/tests/test_request_admission.py b/tests/test_request_admission.py new file mode 100644 index 0000000..783c38c --- /dev/null +++ b/tests/test_request_admission.py @@ -0,0 +1,313 @@ +"""Real SQLite/HTTP, deterministic provider; no paid calls or credentials.""" + +import http.client +import http.server +import json +import threading +from concurrent.futures import ThreadPoolExecutor +from datetime import UTC, datetime, timedelta + +import pytest +from llm_connect.messages_gate import MessagesPolicy, MessagesServer + +from rein_aharness.request_admission import RequestLedger +from rein_aharness.spend_admission import SpendAdmissionError, SpendLedger +from test_spend_admission import ledger as ledger # shared private ledger fixture +from test_spend_admission import run, success + + +@pytest.fixture +def meter(ledger): + child = RequestLedger(ledger) + child.initialize() + ledger.reserve(run()) + return child + + +@pytest.fixture +def policy(): + return MessagesPolicy( + "fixture:not-live-tariff", "fixture-model", 1000, 1000, 1000, 1000 + ) + + +def route(meter, policy, **kwargs): + return meter.bind_route( + run().id, + policy.sha256, + lease_id="fixture-lease", + expires_at=(datetime.now(UTC) + timedelta(seconds=60)).isoformat(), + **kwargs, + ) + + +def test_parent_capacity_no_refund_replay_and_reopen(meter, policy): + token = route(meter, policy) + receipt = meter.reserve_request(token, policy.sha256, 2_000_000) + with pytest.raises(SpendAdmissionError, match="unresolved"): + RequestLedger(meter.parent).reserve_request(token, policy.sha256, 1) + meter.complete_request(receipt, 10) + with pytest.raises(SpendAdmissionError, match="replay"): + meter.complete_request(receipt, 10) + second = meter.reserve_request(token, policy.sha256, 2_000_000) + meter.complete_request(second, 0) + with pytest.raises(SpendAdmissionError, match="capacity"): + meter.reserve_request(token, policy.sha256, 2_000_000) + assert sum(row["liability_microusd"] for row in meter.status()) == 4_000_000 + assert meter.parent.status()["reservations"][0]["liability"] == 5_000_000 + serialized = json.dumps(meter.status()) + assert token not in serialized and "private prompt" not in serialized + + +def test_concurrent_admission_only_one_pending(meter, policy): + token = route(meter, policy) + + def reserve(_): + try: + RequestLedger( + SpendLedger(meter.parent.path, meter.parent.policy) + ).reserve_request(token, policy.sha256, 1_000_000) + return True + except SpendAdmissionError: + return False + + with ThreadPoolExecutor(max_workers=8) as pool: + assert sum(pool.map(reserve, range(8))) == 1 + assert len(meter.status()) == 1 + + +def test_unknown_holds_parent_even_after_success_and_operator_closes(meter, policy): + token = route(meter, policy) + meter.reserve_request(token, policy.sha256, 100) + assert not meter.parent.observe(run().id, success(meter.parent, run())) + with pytest.raises(SpendAdmissionError, match="revoked"): + meter.reserve_request(token, policy.sha256, 100) + assert meter.parent.status()["reservations"][0]["state"] == "held" + meter.parent.reconcile( + run().id, cost_usd="1", receipt="audit:provider-stopped-final" + ) + assert meter.status()[0]["state"] == "charged" + with pytest.raises(SpendAdmissionError): + meter.bind_route( + run().id, + policy.sha256, + lease_id="replacement", + expires_at="2098-01-01T00:00:00Z", + ) + + +def test_unknown_terminal_revokes_and_known_overrun_freezes(meter, policy): + token = route(meter, policy) + receipt = meter.reserve_request(token, policy.sha256, 100) + with pytest.raises(SpendAdmissionError, match="breached"): + meter.complete_request(receipt, 6_000_000) + assert meter.parent.status()["breached"] + assert meter.parent.status()["reservations"][0]["liability"] == 6_000_000 + assert not meter.parent.observe(run().id, {"evidence": {}}) + assert not meter.request_active(receipt) + with pytest.raises(SpendAdmissionError): + meter.reserve_request(token, policy.sha256, 1) + + +def test_wrong_token_policy_expired_lease_and_revocation(meter, policy): + now = datetime.now(UTC) + token = meter.bind_route( + run().id, + policy.sha256, + lease_id="fixture", + expires_at=(now + timedelta(seconds=1)).isoformat(), + now=now, + ) + for credential, digest in [("a" * 43, policy.sha256), (token, "b" * 64)]: + with pytest.raises(SpendAdmissionError): + meter.reserve_request(credential, digest, 1, now=now) + receipt = meter.reserve_request(token, policy.sha256, 1, now=now) + assert not meter.request_active(receipt, now=now + timedelta(seconds=2)) + meter.revoke_route(run().id) + assert not meter.request_active(receipt, now=now + timedelta(seconds=2)) + + +def stream_bytes(*, truncated=False, output=10): + events = [ + { + "type": "message_start", + "message": { + "model": "fixture-model", + "usage": { + "input_tokens": 10, + "output_tokens": 0, + "cache_creation_input_tokens": 20, + "cache_read_input_tokens": 30, + }, + }, + }, + { + "type": "message_delta", + "delta": {"stop_reason": "end_turn"}, + "usage": {"output_tokens": output}, + }, + {"type": "message_stop"}, + ] + if truncated: + events.pop() + return "".join( + "event: " + e["type"] + "\ndata: " + json.dumps(e) + "\n\n" for e in events + ).encode() + + +@pytest.fixture +def fake_provider(): + seen = [] + mode = {"status": 200, "truncated": False, "output": 10} + + class Handler(http.server.BaseHTTPRequestHandler): + def log_message(self, *args): + pass + + def do_POST(self): + body = self.rfile.read(int(self.headers["Content-Length"])) + seen.append( + { + "key": self.headers.get("x-api-key"), + "path": self.path, + "body": json.loads(body), + "authorization": self.headers.get("Authorization"), + } + ) + response = stream_bytes(truncated=mode["truncated"], output=mode["output"]) + self.send_response(mode["status"]) + self.send_header("Content-Type", "text/event-stream") + self.send_header("Content-Length", str(len(response))) + self.end_headers() + if wait := mode.get("wait"): + mode["entered"].set() + wait.wait(5) + self.wfile.write(response) + + server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler) + threading.Thread(target=server.serve_forever, daemon=True).start() + yield server, seen, mode + server.shutdown() + server.server_close() + + +@pytest.fixture +def gateway(meter, policy, fake_provider): + provider, seen, mode = fake_provider + token = route(meter, policy) + server = MessagesServer( + policy, + meter, + provider_key="dummy-owner-key", + upstream_url=f"http://127.0.0.1:{provider.server_port}", + allow_test_http=True, + ) + server.start() + yield server, token, seen, mode + server.stop() + + +def request(gateway, **changes): + server, token, _, _ = gateway + body = { + "model": "fixture-model", + "max_tokens": 1000, + "stream": True, + "messages": [{"role": "user", "content": "private fixture prompt"}], + } + body.update(changes) + connection = http.client.HTTPConnection("127.0.0.1", server.port, timeout=5) + connection.request( + "POST", + "/v1/messages?beta=true", + json.dumps(body), + {"Content-Type": "application/json", "x-api-key": token}, + ) + response = connection.getresponse() + result = response.status, response.read() + connection.close() + return result + + +def test_transport_reserves_before_forward_and_preserves_parent(gateway, meter): + for _ in range(2): + assert request(gateway)[0] == 200 + assert request(gateway)[0] == 400 + assert len(gateway[2]) == 2 + assert all( + r["key"] == "dummy-owner-key" and not r["authorization"] for r in gateway[2] + ) + assert all(r["observed_microusd"] == 70000 for r in meter.status()) + assert sum(r["liability_microusd"] for r in meter.status()) == 4_000_000 + assert meter.parent.observe(run().id, success(meter.parent, run())) + assert request(gateway)[0] == 400 and len(gateway[2]) == 2 + + +@pytest.mark.parametrize( + "changes", + [ + {"max_tokens": True}, + {"max_tokens": 1001}, + {"max_tokens": 0}, + {"max_tokens": 1.5}, + {"model": "other-model"}, + {"stream": False}, + {"service_tier": "auto"}, + {"tools": [{"type": "web_search_20250305", "name": "web_search"}]}, + { + "messages": [ + { + "role": "user", + "content": [ + { + "type": "image", + "source": {"type": "url", "url": "https://example.com"}, + } + ], + } + ] + }, + {"context_management": {}}, + {"cache_control": {"type": "unknown"}}, + ], +) +def test_unsupported_requests_never_forward(gateway, meter, changes): + assert request(gateway, **changes)[0] == 400 + assert gateway[2] == [] and meter.status() == [] + + +@pytest.mark.parametrize("failure", ["truncated", "429", "overrun"]) +def test_uncertain_or_overrun_response_prevents_retry(gateway, meter, failure): + if failure == "truncated": + gateway[3]["truncated"] = True + if failure == "429": + gateway[3]["status"] = 429 + if failure == "overrun": + gateway[3]["output"] = 6000 + request(gateway) + assert request(gateway)[0] == 400 + assert len(gateway[2]) == 1 + assert meter.status()[0]["state"] == "held" + assert not meter.parent.observe(run().id, success(meter.parent, run())) + + +def test_inflight_lease_loss_holds_and_denies_concurrent_forward(gateway, meter): + gateway[3].update(wait=threading.Event(), entered=threading.Event()) + with ThreadPoolExecutor() as pool: + first = pool.submit(request, gateway) + assert gateway[3]["entered"].wait(3) + assert request(gateway)[0] == 400 + meter.revoke_route(run().id) + gateway[3]["wait"].set() + first.result(5) + assert len(gateway[2]) == 1 and meter.status()[0]["state"] == "held" + + +def test_missing_request_schema_never_recreates_parent(ledger, policy): + ledger.reserve(run()) + meter = RequestLedger(ledger) + with pytest.raises(SpendAdmissionError): + route(meter, policy) + with pytest.raises(SpendAdmissionError, match="before dispatch"): + meter.initialize() + assert ledger.status()["reservations"][0]["state"] == "held" diff --git a/workplans/REINAH-WP-0003-governed-runtime-integrity.md b/workplans/REINAH-WP-0003-governed-runtime-integrity.md index f105438..8795b67 100644 --- a/workplans/REINAH-WP-0003-governed-runtime-integrity.md +++ b/workplans/REINAH-WP-0003-governed-runtime-integrity.md @@ -672,6 +672,24 @@ Do not infer network/credential confinement from direct CLI tool rules: indirect Git helpers remain inside the sandbox owner's responsibility. No paid request, credential read or deployment was performed during the fixture proof. +### Provider request admission return — 2026-09-09 + +Implemented child request holds in the existing worker envelope and consumed +llm-connect's opt-in Messages transport (LLM-WP-0009). Real HTTP/SQLite proofs +cover exhaustion, simultaneous admission, unknown outcomes, replay, lease loss, +cache accounting and full-charge recovery. Parent terminal observation now +revokes the route and cannot settle unresolved children. The installed CLI +counterexample makes zero upstream calls through the guard; a permitted two-call +tool session creates its file. Worker full suite: 360 passed; llm-connect: 263 +passed. All provider responses, keys, bounds and FX are fixtures. + +See [request admission](../docs/request-admission.md). T05 remains progress for +protected owner hosting, actual queue lease/token injection and revocation, +provider custody and direct-route denial, accepted tariffs/FX and matching +runtime installation. LLM-WP-0009-T03 retains the transport integration return; +T06 remains wait for admitted real-model/natural-queue evidence. No production +policy, listener, credential, deployment or paid attempt was created. + ## Re-prove one governed profiled run and close residuals ```task