diff --git a/deploy/runtime-contract-lock.json b/deploy/runtime-contract-lock.json index 17677d0..5fa0526 100644 --- a/deploy/runtime-contract-lock.json +++ b/deploy/runtime-contract-lock.json @@ -7,7 +7,7 @@ }, "dependencies": [ { - "commit": "dc77f434a8b279436efa64e279af58372d275d8a", + "commit": "718e6730e405ecbf42da2cd4931f4de3ecbe2bae", "distribution": "llm-connect", "source": "../llm-connect", "version": "0.1.0" @@ -19,7 +19,7 @@ "version": "0.1.0" }, { - "commit": "3e49a98a0e2c4a64539a5ccd674be8cd67ac9845", + "commit": "bfe0e4c4c86003892796ba0fb990ae21d72278e1", "distribution": "sandboxer", "source": "../sand-boxer", "version": "0.0.0" diff --git a/docs/evidence/2026-09-09-owner-route.json b/docs/evidence/2026-09-09-owner-route.json new file mode 100644 index 0000000..dedfef8 --- /dev/null +++ b/docs/evidence/2026-09-09-owner-route.json @@ -0,0 +1,33 @@ +{ + "scope": "real local bwrap owner transport; deterministic provider, synthetic key and lease", + "private_absent": true, + "owner_state_through_proc_absent": true, + "pid_namespace": "pid:[4026532314]", + "network_namespace": "net:[4026532316]", + "interfaces": [ + "lo" + ], + "direct_blocked": [ + true, + true + ], + "proxy_absent": true, + "token_is_opaque": true, + "upstream_requests": 1, + "revoked_request_status": 400, + "request_reservations": [ + { + "receipt": "3933988d-bc84-408d-801c-78ac72f204df", + "run_id": "run-one", + "policy_sha256": "056776459012dbf1c0fe4f8fcdb1b7ad7f9f9ee3b53aac739d5c721c632d1b7d", + "lease_id": "b00c3e0f6297dda6ea6c04e0e83389f02b7a05151e98e023a686ab2c0d3b6a0f", + "liability_microusd": 2000000, + "state": "charged", + "observed_microusd": 70000, + "created_at": "2026-09-09T20:16:44.037006+00:00" + } + ], + "factory_attempts": 0, + "workspace_removed": true, + "socket_removed": true +} diff --git a/docs/owner-messages-route.md b/docs/owner-messages-route.md new file mode 100644 index 0000000..c8bf6be --- /dev/null +++ b/docs/owner-messages-route.md @@ -0,0 +1,51 @@ +# Same-host metered request owner + +Source API: `rein_aharness.messages_owner.MessagesOwner`. The trusted host bootstrap +constructs it with an accepted immutable `MessagesPolicy` and an explicitly supplied +provider key, then sets `OpsRunConfig.messages_owner`. This is an in-process owner +capability, not a queue field, serialized profile or remote sandbox API parameter. +The normal CLI does not acquire a key or construct an owner. Set +`AGENT_HARNESS_REQUIRE_REQUEST_ADMISSION=1` in a future admitted service so a missing +bootstrap refuses before claiming work. Parent spend admission must also be configured; +provision the RequestLedger schema explicitly before dispatch. No live bootstrap or +policy is installed by this source increment. + +`process_one` validates the accepted initial heartbeat's run ID, worker, attempt, +claimed state and future lease expiry. It replaces the stale claim expiry with that +accepted expiry. `execute_profiled_run` uses the same checked profile catalog and +parent reservation, then enters the owner context and gives Glas its bound sandbox +manager. The gateway's existing execution, artifact capture and teardown path is +retained. Every exit revokes the token before shutting down the listener. Existing +worker cancellation invokes revocation too; a deadline timer covers loss of +heartbeat connectivity. The route expires at the earlier of the initial accepted +lease and policy expiry. Renewal deliberately does not extend it; start another +admitted demand only through normal parent admission, never rebind an old run. + +The socket is mode 0600 in an ephemeral mode 0700 directory next to the private +ledger. llm-connect listens on AF_UNIX only. Sand-boxer binds exactly that socket +into its isolated namespace and reuses its bounded loopback byte bridge. Only the +opaque token reaches `ANTHROPIC_API_KEY`; `ANTHROPIC_BASE_URL` points to that bridge. +There is no provider key in workload memory, argv, mounts or public sandbox records. +The workload can reuse/encode its own route token, but the owner still enforces its +one run, expiry, policy and durable capacity. Host owner code/state must remain trusted. + +The ephemeral manager accepts one exact bwrap profile/actor/project/run tuple and +one sandbox. It refuses nonempty network egress, provider credential routes, setup +secrets, extra host mounts and owner-state overlap with source/workspace/runtime. +Remote owner transports cannot silently serialize this binding or fall back to +provider credential delivery. Existing unconfigured consumers remain unchanged. +The existing direct-CONNECT proof profile is incompatible with this metered mode; +a separately reviewed empty-egress profile is required for live use. + +Reconciliation semantics remain conservative: unknown provider outcomes retain +child and parent holds; shutdown or a killed socket is not proof of zero charge. +Do not automatically reopen, refund or retry from workload accounting. The same +existing receipt-backed operator reconciliation remains necessary after uncertainty. + +Validation: `tests/test_messages_owner.py`, `tests/test_repository_artifact_bwrap.py`, +and existing request/native-CLI suites. Enable `REIN_REAL_BWRAP=1` for the kernel +proof; `REIN_REAL_CLAUDE=1` retains the separate installed-CLI protocol fixtures. +The owner-route proof uses arbitrary Python in the real sandbox, an external fake +provider and synthetic key. It is a local confinement proof, not a paid provider, +protected-artifact or Railiance acceptance receipt. Remaining delivery is owned by +REINAH-WP-0003-T05/T06, LLM-WP-0009-T03 and SAND-WP-0015-T04 under HFACT T01/T03/T04. diff --git a/docs/request-admission.md b/docs/request-admission.md index c30f9b3..a350d63 100644 --- a/docs/request-admission.md +++ b/docs/request-admission.md @@ -41,3 +41,7 @@ With a USD 0.01 parent the same native request is refused with zero upstream calls; with adequate fixture capacity a two-request tool session succeeds. That verifies request admission and accounting, not a live EUR ceiling or owner/workload secret and network separation. + +Owner lifecycle and sole-route integration are now documented in +[owner-messages-route.md](owner-messages-route.md). This is local source/fixture +evidence; live bootstrap and placement acceptance remain open. diff --git a/rein_aharness/claim_loop.py b/rein_aharness/claim_loop.py index 49460ae..ff5a9a4 100644 --- a/rein_aharness/claim_loop.py +++ b/rein_aharness/claim_loop.py @@ -296,6 +296,10 @@ def process_one( try: spend = worker_spend(cfg) + if cfg.require_request_admission and cfg.messages_owner is None: + raise SpendAdmissionError("required request owner is not configured") + if cfg.messages_owner is not None and spend is None: + raise SpendAdmissionError("request owner requires parent spend admission") if spend is not None: spend.preflight() except SpendAdmissionError as exc: @@ -320,7 +324,15 @@ def process_one( try: # Make active lease ownership observable even for normal short jobs, # and fail closed before dispatch if Activity Core no longer accepts it. - client.heartbeat(run.id, lease_seconds=cfg.lease_seconds) + accepted = client.heartbeat(run.id, lease_seconds=cfg.lease_seconds) + if cfg.messages_owner is not None: + from rein_aharness.messages_owner import accepted_lease + try: + run.lease_until = accepted_lease(run, accepted, cfg.worker_id) + except SpendAdmissionError: + return ProcessResult(claimed=True, run_id=run.id, ok=False, + retry_full_interval=True, + reason="accepted queue lease required for metered route") logger.info("accepted initial heartbeat run_id=%s", run.id) except OpsRunError as exc: rejected = exc.status_code == 409 diff --git a/rein_aharness/glas_execution.py b/rein_aharness/glas_execution.py index 2420bf4..d15f4fc 100644 --- a/rein_aharness/glas_execution.py +++ b/rein_aharness/glas_execution.py @@ -10,6 +10,7 @@ from __future__ import annotations import math from collections.abc import Callable +from contextlib import nullcontext from typing import Any from rein_aharness.execution_cancel import ExecutionCancel, ExecutionCancelled, resolve_cancel @@ -175,6 +176,11 @@ def execute_profiled_run( request = request_factory(**_request_kwargs(run, config, report_to_hub)) kwargs = {"artifact_capture": transfer.capture} if transfer else {} spend = worker_spend(config) + owner = config.messages_owner + if config.require_request_admission and owner is None: + raise SpendAdmissionError("required request owner is not configured") + if owner is not None and (spend is None or guard is None): + raise SpendAdmissionError("request owner requires spend and cancellation") if spend is not None: from glas_harness.profiles import ProfileCatalog catalog = ProfileCatalog() @@ -186,7 +192,11 @@ def execute_profiled_run( if guard is not None: guard.check() spend.reserve(run) - result = gateway(request, **kwargs) + route = owner.activate(run, config, spend, profile, guard) if owner else nullcontext(None) + with route as owner_manager: + if owner_manager is not None: + kwargs["manager"] = owner_manager + result = gateway(request, **kwargs) raw = result.model_dump(mode="json") if hasattr(result, "model_dump") else result except ExecutionCancelled: raise diff --git a/rein_aharness/messages_owner.py b/rein_aharness/messages_owner.py new file mode 100644 index 0000000..3e2d479 --- /dev/null +++ b/rein_aharness/messages_owner.py @@ -0,0 +1,122 @@ +"""Explicit same-host request owner, activated only after worker admission. + +The bootstrap supplies the accepted Messages policy and provider credential. +No environment discovery, custody acquisition or default production activation. +""" + +from __future__ import annotations + +from contextlib import contextmanager +from dataclasses import dataclass, field +from datetime import UTC, datetime +import tempfile +import threading +from pathlib import Path + +from llm_connect.messages_gate import MessagesPolicy, MessagesServer +from rein_aharness.execution_cancel import ExecutionCancel +from rein_aharness.request_admission import RequestLedger +from rein_aharness.spend_admission import SpendAdmissionError, digest, timestamp + + +def accepted_lease(run, accepted, worker_id): + """Accept a queue response, never a caller-declared lease duration.""" + if ( + accepted.id != run.id + or accepted.claim_owner != worker_id + or accepted.attempt != run.attempt + or type(accepted.attempt) is not int or accepted.attempt < 1 + or accepted.state != "claimed" + or not isinstance(accepted.lease_until, str) + or timestamp(accepted.lease_until) <= datetime.now(UTC) + ): + raise SpendAdmissionError("accepted queue lease required for metered route") + return accepted.lease_until + + +@dataclass(frozen=True) +class MessagesOwner: + policy: MessagesPolicy + provider_key: str = field(repr=False) + upstream_url: str = "https://api.anthropic.com" + allow_test_http: bool = False + + @contextmanager + def activate(self, run, config, spend, profile, cancel: ExecutionCancel): + from sandboxer.core.manager import SandboxManager + from sandboxer.extensions.messages_route import OwnerMessagesRoute + + if spend is None or cancel is None: + raise SpendAdmissionError("metered owner requires spend and cancellation") + # The claim loop overwrites this only with its accepted initial heartbeat. + expiry = accepted_lease(run, run, config.worker_id) + if profile.credential_route_refs or profile.model.model != self.policy.model: + raise SpendAdmissionError("metered owner model or credential route mismatch") + deadline = min(timestamp(expiry), timestamp(spend.policy.expires_at)) + meter = RequestLedger(spend) + cancel.check() + token = meter.bind_route( + run.id, self.policy.sha256, + lease_id=digest({"run_id": run.id, "worker_id": config.worker_id, + "attempt": run.attempt, "lease_until": expiry}), + expires_at=deadline.isoformat(), + ) + stopped = threading.Event() + + def revoke(): + # This latch denies local forwards even if durable revocation fails. + stopped.set() + meter.revoke_route(run.id) + + class ActiveMeter: + def reserve_request(self, *args): + if stopped.is_set() or cancel.cancelled: + raise SpendAdmissionError("metered owner stopped") + return meter.reserve_request(*args) + + def request_active(self, receipt): + return not stopped.is_set() and not cancel.cancelled and meter.request_active(receipt) + + def complete_request(self, *args): + return meter.complete_request(*args) + + cancel.register_stop(revoke) + timer = threading.Timer( + max(0, (deadline - datetime.now(UTC)).total_seconds()), + lambda: cancel.cancel("timeout"), + ) + timer.daemon = True + server = None + try: + # Keep the socket with the private ledger, outside source/runtime/workspace. + with tempfile.TemporaryDirectory(prefix="messages-", dir=spend.path.parent) as directory: + socket_path = Path(directory) / "route.sock" + server = MessagesServer( + self.policy, ActiveMeter(), provider_key=self.provider_key, + upstream_url=self.upstream_url, allow_test_http=self.allow_test_http, + unix_path=socket_path, + ) + route = OwnerMessagesRoute( + socket_path, token, profile.sandbox_profile, "agt", + config.execution_project, run.id, + private_paths=(spend.path.parent,), + ) + server.start() + timer.start() + try: + cancel.check() + yield SandboxManager(messages_route=route) + finally: + # Before teardown of the listener, even on a refused/failed gateway. + try: + revoke() + finally: + server.stop() + server = None + finally: + timer.cancel() + try: + revoke() + finally: + if server is not None: + server.stop() diff --git a/rein_aharness/ops_run_client.py b/rein_aharness/ops_run_client.py index ef7661e..3dc988f 100644 --- a/rein_aharness/ops_run_client.py +++ b/rein_aharness/ops_run_client.py @@ -141,6 +141,8 @@ class OpsRunConfig: require_spend_admission: bool = False spend_policy_path: str | None = None spend_ledger_path: str | None = None + require_request_admission: bool = False + messages_owner: Any = field(default=None, repr=False) @classmethod def from_env(cls) -> "OpsRunConfig": @@ -181,6 +183,7 @@ class OpsRunConfig: require_spend_admission=bool(os.environ.get("AGENT_HARNESS_REQUIRE_SPEND_ADMISSION", "")), spend_policy_path=os.environ.get("AGENT_HARNESS_SPEND_POLICY"), spend_ledger_path=os.environ.get("AGENT_HARNESS_SPEND_LEDGER"), + require_request_admission=bool(os.environ.get("AGENT_HARNESS_REQUIRE_REQUEST_ADMISSION", "")), ) diff --git a/scripts/verify-recovery-contracts.sh b/scripts/verify-recovery-contracts.sh index 242f0a7..70c1ef2 100755 --- a/scripts/verify-recovery-contracts.sh +++ b/scripts/verify-recovery-contracts.sh @@ -12,6 +12,7 @@ PYTHONPATH="${REPO_ROOT}:${REPO_ROOT}/../llm-connect" \ tests/test_native_limits.py \ tests/test_spend_admission.py \ tests/test_request_admission.py \ + tests/test_messages_owner.py \ tests/test_claim_loop.py::test_process_one_initial_heartbeat_rejection_refuses_dispatch \ tests/test_claim_loop.py::test_process_one_lease_loss_cancels_registered_adapter_process \ tests/test_claim_loop.py::test_profiled_close_failure_happens_after_repository_lock_release \ diff --git a/scripts/verify-runtime-contracts.sh b/scripts/verify-runtime-contracts.sh index 7560838..228299a 100755 --- a/scripts/verify-runtime-contracts.sh +++ b/scripts/verify-runtime-contracts.sh @@ -13,6 +13,7 @@ PYTHONPATH="${REPO_ROOT}:${REPO_ROOT}/../llm-connect" \ tests/test_native_limits.py \ tests/test_spend_admission.py \ tests/test_request_admission.py \ + tests/test_messages_owner.py \ tests/test_glas_execution.py \ tests/test_ops_run_client.py \ tests/test_claim_loop.py \ diff --git a/tests/test_messages_owner.py b/tests/test_messages_owner.py new file mode 100644 index 0000000..53ed6fc --- /dev/null +++ b/tests/test_messages_owner.py @@ -0,0 +1,191 @@ +"""Owner/worker lifecycle; real namespace proof remains explicitly opt-in.""" + +from dataclasses import replace +from datetime import UTC, datetime, timedelta +import json +import os +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest + +from rein_aharness.execution_cancel import ExecutionCancel +from rein_aharness.messages_owner import MessagesOwner, accepted_lease +from rein_aharness.ops_run_client import ActivityCoreOpsClient, OpsRunConfig +from rein_aharness.request_admission import RequestLedger +from rein_aharness.spend_admission import SpendAdmissionError +from test_spend_admission import ledger as ledger, run, configured +from test_request_admission import policy as policy, fake_provider as fake_provider + + +def claimed(): + return replace(run(), state="claimed", lease_until=(datetime.now(UTC) + timedelta(seconds=60)).isoformat()) + + +@pytest.mark.parametrize("fields", [ + {"id": "other"}, {"claim_owner": "other"}, {"attempt": 2}, + {"state": "open"}, {"lease_until": None}, {"lease_until": "invalid"}, + {"lease_until": "2020-01-01T00:00:00Z"}, +]) +def test_refuse_unaccepted_lease(fields): + item = claimed() + with pytest.raises(SpendAdmissionError): + accepted_lease(item, replace(item, **fields), "worker-1") + + +def test_worker_requires_owner_before_claim(tmp_path, monkeypatch): + from rein_aharness.claim_loop import process_one + from rein_aharness.close_outbox import CloseOutbox + client = MagicMock(spec=ActivityCoreOpsClient) + client.config = OpsRunConfig(require_request_admission=True) + result = process_one(client, outbox=CloseOutbox(state_dir=tmp_path / "close")) + assert not result.claimed and "not configured" in result.reason + client.claim.assert_not_called() + + +@pytest.mark.parametrize("cancel_reason", ["lease-loss", "signal", "timeout", None]) +def test_owner_closes_and_never_rebinds(ledger, policy, fake_provider, cancel_reason, monkeypatch, tmp_path): + monkeypatch.setenv("XDG_DATA_HOME", str(tmp_path / "data")) + meter = RequestLedger(ledger) + meter.initialize() + item = claimed() + ledger.reserve(item) + cfg = configured(ledger) + provider, seen, _ = fake_provider + owner = MessagesOwner(policy, "dummy-provider-key", f"http://127.0.0.1:{provider.server_port}", True) + profile = SimpleNamespace(credential_route_refs=[], model=SimpleNamespace(model="fixture-model"), + sandbox_profile="profile.bwrap-local") + cancel = ExecutionCancel() + with owner.activate(item, cfg, ledger, profile, cancel) as manager: + binding = manager._messages_route + assert binding.socket_path.is_socket() + if cancel_reason: + cancel.cancel(cancel_reason) + with pytest.raises(SpendAdmissionError, match="revoked"): + meter.reserve_request(binding.token, policy.sha256, 1) + assert not binding.socket_path.exists() and not seen + with pytest.raises(SpendAdmissionError, match="revoked"): + meter.reserve_request(binding.token, policy.sha256, 1) + assert ledger.status()["reservations"][0]["state"] == "held" + + +@pytest.mark.skipif(os.environ.get("REIN_REAL_BWRAP") != "1", reason="opt-in kernel namespace proof") +def test_real_owner_route_confinement_and_revocation(ledger, policy, fake_provider, tmp_path, monkeypatch): + from sandboxer.models import SandboxCreateRequest, SandboxExecRequest, Consumer + + monkeypatch.setenv("SANDBOXER_NO_STATE_HUB", "1") + monkeypatch.setenv("XDG_DATA_HOME", str(tmp_path / "data")) + meter = RequestLedger(ledger) + meter.initialize() + item = claimed() + ledger.reserve(item) + cfg = configured(ledger) + provider, seen, _ = fake_provider + owner = MessagesOwner(policy, "synthetic-provider-key-owner-only", f"http://127.0.0.1:{provider.server_port}", True) + profile = SimpleNamespace(credential_route_refs=[], model=SimpleNamespace(model="fixture-model"), + sandbox_profile="profile.bwrap-local") + consumer = Consumer(actor="agt", project=cfg.execution_project, run_id=item.id) + cancel = ExecutionCancel() + source = tmp_path / "source" + source.mkdir() + (source / "README.md").write_text("No owner state belongs in this tree.\n") + with owner.activate(item, cfg, ledger, profile, cancel) as manager: + binding = manager._messages_route + status = manager.create(SandboxCreateRequest(profile=profile.sandbox_profile, + inputs={"repo": str(source)}, consumer=consumer)) + try: + assert status.state.value == "ready", status.error + def execute(code): + return manager.execute(status.sandbox_id, SandboxExecRequest( + command=["python3", "-c", code], consumer=consumer, timeout_seconds=10)) + probe = execute("""import os, json, socket +from pathlib import Path +blocked = [] +for address in [("127.0.0.1", PROVIDER_PORT), ("1.1.1.1", 443)]: + try: + s = socket.create_connection(address, timeout=0.2); s.close(); blocked.append(False) + except OSError: blocked.append(True) +print(json.dumps({"private_absent": not Path(PRIVATE).exists(), + "owner_state_through_proc_absent": not Path(OWNER_PROCESS).exists(), + "pid_namespace": os.readlink("/proc/self/ns/pid"), + "network_namespace": os.readlink("/proc/self/ns/net"), + "interfaces": sorted(p.name for p in Path("/sys/class/net").glob("*")) if Path("/sys/class/net").exists() else [line.split(':')[0].strip() for line in Path('/proc/net/dev').read_text().splitlines()[2:]], + "direct_blocked": blocked, "proxy_absent": not any("proxy" in k.lower() for k in os.environ), + "token_is_opaque": len(os.environ["ANTHROPIC_API_KEY"]) == 43})) +""".replace("PROVIDER_PORT", str(provider.server_port)).replace("PRIVATE", repr(str(ledger.path.parent))).replace("OWNER_PROCESS", repr(f"/proc/{os.getpid()}/root/{str(ledger.path.parent).lstrip(chr(47))}"))) + assert probe.exit_code == 0, probe.stderr + evidence = json.loads(probe.stdout) + assert evidence["private_absent"] and evidence["owner_state_through_proc_absent"], evidence + assert evidence["direct_blocked"] == [True, True] and evidence["proxy_absent"] + assert evidence["token_is_opaque"] and evidence["interfaces"] == ["lo"] + assert evidence["network_namespace"] != os.readlink("/proc/self/ns/net") + assert evidence["pid_namespace"] != os.readlink("/proc/self/ns/pid") + request_code = """import os,json,http.client,urllib.parse +u=urllib.parse.urlsplit(os.environ['ANTHROPIC_BASE_URL']) +c=http.client.HTTPConnection(u.hostname,u.port,timeout=5) +c.request('POST','/v1/messages',json.dumps({'model':'fixture-model','max_tokens':1000,'stream':True,'messages':[{'role':'user','content':'synthetic request'}]}),{'Content-Type':'application/json','x-api-key':os.environ['ANTHROPIC_API_KEY']}) +r=c.getresponse(); print(json.dumps({'status':r.status,'stopped':b'message_stop' in r.read()})); c.close() +""" + response = execute(request_code) + assert response.exit_code == 0, response.stderr + assert json.loads(response.stdout) == {"status": 200, "stopped": True} + assert len(seen) == 1 and seen[0]["key"] == "synthetic-provider-key-owner-only" + assert meter.status()[0]["state"] == "charged" + # This hook is the same callback used by the worker heartbeat monitor. + cancel.cancel("lease-loss") + response = execute(request_code) + assert json.loads(response.stdout)["status"] == 400 + assert len(seen) == 1 + assert binding.token not in json.dumps(status.model_dump(mode="json")) + assert "synthetic-provider-key-owner-only" not in probe.stdout + response.stdout + if directory := os.environ.get("REIN_OWNER_PROOF_OUTPUT"): + out = Path(directory); out.mkdir(parents=True, exist_ok=True) + (out / "owner-route.json").write_text(json.dumps({ + "scope": "real local bwrap owner transport; deterministic provider, synthetic key and lease", + **evidence, "upstream_requests": len(seen), "revoked_request_status": 400, + "request_reservations": meter.status(), "factory_attempts": 0, + }, indent=2) + "\n") + finally: + manager.destroy(status.sandbox_id) + assert not Path(status.inputs["workspace_dir"]).exists() + assert not binding.socket_path.exists() + + +@pytest.mark.parametrize("valid", [False, True]) +def test_claim_loop_uses_accepted_heartbeat_lease(ledger, policy, fake_provider, tmp_path, monkeypatch, valid): + from rein_aharness.claim_loop import process_one + from rein_aharness.close_outbox import CloseOutbox + client = MagicMock(spec=ActivityCoreOpsClient) + client.config = configured(ledger) + client.config.messages_owner = MessagesOwner(policy, "dummy") + item = claimed() + client.claim.return_value = [item] + expiry = (datetime.now(UTC) + timedelta(seconds=30)).isoformat() if valid else None + client.heartbeat.return_value = replace(item, lease_until=expiry) + process = MagicMock(return_value=SimpleNamespace(claimed=True)) + monkeypatch.setattr("rein_aharness.claim_loop._process_profiled_run", process) + result = process_one(client, outbox=CloseOutbox(state_dir=tmp_path / "close")) + if valid: + assert item.lease_until == expiry + process.assert_called_once() + else: + assert "accepted queue lease" in result.reason + process.assert_not_called() + + +def test_lease_expiry_revokes_without_a_heartbeat(ledger, policy, tmp_path, monkeypatch): + monkeypatch.setenv("XDG_DATA_HOME", str(tmp_path / "data")) + meter = RequestLedger(ledger) + meter.initialize() + item = claimed() + item.lease_until = (datetime.now(UTC) + timedelta(seconds=0.5)).isoformat() + ledger.reserve(item) + profile = SimpleNamespace(credential_route_refs=[], model=SimpleNamespace(model="fixture-model"), + sandbox_profile="profile.bwrap-local") + cancel = ExecutionCancel() + with MessagesOwner(policy, "dummy").activate(item, configured(ledger), ledger, profile, cancel) as manager: + binding = manager._messages_route + assert cancel.wait(2), "accepted lease deadline was not enforced" + with pytest.raises(SpendAdmissionError): + meter.reserve_request(binding.token, policy.sha256, 1) diff --git a/tests/test_repository_artifact_bwrap.py b/tests/test_repository_artifact_bwrap.py index 1bd0cc6..3f7506f 100644 --- a/tests/test_repository_artifact_bwrap.py +++ b/tests/test_repository_artifact_bwrap.py @@ -2,6 +2,7 @@ from __future__ import annotations import json +from datetime import UTC, datetime, timedelta import os from pathlib import Path from unittest.mock import MagicMock @@ -31,6 +32,10 @@ from rein_aharness.ops_run_client import ( from test_repository_artifact import commit, git from rein_aharness.repository_grant import RepositoryGrant from rein_aharness.spend_admission import SpendLedger, SpendPolicy, digest +from rein_aharness.messages_owner import MessagesOwner +from rein_aharness.request_admission import RequestLedger +from llm_connect.messages_gate import MessagesPolicy +from test_request_admission import fake_provider as fake_provider pytestmark = pytest.mark.skipif( os.environ.get("REIN_REAL_BWRAP") != "1", reason="opt-in kernel namespace proof" @@ -42,8 +47,9 @@ class DeterministicRein(Rein): workspace = None head = None - def __init__(self, source, baseline): + def __init__(self, source, baseline, metered=False): self.source, self.baseline = source, baseline + self.metered = metered def start_session(self, profile, inputs, sandbox): transport = transport_from_sandbox(sandbox) @@ -52,6 +58,14 @@ class DeterministicRein(Rein): def dispatch_tool(self, transport, tool_call): self.calls += 1 + if self.metered: + response = transport.run(["python3", "-c", """import os,json,http.client,urllib.parse +u=urllib.parse.urlsplit(os.environ['ANTHROPIC_BASE_URL']) +c=http.client.HTTPConnection(u.hostname,u.port,timeout=5) +c.request('POST','/v1/messages',json.dumps({'model':'fixture-model','max_tokens':1000,'stream':True,'messages':[{'role':'user','content':'fixture only'}]}),{'Content-Type':'application/json','x-api-key':os.environ['ANTHROPIC_API_KEY']}) +r=c.getresponse(); assert r.status==200; assert b'message_stop' in r.read(); c.close() +"""], timeout=10) + assert response.returncode == 0, response.stderr code = """from pathlib import Path import subprocess Path('result.txt').write_text('sandbox-result-only\\n') @@ -79,8 +93,8 @@ print(git('rev-parse','HEAD')) assert git(self.source, "rev-parse", "HEAD") == self.baseline -@pytest.mark.parametrize("with_spend", [False, True]) -def test_real_bwrap_worker_import_and_close_replay(tmp_path, monkeypatch, with_spend): +@pytest.mark.parametrize("with_spend", [False, True, "requests"]) +def test_real_bwrap_worker_import_and_close_replay(tmp_path, monkeypatch, with_spend, fake_provider): monkeypatch.setenv("XDG_DATA_HOME", str(tmp_path / "data")) monkeypatch.setenv("REIN_AHARNESS_STATE_DIR", str(tmp_path / "state")) monkeypatch.setenv("SANDBOXER_NO_STATE_HUB", "1") @@ -108,6 +122,7 @@ def test_real_bwrap_worker_import_and_close_replay(tmp_path, monkeypatch, with_s repository_grant=grant, claim_owner="fixture-worker", attempt=1, + lease_until=(datetime.now(UTC) + timedelta(seconds=90)).isoformat(), ) client = MagicMock(spec=ActivityCoreOpsClient) client.config = OpsRunConfig( @@ -141,6 +156,12 @@ def test_real_bwrap_worker_import_and_close_replay(tmp_path, monkeypatch, with_s ) } ) + if with_spend == "requests": + admitted, _ = catalog.resolve(run.harness_profile_ref) + catalog.profiles()[(admitted.id, admitted.version)] = admitted.model_copy(update={ + "model": admitted.model.model_copy(update={"model": "fixture-model"}), + "sandbox_profile": "profile.bwrap-local", "credential_route_refs": [], + }) spend = None if with_spend: private = tmp_path / "spend-state" @@ -166,12 +187,20 @@ def test_real_bwrap_worker_import_and_close_replay(tmp_path, monkeypatch, with_s client.config.spend_policy_path = str(policy_path) client.config.spend_ledger_path = str(spend.path) monkeypatch.setattr("glas_harness.profiles.ProfileCatalog", lambda: catalog) + if with_spend == "requests": + RequestLedger(spend).initialize() + provider, seen, _ = fake_provider + client.config.messages_owner = MessagesOwner( + MessagesPolicy("fixture:upper-rate", "fixture-model", 1000, 1000, 1000, 1000), + "fixture-owner-key", f"http://127.0.0.1:{provider.server_port}", True, + ) + client.config.require_request_admission = True manager = SandboxManager(store=SandboxStore(tmp_path / "sandboxes.json")) - rein = DeterministicRein(source, baseline) + rein = DeterministicRein(source, baseline, metered=with_spend == "requests") monkeypatch.setattr( "glas_harness.gateway.run_execution", lambda request, **kwargs: run_execution( - request, catalog=kwargs.pop("catalog", catalog), rein=rein, manager=manager, **kwargs + request, catalog=kwargs.pop("catalog", catalog), rein=rein, manager=kwargs.pop("manager", manager), **kwargs ), ) outbox = CloseOutbox(state_dir=tmp_path / "state") @@ -205,3 +234,10 @@ def test_real_bwrap_worker_import_and_close_replay(tmp_path, monkeypatch, with_s assert client.complete.call_count == 2 assert client.heartbeat.call_count >= 1 assert git(source, "rev-list", "--count", "HEAD") == "2" + + if with_spend == "requests": + assert len(seen) == 1 and seen[0]["key"] == "fixture-owner-key" + assert RequestLedger(spend).status()[0]["state"] == "charged" + with spend._db() as db: + route = db.execute("SELECT * FROM request_routes").fetchone() + assert route["revoked"] == 1 and route["expires_at"] == run.lease_until diff --git a/workplans/REINAH-WP-0003-governed-runtime-integrity.md b/workplans/REINAH-WP-0003-governed-runtime-integrity.md index 8795b67..adf35f3 100644 --- a/workplans/REINAH-WP-0003-governed-runtime-integrity.md +++ b/workplans/REINAH-WP-0003-governed-runtime-integrity.md @@ -736,3 +736,32 @@ truth, and no residual exists only in prose. repo has an authoritative validated classification. 6. Production deployment, recovery checks, CI, package versions, and docs all describe and exercise the same supported topology. + +### Local owner-route integration return — 2026-09-09 + +`MessagesOwner` now starts an owner-only Unix Messages listener after the worker +reserves its parent envelope. The initial accepted Activity Core heartbeat supplies +its exact expiry, run, worker and attempt binding. Lease loss, timeout, signals, +gateway exceptions and normal exit revoke the route. A timer also enforces the +initial lease deadline; heartbeat renewal does not extend this first route. +Only the opaque token and namespace-local base URL reach the child. Provider key, +ledger and owner socket directory remain outside the workload's mounts and PID +namespace. The sandbox refuses direct egress, alternate credential delivery, +extra host mounts, consumer mismatches and additional sandboxes for this binding. + +Real local bwrap tests prove sole-route forwarding, direct host/public-IP denial, +private-state absence, revocation with no second forward, and teardown. The actual +worker/Glas/bwrap/ledger path also imports its permitted fixture commit and replays +a lost terminal close without repeating the request or authoring. Queue, provider, +credential and authoring remain deterministic fixtures; factory attempts remain 0. +Worker suite: 377 passed, including installed CLI and real namespace tests. +Sand-boxer required `make check`: lint clean, 199 tests passed. LLM suite: 264 +passed; changed transport adds no lint/type diagnostics, with existing full-repo +177 Ruff/36 mypy diagnostics still requiring pre-release disposition. + +This closes local source route/lease/token/confinement wiring. Remaining return: +admitted owner bootstrap that supplies the provider key to `MessagesOwner`, matched +protected runtime/CLI artifact, Railiance host/profile/consumer/custody/recovery +admission, live provider compatibility and accepted bounds/tariffs/FX, then G0 and +natural model/queue evidence. No protected runtime was installed or promoted, +no existing CCR changed, no secret read or paid execution took place.