feat: import governed sandbox commits and enforce native CLI limits
Some checks are pending
Governed runtime contract / contract (push) Waiting to run

Assistant: codex
Assistant-Model: gpt-5.6-luna
Assistant-Session: 01a07ff8-19d0-7820-b4d0-1353833cb7fc
This commit is contained in:
tegwick 2026-09-09 14:08:10 +02:00
parent 1429db5ad4
commit c63caf5568
16 changed files with 1236 additions and 7 deletions

137
tests/test_native_limits.py Normal file
View file

@ -0,0 +1,137 @@
from __future__ import annotations
import json
import subprocess
from unittest.mock import MagicMock, patch
import pytest
from llm_connect.models import BudgetTracker, RunConfig
from rein_aharness.adapter import AgenticClaudeCodeAdapter
from rein_aharness.native_limits import NativeLimitError, terminal_accounting
from rein_aharness.taskspec import TaskSpec, TaskSpecError
from test_adapter import _fake_proc
def terminal(**overrides):
return {
"type": "result",
"subtype": "success",
"is_error": False,
"result": "done",
"num_turns": 2,
"total_cost_usd": 0.1,
"usage": {"input_tokens": 10, "output_tokens": 2, "cache_read_input_tokens": 3},
**overrides,
}
@pytest.mark.parametrize("stream", [False, True])
def test_native_controls_reach_cli_and_account_usage(tmp_path, stream):
adapter = AgenticClaudeCodeAdapter(
workdir=tmp_path, on_tool_event=(lambda e: None) if stream else None
)
config = RunConfig(
timeout_seconds=30,
model_params={"max_budget_usd": 0.25, "max_turns": 3},
budget_tracker=BudgetTracker(total=100),
)
payload = json.dumps(terminal())
proc = _fake_proc([payload + "\n"]) if stream else MagicMock()
if not stream:
proc.communicate.return_value = (payload, "")
proc.returncode = 0
with (
patch(
"rein_aharness.adapter.subprocess.run",
return_value=subprocess.CompletedProcess(
[], 0, "2.1.263 (Claude Code)\n", ""
),
),
patch("rein_aharness.adapter.subprocess.Popen", return_value=proc) as invoke,
):
result = adapter.execute_prompt("task", config)
argv = invoke.call_args.args[0]
assert argv[argv.index("--max-budget-usd") + 1] == "0.25"
assert argv[argv.index("--max-turns") + 1] == "3"
assert result.usage["total_tokens"] == config.budget_tracker.spent == 15
assert result.metadata["cost_usd"] == 0.1
@pytest.mark.parametrize(
"changes",
[
{"total_cost_usd": None},
{"total_cost_usd": float("nan")},
{"total_cost_usd": True},
{"total_cost_usd": 0.3},
{"num_turns": 4},
{"usage": {}},
{"subtype": "error_max_budget_usd", "is_error": True},
{"subtype": "error_max_turns", "is_error": True},
],
)
def test_unaccounted_or_exhausted_result_cannot_succeed(changes):
with pytest.raises(NativeLimitError):
terminal_accounting(terminal(**changes), max_budget_usd=0.25, max_turns=3)
def test_error_preserves_only_bounded_cost():
with pytest.raises(NativeLimitError) as caught:
terminal_accounting(
terminal(subtype="error_max_budget_usd", is_error=True, errors=["secret"]),
max_budget_usd=0.25,
max_turns=3,
)
assert caught.value.cost_usd == 0.1 and "secret" not in str(caught.value)
@pytest.mark.parametrize("version", ["2.1.216 (Claude Code)", "unrecognized"])
def test_old_cli_refuses_before_prompt(tmp_path, version):
adapter = AgenticClaudeCodeAdapter(workdir=tmp_path)
with (
patch(
"rein_aharness.adapter.subprocess.run",
return_value=subprocess.CompletedProcess([], 0, version, ""),
),
patch("rein_aharness.adapter.subprocess.Popen") as invoke,
):
with pytest.raises(NativeLimitError, match="2.1.217"):
adapter.execute_prompt(
"task", RunConfig(model_params={"max_budget_usd": 0.25})
)
invoke.assert_not_called()
@pytest.mark.parametrize(
"limits",
[
{"max_budget_usd": True},
{"max_budget_usd": -1},
{"max_budget_usd": float("inf")},
{"max_turns": True},
{"max_turns": 1.5},
],
)
def test_invalid_task_limits_refuse(tmp_path, limits):
with pytest.raises(TaskSpecError):
TaskSpec(title="task", description="d", target_repo=tmp_path, **limits)
def test_runner_forwards_task_limits(tmp_path):
from test_runner import CommittingAdapter, _make_repo
from rein_aharness.runner import run_task
repo = _make_repo(tmp_path)
adapter = CommittingAdapter(repo)
result = run_task(
TaskSpec(
title="task",
description="d",
target_repo=repo,
max_budget_usd=0.25,
max_turns=3,
),
adapter=adapter,
report_to_hub=False,
write_metrics=False,
)
assert result.ok
assert adapter.configs[0].model_params == {"max_budget_usd": 0.25, "max_turns": 3}

View file

@ -0,0 +1,312 @@
"""Actual Git export/import across separate workspaces; no gateway writes source."""
from __future__ import annotations
import os
import shutil
import subprocess
from types import SimpleNamespace
from unittest.mock import patch
import pytest
from rein_aharness.execution_cancel import ExecutionCancel, ExecutionCancelled
from rein_aharness.repository_artifact import (
RepositoryArtifactError,
RepositoryArtifactTransfer,
)
from rein_aharness.repository_grant import RepositoryGrant
from rein_aharness.repository_transaction import (
RepositoryAcceptanceError,
RepositoryTransaction,
)
def git(repo, *args):
return (
subprocess.check_output(
["git", "-C", str(repo), *args], stderr=subprocess.DEVNULL
)
.decode()
.strip()
)
def commit(repo, name="result.txt", content="accepted change\n"):
path = repo / name
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(content)
git(repo, "add", name)
git(
repo,
"-c",
"user.name=Fixture",
"-c",
"user.email=fixture@example.invalid",
"commit",
"-qm",
"bounded result",
)
return git(repo, "rev-parse", "HEAD")
def fixture(tmp_path):
source = tmp_path / "source"
source.mkdir()
git(source, "init", "-q")
commit(source, "README.md", "baseline\n")
sandbox = tmp_path / "sandbox"
git(tmp_path, "clone", "--quiet", "--no-local", str(source), str(sandbox))
grant = RepositoryGrant.from_mapping(
{
"version": "1",
"allowed_paths": ["result.txt"],
"commit_count": {"min": 1, "max": 1},
"publish": False,
}
)
return source, sandbox, grant
class Transport:
"""Local transport double only; exporter and Git run unmodified."""
kind = "local_namespace"
def __init__(self, workspace):
self.workspace = str(workspace)
def run(self, argv, *, timeout):
return subprocess.run(
argv, cwd=self.workspace, capture_output=True, text=True, timeout=timeout
)
def capture(transfer, sandbox, head):
with patch(
"glas_harness.transport.transport_from_sandbox", return_value=Transport(sandbox)
):
transfer.capture(object(), SimpleNamespace(commit_sha=head))
def test_commit_survives_sandbox_teardown_and_import_preserves_identity(tmp_path):
source, sandbox, grant = fixture(tmp_path)
with RepositoryTransaction(source, state_dir=tmp_path / "state") as tx:
baseline = tx.baseline.head
transfer = RepositoryArtifactTransfer(tx, grant)
head = commit(sandbox)
capture(transfer, sandbox, head)
assert git(source, "rev-parse", "HEAD") == baseline
assert not (source / "result.txt").exists()
shutil.rmtree(sandbox)
result = transfer.import_after_teardown(head)
assert result["head"] == head and result["imported"]
assert git(source, "rev-parse", "HEAD") == head
assert (source / "result.txt").read_text() == "accepted change\n"
assert not git(source, "status", "--porcelain")
assert tx.acceptance.changed_paths == ("result.txt",)
assert "bundle" not in result
@pytest.mark.parametrize(
"fault",
[
"ungranted",
"two_commits",
"wrong_reported_head",
"dirty",
"symlink",
"expanded_size",
],
)
def test_invalid_artifact_cannot_change_source(tmp_path, fault):
source, sandbox, grant = fixture(tmp_path)
with RepositoryTransaction(source, state_dir=tmp_path / "state") as tx:
baseline = tx.baseline.head
transfer = RepositoryArtifactTransfer(tx, grant)
head = commit(
sandbox,
"outside.txt" if fault == "ungranted" else "result.txt",
"x" * (3 * 1024 * 1024) if fault == "expanded_size" else "result\n",
)
if fault == "two_commits":
head = commit(sandbox, content="again\n")
if fault == "dirty":
(sandbox / "untracked.txt").write_text("dirty")
if fault == "symlink":
git(sandbox, "rm", "result.txt")
os.symlink("/tmp/outside", sandbox / "result.txt")
git(sandbox, "add", "result.txt")
git(
sandbox,
"-c",
"user.name=Fixture",
"-c",
"user.email=fixture@example.invalid",
"commit",
"--amend",
"--no-edit",
"-q",
)
head = git(sandbox, "rev-parse", "HEAD")
with pytest.raises((RepositoryArtifactError, RepositoryAcceptanceError)):
capture(transfer, sandbox, head)
transfer.import_after_teardown(
"f" * 40 if fault == "wrong_reported_head" else head
)
assert git(source, "rev-parse", "HEAD") == baseline
assert not (source / "result.txt").exists()
assert not (source / "outside.txt").exists()
def test_lease_loss_after_capture_refuses_import(tmp_path):
source, sandbox, grant = fixture(tmp_path)
cancel = ExecutionCancel()
with RepositoryTransaction(source, state_dir=tmp_path / "state") as tx:
transfer = RepositoryArtifactTransfer(tx, grant, cancel=cancel)
head = commit(sandbox)
capture(transfer, sandbox, head)
cancel.cancel("lease-loss")
with pytest.raises(ExecutionCancelled):
transfer.import_after_teardown(head)
assert git(source, "rev-parse", "HEAD") == tx.baseline.head
def test_changed_source_baseline_refuses_import(tmp_path):
source, sandbox, grant = fixture(tmp_path)
with RepositoryTransaction(source, state_dir=tmp_path / "state") as tx:
transfer = RepositoryArtifactTransfer(tx, grant)
head = commit(sandbox)
capture(transfer, sandbox, head)
human_head = commit(source, "human.txt", "new human work\n")
with pytest.raises(RepositoryAcceptanceError, match="baseline-changed"):
transfer.import_after_teardown(head)
assert git(source, "rev-parse", "HEAD") == human_head
assert (source / "human.txt").read_text() == "new human work\n"
assert not (source / "result.txt").exists()
def test_gateway_failure_after_capture_does_not_import(tmp_path):
from rein_aharness.glas_execution import execute_profiled_run
from rein_aharness.ops_run_client import OpsRun, OpsRunConfig
source, sandbox, grant = fixture(tmp_path)
run = OpsRun(
id="run",
activity_definition_id="def",
idempotency_key="key",
target_repo=str(source),
title="fixture",
description="fixture",
state="claimed",
harness_profile_ref="harness.agent-dev-local@1.0.0",
repository_grant=grant,
)
with RepositoryTransaction(source, state_dir=tmp_path / "state") as tx:
head = commit(sandbox)
def gateway(request, *, artifact_capture):
with patch(
"glas_harness.transport.transport_from_sandbox",
return_value=Transport(sandbox),
):
artifact_capture(object(), SimpleNamespace(commit_sha=head))
return {
"ok": False,
"evidence": {
"commit_sha": head,
"outcome": "failed",
"sandbox_destroy": "failed",
},
}
result = execute_profiled_run(
run,
OpsRunConfig(repo_roots=(str(tmp_path),)),
report_to_hub=False,
transaction=tx,
request_factory=lambda **kw: kw,
gateway=gateway,
)
assert not result["ok"]
assert git(source, "rev-parse", "HEAD") == tx.baseline.head
def test_success_without_artifact_capture_refuses(tmp_path):
from rein_aharness.glas_execution import execute_profiled_run
from rein_aharness.ops_run_client import OpsRun, OpsRunConfig
source, sandbox, grant = fixture(tmp_path)
run = OpsRun(
id="run",
activity_definition_id="def",
idempotency_key="key",
target_repo=str(source),
title="fixture",
description="fixture",
state="claimed",
harness_profile_ref="harness.agent-dev-local@1.0.0",
repository_grant=grant,
)
with RepositoryTransaction(source, state_dir=tmp_path / "state") as tx:
with pytest.raises(RepositoryArtifactError, match="captured artifact"):
execute_profiled_run(
run,
OpsRunConfig(repo_roots=(str(tmp_path),)),
transaction=tx,
request_factory=lambda **kw: kw,
gateway=lambda *a, **kw: {
"ok": True,
"evidence": {
"commit_sha": "a" * 40,
"session_cleanup": "succeeded",
"sandbox_destroy": "succeeded",
},
},
)
assert git(source, "rev-parse", "HEAD") == tx.baseline.head
def test_missing_repository_ownership_refuses():
grant = RepositoryGrant.from_mapping(
{
"version": "1",
"allowed_paths": ["result.txt"],
"commit_count": {"min": 1, "max": 1},
"publish": False,
}
)
with pytest.raises(RepositoryArtifactError, match="active repository transaction"):
RepositoryArtifactTransfer(None, grant)
@pytest.mark.parametrize("corruption", ["truncated", "substituted_head"])
def test_corrupted_capture_cannot_change_source(tmp_path, corruption):
source, sandbox, grant = fixture(tmp_path)
with RepositoryTransaction(source, state_dir=tmp_path / "state") as tx:
transfer = RepositoryArtifactTransfer(tx, grant)
head = commit(sandbox)
capture(transfer, sandbox, head)
if corruption == "truncated":
transfer._bundle = transfer._bundle[:100]
else:
transfer._head = "f" * 40
with pytest.raises(RepositoryArtifactError):
transfer.import_after_teardown(transfer._head)
assert git(source, "rev-parse", "HEAD") == tx.baseline.head
assert not (source / "result.txt").exists()
def test_ignored_sandbox_cache_is_not_imported(tmp_path):
source, sandbox, grant = fixture(tmp_path)
commit(source, ".gitignore", "cache/\n")
git(sandbox, "pull", "--ff-only", "--quiet")
(sandbox / "cache").mkdir()
(sandbox / "cache" / "runtime.txt").write_text("discarded runtime cache")
with RepositoryTransaction(source, state_dir=tmp_path / "state") as tx:
transfer = RepositoryArtifactTransfer(tx, grant)
head = commit(sandbox)
capture(transfer, sandbox, head)
transfer.import_after_teardown(head)
assert not (source / "cache").exists()
assert tx.acceptance.changed_paths == ("result.txt",)

View file

@ -0,0 +1,171 @@
"""Opt-in real sandbox/Glas/worker proof; queue and task authoring are fixtures."""
from __future__ import annotations
import json
import os
from pathlib import Path
from unittest.mock import MagicMock
import pytest
from glas_harness.contract import (
ExecutionSummary,
OperationalReadiness,
Rein,
ToolResult,
)
from glas_harness.gateway import run_execution
from glas_harness.profiles import ProfileCatalog
from glas_harness.transport import transport_from_sandbox
from sandboxer.core.manager import SandboxManager
from sandboxer.lifecycle.store import SandboxStore
from rein_aharness.claim_loop import process_one
from rein_aharness.close_outbox import CloseOutbox
from rein_aharness.metrics import external_metrics_dir
from rein_aharness.ops_run_client import (
ActivityCoreOpsClient,
OpsRun,
OpsRunConfig,
OpsRunError,
)
from test_repository_artifact import commit, git
from rein_aharness.repository_grant import RepositoryGrant
pytestmark = pytest.mark.skipif(
os.environ.get("REIN_REAL_BWRAP") != "1", reason="opt-in kernel namespace proof"
)
class DeterministicRein(Rein):
calls = 0
workspace = None
head = None
def __init__(self, source, baseline):
self.source, self.baseline = source, baseline
def start_session(self, profile, inputs, sandbox):
transport = transport_from_sandbox(sandbox)
self.workspace = Path(transport.workspace)
return transport
def dispatch_tool(self, transport, tool_call):
self.calls += 1
code = """from pathlib import Path
import subprocess
Path('result.txt').write_text('sandbox-result-only\\n')
def git(*args):
p=subprocess.run(['git',*args],check=True,capture_output=True,text=True)
return p.stdout.strip()
git('add','result.txt')
git('-c','user.name=Fixture','-c','user.email=fixture@example.invalid','commit','-qm','bounded result')
print(git('rev-parse','HEAD'))
"""
result = transport.run(["python3", "-c", code], timeout=30)
assert result.returncode == 0, result.stderr
self.head = result.stdout.strip()
assert git(self.source, "rev-parse", "HEAD") == self.baseline
assert not (self.source / "result.txt").exists()
return ToolResult(ok=True, output="sandbox-result-only", tokens_spent=0)
def end_session(self, session):
return ExecutionSummary(
committed=True, commit_sha=self.head, outcome="succeeded", tokens_spent=0
)
def cleanup_session(self, session):
assert self.workspace.exists()
assert git(self.source, "rev-parse", "HEAD") == self.baseline
def test_real_bwrap_worker_import_and_close_replay(tmp_path, monkeypatch):
monkeypatch.setenv("XDG_DATA_HOME", str(tmp_path / "data"))
monkeypatch.setenv("REIN_AHARNESS_STATE_DIR", str(tmp_path / "state"))
monkeypatch.setenv("SANDBOXER_NO_STATE_HUB", "1")
source = tmp_path / "source"
source.mkdir()
git(source, "init", "-q")
baseline = commit(source, "README.md", "baseline\n")
grant = RepositoryGrant.from_mapping(
{
"version": "1",
"allowed_paths": ["result.txt"],
"commit_count": {"min": 1, "max": 1},
"publish": False,
}
)
run = OpsRun(
id="fixture-run",
activity_definition_id="fixture-definition",
idempotency_key="fixture-key",
target_repo=str(source),
title="deterministic fixture",
description="no inference",
state="claimed",
harness_profile_ref="harness.agent-dev-local@1.0.0",
repository_grant=grant,
claim_owner="fixture-worker",
attempt=1,
)
client = MagicMock(spec=ActivityCoreOpsClient)
client.config = OpsRunConfig(
worker_id="fixture-worker", lease_seconds=90, repo_roots=(str(tmp_path),)
)
client.claim.return_value = [run]
client.heartbeat.return_value = run
client.complete.side_effect = [
OpsRunError("response lost", action="complete"),
OpsRun(
id=run.id,
activity_definition_id="fixture-definition",
idempotency_key="fixture-key",
target_repo=str(source),
title=run.title,
description=run.description,
state="succeeded",
close_disposition="reconciled",
),
]
catalog = ProfileCatalog()
profile, _ = catalog.resolve(run.harness_profile_ref)
catalog.profiles()[(profile.id, profile.version)] = profile.model_copy(
update={
"operational_readiness": OperationalReadiness(
status="ready",
reason="isolated test fixture",
owner="tests",
evidence_ref="test:artifact-bwrap",
)
}
)
manager = SandboxManager(store=SandboxStore(tmp_path / "sandboxes.json"))
rein = DeterministicRein(source, baseline)
monkeypatch.setattr(
"glas_harness.gateway.run_execution",
lambda request, **kwargs: run_execution(
request, catalog=catalog, rein=rein, manager=manager, **kwargs
),
)
outbox = CloseOutbox(state_dir=tmp_path / "state")
first = process_one(client, outbox=outbox, report_to_hub=False)
assert first.reason == "close evidence remains pending", first
assert not rein.workspace.exists()
assert git(source, "rev-parse", "HEAD") == rein.head
assert (source / "result.txt").read_text() == "sandbox-result-only\n"
assert not git(source, "status", "--porcelain")
assert outbox.status()["pending"] == 1
record = json.loads(
(external_metrics_dir(source, "rein-aharness") / "executions.jsonl")
.read_text()
.strip()
)
assert record["success"]
durable = json.dumps(client.complete.call_args.kwargs["result"])
assert rein.head in durable
assert "sandbox-result-only" not in durable and '"bundle"' not in durable
client.claim.return_value = []
second = process_one(client, outbox=outbox, report_to_hub=False)
assert second.empty and rein.calls == 1
assert client.complete.call_count == 2
assert client.heartbeat.call_count >= 1
assert git(source, "rev-list", "--count", "HEAD") == "2"
assert outbox.status() == {"pending": 0, "delivered": 1, "quarantined": 0}