Constrain controlled CLI sessions and prove native budget overshoot
Some checks failed
Governed runtime contract / contract (push) Failing after 27s
Some checks failed
Governed runtime contract / contract (push) Failing after 27s
Assistant: codex Assistant-Model: gpt-5.6-luna Assistant-Session: 01a07ff8-19d0-7820-b4d0-1353833cb7fc
This commit is contained in:
parent
e0b3ff99a2
commit
4ae245a88f
14 changed files with 752 additions and 10 deletions
228
tests/native_cli_fixture.py
Normal file
228
tests/native_cli_fixture.py
Normal file
|
|
@ -0,0 +1,228 @@
|
|||
"""Runs inside a network namespace: deterministic HTTP responses, no inference."""
|
||||
|
||||
import http.server
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import threading
|
||||
from pathlib import Path
|
||||
|
||||
root = Path("/work")
|
||||
settings = json.loads((root / "case.json").read_text())
|
||||
seen = []
|
||||
|
||||
|
||||
class Handler(http.server.BaseHTTPRequestHandler):
|
||||
def log_message(self, *args):
|
||||
pass
|
||||
|
||||
def do_POST(self):
|
||||
body = json.loads(self.rfile.read(int(self.headers.get("Content-Length", 0))))
|
||||
results = []
|
||||
for message in body.get("messages", []):
|
||||
if isinstance(message.get("content"), list):
|
||||
results.extend(
|
||||
{
|
||||
"id": block.get("tool_use_id"),
|
||||
"is_error": block.get("is_error", False),
|
||||
}
|
||||
for block in message["content"]
|
||||
if block.get("type") == "tool_result"
|
||||
)
|
||||
seen.append(
|
||||
{
|
||||
"path": self.path,
|
||||
"model": body.get("model"),
|
||||
"max_tokens": body.get("max_tokens"),
|
||||
"tools": [tool.get("name") for tool in body.get("tools", [])],
|
||||
"tool_results": results,
|
||||
"ambient_context_loaded": "AMBIENT_CONTEXT_SENTINEL"
|
||||
in json.dumps(body),
|
||||
}
|
||||
)
|
||||
if "count_tokens" in self.path:
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.end_headers()
|
||||
self.wfile.write(b'{"input_tokens":100}')
|
||||
return
|
||||
message_number = sum("count_tokens" not in request["path"] for request in seen)
|
||||
if settings["case"] == "tools" and message_number == 1:
|
||||
blocks = [
|
||||
{
|
||||
"type": "tool_use",
|
||||
"id": "tool_allowed",
|
||||
"name": "Bash",
|
||||
"input": {
|
||||
"command": "git status",
|
||||
"description": "Read fixture status",
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "tool_use",
|
||||
"id": "tool_create",
|
||||
"name": "Edit",
|
||||
"input": {
|
||||
"file_path": "/work/target/result.txt",
|
||||
"old_string": "",
|
||||
"new_string": "fixture-created\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "tool_use",
|
||||
"id": "tool_denied",
|
||||
"name": "Bash",
|
||||
"input": {
|
||||
"command": "printf bypass > /work/denied-ran",
|
||||
"description": "Fixture forbidden operation",
|
||||
},
|
||||
},
|
||||
]
|
||||
stop = "tool_use"
|
||||
else:
|
||||
blocks = [{"type": "text", "text": "fixture complete"}]
|
||||
stop = "end_turn"
|
||||
usage = {
|
||||
"input_tokens": 60000 if settings["case"] == "overrun" else 100,
|
||||
"output_tokens": 0,
|
||||
"cache_creation_input_tokens": 0,
|
||||
"cache_read_input_tokens": 0,
|
||||
}
|
||||
events = [
|
||||
(
|
||||
"message_start",
|
||||
{
|
||||
"type": "message_start",
|
||||
"message": {
|
||||
"id": "msg_fixture_" + str(message_number),
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "claude-sonnet-4-6",
|
||||
"content": [],
|
||||
"stop_reason": None,
|
||||
"stop_sequence": None,
|
||||
"usage": usage,
|
||||
},
|
||||
},
|
||||
)
|
||||
]
|
||||
for index, block in enumerate(blocks):
|
||||
if block["type"] == "text":
|
||||
start = {"type": "text", "text": ""}
|
||||
delta = {"type": "text_delta", "text": block["text"]}
|
||||
else:
|
||||
start = {**block, "input": {}}
|
||||
delta = {
|
||||
"type": "input_json_delta",
|
||||
"partial_json": json.dumps(block["input"]),
|
||||
}
|
||||
events += [
|
||||
(
|
||||
"content_block_start",
|
||||
{
|
||||
"type": "content_block_start",
|
||||
"index": index,
|
||||
"content_block": start,
|
||||
},
|
||||
),
|
||||
(
|
||||
"content_block_delta",
|
||||
{"type": "content_block_delta", "index": index, "delta": delta},
|
||||
),
|
||||
("content_block_stop", {"type": "content_block_stop", "index": index}),
|
||||
]
|
||||
events += [
|
||||
(
|
||||
"message_delta",
|
||||
{
|
||||
"type": "message_delta",
|
||||
"delta": {"stop_reason": stop, "stop_sequence": None},
|
||||
"usage": {**usage, "output_tokens": 10},
|
||||
},
|
||||
),
|
||||
("message_stop", {"type": "message_stop"}),
|
||||
]
|
||||
encoded = "".join(
|
||||
"event: " + name + "\ndata: " + json.dumps(value) + "\n\n"
|
||||
for name, value in events
|
||||
).encode()
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "text/event-stream")
|
||||
self.send_header("Content-Length", str(len(encoded)))
|
||||
self.end_headers()
|
||||
self.wfile.write(encoded)
|
||||
|
||||
|
||||
server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler)
|
||||
threading.Thread(target=server.serve_forever, daemon=True).start()
|
||||
(root / "target" / ".claude").mkdir(parents=True)
|
||||
(root / "target" / ".claude" / "settings.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"permissions": {"allow": ["Bash"]},
|
||||
"hooks": {
|
||||
"SessionStart": [
|
||||
{"hooks": [{"type": "command", "command": "touch /work/hook-ran"}]}
|
||||
]
|
||||
},
|
||||
}
|
||||
)
|
||||
)
|
||||
(root / "target" / ".mcp.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"mcpServers": {
|
||||
"fixture": {"command": "/bin/sh", "args": ["-c", "touch /work/mcp-ran"]}
|
||||
}
|
||||
}
|
||||
)
|
||||
)
|
||||
(root / "target" / "CLAUDE.md").write_text("AMBIENT_CONTEXT_SENTINEL\n")
|
||||
subprocess.run(["git", "init", "-q", str(root / "target")], check=True)
|
||||
env = dict(
|
||||
os.environ,
|
||||
ANTHROPIC_BASE_URL="http://127.0.0.1:" + str(server.server_port),
|
||||
ANTHROPIC_API_KEY="fixture-no-provider-secret",
|
||||
CLAUDE_CONFIG_DIR="/work/config",
|
||||
CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC="1",
|
||||
DISABLE_AUTOUPDATER="1",
|
||||
)
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
settings["argv"],
|
||||
input="Run the deterministic fixture.",
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
env=env,
|
||||
cwd="/work/target",
|
||||
timeout=45,
|
||||
)
|
||||
terminal = json.loads(proc.stdout)
|
||||
output = {
|
||||
"returncode": proc.returncode,
|
||||
"terminal": {
|
||||
key: terminal.get(key)
|
||||
for key in (
|
||||
"type",
|
||||
"subtype",
|
||||
"is_error",
|
||||
"total_cost_usd",
|
||||
"num_turns",
|
||||
"usage",
|
||||
"permission_denials",
|
||||
)
|
||||
},
|
||||
"requests": seen,
|
||||
"markers": {
|
||||
name: (root / name).exists()
|
||||
for name in ("hook-ran", "mcp-ran", "denied-ran")
|
||||
},
|
||||
"created_file": (root / "target" / "result.txt").read_text()
|
||||
if (root / "target" / "result.txt").exists()
|
||||
else None,
|
||||
"network_namespace": os.readlink("/proc/self/ns/net"),
|
||||
}
|
||||
print(json.dumps(output))
|
||||
finally:
|
||||
server.shutdown()
|
||||
Loading…
Add table
Add a link
Reference in a new issue