Assistant: codex Assistant-Model: gpt-5.6-sol Assistant-Session: 01a06ba0-10aa-7ea0-b20a-4f3fac39efe9
257 lines
7.7 KiB
Python
257 lines
7.7 KiB
Python
"""Read-only startup readiness checks for the authoritative claim worker."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import tempfile
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
from typing import Any, Mapping
|
|
|
|
from rein_aharness.close_outbox import CloseOutbox
|
|
from rein_aharness.ops_run_client import ActivityCoreOpsClient, OpsRunError
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ReadinessCheck:
|
|
name: str
|
|
ok: bool
|
|
detail: str
|
|
|
|
def as_dict(self) -> dict[str, Any]:
|
|
return {"name": self.name, "ok": self.ok, "detail": self.detail}
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ReadinessReport:
|
|
checks: tuple[ReadinessCheck, ...]
|
|
|
|
@property
|
|
def ok(self) -> bool:
|
|
return all(check.ok for check in self.checks)
|
|
|
|
def as_dict(self) -> dict[str, Any]:
|
|
return {
|
|
"ok": self.ok,
|
|
"checks": [check.as_dict() for check in self.checks],
|
|
}
|
|
|
|
|
|
def run_readiness_checks(
|
|
client: ActivityCoreOpsClient | None = None,
|
|
outbox: CloseOutbox | None = None,
|
|
*,
|
|
environ: Mapping[str, str] | None = None,
|
|
check_activity_core: bool = True,
|
|
) -> ReadinessReport:
|
|
"""Validate required boundaries without claiming or executing workload code."""
|
|
env = os.environ if environ is None else environ
|
|
client = client or ActivityCoreOpsClient()
|
|
checks: list[ReadinessCheck] = []
|
|
|
|
try:
|
|
store = outbox or CloseOutbox()
|
|
status = store.status()
|
|
except (OSError, RuntimeError) as exc:
|
|
checks.append(
|
|
ReadinessCheck(
|
|
"close_outbox",
|
|
False,
|
|
f"durable state unavailable ({type(exc).__name__})",
|
|
)
|
|
)
|
|
else:
|
|
quarantined = status["quarantined"]
|
|
checks.append(
|
|
ReadinessCheck(
|
|
"close_outbox",
|
|
quarantined == 0,
|
|
(
|
|
f"pending={status['pending']} delivered={status['delivered']} "
|
|
f"quarantined={quarantined}"
|
|
),
|
|
)
|
|
)
|
|
|
|
checks.extend(_repository_checks(client, env))
|
|
checks.extend(_profile_checks(env))
|
|
|
|
if check_activity_core:
|
|
try:
|
|
client.list_open(limit=1)
|
|
except (OpsRunError, OSError, ValueError) as exc:
|
|
checks.append(
|
|
ReadinessCheck(
|
|
"activity_core",
|
|
False,
|
|
f"read-only queue probe failed ({type(exc).__name__})",
|
|
)
|
|
)
|
|
else:
|
|
checks.append(
|
|
ReadinessCheck("activity_core", True, "read-only queue probe passed")
|
|
)
|
|
else:
|
|
checks.append(ReadinessCheck("activity_core", True, "offline check skipped"))
|
|
|
|
return ReadinessReport(tuple(checks))
|
|
|
|
|
|
def _repository_checks(
|
|
client: ActivityCoreOpsClient,
|
|
env: Mapping[str, str],
|
|
) -> list[ReadinessCheck]:
|
|
checks: list[ReadinessCheck] = []
|
|
for name, configured in sorted(client.config.repo_map.items()):
|
|
path = Path(configured).expanduser()
|
|
ok = path.is_dir() and os.access(path, os.R_OK | os.W_OK | os.X_OK)
|
|
checks.append(
|
|
ReadinessCheck(
|
|
f"repository:{name[:80]}",
|
|
ok,
|
|
"workspace accessible" if ok else "configured workspace unavailable",
|
|
)
|
|
)
|
|
|
|
state_dir = env.get("REIN_AHARNESS_STATE_DIR", "").strip()
|
|
if state_dir:
|
|
path = Path(state_dir).expanduser()
|
|
ok = path.is_dir() and os.access(path, os.R_OK | os.W_OK | os.X_OK)
|
|
checks.append(
|
|
ReadinessCheck(
|
|
"runtime_state",
|
|
ok,
|
|
"state directory accessible" if ok else "state directory unavailable",
|
|
)
|
|
)
|
|
return checks
|
|
|
|
|
|
def _profile_checks(env: Mapping[str, str]) -> list[ReadinessCheck]:
|
|
raw = env.get("AGENT_HARNESS_REQUIRED_PROFILE_REFS", "")
|
|
references = tuple(item.strip() for item in raw.split(",") if item.strip())
|
|
if not references:
|
|
return [
|
|
ReadinessCheck(
|
|
"profiled_runtime",
|
|
True,
|
|
"no production profile references enabled",
|
|
)
|
|
]
|
|
|
|
checks: list[ReadinessCheck] = []
|
|
pinned_references: list[str] = []
|
|
for reference in references:
|
|
if "@" not in reference:
|
|
checks.append(
|
|
ReadinessCheck(
|
|
f"profile:{reference[:80]}",
|
|
False,
|
|
"production profile reference must pin an exact version",
|
|
)
|
|
)
|
|
else:
|
|
pinned_references.append(reference)
|
|
if not pinned_references:
|
|
return checks
|
|
|
|
try:
|
|
from glas_harness.profiles import ProfileCatalog, ProfileError
|
|
import sandboxer # noqa: F401
|
|
except ImportError:
|
|
return checks + [
|
|
ReadinessCheck(
|
|
"profiled_runtime",
|
|
False,
|
|
"glas-harness or sandboxer is not importable",
|
|
)
|
|
]
|
|
|
|
catalog = ProfileCatalog()
|
|
needs_bwrap = False
|
|
for reference in pinned_references:
|
|
try:
|
|
profile, descriptor = catalog.resolve(reference)
|
|
except (ProfileError, OSError, ValueError) as exc:
|
|
checks.append(
|
|
ReadinessCheck(
|
|
f"profile:{reference[:80]}",
|
|
False,
|
|
f"catalog resolution failed ({type(exc).__name__})",
|
|
)
|
|
)
|
|
continue
|
|
readiness = profile.operational_readiness
|
|
ok = readiness.status == "ready"
|
|
checks.append(
|
|
ReadinessCheck(
|
|
f"profile:{reference[:80]}",
|
|
ok,
|
|
(
|
|
f"readiness={readiness.status} "
|
|
f"rein={descriptor.id}@{descriptor.version} "
|
|
f"sandbox={profile.sandbox_profile}"
|
|
),
|
|
)
|
|
)
|
|
needs_bwrap = needs_bwrap or (
|
|
ok and profile.sandbox_profile == "profile.bwrap-local"
|
|
)
|
|
|
|
if needs_bwrap:
|
|
checks.append(_probe_bwrap(env))
|
|
return checks
|
|
|
|
|
|
def _probe_bwrap(env: Mapping[str, str]) -> ReadinessCheck:
|
|
configured = env.get("SANDBOXER_BWRAP_BIN", "").strip()
|
|
executable = configured or shutil.which("bwrap")
|
|
if not executable:
|
|
return ReadinessCheck("bwrap", False, "bubblewrap executable not found")
|
|
|
|
with tempfile.TemporaryDirectory(prefix="rein-readiness-") as workspace:
|
|
argv = [
|
|
executable,
|
|
"--die-with-parent",
|
|
"--unshare-user",
|
|
"--unshare-pid",
|
|
"--unshare-ipc",
|
|
"--unshare-uts",
|
|
"--unshare-cgroup",
|
|
"--unshare-net",
|
|
"--tmpfs",
|
|
"/",
|
|
"--proc",
|
|
"/proc",
|
|
"--dev",
|
|
"/dev",
|
|
]
|
|
for path in ("/usr", "/bin", "/lib", "/lib64", "/etc"):
|
|
if Path(path).exists():
|
|
argv.extend(("--ro-bind", path, path))
|
|
argv.extend(
|
|
("--bind", workspace, workspace, "--chdir", workspace, "/bin/true")
|
|
)
|
|
try:
|
|
probe = subprocess.run(
|
|
argv,
|
|
check=False,
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=10,
|
|
)
|
|
except (OSError, subprocess.TimeoutExpired) as exc:
|
|
return ReadinessCheck(
|
|
"bwrap",
|
|
False,
|
|
f"namespace/AppArmor probe failed ({type(exc).__name__})",
|
|
)
|
|
if probe.returncode != 0:
|
|
return ReadinessCheck(
|
|
"bwrap",
|
|
False,
|
|
f"namespace/AppArmor probe exited {probe.returncode}",
|
|
)
|
|
return ReadinessCheck("bwrap", True, "namespace/AppArmor probe passed")
|