feat(deploy): gate claims on pinned runtime readiness
Assistant: codex Assistant-Model: gpt-5.6-sol Assistant-Session: 01a06ba0-10aa-7ea0-b20a-4f3fac39efe9
This commit is contained in:
parent
d00ffcb402
commit
4310c15eac
17 changed files with 757 additions and 10 deletions
|
|
@ -798,6 +798,15 @@ def run_claim_loop(
|
|||
signal.signal(signal.SIGTERM, _handle_sig)
|
||||
|
||||
client = ActivityCoreOpsClient()
|
||||
from rein_aharness.readiness import run_readiness_checks
|
||||
|
||||
readiness = run_readiness_checks(client)
|
||||
if not readiness.ok:
|
||||
failed = ", ".join(
|
||||
check.name for check in readiness.checks if not check.ok
|
||||
)
|
||||
logger.error("claim-loop readiness failed: %s", failed)
|
||||
return 2
|
||||
logger.info(
|
||||
"claim-loop start worker_id=%s url=%s labels=%s interval=%ss once=%s",
|
||||
client.config.worker_id,
|
||||
|
|
|
|||
|
|
@ -190,6 +190,14 @@ def _cmd_close_outbox(args: argparse.Namespace) -> int:
|
|||
return 1 if payload["pending"] or payload["quarantined"] else 0
|
||||
|
||||
|
||||
def _cmd_preflight(args: argparse.Namespace) -> int:
|
||||
from rein_aharness.readiness import run_readiness_checks
|
||||
|
||||
report = run_readiness_checks(check_activity_core=not args.offline)
|
||||
print(json.dumps(report.as_dict(), indent=2, sort_keys=True))
|
||||
return 0 if report.ok else 1
|
||||
|
||||
|
||||
def _cmd_run(args: argparse.Namespace) -> int:
|
||||
from rein_aharness.intake import IntakeError, IssueCoreClient, poll_next
|
||||
|
||||
|
|
@ -565,6 +573,16 @@ def main(argv: list[str] | None = None) -> int:
|
|||
help="Maximum pending entries to replay (default: 100)",
|
||||
)
|
||||
|
||||
preflight = sub.add_parser(
|
||||
"preflight",
|
||||
help="Check queue, workspace, state, and enabled profile readiness",
|
||||
)
|
||||
preflight.add_argument(
|
||||
"--offline",
|
||||
action="store_true",
|
||||
help="Skip the read-only Activity Core queue probe",
|
||||
)
|
||||
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
if args.command == "validate":
|
||||
|
|
@ -582,6 +600,9 @@ def main(argv: list[str] | None = None) -> int:
|
|||
if args.command == "close-outbox":
|
||||
return _cmd_close_outbox(args)
|
||||
|
||||
if args.command == "preflight":
|
||||
return _cmd_preflight(args)
|
||||
|
||||
if args.command == "run":
|
||||
return _cmd_run(args)
|
||||
|
||||
|
|
|
|||
257
rein_aharness/readiness.py
Normal file
257
rein_aharness/readiness.py
Normal file
|
|
@ -0,0 +1,257 @@
|
|||
"""Read-only startup readiness checks for the authoritative claim worker."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, Mapping
|
||||
|
||||
from rein_aharness.close_outbox import CloseOutbox
|
||||
from rein_aharness.ops_run_client import ActivityCoreOpsClient, OpsRunError
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ReadinessCheck:
|
||||
name: str
|
||||
ok: bool
|
||||
detail: str
|
||||
|
||||
def as_dict(self) -> dict[str, Any]:
|
||||
return {"name": self.name, "ok": self.ok, "detail": self.detail}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ReadinessReport:
|
||||
checks: tuple[ReadinessCheck, ...]
|
||||
|
||||
@property
|
||||
def ok(self) -> bool:
|
||||
return all(check.ok for check in self.checks)
|
||||
|
||||
def as_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"ok": self.ok,
|
||||
"checks": [check.as_dict() for check in self.checks],
|
||||
}
|
||||
|
||||
|
||||
def run_readiness_checks(
|
||||
client: ActivityCoreOpsClient | None = None,
|
||||
outbox: CloseOutbox | None = None,
|
||||
*,
|
||||
environ: Mapping[str, str] | None = None,
|
||||
check_activity_core: bool = True,
|
||||
) -> ReadinessReport:
|
||||
"""Validate required boundaries without claiming or executing workload code."""
|
||||
env = os.environ if environ is None else environ
|
||||
client = client or ActivityCoreOpsClient()
|
||||
checks: list[ReadinessCheck] = []
|
||||
|
||||
try:
|
||||
store = outbox or CloseOutbox()
|
||||
status = store.status()
|
||||
except (OSError, RuntimeError) as exc:
|
||||
checks.append(
|
||||
ReadinessCheck(
|
||||
"close_outbox",
|
||||
False,
|
||||
f"durable state unavailable ({type(exc).__name__})",
|
||||
)
|
||||
)
|
||||
else:
|
||||
quarantined = status["quarantined"]
|
||||
checks.append(
|
||||
ReadinessCheck(
|
||||
"close_outbox",
|
||||
quarantined == 0,
|
||||
(
|
||||
f"pending={status['pending']} delivered={status['delivered']} "
|
||||
f"quarantined={quarantined}"
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
checks.extend(_repository_checks(client, env))
|
||||
checks.extend(_profile_checks(env))
|
||||
|
||||
if check_activity_core:
|
||||
try:
|
||||
client.list_open(limit=1)
|
||||
except (OpsRunError, OSError, ValueError) as exc:
|
||||
checks.append(
|
||||
ReadinessCheck(
|
||||
"activity_core",
|
||||
False,
|
||||
f"read-only queue probe failed ({type(exc).__name__})",
|
||||
)
|
||||
)
|
||||
else:
|
||||
checks.append(
|
||||
ReadinessCheck("activity_core", True, "read-only queue probe passed")
|
||||
)
|
||||
else:
|
||||
checks.append(ReadinessCheck("activity_core", True, "offline check skipped"))
|
||||
|
||||
return ReadinessReport(tuple(checks))
|
||||
|
||||
|
||||
def _repository_checks(
|
||||
client: ActivityCoreOpsClient,
|
||||
env: Mapping[str, str],
|
||||
) -> list[ReadinessCheck]:
|
||||
checks: list[ReadinessCheck] = []
|
||||
for name, configured in sorted(client.config.repo_map.items()):
|
||||
path = Path(configured).expanduser()
|
||||
ok = path.is_dir() and os.access(path, os.R_OK | os.W_OK | os.X_OK)
|
||||
checks.append(
|
||||
ReadinessCheck(
|
||||
f"repository:{name[:80]}",
|
||||
ok,
|
||||
"workspace accessible" if ok else "configured workspace unavailable",
|
||||
)
|
||||
)
|
||||
|
||||
state_dir = env.get("REIN_AHARNESS_STATE_DIR", "").strip()
|
||||
if state_dir:
|
||||
path = Path(state_dir).expanduser()
|
||||
ok = path.is_dir() and os.access(path, os.R_OK | os.W_OK | os.X_OK)
|
||||
checks.append(
|
||||
ReadinessCheck(
|
||||
"runtime_state",
|
||||
ok,
|
||||
"state directory accessible" if ok else "state directory unavailable",
|
||||
)
|
||||
)
|
||||
return checks
|
||||
|
||||
|
||||
def _profile_checks(env: Mapping[str, str]) -> list[ReadinessCheck]:
|
||||
raw = env.get("AGENT_HARNESS_REQUIRED_PROFILE_REFS", "")
|
||||
references = tuple(item.strip() for item in raw.split(",") if item.strip())
|
||||
if not references:
|
||||
return [
|
||||
ReadinessCheck(
|
||||
"profiled_runtime",
|
||||
True,
|
||||
"no production profile references enabled",
|
||||
)
|
||||
]
|
||||
|
||||
checks: list[ReadinessCheck] = []
|
||||
pinned_references: list[str] = []
|
||||
for reference in references:
|
||||
if "@" not in reference:
|
||||
checks.append(
|
||||
ReadinessCheck(
|
||||
f"profile:{reference[:80]}",
|
||||
False,
|
||||
"production profile reference must pin an exact version",
|
||||
)
|
||||
)
|
||||
else:
|
||||
pinned_references.append(reference)
|
||||
if not pinned_references:
|
||||
return checks
|
||||
|
||||
try:
|
||||
from glas_harness.profiles import ProfileCatalog, ProfileError
|
||||
import sandboxer # noqa: F401
|
||||
except ImportError:
|
||||
return checks + [
|
||||
ReadinessCheck(
|
||||
"profiled_runtime",
|
||||
False,
|
||||
"glas-harness or sandboxer is not importable",
|
||||
)
|
||||
]
|
||||
|
||||
catalog = ProfileCatalog()
|
||||
needs_bwrap = False
|
||||
for reference in pinned_references:
|
||||
try:
|
||||
profile, descriptor = catalog.resolve(reference)
|
||||
except (ProfileError, OSError, ValueError) as exc:
|
||||
checks.append(
|
||||
ReadinessCheck(
|
||||
f"profile:{reference[:80]}",
|
||||
False,
|
||||
f"catalog resolution failed ({type(exc).__name__})",
|
||||
)
|
||||
)
|
||||
continue
|
||||
readiness = profile.operational_readiness
|
||||
ok = readiness.status == "ready"
|
||||
checks.append(
|
||||
ReadinessCheck(
|
||||
f"profile:{reference[:80]}",
|
||||
ok,
|
||||
(
|
||||
f"readiness={readiness.status} "
|
||||
f"rein={descriptor.id}@{descriptor.version} "
|
||||
f"sandbox={profile.sandbox_profile}"
|
||||
),
|
||||
)
|
||||
)
|
||||
needs_bwrap = needs_bwrap or (
|
||||
ok and profile.sandbox_profile == "profile.bwrap-local"
|
||||
)
|
||||
|
||||
if needs_bwrap:
|
||||
checks.append(_probe_bwrap(env))
|
||||
return checks
|
||||
|
||||
|
||||
def _probe_bwrap(env: Mapping[str, str]) -> ReadinessCheck:
|
||||
configured = env.get("SANDBOXER_BWRAP_BIN", "").strip()
|
||||
executable = configured or shutil.which("bwrap")
|
||||
if not executable:
|
||||
return ReadinessCheck("bwrap", False, "bubblewrap executable not found")
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="rein-readiness-") as workspace:
|
||||
argv = [
|
||||
executable,
|
||||
"--die-with-parent",
|
||||
"--unshare-user",
|
||||
"--unshare-pid",
|
||||
"--unshare-ipc",
|
||||
"--unshare-uts",
|
||||
"--unshare-cgroup",
|
||||
"--unshare-net",
|
||||
"--tmpfs",
|
||||
"/",
|
||||
"--proc",
|
||||
"/proc",
|
||||
"--dev",
|
||||
"/dev",
|
||||
]
|
||||
for path in ("/usr", "/bin", "/lib", "/lib64", "/etc"):
|
||||
if Path(path).exists():
|
||||
argv.extend(("--ro-bind", path, path))
|
||||
argv.extend(
|
||||
("--bind", workspace, workspace, "--chdir", workspace, "/bin/true")
|
||||
)
|
||||
try:
|
||||
probe = subprocess.run(
|
||||
argv,
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=10,
|
||||
)
|
||||
except (OSError, subprocess.TimeoutExpired) as exc:
|
||||
return ReadinessCheck(
|
||||
"bwrap",
|
||||
False,
|
||||
f"namespace/AppArmor probe failed ({type(exc).__name__})",
|
||||
)
|
||||
if probe.returncode != 0:
|
||||
return ReadinessCheck(
|
||||
"bwrap",
|
||||
False,
|
||||
f"namespace/AppArmor probe exited {probe.returncode}",
|
||||
)
|
||||
return ReadinessCheck("bwrap", True, "namespace/AppArmor probe passed")
|
||||
Loading…
Add table
Add a link
Reference in a new issue