From 4310c15eac80b13e300a94afcfbcdcac879b5df2 Mon Sep 17 00:00:00 2001 From: tegwick Date: Fri, 4 Sep 2026 20:08:37 +0200 Subject: [PATCH] feat(deploy): gate claims on pinned runtime readiness Assistant: codex Assistant-Model: gpt-5.6-sol Assistant-Session: 01a06ba0-10aa-7ea0-b20a-4f3fac39efe9 --- Makefile | 5 +- README.md | 2 + deploy/README.md | 28 ++ deploy/runtime-contract-lock.json | 28 ++ deploy/scripts/install-claim-loop-user.sh | 12 +- deploy/scripts/install-pinned-runtime.sh | 18 ++ deploy/systemd/claim-loop.env.example | 3 + .../systemd/rein-aharness-claim-loop.service | 1 + docs/ops-run-claim-loop.md | 36 ++- rein_aharness/claim_loop.py | 9 + rein_aharness/cli.py | 21 ++ rein_aharness/readiness.py | 257 ++++++++++++++++++ scripts/verify-runtime-contracts.sh | 17 ++ scripts/verify_runtime_lock.py | 116 ++++++++ tests/test_readiness.py | 86 ++++++ tests/test_runtime_lock.py | 90 ++++++ ...INAH-WP-0003-governed-runtime-integrity.md | 38 ++- 17 files changed, 757 insertions(+), 10 deletions(-) create mode 100644 deploy/runtime-contract-lock.json create mode 100755 deploy/scripts/install-pinned-runtime.sh create mode 100644 rein_aharness/readiness.py create mode 100755 scripts/verify-runtime-contracts.sh create mode 100755 scripts/verify_runtime_lock.py create mode 100644 tests/test_readiness.py create mode 100644 tests/test_runtime_lock.py diff --git a/Makefile b/Makefile index 49bf147..5af37cc 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: test image image-export deploy-rsync help +.PHONY: test contract-test image image-export deploy-rsync help UV ?= $(shell command -v uv 2>/dev/null || echo uv) IMAGE ?= rein-aharness:railiance01 @@ -11,6 +11,9 @@ help: ## Show targets test: ## Run unit tests PYTHONPATH=".:$$HOME/llm-connect" python3 -m pytest tests/ -q +contract-test: ## Run non-skippable cross-package runtime contract tests + ./scripts/verify-runtime-contracts.sh + image: ## Build container image (vendors ../llm-connect when present) rm -rf llm_connect_vendor mkdir -p llm_connect_vendor diff --git a/README.md b/README.md index 4af34c2..b6889bb 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,8 @@ export AGENT_HARNESS_REPO_MAP='{"freedom-intelligence":"~/freedom-intelligence", rein-aharness poll --source=ops-run --no-claim rein-aharness run --from-ops-run rein-aharness claim-loop +rein-aharness preflight # read-only startup gate; never claims +rein-aharness close-outbox status # Install user service: ./deploy/scripts/install-claim-loop-user.sh # Docs: docs/ops-run-claim-loop.md diff --git a/deploy/README.md b/deploy/README.md index 578f9a3..5078c1c 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -53,7 +53,9 @@ semantics. | Path | Role | |------|------| | `Containerfile` | Image: Python CLI + git + openssh; optional vendored llm-connect | +| `runtime-contract-lock.json` | Exact sibling source revisions and cross-service contract pins | | `deploy/k8s/railiance/` | Namespace, ConfigMap, Deployment, smoke Job | +| `deploy/scripts/install-pinned-runtime.sh` | Frozen, non-editable host runtime installation | | `deploy/scripts/railiance-smoke.sh` | Host e2e: clone sandbox → commit → push → hub | | `rein-aharness smoke` | Deterministic smoke (no Claude Code required) | @@ -62,10 +64,28 @@ semantics. - Lane 2 deploy key on host + Forgejo write on `coulomb/executor-sandbox` - Lane 3 AppRole under `~/.local/rein-aharness/approle-binky-mail` - `source ~/.local/rein-aharness/env` +- `uv` for the frozen host runtime installation - Hub: `http://127.0.0.1:18000` (ops-bridge) or in-cluster `state-hub.state-hub.svc` - While profile-absent tenant definitions remain, set a reviewed ISO expiry in `AGENT_HARNESS_LEGACY_APPROACHES_UNTIL`. The checked-in example expires 2026-12-31; missing or expired values refuse compatibility dispatch. +- Before enabling a profiled definition, list its exact `profile@version` in + `AGENT_HARNESS_REQUIRED_PROFILE_REFS`. Service startup then requires Glas + operational readiness and exercises bwrap/AppArmor for local profiles. +- Treat any `preflight` failure or close-outbox quarantine as not ready. Pending + close evidence is replayed by the first claim-loop cycle before a new claim. + +Install or refresh the worker environment only from the checked-in locks: + +```bash +./deploy/scripts/install-pinned-runtime.sh +make contract-test +./deploy/scripts/install-claim-loop-user.sh +``` + +The installer rejects a missing, dirty, or revision-mismatched llm-connect, +Glas, or sand-boxer sibling and uses `uv sync --frozen --no-editable` so the +service does not depend on mutable editable checkout state. ## Build & load image (workstation → railiance01) @@ -96,6 +116,14 @@ ssh railiance01 'bash ~/rein-aharness/deploy/scripts/railiance-smoke.sh' Expect: local commit + push to `executor-sandbox`, hub event `harness_smoke`, `.kaizen/metrics/coach/` on the sandbox checkout. +Before starting or restarting the authoritative service, run the same gate the +unit uses: + +```bash +ssh railiance01 '~/bin/rein-aharness-claim preflight' +ssh railiance01 '~/bin/rein-aharness-claim close-outbox status' +``` + ## Personal follow-ups (not T06) - At **binky cutover only**: attach the same deploy key to `coulomb/binky-control` diff --git a/deploy/runtime-contract-lock.json b/deploy/runtime-contract-lock.json new file mode 100644 index 0000000..35ef6d8 --- /dev/null +++ b/deploy/runtime-contract-lock.json @@ -0,0 +1,28 @@ +{ + "schema_version": "1", + "contracts": { + "activity_core_contract_commit": "b63131e863be38ba1274d1077543aaf9320500b3", + "activity_core_schema": "0010", + "glas_contract_version": "1.0" + }, + "dependencies": [ + { + "commit": "00560945f81ba6ff1f5cacd9fe99c7fe756cc4b1", + "distribution": "llm-connect", + "source": "../llm-connect", + "version": "0.1.0" + }, + { + "commit": "02b29af9ca87867e0d398c06139d771a19728b70", + "distribution": "glas-harness", + "source": "../glas-harness", + "version": "0.1.0" + }, + { + "commit": "5c760100264828865c45a7e4e199d43e4603d57a", + "distribution": "sandboxer", + "source": "../sand-boxer", + "version": "0.0.0" + } + ] +} diff --git a/deploy/scripts/install-claim-loop-user.sh b/deploy/scripts/install-claim-loop-user.sh index f55a637..e152692 100755 --- a/deploy/scripts/install-claim-loop-user.sh +++ b/deploy/scripts/install-claim-loop-user.sh @@ -9,6 +9,9 @@ ENV_DIR="${HOME}/.config/rein-aharness" BIN_DIR="${HOME}/bin" mkdir -p "${UNIT_DIR}" "${ENV_DIR}" "${BIN_DIR}" +# Refuse to install a service unit over accidental mutable sibling revisions. +python3 "${ROOT}/scripts/verify_runtime_lock.py" + cp -f "${ROOT}/deploy/systemd/rein-aharness-claim-loop.service" "${UNIT_DIR}/" # Durable env loader + k8s:// ClusterIP resolution (no port-forward) install -m 755 "${ROOT}/deploy/scripts/rein-aharness-claim" "${BIN_DIR}/rein-aharness-claim" @@ -24,10 +27,13 @@ fi # Ensure package entrypoint exists if [[ ! -x "${ROOT}/.venv/bin/rein-aharness" ]]; then - echo "warning: ${ROOT}/.venv/bin/rein-aharness missing; install the worker environment first" + echo "error: ${ROOT}/.venv/bin/rein-aharness missing" >&2 + echo "run ${ROOT}/deploy/scripts/install-pinned-runtime.sh first" >&2 + exit 2 elif ! "${ROOT}/.venv/bin/python" -c 'import glas_harness, sandboxer' >/dev/null 2>&1; then - echo "warning: profiled ops runs require sibling runtimes; install with:" - echo " ${ROOT}/.venv/bin/pip install -e ${ROOT}/../sand-boxer -e ${ROOT}/../glas-harness" + echo "error: pinned profiled runtime packages are unavailable" >&2 + echo "run ${ROOT}/deploy/scripts/install-pinned-runtime.sh first" >&2 + exit 2 fi systemctl --user daemon-reload diff --git a/deploy/scripts/install-pinned-runtime.sh b/deploy/scripts/install-pinned-runtime.sh new file mode 100755 index 0000000..930ae94 --- /dev/null +++ b/deploy/scripts/install-pinned-runtime.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# Reproduce the host worker from uv.lock plus exact, clean sibling revisions. +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +cd "${REPO_ROOT}" + +command -v uv >/dev/null 2>&1 || { + echo "install-pinned-runtime: uv is required" >&2 + exit 2 +} + +python3 scripts/verify_runtime_lock.py +uv sync --frozen --no-editable --extra glas --extra llm +.venv/bin/python -c 'import glas_harness, llm_connect, rein_aharness, sandboxer' +.venv/bin/rein-aharness preflight --offline + +echo "Pinned non-editable runtime installed from uv.lock and runtime-contract-lock.json" diff --git a/deploy/systemd/claim-loop.env.example b/deploy/systemd/claim-loop.env.example index 6ea660b..9722227 100644 --- a/deploy/systemd/claim-loop.env.example +++ b/deploy/systemd/claim-loop.env.example @@ -9,6 +9,9 @@ AGENT_HARNESS_OPS_LABELS=automated AGENT_HARNESS_OPS_LABELS_MODE=any AGENT_HARNESS_OPS_LEASE_SECONDS=900 AGENT_HARNESS_CLAIM_INTERVAL=30 +# Comma-separated, exact profile@version refs. Leave empty while no profiled +# production definition is enabled; non-ready profiles fail startup preflight. +AGENT_HARNESS_REQUIRED_PROFILE_REFS= AGENT_HARNESS_LEGACY_APPROACHES_UNTIL=2026-12-31 AGENT_HARNESS_REPO_MAP={"freedom-intelligence":"/home/tegwick/freedom-intelligence","binky-control":"/home/tegwick/binky-control"} AGENT_HARNESS_REPO_ROOTS=/home/tegwick:/home/tegwick/work diff --git a/deploy/systemd/rein-aharness-claim-loop.service b/deploy/systemd/rein-aharness-claim-loop.service index acdd2a7..0147aee 100644 --- a/deploy/systemd/rein-aharness-claim-loop.service +++ b/deploy/systemd/rein-aharness-claim-loop.service @@ -15,6 +15,7 @@ Environment=AGENT_HARNESS_LEGACY_APPROACHES_UNTIL=2026-12-31 Environment=PATH=%h/.local/bin:/usr/local/bin:/usr/bin:/bin:%h/bin # Wrapper loads claim-loop.env (JSON-safe) and resolves k8s:// ClusterIPs # Do not use EnvironmentFile= for claim-loop.env — REPO_MAP JSON breaks systemd parser +ExecStartPre=%h/bin/rein-aharness-claim preflight ExecStart=%h/bin/rein-aharness-claim claim-loop Restart=on-failure RestartSec=15 diff --git a/docs/ops-run-claim-loop.md b/docs/ops-run-claim-loop.md index 22d00a0..d11884d 100644 --- a/docs/ops-run-claim-loop.md +++ b/docs/ops-run-claim-loop.md @@ -90,6 +90,9 @@ rein-aharness poll --source=ops-run rein-aharness claim-loop rein-aharness claim-loop --once --verbose +# Read-only startup readiness (never claims) +rein-aharness preflight + # Required close evidence and recovery rein-aharness close-outbox status rein-aharness close-outbox replay @@ -109,8 +112,7 @@ rein-aharness run --from-issue-core ```bash # From rein-aharness checkout on railiance01 -python3 -m venv .venv -.venv/bin/pip install -e . -e ../llm-connect -e ../sand-boxer -e ../glas-harness +./deploy/scripts/install-pinned-runtime.sh ./deploy/scripts/install-claim-loop-user.sh # Or manually: @@ -122,6 +124,36 @@ systemctl --user enable --now rein-aharness-claim-loop.service journalctl --user -u rein-aharness-claim-loop -f ``` +The unit runs `rein-aharness preflight` through the same environment-loading +wrapper before every start. It probes Activity Core with a read-only open-run +list, verifies configured repository workspaces and private runtime state, and +refuses unresolved close-evidence quarantine without claiming or executing +work. Pending close evidence is allowed through startup because the first +claim-loop cycle replays it before any new claim; quarantine requires operator +review. + +Set `AGENT_HARNESS_REQUIRED_PROFILE_REFS` to the comma-separated exact +`profile@version` references enabled for production. Every named profile must +resolve through the installed Glas catalog with `operational_readiness: ready`. +A ready `profile.bwrap-local` also executes a minimal bwrap namespace probe, +which detects missing bubblewrap and host user-namespace/AppArmor refusal. +Leaving the variable empty declares that no profiled production definition is +enabled; this does not make a blocked profile executable or introduce fallback. + +Use `rein-aharness preflight --offline` only while installing or diagnosing +local files. It skips the Activity Core probe and is not the systemd gate. + +`install-pinned-runtime.sh` first verifies clean sibling checkouts against +`deploy/runtime-contract-lock.json`, then applies the checked-in `uv.lock` with +`--frozen --no-editable`. This prevents the authoritative service from running +against whichever mutable editable sibling happened to be present. Updating a +runtime contract requires reviewing and committing both locks together. + +`make contract-test` is the non-skippable cross-package release gate. It imports +Glas, sand-boxer, and llm-connect before running the contract/claim suites, so +the optional `pytest.importorskip` development behavior cannot turn a missing +production dependency into a green release result. + ### Host access to cluster services (no port-forward) On railiance01 (single-node k3s), set **k8s://** pseudo-URLs in diff --git a/rein_aharness/claim_loop.py b/rein_aharness/claim_loop.py index 5fb7800..04a46f4 100644 --- a/rein_aharness/claim_loop.py +++ b/rein_aharness/claim_loop.py @@ -798,6 +798,15 @@ def run_claim_loop( signal.signal(signal.SIGTERM, _handle_sig) client = ActivityCoreOpsClient() + from rein_aharness.readiness import run_readiness_checks + + readiness = run_readiness_checks(client) + if not readiness.ok: + failed = ", ".join( + check.name for check in readiness.checks if not check.ok + ) + logger.error("claim-loop readiness failed: %s", failed) + return 2 logger.info( "claim-loop start worker_id=%s url=%s labels=%s interval=%ss once=%s", client.config.worker_id, diff --git a/rein_aharness/cli.py b/rein_aharness/cli.py index f6f2011..87cf04d 100644 --- a/rein_aharness/cli.py +++ b/rein_aharness/cli.py @@ -190,6 +190,14 @@ def _cmd_close_outbox(args: argparse.Namespace) -> int: return 1 if payload["pending"] or payload["quarantined"] else 0 +def _cmd_preflight(args: argparse.Namespace) -> int: + from rein_aharness.readiness import run_readiness_checks + + report = run_readiness_checks(check_activity_core=not args.offline) + print(json.dumps(report.as_dict(), indent=2, sort_keys=True)) + return 0 if report.ok else 1 + + def _cmd_run(args: argparse.Namespace) -> int: from rein_aharness.intake import IntakeError, IssueCoreClient, poll_next @@ -565,6 +573,16 @@ def main(argv: list[str] | None = None) -> int: help="Maximum pending entries to replay (default: 100)", ) + preflight = sub.add_parser( + "preflight", + help="Check queue, workspace, state, and enabled profile readiness", + ) + preflight.add_argument( + "--offline", + action="store_true", + help="Skip the read-only Activity Core queue probe", + ) + args = parser.parse_args(argv) if args.command == "validate": @@ -582,6 +600,9 @@ def main(argv: list[str] | None = None) -> int: if args.command == "close-outbox": return _cmd_close_outbox(args) + if args.command == "preflight": + return _cmd_preflight(args) + if args.command == "run": return _cmd_run(args) diff --git a/rein_aharness/readiness.py b/rein_aharness/readiness.py new file mode 100644 index 0000000..36bf3c0 --- /dev/null +++ b/rein_aharness/readiness.py @@ -0,0 +1,257 @@ +"""Read-only startup readiness checks for the authoritative claim worker.""" + +from __future__ import annotations + +import os +import shutil +import subprocess +import tempfile +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Mapping + +from rein_aharness.close_outbox import CloseOutbox +from rein_aharness.ops_run_client import ActivityCoreOpsClient, OpsRunError + + +@dataclass(frozen=True) +class ReadinessCheck: + name: str + ok: bool + detail: str + + def as_dict(self) -> dict[str, Any]: + return {"name": self.name, "ok": self.ok, "detail": self.detail} + + +@dataclass(frozen=True) +class ReadinessReport: + checks: tuple[ReadinessCheck, ...] + + @property + def ok(self) -> bool: + return all(check.ok for check in self.checks) + + def as_dict(self) -> dict[str, Any]: + return { + "ok": self.ok, + "checks": [check.as_dict() for check in self.checks], + } + + +def run_readiness_checks( + client: ActivityCoreOpsClient | None = None, + outbox: CloseOutbox | None = None, + *, + environ: Mapping[str, str] | None = None, + check_activity_core: bool = True, +) -> ReadinessReport: + """Validate required boundaries without claiming or executing workload code.""" + env = os.environ if environ is None else environ + client = client or ActivityCoreOpsClient() + checks: list[ReadinessCheck] = [] + + try: + store = outbox or CloseOutbox() + status = store.status() + except (OSError, RuntimeError) as exc: + checks.append( + ReadinessCheck( + "close_outbox", + False, + f"durable state unavailable ({type(exc).__name__})", + ) + ) + else: + quarantined = status["quarantined"] + checks.append( + ReadinessCheck( + "close_outbox", + quarantined == 0, + ( + f"pending={status['pending']} delivered={status['delivered']} " + f"quarantined={quarantined}" + ), + ) + ) + + checks.extend(_repository_checks(client, env)) + checks.extend(_profile_checks(env)) + + if check_activity_core: + try: + client.list_open(limit=1) + except (OpsRunError, OSError, ValueError) as exc: + checks.append( + ReadinessCheck( + "activity_core", + False, + f"read-only queue probe failed ({type(exc).__name__})", + ) + ) + else: + checks.append( + ReadinessCheck("activity_core", True, "read-only queue probe passed") + ) + else: + checks.append(ReadinessCheck("activity_core", True, "offline check skipped")) + + return ReadinessReport(tuple(checks)) + + +def _repository_checks( + client: ActivityCoreOpsClient, + env: Mapping[str, str], +) -> list[ReadinessCheck]: + checks: list[ReadinessCheck] = [] + for name, configured in sorted(client.config.repo_map.items()): + path = Path(configured).expanduser() + ok = path.is_dir() and os.access(path, os.R_OK | os.W_OK | os.X_OK) + checks.append( + ReadinessCheck( + f"repository:{name[:80]}", + ok, + "workspace accessible" if ok else "configured workspace unavailable", + ) + ) + + state_dir = env.get("REIN_AHARNESS_STATE_DIR", "").strip() + if state_dir: + path = Path(state_dir).expanduser() + ok = path.is_dir() and os.access(path, os.R_OK | os.W_OK | os.X_OK) + checks.append( + ReadinessCheck( + "runtime_state", + ok, + "state directory accessible" if ok else "state directory unavailable", + ) + ) + return checks + + +def _profile_checks(env: Mapping[str, str]) -> list[ReadinessCheck]: + raw = env.get("AGENT_HARNESS_REQUIRED_PROFILE_REFS", "") + references = tuple(item.strip() for item in raw.split(",") if item.strip()) + if not references: + return [ + ReadinessCheck( + "profiled_runtime", + True, + "no production profile references enabled", + ) + ] + + checks: list[ReadinessCheck] = [] + pinned_references: list[str] = [] + for reference in references: + if "@" not in reference: + checks.append( + ReadinessCheck( + f"profile:{reference[:80]}", + False, + "production profile reference must pin an exact version", + ) + ) + else: + pinned_references.append(reference) + if not pinned_references: + return checks + + try: + from glas_harness.profiles import ProfileCatalog, ProfileError + import sandboxer # noqa: F401 + except ImportError: + return checks + [ + ReadinessCheck( + "profiled_runtime", + False, + "glas-harness or sandboxer is not importable", + ) + ] + + catalog = ProfileCatalog() + needs_bwrap = False + for reference in pinned_references: + try: + profile, descriptor = catalog.resolve(reference) + except (ProfileError, OSError, ValueError) as exc: + checks.append( + ReadinessCheck( + f"profile:{reference[:80]}", + False, + f"catalog resolution failed ({type(exc).__name__})", + ) + ) + continue + readiness = profile.operational_readiness + ok = readiness.status == "ready" + checks.append( + ReadinessCheck( + f"profile:{reference[:80]}", + ok, + ( + f"readiness={readiness.status} " + f"rein={descriptor.id}@{descriptor.version} " + f"sandbox={profile.sandbox_profile}" + ), + ) + ) + needs_bwrap = needs_bwrap or ( + ok and profile.sandbox_profile == "profile.bwrap-local" + ) + + if needs_bwrap: + checks.append(_probe_bwrap(env)) + return checks + + +def _probe_bwrap(env: Mapping[str, str]) -> ReadinessCheck: + configured = env.get("SANDBOXER_BWRAP_BIN", "").strip() + executable = configured or shutil.which("bwrap") + if not executable: + return ReadinessCheck("bwrap", False, "bubblewrap executable not found") + + with tempfile.TemporaryDirectory(prefix="rein-readiness-") as workspace: + argv = [ + executable, + "--die-with-parent", + "--unshare-user", + "--unshare-pid", + "--unshare-ipc", + "--unshare-uts", + "--unshare-cgroup", + "--unshare-net", + "--tmpfs", + "/", + "--proc", + "/proc", + "--dev", + "/dev", + ] + for path in ("/usr", "/bin", "/lib", "/lib64", "/etc"): + if Path(path).exists(): + argv.extend(("--ro-bind", path, path)) + argv.extend( + ("--bind", workspace, workspace, "--chdir", workspace, "/bin/true") + ) + try: + probe = subprocess.run( + argv, + check=False, + capture_output=True, + text=True, + timeout=10, + ) + except (OSError, subprocess.TimeoutExpired) as exc: + return ReadinessCheck( + "bwrap", + False, + f"namespace/AppArmor probe failed ({type(exc).__name__})", + ) + if probe.returncode != 0: + return ReadinessCheck( + "bwrap", + False, + f"namespace/AppArmor probe exited {probe.returncode}", + ) + return ReadinessCheck("bwrap", True, "namespace/AppArmor probe passed") diff --git a/scripts/verify-runtime-contracts.sh b/scripts/verify-runtime-contracts.sh new file mode 100755 index 0000000..ad1db9f --- /dev/null +++ b/scripts/verify-runtime-contracts.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# CI/release gate: absence of optional runtime packages is a hard failure. +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" +cd "${REPO_ROOT}" + +PYTHON_BIN="${REIN_CONTRACT_PYTHON:-${REPO_ROOT}/.venv/bin/python}" +"${PYTHON_BIN}" scripts/verify_runtime_lock.py +"${PYTHON_BIN}" -c 'import glas_harness.contract, llm_connect, sandboxer.models' +PYTHONPATH="${REPO_ROOT}:${REPO_ROOT}/../llm-connect" \ + "${PYTHON_BIN}" -m pytest \ + tests/test_glas_execution.py \ + tests/test_ops_run_client.py \ + tests/test_claim_loop.py \ + tests/test_readiness.py \ + -q diff --git a/scripts/verify_runtime_lock.py b/scripts/verify_runtime_lock.py new file mode 100755 index 0000000..72f168f --- /dev/null +++ b/scripts/verify_runtime_lock.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +"""Verify production sibling sources against deploy/runtime-contract-lock.json.""" + +from __future__ import annotations + +import json +import re +import subprocess +import sys +import tomllib +from pathlib import Path +from typing import Any + + +def _git(path: Path, *args: str) -> str: + return subprocess.check_output( + ["git", "-C", str(path), *args], + text=True, + stderr=subprocess.DEVNULL, + timeout=10, + ).strip() + + +def verify(lock_path: Path) -> dict[str, Any]: + lock_path = lock_path.resolve() + repo_root = lock_path.parent.parent + payload = json.loads(lock_path.read_text(encoding="utf-8")) + if set(payload) != {"schema_version", "contracts", "dependencies"}: + raise ValueError("runtime lock has invalid top-level fields") + if payload.get("schema_version") != "1": + raise ValueError("unsupported runtime lock schema") + contracts = payload.get("contracts") + if not isinstance(contracts, dict) or set(contracts) != { + "activity_core_contract_commit", + "activity_core_schema", + "glas_contract_version", + }: + raise ValueError("runtime lock has invalid contract fields") + activity_commit = str(contracts["activity_core_contract_commit"]) + if re.fullmatch(r"[0-9a-f]{40}", activity_commit) is None: + raise ValueError("Activity Core contract commit must be a full Git object id") + activity_schema = str(contracts["activity_core_schema"]) + if re.fullmatch(r"[0-9]{4}", activity_schema) is None: + raise ValueError("Activity Core schema must be a four-digit migration id") + glas_contract = str(contracts["glas_contract_version"]) + if re.fullmatch(r"[0-9]+\.[0-9]+", glas_contract) is None: + raise ValueError("Glas contract version must pin major.minor") + dependencies = payload.get("dependencies") + if not isinstance(dependencies, list) or not dependencies: + raise ValueError("runtime lock has no dependencies") + + results: list[dict[str, Any]] = [] + for entry in dependencies: + if not isinstance(entry, dict) or set(entry) != { + "commit", + "distribution", + "source", + "version", + }: + raise ValueError("runtime dependency entry has invalid fields") + source = (repo_root / str(entry["source"])).resolve() + if not source.is_dir(): + raise ValueError(f"missing source checkout for {entry['distribution']}") + head = _git(source, "rev-parse", "HEAD") + if head != entry["commit"]: + raise ValueError( + f"{entry['distribution']} revision mismatch: " + f"expected {entry['commit']} actual {head}" + ) + if _git(source, "status", "--porcelain"): + raise ValueError(f"{entry['distribution']} source checkout is dirty") + project = tomllib.loads( + (source / "pyproject.toml").read_text(encoding="utf-8") + ).get("project", {}) + if project.get("name") != entry["distribution"]: + raise ValueError(f"{entry['distribution']} package name mismatch") + if project.get("version") != entry["version"]: + raise ValueError(f"{entry['distribution']} package version mismatch") + results.append( + { + "distribution": entry["distribution"], + "version": entry["version"], + "commit": head, + "clean": True, + } + ) + return {"ok": True, "schema_version": "1", "dependencies": results} + + +def main(argv: list[str] | None = None) -> int: + args = list(sys.argv[1:] if argv is None else argv) + if len(args) > 1: + print("usage: verify_runtime_lock.py [lock-file]", file=sys.stderr) + return 2 + default = Path(__file__).resolve().parents[1] / "deploy/runtime-contract-lock.json" + lock_path = Path(args[0]) if args else default + try: + report = verify(lock_path) + except (OSError, ValueError, subprocess.SubprocessError, json.JSONDecodeError) as exc: + print( + json.dumps( + { + "ok": False, + "error": f"runtime lock verification failed ({type(exc).__name__})", + "detail": str(exc)[:500], + }, + sort_keys=True, + ) + ) + return 1 + print(json.dumps(report, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_readiness.py b/tests/test_readiness.py new file mode 100644 index 0000000..8f6d51a --- /dev/null +++ b/tests/test_readiness.py @@ -0,0 +1,86 @@ +from __future__ import annotations + +from pathlib import Path + +from rein_aharness.close_outbox import CloseOutbox +from rein_aharness.ops_run_client import OpsRunConfig, OpsRunError +from rein_aharness.readiness import run_readiness_checks + + +class _Client: + def __init__(self, *, repo_map: dict[str, str] | None = None, error=None): + self.config = OpsRunConfig(repo_map=repo_map or {}) + self.error = error + self.list_calls = 0 + + def list_open(self, *, limit: int = 50): + self.list_calls += 1 + assert limit == 1 + if self.error is not None: + raise self.error + return [] + + +def test_readiness_is_read_only_and_passes_minimal_runtime(tmp_path: Path) -> None: + client = _Client() + report = run_readiness_checks( + client, # type: ignore[arg-type] + CloseOutbox(state_dir=tmp_path / "state"), + environ={}, + ) + + assert report.ok + assert client.list_calls == 1 + assert [check.name for check in report.checks] == [ + "close_outbox", + "profiled_runtime", + "activity_core", + ] + + +def test_readiness_fails_before_claim_when_queue_probe_is_unavailable( + tmp_path: Path, +) -> None: + client = _Client(error=OpsRunError("transport", action="list")) + + report = run_readiness_checks( + client, # type: ignore[arg-type] + CloseOutbox(state_dir=tmp_path / "state"), + environ={}, + ) + + assert not report.ok + failure = next(check for check in report.checks if check.name == "activity_core") + assert failure.detail == "read-only queue probe failed (OpsRunError)" + + +def test_readiness_refuses_quarantine_and_missing_workspace(tmp_path: Path) -> None: + outbox = CloseOutbox(state_dir=tmp_path / "state") + (outbox.quarantine_dir / "entry.deadbeef.json").write_text("{}", encoding="utf-8") + client = _Client(repo_map={"missing": str(tmp_path / "absent")}) + + report = run_readiness_checks( + client, # type: ignore[arg-type] + outbox, + environ={}, + check_activity_core=False, + ) + + assert not report.ok + failed = {check.name for check in report.checks if not check.ok} + assert failed == {"close_outbox", "repository:missing"} + + +def test_readiness_requires_exact_profile_versions_without_optional_imports( + tmp_path: Path, +) -> None: + report = run_readiness_checks( + _Client(), # type: ignore[arg-type] + CloseOutbox(state_dir=tmp_path / "state"), + environ={"AGENT_HARNESS_REQUIRED_PROFILE_REFS": "harness.agent-dev-local"}, + check_activity_core=False, + ) + + assert not report.ok + failure = next(check for check in report.checks if check.name.startswith("profile:")) + assert failure.detail == "production profile reference must pin an exact version" diff --git a/tests/test_runtime_lock.py b/tests/test_runtime_lock.py new file mode 100644 index 0000000..1c4032b --- /dev/null +++ b/tests/test_runtime_lock.py @@ -0,0 +1,90 @@ +from __future__ import annotations + +import json +import subprocess +from pathlib import Path + +import pytest + +from scripts.verify_runtime_lock import verify + + +def _git(repo: Path, *args: str) -> str: + return subprocess.check_output(["git", *args], cwd=repo, text=True).strip() + + +def _fixture(tmp_path: Path) -> tuple[Path, Path]: + root = tmp_path / "root" + worker = root / "rein-aharness" + source = root / "dependency" + (worker / "deploy").mkdir(parents=True) + source.mkdir(parents=True) + (source / "pyproject.toml").write_text( + '[project]\nname = "example-runtime"\nversion = "1.2.3"\n', + encoding="utf-8", + ) + subprocess.run(["git", "init", "-q"], cwd=source, check=True) + subprocess.run(["git", "add", "pyproject.toml"], cwd=source, check=True) + subprocess.run( + [ + "git", + "-c", + "user.name=Test", + "-c", + "user.email=test@example.invalid", + "commit", + "-qm", + "initial", + ], + cwd=source, + check=True, + ) + lock = worker / "deploy" / "runtime-contract-lock.json" + lock.write_text( + json.dumps( + { + "schema_version": "1", + "contracts": { + "activity_core_contract_commit": "a" * 40, + "activity_core_schema": "0010", + "glas_contract_version": "1.0", + }, + "dependencies": [ + { + "distribution": "example-runtime", + "version": "1.2.3", + "source": "../dependency", + "commit": _git(source, "rev-parse", "HEAD"), + } + ], + } + ), + encoding="utf-8", + ) + return lock, source + + +def test_runtime_lock_accepts_exact_clean_source(tmp_path: Path) -> None: + lock, _source = _fixture(tmp_path) + + report = verify(lock) + + assert report["ok"] is True + assert report["dependencies"][0]["distribution"] == "example-runtime" + assert report["dependencies"][0]["clean"] is True + + +def test_runtime_lock_rejects_revision_or_dirty_source(tmp_path: Path) -> None: + lock, source = _fixture(tmp_path) + payload = json.loads(lock.read_text(encoding="utf-8")) + payload["dependencies"][0]["commit"] = "0" * 40 + lock.write_text(json.dumps(payload), encoding="utf-8") + + with pytest.raises(ValueError, match="revision mismatch"): + verify(lock) + + payload["dependencies"][0]["commit"] = _git(source, "rev-parse", "HEAD") + lock.write_text(json.dumps(payload), encoding="utf-8") + (source / "untracked.txt").write_text("dirty\n", encoding="utf-8") + with pytest.raises(ValueError, match="source checkout is dirty"): + verify(lock) diff --git a/workplans/REINAH-WP-0003-governed-runtime-integrity.md b/workplans/REINAH-WP-0003-governed-runtime-integrity.md index b7a069b..c1cd8c2 100644 --- a/workplans/REINAH-WP-0003-governed-runtime-integrity.md +++ b/workplans/REINAH-WP-0003-governed-runtime-integrity.md @@ -555,10 +555,40 @@ Deployment is now explicitly labeled `packaging-smoke`; its source comments and deployment guide state that `sleep infinity` is neither ready to claim nor a failover worker and must never receive the Activity Core worker credential. -T05 remains `progress`. Reproducible pinned sibling packages, startup readiness, -recovery/outbox controls, and non-skipped cross-package CI still need -implementation; the currently catalogued local Glas profiles are also -operationally `blocked` on `GLAS-IN-0002`. +T05 remains `progress`. Reproducible pinned sibling artifacts, full recovery +drills, and non-skipped cross-package CI still need implementation; the +currently catalogued local Glas profiles are also operationally `blocked` on +`GLAS-IN-0002`. + +### Pre-claim readiness and recovery gate — 2026-09-04 + +Added `rein-aharness preflight` and made it an `ExecStartPre` requirement of +the authoritative host user service. The gate performs a read-only Activity +Core queue probe, verifies configured repository workspaces and private state, +and refuses unresolved close-evidence quarantine without claiming or executing +work. Pending close evidence remains recoverable: the first claim-loop cycle +replays it before attempting any new claim. + +Production profile enablement is now declared as exact comma-separated +`AGENT_HARNESS_REQUIRED_PROFILE_REFS`. Each reference must be version-pinned, +resolve through the installed Glas catalog, and declare operational readiness +`ready`. A ready `profile.bwrap-local` must also pass an executable bwrap +namespace probe, covering missing binaries and user-namespace/AppArmor host +refusal. The current example leaves the list empty because the catalogued local +profiles remain blocked on `GLAS-IN-0002`; the gate does not misrepresent them +as ready. + +Added a reviewed runtime lock for exact llm-connect, Glas, and sand-boxer source +revisions plus the Glas `1.0` and Activity Core migration `0010` contract pins. +The host installer rejects missing, dirty, or mismatched sibling checkouts and +uses the frozen uv lock with `--no-editable`. `make contract-test` first imports +all three runtime packages and then runs the cross-package contract/claim +suites, making a missing optional dependency a hard release failure. + +T05 remains `progress`: the non-skippable gate still needs CI-runner wiring, +the compatible Activity Core artifact and migration need deployment evidence, +and the documented crash/lease/API-close/sandbox cleanup recovery drills remain +to be executed against the pinned host artifact. ## Re-prove one governed profiled run and close residuals