"""Claim loop process_one tests (REIN-A-0002-T03).""" from __future__ import annotations import json import subprocess import time from pathlib import Path from unittest.mock import MagicMock, patch import pytest from rein_aharness.approaches import ApproachResult, APPROACH_FI_RESEARCH_BRIEF from rein_aharness.claim_loop import ( _cancel_active_run, _set_active_run_cancel, process_one, poll_peek, ) from rein_aharness.close_outbox import CloseOutbox, CloseRequest from rein_aharness.execution_cancel import ExecutionCancel, active_cancel from rein_aharness.glas_execution import GLAS_APPROACH, GlasExecutionError from rein_aharness.metrics import external_metrics_dir from rein_aharness.ops_run_client import ( ActivityCoreOpsClient, OpsRun, OpsRunConfig, OpsRunError, ) from rein_aharness.repository_grant import RepositoryGrant from rein_aharness.repository_transaction import RepositoryTransaction def _claimed_run() -> OpsRun: return OpsRun( id="run-1", activity_definition_id="def", idempotency_key="k", target_repo="freedom-intelligence", title="FI daily", description="d", labels=["automated", "research-brief"], state="claimed", claim_owner="worker-1", attempt=1, ) def _profiled_case( tmp_path: Path, ) -> tuple[Path, OpsRun, MagicMock]: repo = tmp_path / "freedom-intelligence" repo.mkdir() subprocess.run(["git", "init", "-q"], cwd=repo, check=True) (repo / "README.md").write_text("controlled target\n", encoding="utf-8") subprocess.run(["git", "add", "."], cwd=repo, check=True) subprocess.run( [ "git", "-c", "user.email=test@example.invalid", "-c", "user.name=test", "commit", "-qm", "baseline", ], cwd=repo, check=True, ) run = _claimed_run() run.harness_profile_ref = "harness.agent-dev-local@1.0.0" client = MagicMock(spec=ActivityCoreOpsClient) client.config = OpsRunConfig( worker_id="w", lease_seconds=90, repo_roots=(str(tmp_path),), ) client.claim.return_value = [run] return repo, run, client def _grant(*paths: str) -> RepositoryGrant: return RepositoryGrant.from_mapping( { "version": "1", "allowed_paths": list(paths or ("docs/",)), "commit_count": {"min": 1, "max": 1}, "publish": False, } ) def _commit_file(repo: Path, relative: str, value: str = "result\n") -> str: path = repo / relative path.parent.mkdir(parents=True, exist_ok=True) path.write_text(value, encoding="utf-8") subprocess.run(["git", "add", relative], cwd=repo, check=True) subprocess.run( [ "git", "-c", "user.email=test@example.invalid", "-c", "user.name=test", "commit", "-qm", "profiled result", ], cwd=repo, check=True, ) return subprocess.run( ["git", "rev-parse", "HEAD"], cwd=repo, check=True, capture_output=True, text=True, ).stdout.strip() def _accept_initial_heartbeat_then_reject(run: OpsRun): calls = 0 def heartbeat(*_args, **_kwargs): nonlocal calls calls += 1 if calls == 1: return run raise OpsRunError( "lease rejected", action="heartbeat", status_code=409, code="expired_lease", ) return heartbeat def test_process_one_empty() -> None: client = MagicMock(spec=ActivityCoreOpsClient) client.config = OpsRunConfig(worker_id="w", lease_seconds=90) client.claim.return_value = [] r = process_one(client) assert r.empty is True assert r.claimed is False def test_process_one_claim_error_requests_full_backoff() -> None: client = MagicMock(spec=ActivityCoreOpsClient) client.config = OpsRunConfig(worker_id="w", lease_seconds=90) client.claim.side_effect = OpsRunError("upstream failed") result = process_one(client) assert result.claimed is False assert result.empty is False assert result.retry_full_interval is True def test_process_one_success_completes() -> None: client = MagicMock(spec=ActivityCoreOpsClient) client.config = OpsRunConfig(worker_id="w", lease_seconds=90) client.claim.return_value = [_claimed_run()] client.complete.return_value = OpsRun( id="run-1", activity_definition_id="def", idempotency_key="k", target_repo="freedom-intelligence", title="FI daily", description="", state="succeeded", labels=["automated", "research-brief"], ) ar = ApproachResult( ok=True, approach=APPROACH_FI_RESEARCH_BRIEF, result={"path": "briefs/x.md"}, reason="ok", ) with patch("rein_aharness.claim_loop.execute_approach", return_value=ar): r = process_one(client) assert r.claimed is True assert r.ok is True assert r.approach == APPROACH_FI_RESEARCH_BRIEF client.complete.assert_called_once() client.fail.assert_not_called() def test_process_one_sends_initial_heartbeat_before_dispatch() -> None: client = MagicMock(spec=ActivityCoreOpsClient) client.config = OpsRunConfig(worker_id="w", lease_seconds=90) run = _claimed_run() events: list[str] = [] client.claim.side_effect = lambda **_kwargs: events.append("claim") or [run] client.heartbeat.side_effect = ( lambda *_args, **_kwargs: events.append("heartbeat") or run ) client.complete.side_effect = lambda *_args, **_kwargs: ( events.append("complete") or run ) result = ApproachResult( ok=True, approach=APPROACH_FI_RESEARCH_BRIEF, reason="ok", ) def execute(*_args, **_kwargs): events.append("execute") return result with patch("rein_aharness.claim_loop.execute_approach", side_effect=execute): processed = process_one(client) assert processed.ok is True assert events == ["claim", "heartbeat", "execute", "complete"] client.heartbeat.assert_called_once_with(run.id, lease_seconds=90) def test_process_one_initial_heartbeat_rejection_refuses_dispatch() -> None: client = MagicMock(spec=ActivityCoreOpsClient) client.config = OpsRunConfig(worker_id="w", lease_seconds=90) run = _claimed_run() client.claim.return_value = [run] client.heartbeat.side_effect = OpsRunError( "lease rejected", action="heartbeat", status_code=409, code="expired_lease", ) with patch("rein_aharness.claim_loop.execute_approach") as execute: result = process_one(client) assert result.claimed is True assert result.ok is False assert result.retry_full_interval is True assert result.reason == "initial heartbeat rejected: lease rejected" assert result.detail == { "lease_loss": { "stage": "pre-dispatch-heartbeat", "status_code": 409, "code": "expired_lease", } } execute.assert_not_called() client.complete.assert_not_called() client.fail.assert_not_called() def test_process_one_failure_reopens() -> None: client = MagicMock(spec=ActivityCoreOpsClient) client.config = OpsRunConfig(worker_id="w", lease_seconds=90) client.claim.return_value = [_claimed_run()] client.fail.return_value = OpsRun( id="run-1", activity_definition_id="def", idempotency_key="k", target_repo="freedom-intelligence", title="t", description="", state="open", labels=["automated", "research-brief"], ) ar = ApproachResult( ok=False, approach=APPROACH_FI_RESEARCH_BRIEF, reason="llm timeout", reopen=True, ) with patch("rein_aharness.claim_loop.execute_approach", return_value=ar): r = process_one(client) assert r.ok is False client.fail.assert_called_once() assert client.fail.call_args.kwargs["reopen"] is True def test_grant_without_authoritative_profile_refuses_before_execution() -> None: client = MagicMock(spec=ActivityCoreOpsClient) client.config = OpsRunConfig(worker_id="w", lease_seconds=90) run = _claimed_run() run.repository_grant = _grant("docs/") client.claim.return_value = [run] client.fail.return_value = OpsRun( id=run.id, activity_definition_id="def", idempotency_key="k", target_repo=run.target_repo, title=run.title, description="", state="failed", ) with patch("rein_aharness.claim_loop.execute_approach") as execute: result = process_one(client) assert result.ok is False assert "requires an authoritative harness_profile_ref" in result.reason execute.assert_not_called() assert client.fail.call_args.kwargs["reopen"] is False def test_process_one_refuses_close_after_lease_loss() -> None: client = MagicMock(spec=ActivityCoreOpsClient) client.config = OpsRunConfig(worker_id="w", lease_seconds=90) client.claim.return_value = [_claimed_run()] client.heartbeat.side_effect = _accept_initial_heartbeat_then_reject( client.claim.return_value[0] ) ar = ApproachResult( ok=True, approach=APPROACH_FI_RESEARCH_BRIEF, result={"path": "briefs/x.md"}, reason="ok", ) def slow_execute(*_args, **_kwargs): time.sleep(0.05) return ar with ( patch("rein_aharness.claim_loop._heartbeat_interval", return_value=0.01), patch("rein_aharness.claim_loop.execute_approach", side_effect=slow_execute), ): result = process_one(client) assert result.ok is False assert result.reason.startswith("lease lost") assert "lease_loss" in result.detail client.complete.assert_not_called() client.fail.assert_not_called() def test_process_one_lease_loss_cancels_registered_adapter_process() -> None: client = MagicMock(spec=ActivityCoreOpsClient) client.config = OpsRunConfig(worker_id="w", lease_seconds=90) client.claim.return_value = [_claimed_run()] client.heartbeat.side_effect = _accept_initial_heartbeat_then_reject( client.claim.return_value[0] ) proc = MagicMock() proc.poll.return_value = None def slow_execute(*_args, **_kwargs): cancel = active_cancel() assert cancel is not None cancel.register_process(proc) deadline = time.monotonic() + 1.0 while not cancel.cancelled and time.monotonic() < deadline: time.sleep(0.005) cancel.check() with ( patch("rein_aharness.claim_loop._heartbeat_interval", return_value=0.01), patch("rein_aharness.claim_loop.execute_approach", side_effect=slow_execute), ): result = process_one(client) assert result.ok is False assert result.reason.startswith("lease lost") proc.kill.assert_called() client.complete.assert_not_called() client.fail.assert_not_called() def test_process_one_skips_close_after_signal_cancel() -> None: client = MagicMock(spec=ActivityCoreOpsClient) client.config = OpsRunConfig(worker_id="w", lease_seconds=90) client.claim.return_value = [_claimed_run()] ar = ApproachResult( ok=True, approach=APPROACH_FI_RESEARCH_BRIEF, result={"path": "briefs/x.md"}, reason="ok", ) def execute_then_signal(*_args, **_kwargs): cancel = active_cancel() assert cancel is not None cancel.cancel("signal") return ar with patch("rein_aharness.claim_loop.execute_approach", side_effect=execute_then_signal): result = process_one(client) assert result.ok is False assert result.reason == "execution cancelled (signal)" assert result.detail == {"cancellation": {"cancelled": "true", "reason": "signal"}} client.complete.assert_not_called() client.fail.assert_not_called() def test_shutdown_signal_cancels_active_run() -> None: cancel = ExecutionCancel() proc = MagicMock() proc.poll.return_value = None cancel.register_process(proc) _set_active_run_cancel(cancel) try: _cancel_active_run("signal") finally: _set_active_run_cancel(None) assert cancel.reason == "signal" proc.kill.assert_called_once() def test_process_one_records_adapter_exception_without_unbound_result() -> None: client = MagicMock(spec=ActivityCoreOpsClient) client.config = OpsRunConfig(worker_id="w", lease_seconds=90) client.claim.return_value = [_claimed_run()] with patch( "rein_aharness.claim_loop.execute_approach", side_effect=RuntimeError("provider output must not be retained"), ): result = process_one(client) assert result.ok is False assert result.reason == "approach failed (RuntimeError)" assert result.detail == {"execution_error_type": "RuntimeError"} client.complete.assert_not_called() client.fail.assert_not_called() def test_profiled_exception_after_lease_loss_skips_close(tmp_path: Path) -> None: repo, _run, client = _profiled_case(tmp_path) client.heartbeat.side_effect = _accept_initial_heartbeat_then_reject( client.claim.return_value[0] ) def slow_profile(*_args, **_kwargs): time.sleep(0.05) raise GlasExecutionError("profile failed") with ( patch("rein_aharness.claim_loop._heartbeat_interval", return_value=0.01), patch("rein_aharness.claim_loop.execute_profiled_run", side_effect=slow_profile), ): result = process_one(client) assert result.ok is False assert result.reason.startswith("lease lost") client.complete.assert_not_called() client.fail.assert_not_called() with RepositoryTransaction(repo) as retry: assert retry.locked is True def test_poll_peek() -> None: client = MagicMock(spec=ActivityCoreOpsClient) client.list_open.return_value = [_claimed_run()] rows = poll_peek(client) assert len(rows) == 1 assert rows[0]["approach"] == APPROACH_FI_RESEARCH_BRIEF def test_profiled_run_uses_glas_and_completes_with_full_result( tmp_path: Path, ) -> None: _repo, run, client = _profiled_case(tmp_path) run.approach_hint = "fi-research-brief" client.complete.return_value = OpsRun( id=run.id, activity_definition_id="def", idempotency_key="k", target_repo=run.target_repo, title=run.title, description="", state="succeeded", ) gateway_result = { "ok": True, "evidence": { "outcome": "succeeded", "profile_ref": run.harness_profile_ref, "sandbox_id": "sbx-1", }, "tool_output": "not copied into ProcessResult.detail", "tool_error": None, } with ( patch("rein_aharness.claim_loop.execute_profiled_run", return_value=gateway_result), patch("rein_aharness.claim_loop.select_approach") as select, patch("rein_aharness.claim_loop.execute_approach") as execute, ): result = process_one(client) assert result.ok is True assert result.approach == GLAS_APPROACH assert result.detail["execution_evidence"] == gateway_result["evidence"] transaction = result.detail["repository_transaction"] assert transaction["correlation_id"] == run.id assert transaction["baseline"]["clean"] is True client.complete.assert_called_once() complete_run_id = client.complete.call_args.args[0] complete_result = client.complete.call_args.kwargs["result"] assert complete_run_id == run.id assert complete_result["execution_evidence"] == gateway_result["evidence"] assert "tool_output" not in complete_result assert complete_result["repository_transaction"] == transaction client.fail.assert_not_called() select.assert_not_called() execute.assert_not_called() def test_profile_refusal_fails_terminally_without_legacy_fallback( tmp_path: Path, ) -> None: _repo, run, client = _profiled_case(tmp_path) run.harness_profile_ref = "harness.unknown@9.9.9" run.approach_hint = "fi-research-brief" client.fail.return_value = OpsRun( id=run.id, activity_definition_id="def", idempotency_key="k", target_repo=run.target_repo, title=run.title, description="", state="failed", ) with ( patch( "rein_aharness.claim_loop.execute_profiled_run", side_effect=GlasExecutionError("unknown harness profile"), ), patch("rein_aharness.claim_loop.select_approach") as select, patch("rein_aharness.claim_loop.execute_approach") as execute, ): result = process_one(client) assert result.ok is False assert result.ops_state == "failed" assert client.fail.call_args.kwargs["reopen"] is False select.assert_not_called() execute.assert_not_called() def test_profiled_signal_cancellation_releases_lock_and_durably_fails( tmp_path: Path, ) -> None: repo, _run, client = _profiled_case(tmp_path) def cancel_during_gateway(*_args, **_kwargs): _cancel_active_run("signal") return {"ok": True, "evidence": {"outcome": "late-success"}} with patch( "rein_aharness.claim_loop.execute_profiled_run", side_effect=cancel_during_gateway, ): result = process_one(client) assert result.ok is False assert result.reason == "execution cancelled (signal)" client.complete.assert_not_called() client.fail.assert_called_once() assert client.fail.call_args.kwargs["reopen"] is False assert "repository_transaction" in client.fail.call_args.kwargs["result"] with RepositoryTransaction(repo) as retry: assert retry.locked is True def test_profiled_close_failure_happens_after_repository_lock_release( tmp_path: Path, ) -> None: repo, _run, client = _profiled_case(tmp_path) client.complete.side_effect = OpsRunError("complete transport failed") gateway_result = {"ok": True, "evidence": {"outcome": "succeeded"}} with patch( "rein_aharness.claim_loop.execute_profiled_run", return_value=gateway_result, ): result = process_one(client) assert result.ok is False assert result.reason == "close evidence remains pending" assert "repository_transaction" in result.detail with RepositoryTransaction(repo) as retry: assert retry.locked is True def test_granted_profile_accepts_commit_records_metrics_and_durable_close( tmp_path: Path, ) -> None: repo, run, client = _profiled_case(tmp_path) run.repository_grant = _grant("docs/") client.complete.return_value = OpsRun( id=run.id, activity_definition_id="def", idempotency_key="k", target_repo=run.target_repo, title=run.title, description="", state="succeeded", close_disposition="applied", ) outbox = CloseOutbox(state_dir=tmp_path / "state") def execute(*_args, **_kwargs): head = _commit_file(repo, "docs/result.md") return { "ok": True, "evidence": { "outcome": "succeeded", "commit_sha": head, "provider_response": {"secret": "must-not-persist"}, }, "tool_output": "must-not-persist", } with patch("rein_aharness.claim_loop.execute_profiled_run", side_effect=execute): result = process_one(client, outbox=outbox) assert result.ok is True transaction = result.detail["repository_transaction"] assert transaction["acceptance"]["accepted"] is True assert transaction["acceptance"]["changed_paths"] == ["docs/result.md"] assert transaction["repository_grant"]["grant_id"] == run.repository_grant.grant_id assert transaction["metrics"]["session_id"] == transaction["transaction_id"] assert outbox.status() == {"pending": 0, "delivered": 1, "quarantined": 0} close_payload = client.complete.call_args.kwargs["result"] serialized = json.dumps(close_payload) assert "tool_output" not in serialized assert "provider_response" not in serialized assert close_payload["repository_transaction"]["acceptance"]["accepted"] is True ledger = external_metrics_dir(repo, "rein-aharness") / "executions.jsonl" record = json.loads(ledger.read_text(encoding="utf-8").strip()) assert record["success"] is True assert record["session_id"] == transaction["transaction_id"] def test_granted_profile_rejects_out_of_grant_commit_and_closes_failed( tmp_path: Path, ) -> None: repo, run, client = _profiled_case(tmp_path) run.repository_grant = _grant("docs/") client.fail.return_value = OpsRun( id=run.id, activity_definition_id="def", idempotency_key="k", target_repo=run.target_repo, title=run.title, description="", state="failed", close_disposition="applied", ) def execute(*_args, **_kwargs): head = _commit_file(repo, "UNRELATED.md") return {"ok": True, "evidence": {"outcome": "succeeded", "commit_sha": head}} with patch("rein_aharness.claim_loop.execute_profiled_run", side_effect=execute): result = process_one(client, outbox=CloseOutbox(state_dir=tmp_path / "state")) assert result.ok is False assert result.reason.startswith("repository acceptance failed: path-not-granted") client.complete.assert_not_called() client.fail.assert_called_once() close_payload = client.fail.call_args.kwargs["result"] assert close_payload["repository_transaction"]["repository_grant"]["grant_id"] assert "acceptance" not in close_payload["repository_transaction"] assert client.fail.call_args.kwargs["reopen"] is False def test_granted_profile_metrics_failure_prevents_successful_close( tmp_path: Path, monkeypatch, ) -> None: repo, run, client = _profiled_case(tmp_path) run.repository_grant = _grant("docs/") client.fail.return_value = OpsRun( id=run.id, activity_definition_id="def", idempotency_key="k", target_repo=run.target_repo, title=run.title, description="", state="failed", ) def execute(*_args, **_kwargs): head = _commit_file(repo, "docs/result.md") return {"ok": True, "evidence": {"outcome": "succeeded", "commit_sha": head}} def fail_metrics(*_args, **_kwargs): raise OSError("state volume unavailable") monkeypatch.setattr( "rein_aharness.claim_loop.metrics.record_external_execution", fail_metrics, ) with patch("rein_aharness.claim_loop.execute_profiled_run", side_effect=execute): result = process_one(client, outbox=CloseOutbox(state_dir=tmp_path / "state")) assert result.ok is False assert result.reason == "required external metrics persistence failed" client.complete.assert_not_called() close_payload = client.fail.call_args.kwargs["result"] assert close_payload["repository_transaction"]["acceptance"]["accepted"] is True assert "metrics" not in close_payload["repository_transaction"] def test_response_lost_close_replays_without_reexecuting_workload( tmp_path: Path, ) -> None: repo, run, client = _profiled_case(tmp_path) run.repository_grant = _grant("docs/") outbox = CloseOutbox(state_dir=tmp_path / "state") reconciled = OpsRun( id=run.id, activity_definition_id="def", idempotency_key="k", target_repo=run.target_repo, title=run.title, description="", state="succeeded", close_disposition="reconciled", ) client.complete.side_effect = [ OpsRunError("complete transport failed", action="complete"), reconciled, ] gateway_calls = 0 def execute(*_args, **_kwargs): nonlocal gateway_calls gateway_calls += 1 head = _commit_file(repo, "docs/result.md") return { "ok": True, "evidence": {"outcome": "succeeded", "commit_sha": head}, } with patch("rein_aharness.claim_loop.execute_profiled_run", side_effect=execute): first = process_one(client, outbox=outbox) assert first.reason == "close evidence remains pending" assert outbox.status()["pending"] == 1 client.claim.return_value = [] second = process_one(client, outbox=outbox) assert second.empty is True assert gateway_calls == 1 assert client.complete.call_count == 2 assert outbox.status() == {"pending": 0, "delivered": 1, "quarantined": 0} commit_count = subprocess.run( ["git", "rev-list", "--count", "HEAD"], cwd=repo, check=True, capture_output=True, text=True, ).stdout.strip() assert commit_count == "2" def test_pending_close_failure_blocks_new_claim(tmp_path: Path) -> None: client = MagicMock(spec=ActivityCoreOpsClient) client.config = OpsRunConfig(worker_id="w", lease_seconds=90) client.complete.side_effect = OpsRunError("transport failed", action="complete") outbox = CloseOutbox(state_dir=tmp_path / "state") outbox.enqueue( CloseRequest( run_id="run-1", transaction_id="tx-1", worker_id="w", action="complete", result={"ok": True}, ) ) result = process_one(client, outbox=outbox) assert result.claimed is False assert result.retry_full_interval is True assert "remains pending" in result.reason client.claim.assert_not_called() @pytest.mark.parametrize( "code", ( "not_found", "wrong_owner", "expired_lease", "state_conflict", "terminal_conflict", "evidence_conflict", ), ) def test_permanent_close_refusal_is_quarantined_before_claim( tmp_path: Path, code: str, ) -> None: client = MagicMock(spec=ActivityCoreOpsClient) client.config = OpsRunConfig(worker_id="w", lease_seconds=90) client.complete.side_effect = OpsRunError( "conflict", action="complete", status_code=404 if code == "not_found" else 409, code=code, ) client.claim.return_value = [] outbox = CloseOutbox(state_dir=tmp_path / "state") outbox.enqueue( CloseRequest( run_id="run-1", transaction_id="tx-1", worker_id="w", action="complete", result={"ok": True}, ) ) result = process_one(client, outbox=outbox) assert result.empty is True assert outbox.status() == {"pending": 0, "delivered": 0, "quarantined": 1} client.claim.assert_called_once() def test_poll_peek_reports_authoritative_profile_route() -> None: client = MagicMock(spec=ActivityCoreOpsClient) run = _claimed_run() run.harness_profile_ref = "harness.agent-dev-local@1.0.0" client.list_open.return_value = [run] with patch("rein_aharness.claim_loop.select_approach") as select: rows = poll_peek(client) assert rows[0]["approach"] == GLAS_APPROACH assert rows[0]["harness_profile_ref"] == run.harness_profile_ref select.assert_not_called()