Some checks failed
Governed runtime contract / contract (push) Failing after 1s
Assistant: codex Assistant-Model: gpt-5.6-sol Assistant-Session: 01a06ba0-10aa-7ea0-b20a-4f3fac39efe9
834 lines
27 KiB
Python
834 lines
27 KiB
Python
"""Claim loop process_one tests (REIN-A-0002-T03)."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import subprocess
|
|
import time
|
|
from pathlib import Path
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
import pytest
|
|
|
|
from rein_aharness.approaches import ApproachResult, APPROACH_FI_RESEARCH_BRIEF
|
|
from rein_aharness.claim_loop import (
|
|
_cancel_active_run,
|
|
_set_active_run_cancel,
|
|
process_one,
|
|
poll_peek,
|
|
)
|
|
from rein_aharness.close_outbox import CloseOutbox, CloseRequest
|
|
from rein_aharness.execution_cancel import ExecutionCancel, active_cancel
|
|
from rein_aharness.glas_execution import GLAS_APPROACH, GlasExecutionError
|
|
from rein_aharness.metrics import external_metrics_dir
|
|
from rein_aharness.ops_run_client import (
|
|
ActivityCoreOpsClient,
|
|
OpsRun,
|
|
OpsRunConfig,
|
|
OpsRunError,
|
|
)
|
|
from rein_aharness.repository_grant import RepositoryGrant
|
|
from rein_aharness.repository_transaction import RepositoryTransaction
|
|
|
|
|
|
def _claimed_run() -> OpsRun:
|
|
return OpsRun(
|
|
id="run-1",
|
|
activity_definition_id="def",
|
|
idempotency_key="k",
|
|
target_repo="freedom-intelligence",
|
|
title="FI daily",
|
|
description="d",
|
|
labels=["automated", "research-brief"],
|
|
state="claimed",
|
|
claim_owner="worker-1",
|
|
attempt=1,
|
|
)
|
|
|
|
|
|
def _profiled_case(
|
|
tmp_path: Path,
|
|
) -> tuple[Path, OpsRun, MagicMock]:
|
|
repo = tmp_path / "freedom-intelligence"
|
|
repo.mkdir()
|
|
subprocess.run(["git", "init", "-q"], cwd=repo, check=True)
|
|
(repo / "README.md").write_text("controlled target\n", encoding="utf-8")
|
|
subprocess.run(["git", "add", "."], cwd=repo, check=True)
|
|
subprocess.run(
|
|
[
|
|
"git",
|
|
"-c",
|
|
"user.email=test@example.invalid",
|
|
"-c",
|
|
"user.name=test",
|
|
"commit",
|
|
"-qm",
|
|
"baseline",
|
|
],
|
|
cwd=repo,
|
|
check=True,
|
|
)
|
|
|
|
run = _claimed_run()
|
|
run.harness_profile_ref = "harness.agent-dev-local@1.0.0"
|
|
client = MagicMock(spec=ActivityCoreOpsClient)
|
|
client.config = OpsRunConfig(
|
|
worker_id="w",
|
|
lease_seconds=90,
|
|
repo_roots=(str(tmp_path),),
|
|
)
|
|
client.claim.return_value = [run]
|
|
return repo, run, client
|
|
|
|
|
|
def _grant(*paths: str) -> RepositoryGrant:
|
|
return RepositoryGrant.from_mapping(
|
|
{
|
|
"version": "1",
|
|
"allowed_paths": list(paths or ("docs/",)),
|
|
"commit_count": {"min": 1, "max": 1},
|
|
"publish": False,
|
|
}
|
|
)
|
|
|
|
|
|
def _commit_file(repo: Path, relative: str, value: str = "result\n") -> str:
|
|
path = repo / relative
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(value, encoding="utf-8")
|
|
subprocess.run(["git", "add", relative], cwd=repo, check=True)
|
|
subprocess.run(
|
|
[
|
|
"git",
|
|
"-c",
|
|
"user.email=test@example.invalid",
|
|
"-c",
|
|
"user.name=test",
|
|
"commit",
|
|
"-qm",
|
|
"profiled result",
|
|
],
|
|
cwd=repo,
|
|
check=True,
|
|
)
|
|
return subprocess.run(
|
|
["git", "rev-parse", "HEAD"],
|
|
cwd=repo,
|
|
check=True,
|
|
capture_output=True,
|
|
text=True,
|
|
).stdout.strip()
|
|
|
|
|
|
def _accept_initial_heartbeat_then_reject(run: OpsRun):
|
|
calls = 0
|
|
|
|
def heartbeat(*_args, **_kwargs):
|
|
nonlocal calls
|
|
calls += 1
|
|
if calls == 1:
|
|
return run
|
|
raise OpsRunError(
|
|
"lease rejected",
|
|
action="heartbeat",
|
|
status_code=409,
|
|
code="expired_lease",
|
|
)
|
|
|
|
return heartbeat
|
|
|
|
|
|
def test_process_one_empty() -> None:
|
|
client = MagicMock(spec=ActivityCoreOpsClient)
|
|
client.config = OpsRunConfig(worker_id="w", lease_seconds=90)
|
|
client.claim.return_value = []
|
|
r = process_one(client)
|
|
assert r.empty is True
|
|
assert r.claimed is False
|
|
|
|
|
|
def test_process_one_claim_error_requests_full_backoff() -> None:
|
|
client = MagicMock(spec=ActivityCoreOpsClient)
|
|
client.config = OpsRunConfig(worker_id="w", lease_seconds=90)
|
|
client.claim.side_effect = OpsRunError("upstream failed")
|
|
result = process_one(client)
|
|
assert result.claimed is False
|
|
assert result.empty is False
|
|
assert result.retry_full_interval is True
|
|
|
|
|
|
def test_process_one_success_completes() -> None:
|
|
client = MagicMock(spec=ActivityCoreOpsClient)
|
|
client.config = OpsRunConfig(worker_id="w", lease_seconds=90)
|
|
client.claim.return_value = [_claimed_run()]
|
|
client.complete.return_value = OpsRun(
|
|
id="run-1",
|
|
activity_definition_id="def",
|
|
idempotency_key="k",
|
|
target_repo="freedom-intelligence",
|
|
title="FI daily",
|
|
description="",
|
|
state="succeeded",
|
|
labels=["automated", "research-brief"],
|
|
)
|
|
ar = ApproachResult(
|
|
ok=True,
|
|
approach=APPROACH_FI_RESEARCH_BRIEF,
|
|
result={"path": "briefs/x.md"},
|
|
reason="ok",
|
|
)
|
|
with patch("rein_aharness.claim_loop.execute_approach", return_value=ar):
|
|
r = process_one(client)
|
|
assert r.claimed is True
|
|
assert r.ok is True
|
|
assert r.approach == APPROACH_FI_RESEARCH_BRIEF
|
|
client.complete.assert_called_once()
|
|
client.fail.assert_not_called()
|
|
|
|
|
|
def test_process_one_sends_initial_heartbeat_before_dispatch() -> None:
|
|
client = MagicMock(spec=ActivityCoreOpsClient)
|
|
client.config = OpsRunConfig(worker_id="w", lease_seconds=90)
|
|
run = _claimed_run()
|
|
events: list[str] = []
|
|
client.claim.side_effect = lambda **_kwargs: events.append("claim") or [run]
|
|
client.heartbeat.side_effect = (
|
|
lambda *_args, **_kwargs: events.append("heartbeat") or run
|
|
)
|
|
client.complete.side_effect = lambda *_args, **_kwargs: (
|
|
events.append("complete") or run
|
|
)
|
|
result = ApproachResult(
|
|
ok=True,
|
|
approach=APPROACH_FI_RESEARCH_BRIEF,
|
|
reason="ok",
|
|
)
|
|
|
|
def execute(*_args, **_kwargs):
|
|
events.append("execute")
|
|
return result
|
|
|
|
with patch("rein_aharness.claim_loop.execute_approach", side_effect=execute):
|
|
processed = process_one(client)
|
|
|
|
assert processed.ok is True
|
|
assert events == ["claim", "heartbeat", "execute", "complete"]
|
|
client.heartbeat.assert_called_once_with(run.id, lease_seconds=90)
|
|
|
|
|
|
def test_process_one_initial_heartbeat_rejection_refuses_dispatch() -> None:
|
|
client = MagicMock(spec=ActivityCoreOpsClient)
|
|
client.config = OpsRunConfig(worker_id="w", lease_seconds=90)
|
|
run = _claimed_run()
|
|
client.claim.return_value = [run]
|
|
client.heartbeat.side_effect = OpsRunError(
|
|
"lease rejected",
|
|
action="heartbeat",
|
|
status_code=409,
|
|
code="expired_lease",
|
|
)
|
|
|
|
with patch("rein_aharness.claim_loop.execute_approach") as execute:
|
|
result = process_one(client)
|
|
|
|
assert result.claimed is True
|
|
assert result.ok is False
|
|
assert result.retry_full_interval is True
|
|
assert result.reason == "initial heartbeat rejected: lease rejected"
|
|
assert result.detail == {
|
|
"lease_loss": {
|
|
"stage": "pre-dispatch-heartbeat",
|
|
"status_code": 409,
|
|
"code": "expired_lease",
|
|
}
|
|
}
|
|
execute.assert_not_called()
|
|
client.complete.assert_not_called()
|
|
client.fail.assert_not_called()
|
|
|
|
|
|
def test_process_one_failure_reopens() -> None:
|
|
client = MagicMock(spec=ActivityCoreOpsClient)
|
|
client.config = OpsRunConfig(worker_id="w", lease_seconds=90)
|
|
client.claim.return_value = [_claimed_run()]
|
|
client.fail.return_value = OpsRun(
|
|
id="run-1",
|
|
activity_definition_id="def",
|
|
idempotency_key="k",
|
|
target_repo="freedom-intelligence",
|
|
title="t",
|
|
description="",
|
|
state="open",
|
|
labels=["automated", "research-brief"],
|
|
)
|
|
ar = ApproachResult(
|
|
ok=False,
|
|
approach=APPROACH_FI_RESEARCH_BRIEF,
|
|
reason="llm timeout",
|
|
reopen=True,
|
|
)
|
|
with patch("rein_aharness.claim_loop.execute_approach", return_value=ar):
|
|
r = process_one(client)
|
|
assert r.ok is False
|
|
client.fail.assert_called_once()
|
|
assert client.fail.call_args.kwargs["reopen"] is True
|
|
|
|
|
|
def test_grant_without_authoritative_profile_refuses_before_execution() -> None:
|
|
client = MagicMock(spec=ActivityCoreOpsClient)
|
|
client.config = OpsRunConfig(worker_id="w", lease_seconds=90)
|
|
run = _claimed_run()
|
|
run.repository_grant = _grant("docs/")
|
|
client.claim.return_value = [run]
|
|
client.fail.return_value = OpsRun(
|
|
id=run.id,
|
|
activity_definition_id="def",
|
|
idempotency_key="k",
|
|
target_repo=run.target_repo,
|
|
title=run.title,
|
|
description="",
|
|
state="failed",
|
|
)
|
|
|
|
with patch("rein_aharness.claim_loop.execute_approach") as execute:
|
|
result = process_one(client)
|
|
|
|
assert result.ok is False
|
|
assert "requires an authoritative harness_profile_ref" in result.reason
|
|
execute.assert_not_called()
|
|
assert client.fail.call_args.kwargs["reopen"] is False
|
|
|
|
|
|
def test_process_one_refuses_close_after_lease_loss() -> None:
|
|
client = MagicMock(spec=ActivityCoreOpsClient)
|
|
client.config = OpsRunConfig(worker_id="w", lease_seconds=90)
|
|
client.claim.return_value = [_claimed_run()]
|
|
client.heartbeat.side_effect = _accept_initial_heartbeat_then_reject(
|
|
client.claim.return_value[0]
|
|
)
|
|
ar = ApproachResult(
|
|
ok=True,
|
|
approach=APPROACH_FI_RESEARCH_BRIEF,
|
|
result={"path": "briefs/x.md"},
|
|
reason="ok",
|
|
)
|
|
|
|
def slow_execute(*_args, **_kwargs):
|
|
time.sleep(0.05)
|
|
return ar
|
|
|
|
with (
|
|
patch("rein_aharness.claim_loop._heartbeat_interval", return_value=0.01),
|
|
patch("rein_aharness.claim_loop.execute_approach", side_effect=slow_execute),
|
|
):
|
|
result = process_one(client)
|
|
|
|
assert result.ok is False
|
|
assert result.reason.startswith("lease lost")
|
|
assert "lease_loss" in result.detail
|
|
client.complete.assert_not_called()
|
|
client.fail.assert_not_called()
|
|
|
|
|
|
def test_process_one_lease_loss_cancels_registered_adapter_process() -> None:
|
|
client = MagicMock(spec=ActivityCoreOpsClient)
|
|
client.config = OpsRunConfig(worker_id="w", lease_seconds=90)
|
|
client.claim.return_value = [_claimed_run()]
|
|
client.heartbeat.side_effect = _accept_initial_heartbeat_then_reject(
|
|
client.claim.return_value[0]
|
|
)
|
|
proc = MagicMock()
|
|
proc.poll.return_value = None
|
|
|
|
def slow_execute(*_args, **_kwargs):
|
|
cancel = active_cancel()
|
|
assert cancel is not None
|
|
cancel.register_process(proc)
|
|
deadline = time.monotonic() + 1.0
|
|
while not cancel.cancelled and time.monotonic() < deadline:
|
|
time.sleep(0.005)
|
|
cancel.check()
|
|
|
|
with (
|
|
patch("rein_aharness.claim_loop._heartbeat_interval", return_value=0.01),
|
|
patch("rein_aharness.claim_loop.execute_approach", side_effect=slow_execute),
|
|
):
|
|
result = process_one(client)
|
|
|
|
assert result.ok is False
|
|
assert result.reason.startswith("lease lost")
|
|
proc.kill.assert_called()
|
|
client.complete.assert_not_called()
|
|
client.fail.assert_not_called()
|
|
|
|
|
|
def test_process_one_skips_close_after_signal_cancel() -> None:
|
|
client = MagicMock(spec=ActivityCoreOpsClient)
|
|
client.config = OpsRunConfig(worker_id="w", lease_seconds=90)
|
|
client.claim.return_value = [_claimed_run()]
|
|
ar = ApproachResult(
|
|
ok=True,
|
|
approach=APPROACH_FI_RESEARCH_BRIEF,
|
|
result={"path": "briefs/x.md"},
|
|
reason="ok",
|
|
)
|
|
|
|
def execute_then_signal(*_args, **_kwargs):
|
|
cancel = active_cancel()
|
|
assert cancel is not None
|
|
cancel.cancel("signal")
|
|
return ar
|
|
|
|
with patch("rein_aharness.claim_loop.execute_approach", side_effect=execute_then_signal):
|
|
result = process_one(client)
|
|
|
|
assert result.ok is False
|
|
assert result.reason == "execution cancelled (signal)"
|
|
assert result.detail == {"cancellation": {"cancelled": "true", "reason": "signal"}}
|
|
client.complete.assert_not_called()
|
|
client.fail.assert_not_called()
|
|
|
|
|
|
def test_shutdown_signal_cancels_active_run() -> None:
|
|
cancel = ExecutionCancel()
|
|
proc = MagicMock()
|
|
proc.poll.return_value = None
|
|
cancel.register_process(proc)
|
|
_set_active_run_cancel(cancel)
|
|
try:
|
|
_cancel_active_run("signal")
|
|
finally:
|
|
_set_active_run_cancel(None)
|
|
|
|
assert cancel.reason == "signal"
|
|
proc.kill.assert_called_once()
|
|
|
|
|
|
def test_process_one_records_adapter_exception_without_unbound_result() -> None:
|
|
client = MagicMock(spec=ActivityCoreOpsClient)
|
|
client.config = OpsRunConfig(worker_id="w", lease_seconds=90)
|
|
client.claim.return_value = [_claimed_run()]
|
|
|
|
with patch(
|
|
"rein_aharness.claim_loop.execute_approach",
|
|
side_effect=RuntimeError("provider output must not be retained"),
|
|
):
|
|
result = process_one(client)
|
|
|
|
assert result.ok is False
|
|
assert result.reason == "approach failed (RuntimeError)"
|
|
assert result.detail == {"execution_error_type": "RuntimeError"}
|
|
client.complete.assert_not_called()
|
|
client.fail.assert_not_called()
|
|
|
|
|
|
def test_profiled_exception_after_lease_loss_skips_close(tmp_path: Path) -> None:
|
|
repo, _run, client = _profiled_case(tmp_path)
|
|
client.heartbeat.side_effect = _accept_initial_heartbeat_then_reject(
|
|
client.claim.return_value[0]
|
|
)
|
|
|
|
def slow_profile(*_args, **_kwargs):
|
|
time.sleep(0.05)
|
|
raise GlasExecutionError("profile failed")
|
|
|
|
with (
|
|
patch("rein_aharness.claim_loop._heartbeat_interval", return_value=0.01),
|
|
patch("rein_aharness.claim_loop.execute_profiled_run", side_effect=slow_profile),
|
|
):
|
|
result = process_one(client)
|
|
|
|
assert result.ok is False
|
|
assert result.reason.startswith("lease lost")
|
|
client.complete.assert_not_called()
|
|
client.fail.assert_not_called()
|
|
with RepositoryTransaction(repo) as retry:
|
|
assert retry.locked is True
|
|
|
|
|
|
def test_poll_peek() -> None:
|
|
client = MagicMock(spec=ActivityCoreOpsClient)
|
|
client.list_open.return_value = [_claimed_run()]
|
|
rows = poll_peek(client)
|
|
assert len(rows) == 1
|
|
assert rows[0]["approach"] == APPROACH_FI_RESEARCH_BRIEF
|
|
|
|
|
|
def test_profiled_run_uses_glas_and_completes_with_full_result(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
_repo, run, client = _profiled_case(tmp_path)
|
|
run.approach_hint = "fi-research-brief"
|
|
client.complete.return_value = OpsRun(
|
|
id=run.id,
|
|
activity_definition_id="def",
|
|
idempotency_key="k",
|
|
target_repo=run.target_repo,
|
|
title=run.title,
|
|
description="",
|
|
state="succeeded",
|
|
)
|
|
gateway_result = {
|
|
"ok": True,
|
|
"evidence": {
|
|
"outcome": "succeeded",
|
|
"profile_ref": run.harness_profile_ref,
|
|
"sandbox_id": "sbx-1",
|
|
},
|
|
"tool_output": "not copied into ProcessResult.detail",
|
|
"tool_error": None,
|
|
}
|
|
with (
|
|
patch("rein_aharness.claim_loop.execute_profiled_run", return_value=gateway_result),
|
|
patch("rein_aharness.claim_loop.select_approach") as select,
|
|
patch("rein_aharness.claim_loop.execute_approach") as execute,
|
|
):
|
|
result = process_one(client)
|
|
|
|
assert result.ok is True
|
|
assert result.approach == GLAS_APPROACH
|
|
assert result.detail["execution_evidence"] == gateway_result["evidence"]
|
|
transaction = result.detail["repository_transaction"]
|
|
assert transaction["correlation_id"] == run.id
|
|
assert transaction["baseline"]["clean"] is True
|
|
client.complete.assert_called_once()
|
|
complete_run_id = client.complete.call_args.args[0]
|
|
complete_result = client.complete.call_args.kwargs["result"]
|
|
assert complete_run_id == run.id
|
|
assert complete_result["execution_evidence"] == gateway_result["evidence"]
|
|
assert "tool_output" not in complete_result
|
|
assert complete_result["repository_transaction"] == transaction
|
|
client.fail.assert_not_called()
|
|
select.assert_not_called()
|
|
execute.assert_not_called()
|
|
|
|
|
|
def test_profile_refusal_fails_terminally_without_legacy_fallback(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
_repo, run, client = _profiled_case(tmp_path)
|
|
run.harness_profile_ref = "harness.unknown@9.9.9"
|
|
run.approach_hint = "fi-research-brief"
|
|
client.fail.return_value = OpsRun(
|
|
id=run.id,
|
|
activity_definition_id="def",
|
|
idempotency_key="k",
|
|
target_repo=run.target_repo,
|
|
title=run.title,
|
|
description="",
|
|
state="failed",
|
|
)
|
|
with (
|
|
patch(
|
|
"rein_aharness.claim_loop.execute_profiled_run",
|
|
side_effect=GlasExecutionError("unknown harness profile"),
|
|
),
|
|
patch("rein_aharness.claim_loop.select_approach") as select,
|
|
patch("rein_aharness.claim_loop.execute_approach") as execute,
|
|
):
|
|
result = process_one(client)
|
|
|
|
assert result.ok is False
|
|
assert result.ops_state == "failed"
|
|
assert client.fail.call_args.kwargs["reopen"] is False
|
|
select.assert_not_called()
|
|
execute.assert_not_called()
|
|
|
|
|
|
def test_profiled_signal_cancellation_releases_lock_and_durably_fails(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
repo, _run, client = _profiled_case(tmp_path)
|
|
|
|
def cancel_during_gateway(*_args, **_kwargs):
|
|
_cancel_active_run("signal")
|
|
return {"ok": True, "evidence": {"outcome": "late-success"}}
|
|
|
|
with patch(
|
|
"rein_aharness.claim_loop.execute_profiled_run",
|
|
side_effect=cancel_during_gateway,
|
|
):
|
|
result = process_one(client)
|
|
|
|
assert result.ok is False
|
|
assert result.reason == "execution cancelled (signal)"
|
|
client.complete.assert_not_called()
|
|
client.fail.assert_called_once()
|
|
assert client.fail.call_args.kwargs["reopen"] is False
|
|
assert "repository_transaction" in client.fail.call_args.kwargs["result"]
|
|
with RepositoryTransaction(repo) as retry:
|
|
assert retry.locked is True
|
|
|
|
|
|
def test_profiled_close_failure_happens_after_repository_lock_release(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
repo, _run, client = _profiled_case(tmp_path)
|
|
client.complete.side_effect = OpsRunError("complete transport failed")
|
|
gateway_result = {"ok": True, "evidence": {"outcome": "succeeded"}}
|
|
|
|
with patch(
|
|
"rein_aharness.claim_loop.execute_profiled_run",
|
|
return_value=gateway_result,
|
|
):
|
|
result = process_one(client)
|
|
|
|
assert result.ok is False
|
|
assert result.reason == "close evidence remains pending"
|
|
assert "repository_transaction" in result.detail
|
|
with RepositoryTransaction(repo) as retry:
|
|
assert retry.locked is True
|
|
|
|
|
|
def test_granted_profile_accepts_commit_records_metrics_and_durable_close(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
repo, run, client = _profiled_case(tmp_path)
|
|
run.repository_grant = _grant("docs/")
|
|
client.complete.return_value = OpsRun(
|
|
id=run.id,
|
|
activity_definition_id="def",
|
|
idempotency_key="k",
|
|
target_repo=run.target_repo,
|
|
title=run.title,
|
|
description="",
|
|
state="succeeded",
|
|
close_disposition="applied",
|
|
)
|
|
outbox = CloseOutbox(state_dir=tmp_path / "state")
|
|
|
|
def execute(*_args, **_kwargs):
|
|
head = _commit_file(repo, "docs/result.md")
|
|
return {
|
|
"ok": True,
|
|
"evidence": {
|
|
"outcome": "succeeded",
|
|
"commit_sha": head,
|
|
"provider_response": {"secret": "must-not-persist"},
|
|
},
|
|
"tool_output": "must-not-persist",
|
|
}
|
|
|
|
with patch("rein_aharness.claim_loop.execute_profiled_run", side_effect=execute):
|
|
result = process_one(client, outbox=outbox)
|
|
|
|
assert result.ok is True
|
|
transaction = result.detail["repository_transaction"]
|
|
assert transaction["acceptance"]["accepted"] is True
|
|
assert transaction["acceptance"]["changed_paths"] == ["docs/result.md"]
|
|
assert transaction["repository_grant"]["grant_id"] == run.repository_grant.grant_id
|
|
assert transaction["metrics"]["session_id"] == transaction["transaction_id"]
|
|
assert outbox.status() == {"pending": 0, "delivered": 1, "quarantined": 0}
|
|
|
|
close_payload = client.complete.call_args.kwargs["result"]
|
|
serialized = json.dumps(close_payload)
|
|
assert "tool_output" not in serialized
|
|
assert "provider_response" not in serialized
|
|
assert close_payload["repository_transaction"]["acceptance"]["accepted"] is True
|
|
|
|
ledger = external_metrics_dir(repo, "rein-aharness") / "executions.jsonl"
|
|
record = json.loads(ledger.read_text(encoding="utf-8").strip())
|
|
assert record["success"] is True
|
|
assert record["session_id"] == transaction["transaction_id"]
|
|
|
|
|
|
def test_granted_profile_rejects_out_of_grant_commit_and_closes_failed(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
repo, run, client = _profiled_case(tmp_path)
|
|
run.repository_grant = _grant("docs/")
|
|
client.fail.return_value = OpsRun(
|
|
id=run.id,
|
|
activity_definition_id="def",
|
|
idempotency_key="k",
|
|
target_repo=run.target_repo,
|
|
title=run.title,
|
|
description="",
|
|
state="failed",
|
|
close_disposition="applied",
|
|
)
|
|
|
|
def execute(*_args, **_kwargs):
|
|
head = _commit_file(repo, "UNRELATED.md")
|
|
return {"ok": True, "evidence": {"outcome": "succeeded", "commit_sha": head}}
|
|
|
|
with patch("rein_aharness.claim_loop.execute_profiled_run", side_effect=execute):
|
|
result = process_one(client, outbox=CloseOutbox(state_dir=tmp_path / "state"))
|
|
|
|
assert result.ok is False
|
|
assert result.reason.startswith("repository acceptance failed: path-not-granted")
|
|
client.complete.assert_not_called()
|
|
client.fail.assert_called_once()
|
|
close_payload = client.fail.call_args.kwargs["result"]
|
|
assert close_payload["repository_transaction"]["repository_grant"]["grant_id"]
|
|
assert "acceptance" not in close_payload["repository_transaction"]
|
|
assert client.fail.call_args.kwargs["reopen"] is False
|
|
|
|
|
|
def test_granted_profile_metrics_failure_prevents_successful_close(
|
|
tmp_path: Path,
|
|
monkeypatch,
|
|
) -> None:
|
|
repo, run, client = _profiled_case(tmp_path)
|
|
run.repository_grant = _grant("docs/")
|
|
client.fail.return_value = OpsRun(
|
|
id=run.id,
|
|
activity_definition_id="def",
|
|
idempotency_key="k",
|
|
target_repo=run.target_repo,
|
|
title=run.title,
|
|
description="",
|
|
state="failed",
|
|
)
|
|
|
|
def execute(*_args, **_kwargs):
|
|
head = _commit_file(repo, "docs/result.md")
|
|
return {"ok": True, "evidence": {"outcome": "succeeded", "commit_sha": head}}
|
|
|
|
def fail_metrics(*_args, **_kwargs):
|
|
raise OSError("state volume unavailable")
|
|
|
|
monkeypatch.setattr(
|
|
"rein_aharness.claim_loop.metrics.record_external_execution",
|
|
fail_metrics,
|
|
)
|
|
with patch("rein_aharness.claim_loop.execute_profiled_run", side_effect=execute):
|
|
result = process_one(client, outbox=CloseOutbox(state_dir=tmp_path / "state"))
|
|
|
|
assert result.ok is False
|
|
assert result.reason == "required external metrics persistence failed"
|
|
client.complete.assert_not_called()
|
|
close_payload = client.fail.call_args.kwargs["result"]
|
|
assert close_payload["repository_transaction"]["acceptance"]["accepted"] is True
|
|
assert "metrics" not in close_payload["repository_transaction"]
|
|
|
|
|
|
def test_response_lost_close_replays_without_reexecuting_workload(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
repo, run, client = _profiled_case(tmp_path)
|
|
run.repository_grant = _grant("docs/")
|
|
outbox = CloseOutbox(state_dir=tmp_path / "state")
|
|
reconciled = OpsRun(
|
|
id=run.id,
|
|
activity_definition_id="def",
|
|
idempotency_key="k",
|
|
target_repo=run.target_repo,
|
|
title=run.title,
|
|
description="",
|
|
state="succeeded",
|
|
close_disposition="reconciled",
|
|
)
|
|
client.complete.side_effect = [
|
|
OpsRunError("complete transport failed", action="complete"),
|
|
reconciled,
|
|
]
|
|
gateway_calls = 0
|
|
|
|
def execute(*_args, **_kwargs):
|
|
nonlocal gateway_calls
|
|
gateway_calls += 1
|
|
head = _commit_file(repo, "docs/result.md")
|
|
return {
|
|
"ok": True,
|
|
"evidence": {"outcome": "succeeded", "commit_sha": head},
|
|
}
|
|
|
|
with patch("rein_aharness.claim_loop.execute_profiled_run", side_effect=execute):
|
|
first = process_one(client, outbox=outbox)
|
|
|
|
assert first.reason == "close evidence remains pending"
|
|
assert outbox.status()["pending"] == 1
|
|
client.claim.return_value = []
|
|
|
|
second = process_one(client, outbox=outbox)
|
|
|
|
assert second.empty is True
|
|
assert gateway_calls == 1
|
|
assert client.complete.call_count == 2
|
|
assert outbox.status() == {"pending": 0, "delivered": 1, "quarantined": 0}
|
|
commit_count = subprocess.run(
|
|
["git", "rev-list", "--count", "HEAD"],
|
|
cwd=repo,
|
|
check=True,
|
|
capture_output=True,
|
|
text=True,
|
|
).stdout.strip()
|
|
assert commit_count == "2"
|
|
|
|
|
|
def test_pending_close_failure_blocks_new_claim(tmp_path: Path) -> None:
|
|
client = MagicMock(spec=ActivityCoreOpsClient)
|
|
client.config = OpsRunConfig(worker_id="w", lease_seconds=90)
|
|
client.complete.side_effect = OpsRunError("transport failed", action="complete")
|
|
outbox = CloseOutbox(state_dir=tmp_path / "state")
|
|
outbox.enqueue(
|
|
CloseRequest(
|
|
run_id="run-1",
|
|
transaction_id="tx-1",
|
|
worker_id="w",
|
|
action="complete",
|
|
result={"ok": True},
|
|
)
|
|
)
|
|
|
|
result = process_one(client, outbox=outbox)
|
|
|
|
assert result.claimed is False
|
|
assert result.retry_full_interval is True
|
|
assert "remains pending" in result.reason
|
|
client.claim.assert_not_called()
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"code",
|
|
(
|
|
"not_found",
|
|
"wrong_owner",
|
|
"expired_lease",
|
|
"state_conflict",
|
|
"terminal_conflict",
|
|
"evidence_conflict",
|
|
),
|
|
)
|
|
def test_permanent_close_refusal_is_quarantined_before_claim(
|
|
tmp_path: Path,
|
|
code: str,
|
|
) -> None:
|
|
client = MagicMock(spec=ActivityCoreOpsClient)
|
|
client.config = OpsRunConfig(worker_id="w", lease_seconds=90)
|
|
client.complete.side_effect = OpsRunError(
|
|
"conflict",
|
|
action="complete",
|
|
status_code=404 if code == "not_found" else 409,
|
|
code=code,
|
|
)
|
|
client.claim.return_value = []
|
|
outbox = CloseOutbox(state_dir=tmp_path / "state")
|
|
outbox.enqueue(
|
|
CloseRequest(
|
|
run_id="run-1",
|
|
transaction_id="tx-1",
|
|
worker_id="w",
|
|
action="complete",
|
|
result={"ok": True},
|
|
)
|
|
)
|
|
|
|
result = process_one(client, outbox=outbox)
|
|
|
|
assert result.empty is True
|
|
assert outbox.status() == {"pending": 0, "delivered": 0, "quarantined": 1}
|
|
client.claim.assert_called_once()
|
|
|
|
|
|
def test_poll_peek_reports_authoritative_profile_route() -> None:
|
|
client = MagicMock(spec=ActivityCoreOpsClient)
|
|
run = _claimed_run()
|
|
run.harness_profile_ref = "harness.agent-dev-local@1.0.0"
|
|
client.list_open.return_value = [run]
|
|
with patch("rein_aharness.claim_loop.select_approach") as select:
|
|
rows = poll_peek(client)
|
|
assert rows[0]["approach"] == GLAS_APPROACH
|
|
assert rows[0]["harness_profile_ref"] == run.harness_profile_ref
|
|
select.assert_not_called()
|