rein-aharness/tests/test_claim_loop.py

"""Claim loop process_one tests (REIN-A-0002-T03)."""

from __future__ import annotations

import json
import subprocess
import time
from pathlib import Path
from unittest.mock import MagicMock, patch

from rein_aharness.approaches import ApproachResult, APPROACH_FI_RESEARCH_BRIEF
from rein_aharness.claim_loop import (
    _cancel_active_run,
    _set_active_run_cancel,
    process_one,
    poll_peek,
)
from rein_aharness.close_outbox import CloseOutbox, CloseRequest
from rein_aharness.execution_cancel import ExecutionCancel, active_cancel
from rein_aharness.glas_execution import GLAS_APPROACH, GlasExecutionError
from rein_aharness.metrics import external_metrics_dir
from rein_aharness.ops_run_client import (
    ActivityCoreOpsClient,
    OpsRun,
    OpsRunConfig,
    OpsRunError,
)
from rein_aharness.repository_grant import RepositoryGrant
from rein_aharness.repository_transaction import RepositoryTransaction


def _claimed_run() -> OpsRun:
    return OpsRun(
        id="run-1",
        activity_definition_id="def",
        idempotency_key="k",
        target_repo="freedom-intelligence",
        title="FI daily",
        description="d",
        labels=["automated", "research-brief"],
        state="claimed",
        claim_owner="worker-1",
        attempt=1,
    )


def _profiled_case(
    tmp_path: Path,
) -> tuple[Path, OpsRun, MagicMock]:
    repo = tmp_path / "freedom-intelligence"
    repo.mkdir()
    subprocess.run(["git", "init", "-q"], cwd=repo, check=True)
    (repo / "README.md").write_text("controlled target\n", encoding="utf-8")
    subprocess.run(["git", "add", "."], cwd=repo, check=True)
    subprocess.run(
        [
            "git",
            "-c",
            "user.email=test@example.invalid",
            "-c",
            "user.name=test",
            "commit",
            "-qm",
            "baseline",
        ],
        cwd=repo,
        check=True,
    )

    run = _claimed_run()
    run.harness_profile_ref = "harness.agent-dev-local@1.0.0"
    client = MagicMock(spec=ActivityCoreOpsClient)
    client.config = OpsRunConfig(
        worker_id="w",
        lease_seconds=90,
        repo_roots=(str(tmp_path),),
    )
    client.claim.return_value = [run]
    return repo, run, client


def _grant(*paths: str) -> RepositoryGrant:
    return RepositoryGrant.from_mapping(
        {
            "version": "1",
            "allowed_paths": list(paths or ("docs/",)),
            "commit_count": {"min": 1, "max": 1},
            "publish": False,
        }
    )


def _commit_file(repo: Path, relative: str, value: str = "result\n") -> str:
    path = repo / relative
    path.parent.mkdir(parents=True, exist_ok=True)
    path.write_text(value, encoding="utf-8")
    subprocess.run(["git", "add", relative], cwd=repo, check=True)
    subprocess.run(
        [
            "git",
            "-c",
            "user.email=test@example.invalid",
            "-c",
            "user.name=test",
            "commit",
            "-qm",
            "profiled result",
        ],
        cwd=repo,
        check=True,
    )
    return subprocess.run(
        ["git", "rev-parse", "HEAD"],
        cwd=repo,
        check=True,
        capture_output=True,
        text=True,
    ).stdout.strip()


def test_process_one_empty() -> None:
    client = MagicMock(spec=ActivityCoreOpsClient)
    client.config = OpsRunConfig(worker_id="w", lease_seconds=90)
    client.claim.return_value = []
    r = process_one(client)
    assert r.empty is True
    assert r.claimed is False


def test_process_one_claim_error_requests_full_backoff() -> None:
    client = MagicMock(spec=ActivityCoreOpsClient)
    client.config = OpsRunConfig(worker_id="w", lease_seconds=90)
    client.claim.side_effect = OpsRunError("upstream failed")
    result = process_one(client)
    assert result.claimed is False
    assert result.empty is False
    assert result.retry_full_interval is True


def test_process_one_success_completes() -> None:
    client = MagicMock(spec=ActivityCoreOpsClient)
    client.config = OpsRunConfig(worker_id="w", lease_seconds=90)
    client.claim.return_value = [_claimed_run()]
    client.complete.return_value = OpsRun(
        id="run-1",
        activity_definition_id="def",
        idempotency_key="k",
        target_repo="freedom-intelligence",
        title="FI daily",
        description="",
        state="succeeded",
        labels=["automated", "research-brief"],
    )
    ar = ApproachResult(
        ok=True,
        approach=APPROACH_FI_RESEARCH_BRIEF,
        result={"path": "briefs/x.md"},
        reason="ok",
    )
    with patch("rein_aharness.claim_loop.execute_approach", return_value=ar):
        r = process_one(client)
    assert r.claimed is True
    assert r.ok is True
    assert r.approach == APPROACH_FI_RESEARCH_BRIEF
    client.complete.assert_called_once()
    client.fail.assert_not_called()


def test_process_one_failure_reopens() -> None:
    client = MagicMock(spec=ActivityCoreOpsClient)
    client.config = OpsRunConfig(worker_id="w", lease_seconds=90)
    client.claim.return_value = [_claimed_run()]
    client.fail.return_value = OpsRun(
        id="run-1",
        activity_definition_id="def",
        idempotency_key="k",
        target_repo="freedom-intelligence",
        title="t",
        description="",
        state="open",
        labels=["automated", "research-brief"],
    )
    ar = ApproachResult(
        ok=False,
        approach=APPROACH_FI_RESEARCH_BRIEF,
        reason="llm timeout",
        reopen=True,
    )
    with patch("rein_aharness.claim_loop.execute_approach", return_value=ar):
        r = process_one(client)
    assert r.ok is False
    client.fail.assert_called_once()
    assert client.fail.call_args.kwargs["reopen"] is True


def test_grant_without_authoritative_profile_refuses_before_execution() -> None:
    client = MagicMock(spec=ActivityCoreOpsClient)
    client.config = OpsRunConfig(worker_id="w", lease_seconds=90)
    run = _claimed_run()
    run.repository_grant = _grant("docs/")
    client.claim.return_value = [run]
    client.fail.return_value = OpsRun(
        id=run.id,
        activity_definition_id="def",
        idempotency_key="k",
        target_repo=run.target_repo,
        title=run.title,
        description="",
        state="failed",
    )

    with patch("rein_aharness.claim_loop.execute_approach") as execute:
        result = process_one(client)

    assert result.ok is False
    assert "requires an authoritative harness_profile_ref" in result.reason
    execute.assert_not_called()
    assert client.fail.call_args.kwargs["reopen"] is False


def test_process_one_refuses_close_after_lease_loss() -> None:
    client = MagicMock(spec=ActivityCoreOpsClient)
    client.config = OpsRunConfig(worker_id="w", lease_seconds=90)
    client.claim.return_value = [_claimed_run()]
    client.heartbeat.side_effect = OpsRunError("lease rejected", status_code=409)
    ar = ApproachResult(
        ok=True,
        approach=APPROACH_FI_RESEARCH_BRIEF,
        result={"path": "briefs/x.md"},
        reason="ok",
    )

    def slow_execute(*_args, **_kwargs):
        time.sleep(0.05)
        return ar

    with (
        patch("rein_aharness.claim_loop._heartbeat_interval", return_value=0.01),
        patch("rein_aharness.claim_loop.execute_approach", side_effect=slow_execute),
    ):
        result = process_one(client)

    assert result.ok is False
    assert result.reason.startswith("lease lost")
    assert "lease_loss" in result.detail
    client.complete.assert_not_called()
    client.fail.assert_not_called()


def test_process_one_lease_loss_cancels_registered_adapter_process() -> None:
    client = MagicMock(spec=ActivityCoreOpsClient)
    client.config = OpsRunConfig(worker_id="w", lease_seconds=90)
    client.claim.return_value = [_claimed_run()]
    client.heartbeat.side_effect = OpsRunError("lease rejected", status_code=409)
    proc = MagicMock()
    proc.poll.return_value = None

    def slow_execute(*_args, **_kwargs):
        cancel = active_cancel()
        assert cancel is not None
        cancel.register_process(proc)
        deadline = time.monotonic() + 1.0
        while not cancel.cancelled and time.monotonic() < deadline:
            time.sleep(0.005)
        cancel.check()

    with (
        patch("rein_aharness.claim_loop._heartbeat_interval", return_value=0.01),
        patch("rein_aharness.claim_loop.execute_approach", side_effect=slow_execute),
    ):
        result = process_one(client)

    assert result.ok is False
    assert result.reason.startswith("lease lost")
    proc.kill.assert_called()
    client.complete.assert_not_called()
    client.fail.assert_not_called()


def test_process_one_skips_close_after_signal_cancel() -> None:
    client = MagicMock(spec=ActivityCoreOpsClient)
    client.config = OpsRunConfig(worker_id="w", lease_seconds=90)
    client.claim.return_value = [_claimed_run()]
    ar = ApproachResult(
        ok=True,
        approach=APPROACH_FI_RESEARCH_BRIEF,
        result={"path": "briefs/x.md"},
        reason="ok",
    )

    def execute_then_signal(*_args, **_kwargs):
        cancel = active_cancel()
        assert cancel is not None
        cancel.cancel("signal")
        return ar

    with patch("rein_aharness.claim_loop.execute_approach", side_effect=execute_then_signal):
        result = process_one(client)

    assert result.ok is False
    assert result.reason == "execution cancelled (signal)"
    assert result.detail == {"cancellation": {"cancelled": "true", "reason": "signal"}}
    client.complete.assert_not_called()
    client.fail.assert_not_called()


def test_shutdown_signal_cancels_active_run() -> None:
    cancel = ExecutionCancel()
    proc = MagicMock()
    proc.poll.return_value = None
    cancel.register_process(proc)
    _set_active_run_cancel(cancel)
    try:
        _cancel_active_run("signal")
    finally:
        _set_active_run_cancel(None)

    assert cancel.reason == "signal"
    proc.kill.assert_called_once()


def test_process_one_records_adapter_exception_without_unbound_result() -> None:
    client = MagicMock(spec=ActivityCoreOpsClient)
    client.config = OpsRunConfig(worker_id="w", lease_seconds=90)
    client.claim.return_value = [_claimed_run()]

    with patch(
        "rein_aharness.claim_loop.execute_approach",
        side_effect=RuntimeError("provider output must not be retained"),
    ):
        result = process_one(client)

    assert result.ok is False
    assert result.reason == "approach failed (RuntimeError)"
    assert result.detail == {"execution_error_type": "RuntimeError"}
    client.complete.assert_not_called()
    client.fail.assert_not_called()


def test_profiled_exception_after_lease_loss_skips_close(tmp_path: Path) -> None:
    repo, _run, client = _profiled_case(tmp_path)
    client.heartbeat.side_effect = OpsRunError("lease rejected", status_code=409)

    def slow_profile(*_args, **_kwargs):
        time.sleep(0.05)
        raise GlasExecutionError("profile failed")

    with (
        patch("rein_aharness.claim_loop._heartbeat_interval", return_value=0.01),
        patch("rein_aharness.claim_loop.execute_profiled_run", side_effect=slow_profile),
    ):
        result = process_one(client)

    assert result.ok is False
    assert result.reason.startswith("lease lost")
    client.complete.assert_not_called()
    client.fail.assert_not_called()
    with RepositoryTransaction(repo) as retry:
        assert retry.locked is True


def test_poll_peek() -> None:
    client = MagicMock(spec=ActivityCoreOpsClient)
    client.list_open.return_value = [_claimed_run()]
    rows = poll_peek(client)
    assert len(rows) == 1
    assert rows[0]["approach"] == APPROACH_FI_RESEARCH_BRIEF


def test_profiled_run_uses_glas_and_completes_with_full_result(
    tmp_path: Path,
) -> None:
    _repo, run, client = _profiled_case(tmp_path)
    run.approach_hint = "fi-research-brief"
    client.complete.return_value = OpsRun(
        id=run.id,
        activity_definition_id="def",
        idempotency_key="k",
        target_repo=run.target_repo,
        title=run.title,
        description="",
        state="succeeded",
    )
    gateway_result = {
        "ok": True,
        "evidence": {
            "outcome": "succeeded",
            "profile_ref": run.harness_profile_ref,
            "sandbox_id": "sbx-1",
        },
        "tool_output": "not copied into ProcessResult.detail",
        "tool_error": None,
    }
    with (
        patch("rein_aharness.claim_loop.execute_profiled_run", return_value=gateway_result),
        patch("rein_aharness.claim_loop.select_approach") as select,
        patch("rein_aharness.claim_loop.execute_approach") as execute,
    ):
        result = process_one(client)

    assert result.ok is True
    assert result.approach == GLAS_APPROACH
    assert result.detail["execution_evidence"] == gateway_result["evidence"]
    transaction = result.detail["repository_transaction"]
    assert transaction["correlation_id"] == run.id
    assert transaction["baseline"]["clean"] is True
    client.complete.assert_called_once()
    complete_run_id = client.complete.call_args.args[0]
    complete_result = client.complete.call_args.kwargs["result"]
    assert complete_run_id == run.id
    assert complete_result["execution_evidence"] == gateway_result["evidence"]
    assert "tool_output" not in complete_result
    assert complete_result["repository_transaction"] == transaction
    client.fail.assert_not_called()
    select.assert_not_called()
    execute.assert_not_called()


def test_profile_refusal_fails_terminally_without_legacy_fallback(
    tmp_path: Path,
) -> None:
    _repo, run, client = _profiled_case(tmp_path)
    run.harness_profile_ref = "harness.unknown@9.9.9"
    run.approach_hint = "fi-research-brief"
    client.fail.return_value = OpsRun(
        id=run.id,
        activity_definition_id="def",
        idempotency_key="k",
        target_repo=run.target_repo,
        title=run.title,
        description="",
        state="failed",
    )
    with (
        patch(
            "rein_aharness.claim_loop.execute_profiled_run",
            side_effect=GlasExecutionError("unknown harness profile"),
        ),
        patch("rein_aharness.claim_loop.select_approach") as select,
        patch("rein_aharness.claim_loop.execute_approach") as execute,
    ):
        result = process_one(client)

    assert result.ok is False
    assert result.ops_state == "failed"
    assert client.fail.call_args.kwargs["reopen"] is False
    select.assert_not_called()
    execute.assert_not_called()


def test_profiled_signal_cancellation_releases_lock_and_durably_fails(
    tmp_path: Path,
) -> None:
    repo, _run, client = _profiled_case(tmp_path)

    def cancel_during_gateway(*_args, **_kwargs):
        _cancel_active_run("signal")
        return {"ok": True, "evidence": {"outcome": "late-success"}}

    with patch(
        "rein_aharness.claim_loop.execute_profiled_run",
        side_effect=cancel_during_gateway,
    ):
        result = process_one(client)

    assert result.ok is False
    assert result.reason == "execution cancelled (signal)"
    client.complete.assert_not_called()
    client.fail.assert_called_once()
    assert client.fail.call_args.kwargs["reopen"] is False
    assert "repository_transaction" in client.fail.call_args.kwargs["result"]
    with RepositoryTransaction(repo) as retry:
        assert retry.locked is True


def test_profiled_close_failure_happens_after_repository_lock_release(
    tmp_path: Path,
) -> None:
    repo, _run, client = _profiled_case(tmp_path)
    client.complete.side_effect = OpsRunError("complete transport failed")
    gateway_result = {"ok": True, "evidence": {"outcome": "succeeded"}}

    with patch(
        "rein_aharness.claim_loop.execute_profiled_run",
        return_value=gateway_result,
    ):
        result = process_one(client)

    assert result.ok is False
    assert result.reason == "close evidence remains pending"
    assert "repository_transaction" in result.detail
    with RepositoryTransaction(repo) as retry:
        assert retry.locked is True


def test_granted_profile_accepts_commit_records_metrics_and_durable_close(
    tmp_path: Path,
) -> None:
    repo, run, client = _profiled_case(tmp_path)
    run.repository_grant = _grant("docs/")
    client.complete.return_value = OpsRun(
        id=run.id,
        activity_definition_id="def",
        idempotency_key="k",
        target_repo=run.target_repo,
        title=run.title,
        description="",
        state="succeeded",
        close_disposition="applied",
    )
    outbox = CloseOutbox(state_dir=tmp_path / "state")

    def execute(*_args, **_kwargs):
        head = _commit_file(repo, "docs/result.md")
        return {
            "ok": True,
            "evidence": {
                "outcome": "succeeded",
                "commit_sha": head,
                "provider_response": {"secret": "must-not-persist"},
            },
            "tool_output": "must-not-persist",
        }

    with patch("rein_aharness.claim_loop.execute_profiled_run", side_effect=execute):
        result = process_one(client, outbox=outbox)

    assert result.ok is True
    transaction = result.detail["repository_transaction"]
    assert transaction["acceptance"]["accepted"] is True
    assert transaction["acceptance"]["changed_paths"] == ["docs/result.md"]
    assert transaction["repository_grant"]["grant_id"] == run.repository_grant.grant_id
    assert transaction["metrics"]["session_id"] == transaction["transaction_id"]
    assert outbox.status() == {"pending": 0, "delivered": 1, "quarantined": 0}

    close_payload = client.complete.call_args.kwargs["result"]
    serialized = json.dumps(close_payload)
    assert "tool_output" not in serialized
    assert "provider_response" not in serialized
    assert close_payload["repository_transaction"]["acceptance"]["accepted"] is True

    ledger = external_metrics_dir(repo, "rein-aharness") / "executions.jsonl"
    record = json.loads(ledger.read_text(encoding="utf-8").strip())
    assert record["success"] is True
    assert record["session_id"] == transaction["transaction_id"]


def test_granted_profile_rejects_out_of_grant_commit_and_closes_failed(
    tmp_path: Path,
) -> None:
    repo, run, client = _profiled_case(tmp_path)
    run.repository_grant = _grant("docs/")
    client.fail.return_value = OpsRun(
        id=run.id,
        activity_definition_id="def",
        idempotency_key="k",
        target_repo=run.target_repo,
        title=run.title,
        description="",
        state="failed",
        close_disposition="applied",
    )

    def execute(*_args, **_kwargs):
        head = _commit_file(repo, "UNRELATED.md")
        return {"ok": True, "evidence": {"outcome": "succeeded", "commit_sha": head}}

    with patch("rein_aharness.claim_loop.execute_profiled_run", side_effect=execute):
        result = process_one(client, outbox=CloseOutbox(state_dir=tmp_path / "state"))

    assert result.ok is False
    assert result.reason.startswith("repository acceptance failed: path-not-granted")
    client.complete.assert_not_called()
    client.fail.assert_called_once()
    close_payload = client.fail.call_args.kwargs["result"]
    assert close_payload["repository_transaction"]["repository_grant"]["grant_id"]
    assert "acceptance" not in close_payload["repository_transaction"]
    assert client.fail.call_args.kwargs["reopen"] is False


def test_granted_profile_metrics_failure_prevents_successful_close(
    tmp_path: Path,
    monkeypatch,
) -> None:
    repo, run, client = _profiled_case(tmp_path)
    run.repository_grant = _grant("docs/")
    client.fail.return_value = OpsRun(
        id=run.id,
        activity_definition_id="def",
        idempotency_key="k",
        target_repo=run.target_repo,
        title=run.title,
        description="",
        state="failed",
    )

    def execute(*_args, **_kwargs):
        head = _commit_file(repo, "docs/result.md")
        return {"ok": True, "evidence": {"outcome": "succeeded", "commit_sha": head}}

    def fail_metrics(*_args, **_kwargs):
        raise OSError("state volume unavailable")

    monkeypatch.setattr(
        "rein_aharness.claim_loop.metrics.record_external_execution",
        fail_metrics,
    )
    with patch("rein_aharness.claim_loop.execute_profiled_run", side_effect=execute):
        result = process_one(client, outbox=CloseOutbox(state_dir=tmp_path / "state"))

    assert result.ok is False
    assert result.reason == "required external metrics persistence failed"
    client.complete.assert_not_called()
    close_payload = client.fail.call_args.kwargs["result"]
    assert close_payload["repository_transaction"]["acceptance"]["accepted"] is True
    assert "metrics" not in close_payload["repository_transaction"]


def test_response_lost_close_replays_without_reexecuting_workload(
    tmp_path: Path,
) -> None:
    repo, run, client = _profiled_case(tmp_path)
    run.repository_grant = _grant("docs/")
    outbox = CloseOutbox(state_dir=tmp_path / "state")
    reconciled = OpsRun(
        id=run.id,
        activity_definition_id="def",
        idempotency_key="k",
        target_repo=run.target_repo,
        title=run.title,
        description="",
        state="succeeded",
        close_disposition="reconciled",
    )
    client.complete.side_effect = [
        OpsRunError("complete transport failed", action="complete"),
        reconciled,
    ]
    gateway_calls = 0

    def execute(*_args, **_kwargs):
        nonlocal gateway_calls
        gateway_calls += 1
        head = _commit_file(repo, "docs/result.md")
        return {
            "ok": True,
            "evidence": {"outcome": "succeeded", "commit_sha": head},
        }

    with patch("rein_aharness.claim_loop.execute_profiled_run", side_effect=execute):
        first = process_one(client, outbox=outbox)

    assert first.reason == "close evidence remains pending"
    assert outbox.status()["pending"] == 1
    client.claim.return_value = []

    second = process_one(client, outbox=outbox)

    assert second.empty is True
    assert gateway_calls == 1
    assert client.complete.call_count == 2
    assert outbox.status() == {"pending": 0, "delivered": 1, "quarantined": 0}
    commit_count = subprocess.run(
        ["git", "rev-list", "--count", "HEAD"],
        cwd=repo,
        check=True,
        capture_output=True,
        text=True,
    ).stdout.strip()
    assert commit_count == "2"


def test_pending_close_failure_blocks_new_claim(tmp_path: Path) -> None:
    client = MagicMock(spec=ActivityCoreOpsClient)
    client.config = OpsRunConfig(worker_id="w", lease_seconds=90)
    client.complete.side_effect = OpsRunError("transport failed", action="complete")
    outbox = CloseOutbox(state_dir=tmp_path / "state")
    outbox.enqueue(
        CloseRequest(
            run_id="run-1",
            transaction_id="tx-1",
            worker_id="w",
            action="complete",
            result={"ok": True},
        )
    )

    result = process_one(client, outbox=outbox)

    assert result.claimed is False
    assert result.retry_full_interval is True
    assert "remains pending" in result.reason
    client.claim.assert_not_called()


def test_permanent_close_conflict_is_quarantined_before_claim(tmp_path: Path) -> None:
    client = MagicMock(spec=ActivityCoreOpsClient)
    client.config = OpsRunConfig(worker_id="w", lease_seconds=90)
    client.complete.side_effect = OpsRunError(
        "conflict",
        action="complete",
        status_code=409,
        code="terminal_conflict",
    )
    client.claim.return_value = []
    outbox = CloseOutbox(state_dir=tmp_path / "state")
    outbox.enqueue(
        CloseRequest(
            run_id="run-1",
            transaction_id="tx-1",
            worker_id="w",
            action="complete",
            result={"ok": True},
        )
    )

    result = process_one(client, outbox=outbox)

    assert result.empty is True
    assert outbox.status() == {"pending": 0, "delivered": 0, "quarantined": 1}
    client.claim.assert_called_once()


def test_poll_peek_reports_authoritative_profile_route() -> None:
    client = MagicMock(spec=ActivityCoreOpsClient)
    run = _claimed_run()
    run.harness_profile_ref = "harness.agent-dev-local@1.0.0"
    client.list_open.return_value = [run]
    with patch("rein_aharness.claim_loop.select_approach") as select:
        rows = poll_peek(client)
    assert rows[0]["approach"] == GLAS_APPROACH
    assert rows[0]["harness_profile_ref"] == run.harness_profile_ref
    select.assert_not_called()