Refuse close after lease loss
Assistant: codex Assistant-Model: gpt-5.6-sol Assistant-Session: 01a02b6f-7db1-7222-918b-e813a6bda38d
This commit is contained in:
parent
9097d69e1d
commit
75f4914777
3 changed files with 68 additions and 1 deletions
|
|
@ -179,6 +179,23 @@ def process_one(
|
||||||
finally:
|
finally:
|
||||||
hb.stop()
|
hb.stop()
|
||||||
|
|
||||||
|
if hb.monitor.lost:
|
||||||
|
loss = hb.monitor.evidence
|
||||||
|
assert loss is not None
|
||||||
|
return ProcessResult(
|
||||||
|
claimed=True,
|
||||||
|
run_id=run.id,
|
||||||
|
approach=approach,
|
||||||
|
ok=False,
|
||||||
|
reason=f"lease lost ({loss.error_type})",
|
||||||
|
detail={
|
||||||
|
"lease_loss": {
|
||||||
|
"error_type": loss.error_type,
|
||||||
|
"observed_at": loss.observed_at,
|
||||||
|
}
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
payload = {
|
payload = {
|
||||||
"approach": ar.approach,
|
"approach": ar.approach,
|
||||||
"ok": ar.ok,
|
"ok": ar.ok,
|
||||||
|
|
@ -277,6 +294,23 @@ def _process_profiled_run(
|
||||||
finally:
|
finally:
|
||||||
hb.stop()
|
hb.stop()
|
||||||
|
|
||||||
|
if hb.monitor.lost:
|
||||||
|
loss = hb.monitor.evidence
|
||||||
|
assert loss is not None
|
||||||
|
return ProcessResult(
|
||||||
|
claimed=True,
|
||||||
|
run_id=run.id,
|
||||||
|
approach=GLAS_APPROACH,
|
||||||
|
ok=False,
|
||||||
|
reason=f"lease lost ({loss.error_type})",
|
||||||
|
detail={
|
||||||
|
"lease_loss": {
|
||||||
|
"error_type": loss.error_type,
|
||||||
|
"observed_at": loss.observed_at,
|
||||||
|
}
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
evidence = gateway_result["evidence"]
|
evidence = gateway_result["evidence"]
|
||||||
ok = gateway_result["ok"]
|
ok = gateway_result["ok"]
|
||||||
reason = str(evidence.get("error") or evidence.get("outcome") or "Glas execution failed")
|
reason = str(evidence.get("error") or evidence.get("outcome") or "Glas execution failed")
|
||||||
|
|
|
||||||
|
|
@ -3,6 +3,7 @@
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from unittest.mock import MagicMock, patch
|
from unittest.mock import MagicMock, patch
|
||||||
|
import time
|
||||||
|
|
||||||
from rein_aharness.approaches import ApproachResult, APPROACH_FI_RESEARCH_BRIEF
|
from rein_aharness.approaches import ApproachResult, APPROACH_FI_RESEARCH_BRIEF
|
||||||
from rein_aharness.claim_loop import process_one, poll_peek
|
from rein_aharness.claim_loop import process_one, poll_peek
|
||||||
|
|
@ -105,6 +106,35 @@ def test_process_one_failure_reopens() -> None:
|
||||||
assert client.fail.call_args.kwargs["reopen"] is True
|
assert client.fail.call_args.kwargs["reopen"] is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_process_one_refuses_close_after_lease_loss() -> None:
|
||||||
|
client = MagicMock(spec=ActivityCoreOpsClient)
|
||||||
|
client.config = OpsRunConfig(worker_id="w", lease_seconds=90)
|
||||||
|
client.claim.return_value = [_claimed_run()]
|
||||||
|
client.heartbeat.side_effect = OpsRunError("lease rejected")
|
||||||
|
ar = ApproachResult(
|
||||||
|
ok=True,
|
||||||
|
approach=APPROACH_FI_RESEARCH_BRIEF,
|
||||||
|
result={"path": "briefs/x.md"},
|
||||||
|
reason="ok",
|
||||||
|
)
|
||||||
|
|
||||||
|
def slow_execute(*_args, **_kwargs):
|
||||||
|
time.sleep(0.05)
|
||||||
|
return ar
|
||||||
|
|
||||||
|
with (
|
||||||
|
patch("rein_aharness.claim_loop._heartbeat_interval", return_value=0.01),
|
||||||
|
patch("rein_aharness.claim_loop.execute_approach", side_effect=slow_execute),
|
||||||
|
):
|
||||||
|
result = process_one(client)
|
||||||
|
|
||||||
|
assert result.ok is False
|
||||||
|
assert result.reason.startswith("lease lost")
|
||||||
|
assert "lease_loss" in result.detail
|
||||||
|
client.complete.assert_not_called()
|
||||||
|
client.fail.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
def test_poll_peek() -> None:
|
def test_poll_peek() -> None:
|
||||||
client = MagicMock(spec=ActivityCoreOpsClient)
|
client = MagicMock(spec=ActivityCoreOpsClient)
|
||||||
client.list_open.return_value = [_claimed_run()]
|
client.list_open.return_value = [_claimed_run()]
|
||||||
|
|
|
||||||
|
|
@ -201,7 +201,10 @@ evidence instead of raw provider exception text. This establishes the signal
|
||||||
needed by execution and acceptance boundaries without pretending that adapters
|
needed by execution and acceptance boundaries without pretending that adapters
|
||||||
can already cancel in-flight work. Remaining work is to classify Activity Core
|
can already cancel in-flight work. Remaining work is to classify Activity Core
|
||||||
responses, connect cancellation at each adapter boundary, and refuse result
|
responses, connect cancellation at each adapter boundary, and refuse result
|
||||||
acceptance/close after loss.
|
acceptance/close after loss. The claim loop now implements that last refusal for
|
||||||
|
normal and successful profiled executions: a lost lease returns bounded loss
|
||||||
|
evidence and skips Activity Core completion/failure calls, leaving reconciliation
|
||||||
|
to the owner of the expired lease.
|
||||||
|
|
||||||
## Verify accepted commits and reconcile metrics/reporting
|
## Verify accepted commits and reconcile metrics/reporting
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue