feat(ops-runs): expire claims that keep losing their lease
Lapsed claims were reopened with no attempt ceiling, so a run whose worker lost its lease (including a 409 before dispatch) could cycle forever. Activity Core now reopens a lapsed claim below OPS_RUN_MAX_ATTEMPTS and sets terminal 'expired' at the ceiling. The owner is kept for audit, and /ops-runs/expire-leases reports reopened and expired counts. Answers the REINAH-WP-0003-T05 open point on thread 27cd4c47 (ACTIVITY-WP-ADHOC-2026-10-19). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Assistant: claude-code Assistant-Model: opus Assistant-Process: 244572@bnt-lap001 Assistant-Session: 3b235e1a-ae45-4d0f-ae70-17a9bd78ece8
This commit is contained in:
parent
3b25aea7b0
commit
42347da012
6 changed files with 192 additions and 14 deletions
|
|
@ -632,3 +632,91 @@ def test_label_filter_any_vs_all() -> None:
|
|||
want = {"research-brief", "missing"}
|
||||
assert bool(want & row_labels) # any
|
||||
assert not want.issubset(row_labels) # all
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_expire_stale_claims_reopens_below_and_expires_at_ceiling(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""A lapsed lease returns to open below the ceiling, else terminal expired."""
|
||||
from sqlalchemy.dialects import postgresql
|
||||
|
||||
from activity_core import ops_run_queue as oq
|
||||
|
||||
monkeypatch.setenv("OPS_RUN_MAX_ATTEMPTS", "3")
|
||||
statements: list = []
|
||||
|
||||
async def execute(stmt, *args, **kwargs):
|
||||
statements.append(stmt)
|
||||
res = MagicMock()
|
||||
res.rowcount = 2 if len(statements) == 1 else 5
|
||||
return res
|
||||
|
||||
session = AsyncMock()
|
||||
session.execute = AsyncMock(side_effect=execute)
|
||||
|
||||
counts = await oq.expire_stale_claims(session)
|
||||
assert counts == {"expired": 2, "reopened": 5}
|
||||
assert len(statements) == 2
|
||||
|
||||
compiled = [
|
||||
s.compile(dialect=postgresql.dialect(), compile_kwargs={"literal_binds": False})
|
||||
for s in statements
|
||||
]
|
||||
expire_sql, reopen_sql = (str(c) for c in compiled)
|
||||
expire_params, reopen_params = (c.params for c in compiled)
|
||||
|
||||
# Both statements only touch claimed rows whose lease has lapsed.
|
||||
for sql in (expire_sql, reopen_sql):
|
||||
assert "ops_runs.state = " in sql
|
||||
assert "ops_runs.lease_until IS NOT NULL" in sql
|
||||
assert "ops_runs.lease_until <= " in sql
|
||||
assert "ops_runs.attempt >= " in expire_sql
|
||||
assert "ops_runs.attempt < " in reopen_sql
|
||||
|
||||
assert expire_params["state"] == "expired"
|
||||
assert "claimed" in expire_params.values()
|
||||
assert 3 in expire_params.values()
|
||||
assert expire_params["lease_until"] is None
|
||||
assert "expired by activity-core" in expire_params["result"]["error"]
|
||||
# The original claim owner stays on the expired row for audit.
|
||||
assert "claim_owner" not in expire_params
|
||||
|
||||
assert reopen_params["state"] == "open"
|
||||
assert reopen_params["claim_owner"] is None
|
||||
assert reopen_params["lease_until"] is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reopen_stale_claims_returns_reopened_count() -> None:
|
||||
from activity_core import ops_run_queue as oq
|
||||
|
||||
session = AsyncMock()
|
||||
with patch.object(
|
||||
oq,
|
||||
"expire_stale_claims",
|
||||
AsyncMock(return_value={"reopened": 4, "expired": 1}),
|
||||
):
|
||||
assert await oq.reopen_stale_claims(session) == 4
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_lost_owner_cannot_close_an_expired_row() -> None:
|
||||
"""After activity-core expires a lapsed claim, a late close is refused."""
|
||||
from activity_core import ops_run_queue as oq
|
||||
|
||||
row = MagicMock()
|
||||
row.state = "expired"
|
||||
row.claim_owner = "worker-1"
|
||||
row.lease_until = None
|
||||
row.result = {"error": "lease expired"}
|
||||
row.repository_grant = None
|
||||
row.close_intent_digest = None
|
||||
session = AsyncMock()
|
||||
session.get = AsyncMock(return_value=row)
|
||||
|
||||
outcome = await oq.close_ops_run(
|
||||
session, uuid.uuid4(), worker_id="worker-1", action="fail", error="late"
|
||||
)
|
||||
assert outcome.disposition == "state_conflict"
|
||||
assert row.state == "expired"
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue