feat(ops-runs): expire claims that keep losing their lease

Lapsed claims were reopened with no attempt ceiling, so a run whose worker
lost its lease (including a 409 before dispatch) could cycle forever.
Activity Core now reopens a lapsed claim below OPS_RUN_MAX_ATTEMPTS and sets
terminal 'expired' at the ceiling. The owner is kept for audit, and
/ops-runs/expire-leases reports reopened and expired counts. Answers the
REINAH-WP-0003-T05 open point on thread 27cd4c47 (ACTIVITY-WP-ADHOC-2026-10-19).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

Assistant: claude-code
Assistant-Model: opus
Assistant-Process: 244572@bnt-lap001
Assistant-Session: 3b235e1a-ae45-4d0f-ae70-17a9bd78ece8
This commit is contained in:
tegwick 2026-10-19 15:30:29 +02:00
parent 3b25aea7b0
commit 42347da012
6 changed files with 192 additions and 14 deletions

View file

@ -632,3 +632,91 @@ def test_label_filter_any_vs_all() -> None:
want = {"research-brief", "missing"}
assert bool(want & row_labels) # any
assert not want.issubset(row_labels) # all
@pytest.mark.asyncio
async def test_expire_stale_claims_reopens_below_and_expires_at_ceiling(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""A lapsed lease returns to open below the ceiling, else terminal expired."""
from sqlalchemy.dialects import postgresql
from activity_core import ops_run_queue as oq
monkeypatch.setenv("OPS_RUN_MAX_ATTEMPTS", "3")
statements: list = []
async def execute(stmt, *args, **kwargs):
statements.append(stmt)
res = MagicMock()
res.rowcount = 2 if len(statements) == 1 else 5
return res
session = AsyncMock()
session.execute = AsyncMock(side_effect=execute)
counts = await oq.expire_stale_claims(session)
assert counts == {"expired": 2, "reopened": 5}
assert len(statements) == 2
compiled = [
s.compile(dialect=postgresql.dialect(), compile_kwargs={"literal_binds": False})
for s in statements
]
expire_sql, reopen_sql = (str(c) for c in compiled)
expire_params, reopen_params = (c.params for c in compiled)
# Both statements only touch claimed rows whose lease has lapsed.
for sql in (expire_sql, reopen_sql):
assert "ops_runs.state = " in sql
assert "ops_runs.lease_until IS NOT NULL" in sql
assert "ops_runs.lease_until <= " in sql
assert "ops_runs.attempt >= " in expire_sql
assert "ops_runs.attempt < " in reopen_sql
assert expire_params["state"] == "expired"
assert "claimed" in expire_params.values()
assert 3 in expire_params.values()
assert expire_params["lease_until"] is None
assert "expired by activity-core" in expire_params["result"]["error"]
# The original claim owner stays on the expired row for audit.
assert "claim_owner" not in expire_params
assert reopen_params["state"] == "open"
assert reopen_params["claim_owner"] is None
assert reopen_params["lease_until"] is None
@pytest.mark.asyncio
async def test_reopen_stale_claims_returns_reopened_count() -> None:
from activity_core import ops_run_queue as oq
session = AsyncMock()
with patch.object(
oq,
"expire_stale_claims",
AsyncMock(return_value={"reopened": 4, "expired": 1}),
):
assert await oq.reopen_stale_claims(session) == 4
@pytest.mark.asyncio
async def test_lost_owner_cannot_close_an_expired_row() -> None:
"""After activity-core expires a lapsed claim, a late close is refused."""
from activity_core import ops_run_queue as oq
row = MagicMock()
row.state = "expired"
row.claim_owner = "worker-1"
row.lease_until = None
row.result = {"error": "lease expired"}
row.repository_grant = None
row.close_intent_digest = None
session = AsyncMock()
session.get = AsyncMock(return_value=row)
outcome = await oq.close_ops_run(
session, uuid.uuid4(), worker_id="worker-1", action="fail", error="late"
)
assert outcome.disposition == "state_conflict"
assert row.state == "expired"