Harden ops run identity and leases
Assistant: codex Assistant-Model: gpt-5.6-sol Assistant-Session: 01a028de-e2c8-7732-8521-46a7fc5db82f
This commit is contained in:
parent
36161d346f
commit
f0a897e088
13 changed files with 522 additions and 50 deletions
|
|
@ -232,6 +232,7 @@ async def test_complete_persists_only_normalized_glas_evidence() -> None:
|
|||
row = MagicMock()
|
||||
row.state = "claimed"
|
||||
row.claim_owner = "worker-1"
|
||||
row.lease_until = datetime.now(timezone.utc) + timedelta(minutes=1)
|
||||
row.result = {}
|
||||
session = AsyncMock()
|
||||
session.get = AsyncMock(return_value=row)
|
||||
|
|
@ -275,6 +276,7 @@ async def test_fail_reopen_under_max_attempts(monkeypatch: pytest.MonkeyPatch) -
|
|||
row.id = uuid.uuid4()
|
||||
row.state = "claimed"
|
||||
row.claim_owner = "w1"
|
||||
row.lease_until = datetime.now(timezone.utc) + timedelta(minutes=1)
|
||||
row.attempt = 1
|
||||
row.result = {}
|
||||
session = AsyncMock()
|
||||
|
|
@ -297,6 +299,7 @@ async def test_fail_permanent_at_max_attempts(monkeypatch: pytest.MonkeyPatch) -
|
|||
row.id = uuid.uuid4()
|
||||
row.state = "claimed"
|
||||
row.claim_owner = "w1"
|
||||
row.lease_until = datetime.now(timezone.utc) + timedelta(minutes=1)
|
||||
row.attempt = 3
|
||||
row.result = {}
|
||||
session = AsyncMock()
|
||||
|
|
@ -316,6 +319,7 @@ async def test_fail_persists_redacted_failure_evidence() -> None:
|
|||
row = MagicMock()
|
||||
row.state = "claimed"
|
||||
row.claim_owner = "worker-1"
|
||||
row.lease_until = datetime.now(timezone.utc) + timedelta(minutes=1)
|
||||
row.attempt = 1
|
||||
row.result = {}
|
||||
session = AsyncMock()
|
||||
|
|
@ -347,6 +351,87 @@ async def test_fail_persists_redacted_failure_evidence() -> None:
|
|||
assert "tool_error" not in row.result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("lease_offset", [None, timedelta(0), timedelta(seconds=-1)])
|
||||
async def test_heartbeat_rejects_missing_or_expired_lease(
|
||||
lease_offset: timedelta | None,
|
||||
) -> None:
|
||||
from activity_core import ops_run_queue as oq
|
||||
|
||||
now = datetime(2026, 8, 23, 10, 0, tzinfo=timezone.utc)
|
||||
row = MagicMock()
|
||||
row.state = "claimed"
|
||||
row.claim_owner = "worker-1"
|
||||
row.lease_until = now + lease_offset if lease_offset is not None else None
|
||||
session = AsyncMock()
|
||||
session.get = AsyncMock(return_value=row)
|
||||
|
||||
with patch.object(oq, "_utcnow", return_value=now):
|
||||
heartbeat = await oq.heartbeat_ops_run(
|
||||
session,
|
||||
uuid.uuid4(),
|
||||
worker_id="worker-1",
|
||||
lease_seconds=60,
|
||||
)
|
||||
|
||||
assert heartbeat is None
|
||||
assert row.lease_until == (now + lease_offset if lease_offset is not None else None)
|
||||
session.get.assert_awaited_once()
|
||||
assert session.get.await_args.kwargs == {"with_for_update": True}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_complete_rejects_expired_lease_without_mutation() -> None:
|
||||
from activity_core import ops_run_queue as oq
|
||||
|
||||
now = datetime(2026, 8, 23, 10, 0, tzinfo=timezone.utc)
|
||||
row = MagicMock()
|
||||
row.state = "claimed"
|
||||
row.claim_owner = "worker-1"
|
||||
row.lease_until = now - timedelta(microseconds=1)
|
||||
row.result = {"before": True}
|
||||
session = AsyncMock()
|
||||
session.get = AsyncMock(return_value=row)
|
||||
|
||||
with patch.object(oq, "_utcnow", return_value=now):
|
||||
completed = await oq.complete_ops_run(
|
||||
session,
|
||||
uuid.uuid4(),
|
||||
worker_id="worker-1",
|
||||
result={"ok": True},
|
||||
)
|
||||
|
||||
assert completed is None
|
||||
assert row.state == "claimed"
|
||||
assert row.result == {"before": True}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fail_rejects_wrong_owner_with_active_lease() -> None:
|
||||
from activity_core import ops_run_queue as oq
|
||||
|
||||
now = datetime(2026, 8, 23, 10, 0, tzinfo=timezone.utc)
|
||||
row = MagicMock()
|
||||
row.state = "claimed"
|
||||
row.claim_owner = "worker-1"
|
||||
row.lease_until = now + timedelta(minutes=1)
|
||||
row.result = {"before": True}
|
||||
session = AsyncMock()
|
||||
session.get = AsyncMock(return_value=row)
|
||||
|
||||
with patch.object(oq, "_utcnow", return_value=now):
|
||||
failed = await oq.fail_ops_run(
|
||||
session,
|
||||
uuid.uuid4(),
|
||||
worker_id="worker-2",
|
||||
error="must not persist",
|
||||
)
|
||||
|
||||
assert failed is None
|
||||
assert row.state == "claimed"
|
||||
assert row.result == {"before": True}
|
||||
|
||||
|
||||
def test_label_filter_any_vs_all() -> None:
|
||||
"""Document labels_mode semantics used by claim_ops_runs."""
|
||||
row_labels = {"automated", "research-brief"}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue