Harden ops run identity and leases
All checks were successful
CI Smoke / host-smoke (push) Successful in 0s
CI Smoke / container-smoke (push) Successful in 1s
Build and Publish Container Image / build-and-push (push) Successful in 33s

Assistant: codex
Assistant-Model: gpt-5.6-sol
Assistant-Session: 01a028de-e2c8-7732-8521-46a7fc5db82f
This commit is contained in:
tegwick 2026-08-23 13:01:46 +02:00
parent 36161d346f
commit f0a897e088
13 changed files with 522 additions and 50 deletions

View file

@ -232,6 +232,7 @@ async def test_complete_persists_only_normalized_glas_evidence() -> None:
row = MagicMock()
row.state = "claimed"
row.claim_owner = "worker-1"
row.lease_until = datetime.now(timezone.utc) + timedelta(minutes=1)
row.result = {}
session = AsyncMock()
session.get = AsyncMock(return_value=row)
@ -275,6 +276,7 @@ async def test_fail_reopen_under_max_attempts(monkeypatch: pytest.MonkeyPatch) -
row.id = uuid.uuid4()
row.state = "claimed"
row.claim_owner = "w1"
row.lease_until = datetime.now(timezone.utc) + timedelta(minutes=1)
row.attempt = 1
row.result = {}
session = AsyncMock()
@ -297,6 +299,7 @@ async def test_fail_permanent_at_max_attempts(monkeypatch: pytest.MonkeyPatch) -
row.id = uuid.uuid4()
row.state = "claimed"
row.claim_owner = "w1"
row.lease_until = datetime.now(timezone.utc) + timedelta(minutes=1)
row.attempt = 3
row.result = {}
session = AsyncMock()
@ -316,6 +319,7 @@ async def test_fail_persists_redacted_failure_evidence() -> None:
row = MagicMock()
row.state = "claimed"
row.claim_owner = "worker-1"
row.lease_until = datetime.now(timezone.utc) + timedelta(minutes=1)
row.attempt = 1
row.result = {}
session = AsyncMock()
@ -347,6 +351,87 @@ async def test_fail_persists_redacted_failure_evidence() -> None:
assert "tool_error" not in row.result
@pytest.mark.asyncio
@pytest.mark.parametrize("lease_offset", [None, timedelta(0), timedelta(seconds=-1)])
async def test_heartbeat_rejects_missing_or_expired_lease(
lease_offset: timedelta | None,
) -> None:
from activity_core import ops_run_queue as oq
now = datetime(2026, 8, 23, 10, 0, tzinfo=timezone.utc)
row = MagicMock()
row.state = "claimed"
row.claim_owner = "worker-1"
row.lease_until = now + lease_offset if lease_offset is not None else None
session = AsyncMock()
session.get = AsyncMock(return_value=row)
with patch.object(oq, "_utcnow", return_value=now):
heartbeat = await oq.heartbeat_ops_run(
session,
uuid.uuid4(),
worker_id="worker-1",
lease_seconds=60,
)
assert heartbeat is None
assert row.lease_until == (now + lease_offset if lease_offset is not None else None)
session.get.assert_awaited_once()
assert session.get.await_args.kwargs == {"with_for_update": True}
@pytest.mark.asyncio
async def test_complete_rejects_expired_lease_without_mutation() -> None:
from activity_core import ops_run_queue as oq
now = datetime(2026, 8, 23, 10, 0, tzinfo=timezone.utc)
row = MagicMock()
row.state = "claimed"
row.claim_owner = "worker-1"
row.lease_until = now - timedelta(microseconds=1)
row.result = {"before": True}
session = AsyncMock()
session.get = AsyncMock(return_value=row)
with patch.object(oq, "_utcnow", return_value=now):
completed = await oq.complete_ops_run(
session,
uuid.uuid4(),
worker_id="worker-1",
result={"ok": True},
)
assert completed is None
assert row.state == "claimed"
assert row.result == {"before": True}
@pytest.mark.asyncio
async def test_fail_rejects_wrong_owner_with_active_lease() -> None:
from activity_core import ops_run_queue as oq
now = datetime(2026, 8, 23, 10, 0, tzinfo=timezone.utc)
row = MagicMock()
row.state = "claimed"
row.claim_owner = "worker-1"
row.lease_until = now + timedelta(minutes=1)
row.result = {"before": True}
session = AsyncMock()
session.get = AsyncMock(return_value=row)
with patch.object(oq, "_utcnow", return_value=now):
failed = await oq.fail_ops_run(
session,
uuid.uuid4(),
worker_id="worker-2",
error="must not persist",
)
assert failed is None
assert row.state == "claimed"
assert row.result == {"before": True}
def test_label_filter_any_vs_all() -> None:
"""Document labels_mode semantics used by claim_ops_runs."""
row_labels = {"automated", "research-brief"}