fix: back off after ops claim errors

Assistant: codex
Assistant-Model: gpt-5.6-sol
Assistant-Session: 01a02b6f-7db1-7222-918b-e813a6bda38d
This commit is contained in:
tegwick 2026-08-22 23:59:02 +02:00
parent 81d6222c6b
commit db588010f7
3 changed files with 37 additions and 4 deletions

View file

@ -42,6 +42,7 @@ logger = logging.getLogger("rein_aharness.claim_loop")
class ProcessResult:
claimed: bool
empty: bool = False
retry_full_interval: bool = False
run_id: str | None = None
approach: str | None = None
ok: bool | None = None
@ -104,7 +105,11 @@ def process_one(
try:
claimed = client.claim(limit=1)
except OpsRunError as exc:
return ProcessResult(claimed=False, reason=f"claim error: {exc}")
return ProcessResult(
claimed=False,
retry_full_interval=True,
reason=f"claim error: {exc}",
)
if not claimed:
return ProcessResult(claimed=False, empty=True, reason="queue empty")
@ -396,8 +401,13 @@ def run_claim_loop(
break
if max_iterations is not None and iterations >= max_iterations:
break
# Sleep full interval only when empty; short pause after work
sleep_for = interval_seconds if result.empty else min(2.0, interval_seconds)
# Empty queues and upstream errors both use the configured backoff.
# The short pause is only for a cycle that actually claimed work.
sleep_for = (
interval_seconds
if result.empty or result.retry_full_interval
else min(2.0, interval_seconds)
)
stop.wait(sleep_for)
logger.info("claim-loop stop iterations=%s exit=%s", iterations, exit_code)