coulomb/rein-aharness
4
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Classify heartbeat lease failures

Browse source 
Assistant: codex
Assistant-Model: gpt-5.6-sol
Assistant-Session: 01a02b6f-7db1-7222-918b-e813a6bda38d
This commit is contained in:
tegwick 2026-08-23 14:42:30 +02:00
parent ad4074c20a
commit 84b4089f8a
6 changed files with 84 additions and 14 deletions
Download patch file Download diff file Expand all files Collapse all files

23
rein_aharness/claim_loop.py
View file

@ -80,13 +80,22 @@ class _Heartbeat:
self._client.heartbeat(self._run_id, lease_seconds=self._lease)
logger.info("heartbeat ok run_id=%s", self._run_id)
except OpsRunError as exc:
evidence = self.monitor.mark_lost(type(exc).__name__)
logger.warning(
"heartbeat lost run_id=%s error_type=%s observed_at=%s",
self._run_id,
evidence.error_type,
evidence.observed_at,
)
if exc.lease_rejected:
evidence = self.monitor.mark_lost(
f"{type(exc).__name__}:{exc.status_code}"
)
logger.warning(
"heartbeat lost run_id=%s error_type=%s observed_at=%s",
self._run_id,
evidence.error_type,
evidence.observed_at,
)
else:
logger.warning(
"heartbeat transient failure run_id=%s error_type=%s",
self._run_id,
type(exc).__name__,
)
self._thread = threading.Thread(
target=_loop, name=f"ops-hb-{self._run_id[:8]}", daemon=True