Expose lease loss to execution boundaries

Assistant: codex
Assistant-Model: gpt-5.6-sol
Assistant-Session: 01a02b6f-7db1-7222-918b-e813a6bda38d
This commit is contained in:
tegwick 2026-08-23 14:16:11 +02:00
parent 17f081154c
commit 9097d69e1d
5 changed files with 167 additions and 4 deletions

View file

@ -28,6 +28,7 @@ from rein_aharness.glas_execution import (
GlasExecutionError,
execute_profiled_run,
)
from rein_aharness.lease_monitor import LeaseMonitor
from rein_aharness.ops_run_client import (
ActivityCoreOpsClient,
OpsRun,
@ -68,6 +69,7 @@ class _Heartbeat:
self._lease = lease_seconds
self._stop = threading.Event()
self._thread: threading.Thread | None = None
self.monitor = LeaseMonitor()
def start(self) -> None:
interval = _heartbeat_interval(self._lease)
@ -78,7 +80,13 @@ class _Heartbeat:
self._client.heartbeat(self._run_id, lease_seconds=self._lease)
logger.info("heartbeat ok run_id=%s", self._run_id)
except OpsRunError as exc:
logger.warning("heartbeat failed run_id=%s: %s", self._run_id, exc)
evidence = self.monitor.mark_lost(type(exc).__name__)
logger.warning(
"heartbeat lost run_id=%s error_type=%s observed_at=%s",
self._run_id,
evidence.error_type,
evidence.observed_at,
)
self._thread = threading.Thread(
target=_loop, name=f"ops-hb-{self._run_id[:8]}", daemon=True
@ -89,6 +97,7 @@ class _Heartbeat:
self._stop.set()
if self._thread is not None:
self._thread.join(timeout=5.0)
self.monitor.stop()
def process_one(