Expose lease loss to execution boundaries
Assistant: codex Assistant-Model: gpt-5.6-sol Assistant-Session: 01a02b6f-7db1-7222-918b-e813a6bda38d
This commit is contained in:
parent
17f081154c
commit
9097d69e1d
5 changed files with 167 additions and 4 deletions
|
|
@ -28,6 +28,7 @@ from rein_aharness.glas_execution import (
|
|||
GlasExecutionError,
|
||||
execute_profiled_run,
|
||||
)
|
||||
from rein_aharness.lease_monitor import LeaseMonitor
|
||||
from rein_aharness.ops_run_client import (
|
||||
ActivityCoreOpsClient,
|
||||
OpsRun,
|
||||
|
|
@ -68,6 +69,7 @@ class _Heartbeat:
|
|||
self._lease = lease_seconds
|
||||
self._stop = threading.Event()
|
||||
self._thread: threading.Thread | None = None
|
||||
self.monitor = LeaseMonitor()
|
||||
|
||||
def start(self) -> None:
|
||||
interval = _heartbeat_interval(self._lease)
|
||||
|
|
@ -78,7 +80,13 @@ class _Heartbeat:
|
|||
self._client.heartbeat(self._run_id, lease_seconds=self._lease)
|
||||
logger.info("heartbeat ok run_id=%s", self._run_id)
|
||||
except OpsRunError as exc:
|
||||
logger.warning("heartbeat failed run_id=%s: %s", self._run_id, exc)
|
||||
evidence = self.monitor.mark_lost(type(exc).__name__)
|
||||
logger.warning(
|
||||
"heartbeat lost run_id=%s error_type=%s observed_at=%s",
|
||||
self._run_id,
|
||||
evidence.error_type,
|
||||
evidence.observed_at,
|
||||
)
|
||||
|
||||
self._thread = threading.Thread(
|
||||
target=_loop, name=f"ops-hb-{self._run_id[:8]}", daemon=True
|
||||
|
|
@ -89,6 +97,7 @@ class _Heartbeat:
|
|||
self._stop.set()
|
||||
if self._thread is not None:
|
||||
self._thread.join(timeout=5.0)
|
||||
self.monitor.stop()
|
||||
|
||||
|
||||
def process_one(
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue