coulomb/rein-aharness
4
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Classify heartbeat lease failures

Browse source 
Assistant: codex
Assistant-Model: gpt-5.6-sol
Assistant-Session: 01a02b6f-7db1-7222-918b-e813a6bda38d
This commit is contained in:
tegwick 2026-08-23 14:42:30 +02:00
parent ad4074c20a
commit 84b4089f8a
6 changed files with 84 additions and 14 deletions
Download patch file Download diff file Expand all files Collapse all files

23
rein_aharness/claim_loop.py
View file

@ -80,13 +80,22 @@ class _Heartbeat:
self._client.heartbeat(self._run_id, lease_seconds=self._lease)
logger.info("heartbeat ok run_id=%s", self._run_id)
except OpsRunError as exc:
evidence = self.monitor.mark_lost(type(exc).__name__)
logger.warning(
"heartbeat lost run_id=%s error_type=%s observed_at=%s",
self._run_id,
evidence.error_type,
evidence.observed_at,
)
if exc.lease_rejected:
evidence = self.monitor.mark_lost(
f"{type(exc).__name__}:{exc.status_code}"
)
logger.warning(
"heartbeat lost run_id=%s error_type=%s observed_at=%s",
self._run_id,
evidence.error_type,
evidence.observed_at,
)
else:
logger.warning(
"heartbeat transient failure run_id=%s error_type=%s",
self._run_id,
type(exc).__name__,
)
self._thread = threading.Thread(
target=_loop, name=f"ops-hb-{self._run_id[:8]}", daemon=True

43
rein_aharness/ops_run_client.py
View file

@ -33,7 +33,22 @@ DEFAULT_REPO_ROOTS = ("~", "~/work")
class OpsRunError(RuntimeError):
pass
"""Bounded Activity Core failure with optional HTTP classification."""
def __init__(
self,
message: str,
*,
action: str | None = None,
status_code: int | None = None,
) -> None:
super().__init__(message)
self.action = action
self.status_code = status_code
@property
def lease_rejected(self) -> bool:
return self.status_code in {401, 403, 404, 409, 410, 412, 423}
@dataclass
@ -159,8 +174,14 @@ class ActivityCoreOpsClient:
timeout=self.config.timeout,
)
resp.raise_for_status()
except httpx.HTTPStatusError as exc:
raise OpsRunError(
f"list ops-runs failed: HTTP {exc.response.status_code}",
action="list",
status_code=exc.response.status_code,
) from exc
except httpx.HTTPError as exc:
raise OpsRunError(f"list ops-runs failed: {exc}") from exc
raise OpsRunError("list ops-runs failed: transport error", action="list") from exc
data = resp.json()
items = data.get("items") if isinstance(data, dict) else data
if not isinstance(items, list):
@ -190,8 +211,14 @@ class ActivityCoreOpsClient:
timeout=self.config.timeout,
)
resp.raise_for_status()
except httpx.HTTPStatusError as exc:
raise OpsRunError(
f"claim failed: HTTP {exc.response.status_code}",
action="claim",
status_code=exc.response.status_code,
) from exc
except httpx.HTTPError as exc:
raise OpsRunError(f"claim failed: {exc}") from exc
raise OpsRunError("claim failed: transport error", action="claim") from exc
data = resp.json()
items = data.get("items") if isinstance(data, dict) else []
return [OpsRun.from_api(item) for item in items if isinstance(item, dict)]
@ -247,8 +274,16 @@ class ActivityCoreOpsClient:
timeout=self.config.timeout,
)
resp.raise_for_status()
except httpx.HTTPStatusError as exc:
raise OpsRunError(
f"{action} ops_run {run_id} failed: HTTP {exc.response.status_code}",
action=action,
status_code=exc.response.status_code,
) from exc
except httpx.HTTPError as exc:
raise OpsRunError(f"{action} ops_run {run_id} failed: {exc}") from exc
raise OpsRunError(
f"{action} ops_run {run_id} failed: transport error", action=action
) from exc
return OpsRun.from_api(resp.json())