130 lines
4.6 KiB
Python
130 lines
4.6 KiB
Python
"""Read-only activity-core ops_run projection (STATE-WP-0078)."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import time
|
|
from datetime import datetime, timedelta, timezone
|
|
from typing import Any
|
|
|
|
import httpx
|
|
|
|
from api.config import settings
|
|
from api.schemas.ops_run import OpsRunProjection, OpsRunProjectionItem
|
|
|
|
|
|
_cache: OpsRunProjection | None = None
|
|
_cache_at: float = 0.0
|
|
_lock = asyncio.Lock()
|
|
|
|
|
|
def reset_ops_run_projection_cache() -> None:
|
|
global _cache, _cache_at
|
|
_cache = None
|
|
_cache_at = 0.0
|
|
|
|
|
|
def _parse_datetime(value: Any) -> datetime | None:
|
|
if not value or not isinstance(value, str):
|
|
return None
|
|
try:
|
|
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
|
except ValueError:
|
|
return None
|
|
return parsed if parsed.tzinfo else parsed.replace(tzinfo=timezone.utc)
|
|
|
|
|
|
async def _fetch_ops_runs() -> dict[str, Any]:
|
|
if not settings.activity_core_url:
|
|
raise RuntimeError("ACTIVITY_CORE_URL is not configured")
|
|
headers = {}
|
|
if settings.activity_core_worker_token:
|
|
headers["X-Worker-Token"] = settings.activity_core_worker_token
|
|
async with httpx.AsyncClient(
|
|
base_url=settings.activity_core_url.rstrip("/"),
|
|
timeout=10.0,
|
|
follow_redirects=True,
|
|
trust_env=False,
|
|
headers=headers,
|
|
) as client:
|
|
response = await client.get("/ops-runs", params={"limit": 200})
|
|
response.raise_for_status()
|
|
return response.json()
|
|
|
|
|
|
def _project(payload: dict[str, Any], now: datetime) -> OpsRunProjection:
|
|
raw_items = payload.get("items") or []
|
|
counts = payload.get("counts") or {}
|
|
failed_cutoff = now - timedelta(hours=24)
|
|
stuck_cutoff = now - timedelta(hours=max(0.1, settings.ops_run_sla_hours))
|
|
failed_24h = 0
|
|
stuck = 0
|
|
items: list[OpsRunProjectionItem] = []
|
|
|
|
for raw in raw_items:
|
|
state = str(raw.get("state") or "unknown")
|
|
updated_at = _parse_datetime(raw.get("updated_at"))
|
|
created_at = _parse_datetime(raw.get("created_at"))
|
|
if state == "failed" and updated_at and updated_at >= failed_cutoff:
|
|
failed_24h += 1
|
|
if state in {"open", "claimed"} and (created_at or updated_at):
|
|
if (created_at or updated_at) < stuck_cutoff:
|
|
stuck += 1
|
|
|
|
result = raw.get("result") if isinstance(raw.get("result"), dict) else {}
|
|
last_error = result.get("error") or raw.get("last_error")
|
|
if state in {"open", "claimed", "failed"}:
|
|
items.append(OpsRunProjectionItem(
|
|
id=str(raw.get("id")),
|
|
definition=str(raw.get("activity_definition_id")) if raw.get("activity_definition_id") else None,
|
|
target_repo=raw.get("target_repo"),
|
|
state=state,
|
|
lease={
|
|
"owner": raw.get("claim_owner"),
|
|
"until": raw.get("lease_until"),
|
|
"attempt": raw.get("attempt", 0),
|
|
} if raw.get("claim_owner") or raw.get("lease_until") else None,
|
|
updated_at=updated_at,
|
|
last_error=str(last_error) if last_error else None,
|
|
))
|
|
|
|
items.sort(key=lambda item: item.updated_at or datetime.min.replace(tzinfo=timezone.utc), reverse=True)
|
|
return OpsRunProjection(
|
|
available=True,
|
|
generated_at=now,
|
|
open=int(counts.get("open", sum(item.state == "open" for item in items))),
|
|
claimed=int(counts.get("claimed", sum(item.state == "claimed" for item in items))),
|
|
failed_24h=failed_24h,
|
|
stuck_open_or_claimed=stuck,
|
|
items=items[:50],
|
|
)
|
|
|
|
|
|
async def get_ops_run_projection(*, refresh: bool = False) -> OpsRunProjection:
|
|
global _cache, _cache_at
|
|
now_mono = time.monotonic()
|
|
ttl = max(1.0, settings.ops_run_projection_ttl_seconds)
|
|
if not refresh and _cache is not None and now_mono - _cache_at < ttl:
|
|
return _cache
|
|
|
|
async with _lock:
|
|
now_mono = time.monotonic()
|
|
if not refresh and _cache is not None and now_mono - _cache_at < ttl:
|
|
return _cache
|
|
now = datetime.now(timezone.utc)
|
|
try:
|
|
projection = _project(await _fetch_ops_runs(), now)
|
|
except Exception as exc:
|
|
if _cache is not None and _cache.available:
|
|
projection = _cache.model_copy(update={
|
|
"stale": True,
|
|
"error": f"activity-core refresh failed: {exc}",
|
|
})
|
|
else:
|
|
projection = OpsRunProjection(
|
|
generated_at=now,
|
|
error=f"activity-core unavailable: {exc}",
|
|
)
|
|
_cache = projection
|
|
_cache_at = now_mono
|
|
return projection
|