Prepare State Hub retirement baseline
This commit is contained in:
parent
2217bdd9f5
commit
5927591be8
46 changed files with 32583 additions and 62 deletions
|
|
@ -13,6 +13,10 @@ class Settings(BaseSettings):
|
|||
debug: bool = False
|
||||
state_hub_report_dir: str = "reports/recently-on-scope"
|
||||
state_hub_markitect_cli_path: str | None = None
|
||||
activity_core_url: str | None = None
|
||||
activity_core_worker_token: str | None = None
|
||||
ops_run_projection_ttl_seconds: float = 15.0
|
||||
ops_run_sla_hours: float = 1.0
|
||||
|
||||
|
||||
settings = Settings()
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ from starlette.responses import Response as StarletteResponse
|
|||
from api.database import engine
|
||||
from api.events import shutdown_publisher
|
||||
from api.services.write_idempotency import WriteIdempotencyMiddleware
|
||||
from api.routers import decisions, extension_points, intake, progress, state, suggestions, tasks, technical_debt, topics, workstreams, workstream_dependencies
|
||||
from api.routers import decisions, extension_points, intake, ops_runs, progress, state, suggestions, tasks, technical_debt, topics, workstreams, workstream_dependencies
|
||||
from api.routers import domains, repos, contributions, sbom, policy, domain_goals, repo_goals, messages, capability_requests, tpsc, services
|
||||
from api.routers import token_events
|
||||
from api.routers import interface_changes
|
||||
|
|
@ -135,6 +135,7 @@ app.include_router(execution.router)
|
|||
app.include_router(fabric.router)
|
||||
app.include_router(legacy_meter.router)
|
||||
app.include_router(state.router)
|
||||
app.include_router(ops_runs.router)
|
||||
app.include_router(policy.router)
|
||||
|
||||
|
||||
|
|
|
|||
13
api/routers/ops_runs.py
Normal file
13
api/routers/ops_runs.py
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
from fastapi import APIRouter
|
||||
|
||||
from api.schemas.ops_run import OpsRunProjection
|
||||
from api.services.ops_run_projection import get_ops_run_projection
|
||||
|
||||
|
||||
router = APIRouter(prefix="/ops-runs", tags=["ops-runs"])
|
||||
|
||||
|
||||
@router.get("/summary", response_model=OpsRunProjection)
|
||||
async def get_ops_runs_summary(refresh: bool = False) -> OpsRunProjection:
|
||||
"""Project activity-core queue health; State Hub never claims ops runs."""
|
||||
return await get_ops_run_projection(refresh=refresh)
|
||||
|
|
@ -53,6 +53,7 @@ from api.services.summary_cache import (
|
|||
get_summary_cache,
|
||||
register_summary_cache_invalidation,
|
||||
)
|
||||
from api.services.ops_run_projection import get_ops_run_projection
|
||||
|
||||
|
||||
def _dual_workplan_refs(
|
||||
|
|
@ -111,22 +112,22 @@ async def get_summary(
|
|||
|
||||
if cache_status == "hit-revision" and cached is not None:
|
||||
_summary_cache_headers(response, cache_status="hit-revision", revision=revision_token)
|
||||
return cached
|
||||
return cached.model_copy(update={"ops_runs": await get_ops_run_projection()})
|
||||
|
||||
if cache_status == "progress-section" and cached is not None:
|
||||
result = await apply_progress_section(session, cached, revision)
|
||||
_summary_cache_headers(response, cache_status="hit-revision", revision=revision_token)
|
||||
return result
|
||||
return result.model_copy(update={"ops_runs": await get_ops_run_projection()})
|
||||
|
||||
if cache_status == "stale" and cached is not None:
|
||||
cache.schedule_refresh(revision)
|
||||
_summary_cache_headers(response, cache_status="stale", revision=revision_token)
|
||||
return cached
|
||||
return cached.model_copy(update={"ops_runs": await get_ops_run_projection()})
|
||||
|
||||
result = await build_state_summary(session)
|
||||
cache.store(result, revision)
|
||||
_summary_cache_headers(response, cache_status="miss", revision=revision_token)
|
||||
return result
|
||||
return result.model_copy(update={"ops_runs": await get_ops_run_projection(refresh=force_refresh)})
|
||||
|
||||
|
||||
async def build_state_summary(session: AsyncSession) -> StateSummary:
|
||||
|
|
|
|||
27
api/schemas/ops_run.py
Normal file
27
api/schemas/ops_run.py
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class OpsRunProjectionItem(BaseModel):
|
||||
id: str
|
||||
definition: str | None = None
|
||||
target_repo: str | None = None
|
||||
state: str
|
||||
lease: dict[str, Any] | None = None
|
||||
updated_at: datetime | None = None
|
||||
last_error: str | None = None
|
||||
|
||||
|
||||
class OpsRunProjection(BaseModel):
|
||||
available: bool = False
|
||||
stale: bool = False
|
||||
source: str = "activity-core"
|
||||
generated_at: datetime
|
||||
open: int = 0
|
||||
claimed: int = 0
|
||||
failed_24h: int = 0
|
||||
stuck_open_or_claimed: int = 0
|
||||
items: list[OpsRunProjectionItem] = Field(default_factory=list)
|
||||
error: str | None = None
|
||||
|
|
@ -11,6 +11,7 @@ from api.schemas.task import TaskRead
|
|||
from api.schemas.topic import TopicWithWorkstreams
|
||||
from api.schemas.suggestion import RankedSuggestionDigest
|
||||
from api.schemas.workstream import WorkstreamWithDeps
|
||||
from api.schemas.ops_run import OpsRunProjection
|
||||
|
||||
|
||||
class TopicTotals(BaseModel):
|
||||
|
|
@ -90,6 +91,7 @@ class StateSummary(BaseModel):
|
|||
licence_risk_count: int = 0
|
||||
open_capability_requests: int = 0
|
||||
ranked_suggestions: list[RankedSuggestionDigest] = []
|
||||
ops_runs: OpsRunProjection | None = None
|
||||
|
||||
|
||||
class DashboardWorkplanRow(BaseModel):
|
||||
|
|
|
|||
130
api/services/ops_run_projection.py
Normal file
130
api/services/ops_run_projection.py
Normal file
|
|
@ -0,0 +1,130 @@
|
|||
"""Read-only activity-core ops_run projection (STATE-WP-0078)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
from api.config import settings
|
||||
from api.schemas.ops_run import OpsRunProjection, OpsRunProjectionItem
|
||||
|
||||
|
||||
_cache: OpsRunProjection | None = None
|
||||
_cache_at: float = 0.0
|
||||
_lock = asyncio.Lock()
|
||||
|
||||
|
||||
def reset_ops_run_projection_cache() -> None:
|
||||
global _cache, _cache_at
|
||||
_cache = None
|
||||
_cache_at = 0.0
|
||||
|
||||
|
||||
def _parse_datetime(value: Any) -> datetime | None:
|
||||
if not value or not isinstance(value, str):
|
||||
return None
|
||||
try:
|
||||
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
return None
|
||||
return parsed if parsed.tzinfo else parsed.replace(tzinfo=timezone.utc)
|
||||
|
||||
|
||||
async def _fetch_ops_runs() -> dict[str, Any]:
|
||||
if not settings.activity_core_url:
|
||||
raise RuntimeError("ACTIVITY_CORE_URL is not configured")
|
||||
headers = {}
|
||||
if settings.activity_core_worker_token:
|
||||
headers["X-Worker-Token"] = settings.activity_core_worker_token
|
||||
async with httpx.AsyncClient(
|
||||
base_url=settings.activity_core_url.rstrip("/"),
|
||||
timeout=10.0,
|
||||
follow_redirects=True,
|
||||
trust_env=False,
|
||||
headers=headers,
|
||||
) as client:
|
||||
response = await client.get("/ops-runs", params={"limit": 200})
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
|
||||
|
||||
def _project(payload: dict[str, Any], now: datetime) -> OpsRunProjection:
|
||||
raw_items = payload.get("items") or []
|
||||
counts = payload.get("counts") or {}
|
||||
failed_cutoff = now - timedelta(hours=24)
|
||||
stuck_cutoff = now - timedelta(hours=max(0.1, settings.ops_run_sla_hours))
|
||||
failed_24h = 0
|
||||
stuck = 0
|
||||
items: list[OpsRunProjectionItem] = []
|
||||
|
||||
for raw in raw_items:
|
||||
state = str(raw.get("state") or "unknown")
|
||||
updated_at = _parse_datetime(raw.get("updated_at"))
|
||||
created_at = _parse_datetime(raw.get("created_at"))
|
||||
if state == "failed" and updated_at and updated_at >= failed_cutoff:
|
||||
failed_24h += 1
|
||||
if state in {"open", "claimed"} and (created_at or updated_at):
|
||||
if (created_at or updated_at) < stuck_cutoff:
|
||||
stuck += 1
|
||||
|
||||
result = raw.get("result") if isinstance(raw.get("result"), dict) else {}
|
||||
last_error = result.get("error") or raw.get("last_error")
|
||||
if state in {"open", "claimed", "failed"}:
|
||||
items.append(OpsRunProjectionItem(
|
||||
id=str(raw.get("id")),
|
||||
definition=str(raw.get("activity_definition_id")) if raw.get("activity_definition_id") else None,
|
||||
target_repo=raw.get("target_repo"),
|
||||
state=state,
|
||||
lease={
|
||||
"owner": raw.get("claim_owner"),
|
||||
"until": raw.get("lease_until"),
|
||||
"attempt": raw.get("attempt", 0),
|
||||
} if raw.get("claim_owner") or raw.get("lease_until") else None,
|
||||
updated_at=updated_at,
|
||||
last_error=str(last_error) if last_error else None,
|
||||
))
|
||||
|
||||
items.sort(key=lambda item: item.updated_at or datetime.min.replace(tzinfo=timezone.utc), reverse=True)
|
||||
return OpsRunProjection(
|
||||
available=True,
|
||||
generated_at=now,
|
||||
open=int(counts.get("open", sum(item.state == "open" for item in items))),
|
||||
claimed=int(counts.get("claimed", sum(item.state == "claimed" for item in items))),
|
||||
failed_24h=failed_24h,
|
||||
stuck_open_or_claimed=stuck,
|
||||
items=items[:50],
|
||||
)
|
||||
|
||||
|
||||
async def get_ops_run_projection(*, refresh: bool = False) -> OpsRunProjection:
|
||||
global _cache, _cache_at
|
||||
now_mono = time.monotonic()
|
||||
ttl = max(1.0, settings.ops_run_projection_ttl_seconds)
|
||||
if not refresh and _cache is not None and now_mono - _cache_at < ttl:
|
||||
return _cache
|
||||
|
||||
async with _lock:
|
||||
now_mono = time.monotonic()
|
||||
if not refresh and _cache is not None and now_mono - _cache_at < ttl:
|
||||
return _cache
|
||||
now = datetime.now(timezone.utc)
|
||||
try:
|
||||
projection = _project(await _fetch_ops_runs(), now)
|
||||
except Exception as exc:
|
||||
if _cache is not None and _cache.available:
|
||||
projection = _cache.model_copy(update={
|
||||
"stale": True,
|
||||
"error": f"activity-core refresh failed: {exc}",
|
||||
})
|
||||
else:
|
||||
projection = OpsRunProjection(
|
||||
generated_at=now,
|
||||
error=f"activity-core unavailable: {exc}",
|
||||
)
|
||||
_cache = projection
|
||||
_cache_at = now_mono
|
||||
return projection
|
||||
Loading…
Add table
Add a link
Reference in a new issue