feat: transport authoritative workload projections
Assistant: codex Assistant-Model: gpt-5.6-sol Assistant-Session: 01a0230c-b06c-7641-808a-e191b6d1da49
This commit is contained in:
parent
a090903cfc
commit
ab936a1e98
27 changed files with 1671 additions and 15 deletions
|
|
@ -19,6 +19,12 @@ from hub_core.runtime.repository_navigation_routes import (
|
|||
)
|
||||
from hub_core.runtime.store import InMemoryPortStore, PortStore
|
||||
from hub_core.runtime.validation import ContractValidator
|
||||
from hub_core.runtime.workload_projection import (
|
||||
WorkloadProjectionClient,
|
||||
WorkloadProjectionRejected,
|
||||
WorkloadProjectionService,
|
||||
)
|
||||
from hub_core.runtime.workload_projection_routes import create_workload_projection_router
|
||||
|
||||
|
||||
def create_app(
|
||||
|
|
@ -26,6 +32,7 @@ def create_app(
|
|||
settings: RuntimeSettings | None = None,
|
||||
port_store: PortStore | None = None,
|
||||
repo_projection_client: RepoProjectionClient | None = None,
|
||||
workload_projection_client: WorkloadProjectionClient | None = None,
|
||||
) -> FastAPI:
|
||||
resolved_settings = settings or RuntimeSettings.from_env()
|
||||
resolved_store = port_store or _create_store(resolved_settings)
|
||||
|
|
@ -34,6 +41,10 @@ def create_app(
|
|||
client=repo_projection_client,
|
||||
store=resolved_store,
|
||||
)
|
||||
workload_projection = WorkloadProjectionService(
|
||||
client=workload_projection_client,
|
||||
store=resolved_store,
|
||||
)
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(_: FastAPI):
|
||||
|
|
@ -44,6 +55,11 @@ def create_app(
|
|||
# The readiness dependency reports the rejected or absent
|
||||
# projection while the API remains available for diagnosis.
|
||||
pass
|
||||
if workload_projection_client is not None:
|
||||
try:
|
||||
await workload_projection.refresh()
|
||||
except WorkloadProjectionRejected:
|
||||
pass
|
||||
yield
|
||||
if owns_store and (closer := getattr(resolved_store, "aclose", None)):
|
||||
await closer()
|
||||
|
|
@ -63,6 +79,7 @@ def create_app(
|
|||
)
|
||||
app.state.contract_validator = ContractValidator()
|
||||
app.state.repository_navigation = repository_navigation
|
||||
app.state.workload_projection = workload_projection
|
||||
|
||||
@app.get("/healthz", response_model=HealthResponse, tags=["system"])
|
||||
async def healthz() -> HealthResponse:
|
||||
|
|
@ -76,6 +93,7 @@ def create_app(
|
|||
dependency_checks = {
|
||||
**await resolved_store.readiness_checks(),
|
||||
**await repository_navigation.readiness_checks(),
|
||||
**await workload_projection.readiness_checks(),
|
||||
}
|
||||
ready = resolved_settings.is_ready(resolved_store.backend_name) and all(
|
||||
value in {"ok", "not_applicable"} for value in dependency_checks.values()
|
||||
|
|
@ -90,6 +108,9 @@ def create_app(
|
|||
},
|
||||
)
|
||||
|
||||
# Exact projection routes precede the generic /projections/{projection_id}
|
||||
# route so Starlette dispatch cannot shadow them.
|
||||
app.include_router(create_workload_projection_router())
|
||||
app.include_router(create_ports_router())
|
||||
app.include_router(create_repository_navigation_router())
|
||||
app.include_router(create_compatibility_router())
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ from hub_core.runtime.models import (
|
|||
RegistryRegistration,
|
||||
)
|
||||
from hub_core.runtime.repository_navigation import NavigationProjection
|
||||
from hub_core.runtime.workload_projection import WorkloadProjection
|
||||
from hub_core.runtime.tables import (
|
||||
compat_api_keys,
|
||||
compat_hubs,
|
||||
|
|
@ -33,6 +34,8 @@ from hub_core.runtime.tables import (
|
|||
runtime_repository_navigation_repositories,
|
||||
runtime_repository_navigation_state,
|
||||
runtime_registrations,
|
||||
runtime_workload_projection_records,
|
||||
runtime_workload_projection_state,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -61,6 +64,7 @@ class PostgresPortStore:
|
|||
compat_hubs,
|
||||
compat_api_keys,
|
||||
runtime_repository_navigation_state,
|
||||
runtime_workload_projection_state,
|
||||
):
|
||||
await connection.execute(
|
||||
sa.select(sa.literal(1)).select_from(table).limit(1)
|
||||
|
|
@ -342,6 +346,103 @@ class PostgresPortStore:
|
|||
)
|
||||
)
|
||||
|
||||
async def get_workload_projection(self) -> WorkloadProjection | None:
|
||||
async with self.sessions() as session:
|
||||
state = (
|
||||
await session.execute(
|
||||
sa.select(runtime_workload_projection_state).where(
|
||||
runtime_workload_projection_state.c.projection_id == "workloads"
|
||||
)
|
||||
)
|
||||
).mappings().one_or_none()
|
||||
if state is None:
|
||||
return None
|
||||
rows = (
|
||||
await session.execute(
|
||||
sa.select(runtime_workload_projection_records).order_by(
|
||||
runtime_workload_projection_records.c.rapp_id,
|
||||
runtime_workload_projection_records.c.name,
|
||||
)
|
||||
)
|
||||
).mappings()
|
||||
workloads = tuple(
|
||||
{
|
||||
"rapp_id": row["rapp_id"],
|
||||
"name": row["name"],
|
||||
"declaration_repo": row["declaration_repo"],
|
||||
"declaration_path": row["declaration_path"],
|
||||
"source_git_revision": row["source_git_revision"],
|
||||
"observed_at": row["observed_at"],
|
||||
"ownership_repo": row["ownership_repo"],
|
||||
"readiness_state": row["readiness_state"],
|
||||
"data_classification": row["data_classification"],
|
||||
"criticality": row["criticality"],
|
||||
"deployables": list(row["deployables"] or []),
|
||||
}
|
||||
for row in rows
|
||||
)
|
||||
return WorkloadProjection(
|
||||
projection_status=state["projection_status"],
|
||||
source_snapshot=dict(state["source_snapshot"]),
|
||||
source_checked_at=state["source_checked_at"],
|
||||
rebuilt_at=state["rebuilt_at"],
|
||||
content_hash=state["content_hash"],
|
||||
workloads=workloads,
|
||||
diagnostics=tuple(state["diagnostics"] or []),
|
||||
)
|
||||
|
||||
async def replace_workload_projection(self, projection: WorkloadProjection) -> None:
|
||||
values = {
|
||||
"projection_status": projection.projection_status,
|
||||
"snapshot_id": projection.source_snapshot["snapshot_id"],
|
||||
"source_snapshot": projection.source_snapshot,
|
||||
"source_checked_at": projection.source_checked_at,
|
||||
"rebuilt_at": projection.rebuilt_at,
|
||||
"content_hash": projection.content_hash,
|
||||
"workload_count": len(projection.workloads),
|
||||
"diagnostics": list(projection.diagnostics),
|
||||
}
|
||||
async with self.sessions.begin() as session:
|
||||
await session.execute(runtime_workload_projection_records.delete())
|
||||
if projection.workloads:
|
||||
await session.execute(
|
||||
runtime_workload_projection_records.insert(),
|
||||
[dict(workload) for workload in projection.workloads],
|
||||
)
|
||||
exists = (
|
||||
await session.execute(
|
||||
sa.select(runtime_workload_projection_state.c.projection_id).where(
|
||||
runtime_workload_projection_state.c.projection_id == "workloads"
|
||||
)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if exists is None:
|
||||
await session.execute(
|
||||
runtime_workload_projection_state.insert().values(
|
||||
projection_id="workloads", **values
|
||||
)
|
||||
)
|
||||
else:
|
||||
await session.execute(
|
||||
runtime_workload_projection_state.update()
|
||||
.where(runtime_workload_projection_state.c.projection_id == "workloads")
|
||||
.values(**values)
|
||||
)
|
||||
|
||||
async def mark_workload_projection_stale(
|
||||
self, *, checked_at: datetime, diagnostic: Mapping[str, Any]
|
||||
) -> None:
|
||||
async with self.sessions.begin() as session:
|
||||
await session.execute(
|
||||
runtime_workload_projection_state.update()
|
||||
.where(runtime_workload_projection_state.c.projection_id == "workloads")
|
||||
.values(
|
||||
projection_status="stale",
|
||||
source_checked_at=checked_at,
|
||||
diagnostics=[dict(diagnostic)],
|
||||
)
|
||||
)
|
||||
|
||||
async def _append_event(
|
||||
self,
|
||||
command: EventCommand,
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ from hub_core.runtime.models import (
|
|||
RegistryRegistration,
|
||||
)
|
||||
from hub_core.runtime.repository_navigation import NavigationProjection
|
||||
from hub_core.runtime.workload_projection import WorkloadProjection
|
||||
|
||||
|
||||
class PortStore(Protocol):
|
||||
|
|
@ -55,6 +56,14 @@ class PortStore(Protocol):
|
|||
self, *, checked_at: datetime, diagnostic: Mapping[str, Any]
|
||||
) -> None: ...
|
||||
|
||||
async def get_workload_projection(self) -> WorkloadProjection | None: ...
|
||||
|
||||
async def replace_workload_projection(self, projection: WorkloadProjection) -> None: ...
|
||||
|
||||
async def mark_workload_projection_stale(
|
||||
self, *, checked_at: datetime, diagnostic: Mapping[str, Any]
|
||||
) -> None: ...
|
||||
|
||||
|
||||
class InMemoryPortStore:
|
||||
"""Deterministic ephemeral backend for local runtime and conformance tests.
|
||||
|
|
@ -72,6 +81,7 @@ class InMemoryPortStore:
|
|||
self._progress_events: list[dict[str, Any]] = []
|
||||
self._interaction_events: list[dict[str, Any]] = []
|
||||
self._repository_navigation: NavigationProjection | None = None
|
||||
self._workload_projection: WorkloadProjection | None = None
|
||||
|
||||
async def readiness_checks(self) -> dict[str, str]:
|
||||
return {"database": "not_applicable"}
|
||||
|
|
@ -174,6 +184,31 @@ class InMemoryPortStore:
|
|||
diagnostics=(deepcopy(dict(diagnostic)),),
|
||||
)
|
||||
|
||||
async def get_workload_projection(self) -> WorkloadProjection | None:
|
||||
async with self._lock:
|
||||
return deepcopy(self._workload_projection)
|
||||
|
||||
async def replace_workload_projection(self, projection: WorkloadProjection) -> None:
|
||||
async with self._lock:
|
||||
self._workload_projection = deepcopy(projection)
|
||||
|
||||
async def mark_workload_projection_stale(
|
||||
self, *, checked_at: datetime, diagnostic: Mapping[str, Any]
|
||||
) -> None:
|
||||
async with self._lock:
|
||||
current = self._workload_projection
|
||||
if current is None:
|
||||
return
|
||||
self._workload_projection = WorkloadProjection(
|
||||
projection_status="stale",
|
||||
source_snapshot=deepcopy(current.source_snapshot),
|
||||
source_checked_at=checked_at,
|
||||
rebuilt_at=current.rebuilt_at,
|
||||
content_hash=current.content_hash,
|
||||
workloads=deepcopy(current.workloads),
|
||||
diagnostics=(deepcopy(dict(diagnostic)),),
|
||||
)
|
||||
|
||||
async def _append_event(
|
||||
self,
|
||||
command: EventCommand,
|
||||
|
|
|
|||
|
|
@ -98,6 +98,36 @@ runtime_repository_navigation_facets = sa.Table(
|
|||
sa.Column("repository_ids", sa.JSON(), nullable=False),
|
||||
)
|
||||
|
||||
runtime_workload_projection_state = sa.Table(
|
||||
"runtime_workload_projection_state",
|
||||
runtime_metadata,
|
||||
sa.Column("projection_id", sa.String(80), primary_key=True),
|
||||
sa.Column("projection_status", sa.String(20), nullable=False),
|
||||
sa.Column("snapshot_id", sa.String(64), nullable=False, unique=True),
|
||||
sa.Column("source_snapshot", sa.JSON(), nullable=False),
|
||||
sa.Column("source_checked_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("rebuilt_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("content_hash", sa.String(64), nullable=False),
|
||||
sa.Column("workload_count", sa.Integer(), nullable=False),
|
||||
sa.Column("diagnostics", sa.JSON(), nullable=False),
|
||||
)
|
||||
|
||||
runtime_workload_projection_records = sa.Table(
|
||||
"runtime_workload_projection_records",
|
||||
runtime_metadata,
|
||||
sa.Column("rapp_id", sa.String(120), primary_key=True),
|
||||
sa.Column("name", sa.String(120), primary_key=True),
|
||||
sa.Column("declaration_repo", sa.String(120), nullable=False, index=True),
|
||||
sa.Column("declaration_path", sa.String(260), nullable=False),
|
||||
sa.Column("source_git_revision", sa.String(64), nullable=False),
|
||||
sa.Column("observed_at", sa.String(40), nullable=False),
|
||||
sa.Column("ownership_repo", sa.String(120), nullable=True),
|
||||
sa.Column("readiness_state", sa.String(80), nullable=True),
|
||||
sa.Column("data_classification", sa.String(80), nullable=True),
|
||||
sa.Column("criticality", sa.String(80), nullable=True),
|
||||
sa.Column("deployables", sa.JSON(), nullable=False),
|
||||
)
|
||||
|
||||
compat_hubs = sa.Table(
|
||||
"compat_hubs",
|
||||
runtime_metadata,
|
||||
|
|
|
|||
421
hub_core/runtime/workload_projection.py
Normal file
421
hub_core/runtime/workload_projection.py
Normal file
|
|
@ -0,0 +1,421 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from collections.abc import Mapping
|
||||
from copy import deepcopy
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Literal, Protocol
|
||||
|
||||
from jsonschema import Draft202012Validator, FormatChecker, ValidationError
|
||||
|
||||
from hub_core.contracts import (
|
||||
WORKLOAD_PROJECTION_CONTRACT_ID,
|
||||
WORKLOAD_PROJECTION_CONTRACT_VERSION,
|
||||
workload_projection_contract_root,
|
||||
)
|
||||
|
||||
|
||||
class WorkloadProjectionClient(Protocol):
|
||||
"""Injected `port.repo` workload reader with transport owned by the host."""
|
||||
|
||||
async def fetch_workload_page(self, cursor: str | None) -> Mapping[str, Any]: ...
|
||||
|
||||
|
||||
class WorkloadProjectionStore(Protocol):
|
||||
async def get_workload_projection(self) -> WorkloadProjection | None: ...
|
||||
|
||||
async def replace_workload_projection(self, projection: WorkloadProjection) -> None: ...
|
||||
|
||||
async def mark_workload_projection_stale(
|
||||
self, *, checked_at: datetime, diagnostic: Mapping[str, Any]
|
||||
) -> None: ...
|
||||
|
||||
|
||||
class WorkloadProjectionRejected(ValueError):
|
||||
"""The authoritative workload transfer cannot safely replace the projection."""
|
||||
|
||||
|
||||
class WorkloadCursorMismatch(ValueError):
|
||||
"""A cursor does not belong to the active workload generation and filters."""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class WorkloadProjection:
|
||||
projection_status: Literal["current", "stale"]
|
||||
source_snapshot: dict[str, Any]
|
||||
source_checked_at: datetime
|
||||
rebuilt_at: datetime
|
||||
content_hash: str
|
||||
workloads: tuple[dict[str, Any], ...]
|
||||
diagnostics: tuple[dict[str, Any], ...] = ()
|
||||
|
||||
def to_contract(self) -> dict[str, Any]:
|
||||
return {
|
||||
"contract_id": WORKLOAD_PROJECTION_CONTRACT_ID,
|
||||
"contract_version": WORKLOAD_PROJECTION_CONTRACT_VERSION,
|
||||
"projection_id": "workloads",
|
||||
"projection_status": self.projection_status,
|
||||
"source_snapshot": deepcopy(self.source_snapshot),
|
||||
"source_checked_at": _utc(self.source_checked_at),
|
||||
"rebuilt_at": _utc(self.rebuilt_at),
|
||||
"content_hash": self.content_hash,
|
||||
"workloads": deepcopy(list(self.workloads)),
|
||||
"diagnostics": deepcopy(list(self.diagnostics)),
|
||||
"next_cursor": None,
|
||||
"total_workload_count": len(self.workloads),
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class WorkloadRefreshResult:
|
||||
status: Literal["accepted", "duplicate"]
|
||||
snapshot_id: str
|
||||
content_hash: str
|
||||
workload_count: int
|
||||
|
||||
|
||||
class WorkloadProjectionService:
|
||||
"""Materialize and resolve normalized Repo Manager workload records."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
client: WorkloadProjectionClient | None,
|
||||
store: WorkloadProjectionStore,
|
||||
) -> None:
|
||||
self.client = client
|
||||
self.store = store
|
||||
self._validator = _input_validator()
|
||||
self._lock = asyncio.Lock()
|
||||
|
||||
async def readiness_checks(self) -> dict[str, str]:
|
||||
if self.client is None:
|
||||
return {"workload_projection": "not_applicable"}
|
||||
projection = await self.store.get_workload_projection()
|
||||
if projection is None:
|
||||
return {"workload_projection": "unavailable"}
|
||||
return {
|
||||
"workload_projection": (
|
||||
"ok" if projection.projection_status == "current" else "stale"
|
||||
)
|
||||
}
|
||||
|
||||
async def refresh(self) -> WorkloadRefreshResult:
|
||||
if self.client is None:
|
||||
raise RuntimeError("no port.repo workload projection client is configured")
|
||||
async with self._lock:
|
||||
checked_at = _now()
|
||||
try:
|
||||
pages = await self._fetch_transfer()
|
||||
return await self._apply_transfer(pages, checked_at=checked_at)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
diagnostic = {
|
||||
"severity": "error",
|
||||
"code": "workload_projection.rejected",
|
||||
"message": str(exc)[:1000] or type(exc).__name__,
|
||||
}
|
||||
await self.store.mark_workload_projection_stale(
|
||||
checked_at=checked_at, diagnostic=diagnostic
|
||||
)
|
||||
if isinstance(exc, WorkloadProjectionRejected):
|
||||
raise
|
||||
raise WorkloadProjectionRejected(diagnostic["message"]) from exc
|
||||
|
||||
async def query(
|
||||
self,
|
||||
*,
|
||||
rapp_id: str | None = None,
|
||||
name: str | None = None,
|
||||
deployable: str | None = None,
|
||||
cursor: str | None = None,
|
||||
limit: int = 100,
|
||||
) -> dict[str, Any] | None:
|
||||
if not 1 <= limit <= 500:
|
||||
raise ValueError("limit must be between 1 and 500")
|
||||
filters = _filters(rapp_id=rapp_id, name=name, deployable=deployable)
|
||||
projection = await self.store.get_workload_projection()
|
||||
if projection is None:
|
||||
return None
|
||||
filter_hash = _hash(filters)
|
||||
generation_id = _hash(
|
||||
[projection.source_snapshot["snapshot_id"], projection.content_hash]
|
||||
)
|
||||
offset = (
|
||||
_decode_cursor(cursor, generation_id, filter_hash)
|
||||
if cursor
|
||||
else 0
|
||||
)
|
||||
matches = tuple(
|
||||
record for record in projection.workloads if _matches(record, filters)
|
||||
)
|
||||
if offset > len(matches):
|
||||
raise WorkloadCursorMismatch("cursor offset exceeds the result set")
|
||||
page = matches[offset : offset + limit]
|
||||
next_offset = offset + len(page)
|
||||
result = projection.to_contract()
|
||||
result["workloads"] = deepcopy(list(page))
|
||||
result["total_workload_count"] = len(matches)
|
||||
result["next_cursor"] = (
|
||||
_encode_cursor(next_offset, generation_id, filter_hash)
|
||||
if next_offset < len(matches)
|
||||
else None
|
||||
)
|
||||
return result
|
||||
|
||||
async def resolve(
|
||||
self, *, rapp_id: str, name: str, deployable: str | None = None
|
||||
) -> dict[str, Any] | None:
|
||||
filters = _filters(rapp_id=rapp_id, name=name, deployable=deployable)
|
||||
projection = await self.store.get_workload_projection()
|
||||
if projection is None:
|
||||
return None
|
||||
reference = {"rapp_id": rapp_id, "name": name}
|
||||
if deployable is not None:
|
||||
reference["deployable"] = deployable
|
||||
exact = [
|
||||
record
|
||||
for record in projection.workloads
|
||||
if record["rapp_id"] == rapp_id and record["name"] == name
|
||||
]
|
||||
metadata = {
|
||||
"contract_id": WORKLOAD_PROJECTION_CONTRACT_ID,
|
||||
"contract_version": WORKLOAD_PROJECTION_CONTRACT_VERSION,
|
||||
"reference": reference,
|
||||
"source_snapshot": deepcopy(projection.source_snapshot),
|
||||
"content_hash": projection.content_hash,
|
||||
"projection_status": projection.projection_status,
|
||||
}
|
||||
if len(exact) != 1:
|
||||
return {**metadata, "status": "unknown", "reason": "not_found"}
|
||||
workload = exact[0]
|
||||
if deployable is not None and deployable not in workload["deployables"]:
|
||||
return {
|
||||
**metadata,
|
||||
"status": "unknown",
|
||||
"reason": "deployable_not_declared",
|
||||
}
|
||||
assert _matches(workload, filters)
|
||||
return {**metadata, "status": "resolved", "workload": deepcopy(workload)}
|
||||
|
||||
async def _fetch_transfer(self) -> list[dict[str, Any]]:
|
||||
assert self.client is not None
|
||||
pages: list[dict[str, Any]] = []
|
||||
cursor: str | None = None
|
||||
seen: set[str] = set()
|
||||
while True:
|
||||
raw = dict(await self.client.fetch_workload_page(cursor))
|
||||
try:
|
||||
self._validator.validate(raw)
|
||||
except ValidationError as exc:
|
||||
location = ".".join(str(part) for part in exc.absolute_path) or "root"
|
||||
raise WorkloadProjectionRejected(
|
||||
f"invalid workload page at {location}: {exc.message}"
|
||||
) from exc
|
||||
pages.append(raw)
|
||||
snapshot = raw["snapshot"]
|
||||
next_cursor = snapshot["next_cursor"]
|
||||
if snapshot["final_page"] != (next_cursor is None):
|
||||
raise WorkloadProjectionRejected(
|
||||
"final_page must be true exactly when next_cursor is null"
|
||||
)
|
||||
if snapshot["final_page"]:
|
||||
return pages
|
||||
if next_cursor in seen:
|
||||
raise WorkloadProjectionRejected("workload projection cursor cycle detected")
|
||||
seen.add(next_cursor)
|
||||
cursor = next_cursor
|
||||
if len(pages) >= 10000:
|
||||
raise WorkloadProjectionRejected("workload transfer exceeds 10000 pages")
|
||||
|
||||
async def _apply_transfer(
|
||||
self, pages: list[dict[str, Any]], *, checked_at: datetime
|
||||
) -> WorkloadRefreshResult:
|
||||
workloads = _validate_and_normalize(pages)
|
||||
first = pages[0]
|
||||
snapshot = first["snapshot"]
|
||||
current = await self.store.get_workload_projection()
|
||||
duplicate = bool(
|
||||
current and current.source_snapshot["snapshot_id"] == snapshot["snapshot_id"]
|
||||
)
|
||||
if (
|
||||
current
|
||||
and not duplicate
|
||||
and _parse_time(snapshot["generated_at"])
|
||||
<= _parse_time(current.source_snapshot["generated_at"])
|
||||
):
|
||||
raise WorkloadProjectionRejected(
|
||||
"workload snapshot is older than the active generation"
|
||||
)
|
||||
content_hash = _hash(list(workloads))
|
||||
if duplicate and current and content_hash != current.content_hash:
|
||||
raise WorkloadProjectionRejected(
|
||||
"snapshot_id was reused with different workload content"
|
||||
)
|
||||
projection = WorkloadProjection(
|
||||
projection_status="current",
|
||||
source_snapshot={
|
||||
"snapshot_id": snapshot["snapshot_id"],
|
||||
"source_system": "repo-manager",
|
||||
"source_contract": first["source"]["workload_contract"],
|
||||
"source_revision": snapshot["source_revision"],
|
||||
"generated_at": snapshot["generated_at"],
|
||||
},
|
||||
source_checked_at=checked_at,
|
||||
rebuilt_at=_now(),
|
||||
content_hash=content_hash,
|
||||
workloads=workloads,
|
||||
)
|
||||
await self.store.replace_workload_projection(projection)
|
||||
return WorkloadRefreshResult(
|
||||
status="duplicate" if duplicate else "accepted",
|
||||
snapshot_id=snapshot["snapshot_id"],
|
||||
content_hash=content_hash,
|
||||
workload_count=len(workloads),
|
||||
)
|
||||
|
||||
|
||||
def _input_validator() -> Draft202012Validator:
|
||||
resource = workload_projection_contract_root().joinpath(
|
||||
"schemas", "workload-projection-page.schema.json"
|
||||
)
|
||||
schema = json.loads(resource.read_text(encoding="utf-8"))
|
||||
Draft202012Validator.check_schema(schema)
|
||||
return Draft202012Validator(schema, format_checker=FormatChecker())
|
||||
|
||||
|
||||
def _validate_and_normalize(
|
||||
pages: list[dict[str, Any]],
|
||||
) -> tuple[dict[str, Any], ...]:
|
||||
if not pages:
|
||||
raise WorkloadProjectionRejected("workload transfer contains no pages")
|
||||
first = pages[0]
|
||||
keys = (
|
||||
"snapshot_id",
|
||||
"generated_at",
|
||||
"source_revision",
|
||||
"total_workload_count",
|
||||
)
|
||||
expected = {key: first["snapshot"][key] for key in keys}
|
||||
source = first["source"]
|
||||
previous_cursor: str | None = None
|
||||
records: list[dict[str, Any]] = []
|
||||
identities: set[tuple[str, str]] = set()
|
||||
deployables: set[str] = set()
|
||||
for index, page in enumerate(pages):
|
||||
snapshot = page["snapshot"]
|
||||
if any(snapshot[key] != value for key, value in expected.items()):
|
||||
raise WorkloadProjectionRejected("workload snapshot metadata changed between pages")
|
||||
if page["source"] != source:
|
||||
raise WorkloadProjectionRejected("workload source metadata changed between pages")
|
||||
if snapshot["page_cursor"] != previous_cursor:
|
||||
raise WorkloadProjectionRejected(f"page {index} does not match requested cursor")
|
||||
if any(item["severity"] == "error" for item in page["diagnostics"]):
|
||||
raise WorkloadProjectionRejected("workload transfer contains an error diagnostic")
|
||||
previous_cursor = snapshot["next_cursor"]
|
||||
for raw in page["workloads"]:
|
||||
record = deepcopy(raw)
|
||||
identity = (record["rapp_id"], record["name"])
|
||||
if identity in identities:
|
||||
raise WorkloadProjectionRejected(f"duplicate workload reference {identity!r}")
|
||||
if record["declaration_repo"] != record["rapp_id"]:
|
||||
raise WorkloadProjectionRejected(
|
||||
f"declaration repository does not match {record['rapp_id']}"
|
||||
)
|
||||
expected_path = f"{record['declaration_repo']}/declarations/rapp.yaml"
|
||||
if record["declaration_path"] != expected_path:
|
||||
raise WorkloadProjectionRejected(
|
||||
f"declaration path is not canonical for {record['rapp_id']}"
|
||||
)
|
||||
overlap = deployables.intersection(record["deployables"])
|
||||
if overlap:
|
||||
raise WorkloadProjectionRejected(
|
||||
f"deployable belongs to more than one workload: {sorted(overlap)[0]}"
|
||||
)
|
||||
identities.add(identity)
|
||||
deployables.update(record["deployables"])
|
||||
record["deployables"] = sorted(record["deployables"])
|
||||
records.append(record)
|
||||
normalized = tuple(sorted(records, key=lambda item: (item["rapp_id"], item["name"])))
|
||||
if len(normalized) != expected["total_workload_count"]:
|
||||
raise WorkloadProjectionRejected("total_workload_count does not match records")
|
||||
if list(normalized) != records:
|
||||
raise WorkloadProjectionRejected("workloads are not ordered by exact reference")
|
||||
return normalized
|
||||
|
||||
|
||||
def _filters(
|
||||
*, rapp_id: str | None, name: str | None, deployable: str | None
|
||||
) -> dict[str, str]:
|
||||
values = {"rapp_id": rapp_id, "name": name, "deployable": deployable}
|
||||
for key, value in values.items():
|
||||
if value is None:
|
||||
continue
|
||||
pattern = r"rapp-[a-z0-9]+(?:-[a-z0-9]+)*" if key == "rapp_id" else r"[a-z0-9]+(?:-[a-z0-9]+)*"
|
||||
if len(value) > 120 or re.fullmatch(pattern, value) is None:
|
||||
raise ValueError(f"invalid {key}")
|
||||
return {key: value for key, value in values.items() if value is not None}
|
||||
|
||||
|
||||
def _matches(record: Mapping[str, Any], filters: Mapping[str, str]) -> bool:
|
||||
return all(
|
||||
(value in record["deployables"] if key == "deployable" else record[key] == value)
|
||||
for key, value in filters.items()
|
||||
)
|
||||
|
||||
|
||||
def _encode_cursor(offset: int, generation_id: str, filter_hash: str) -> str:
|
||||
payload = json.dumps(
|
||||
{"generation_id": generation_id, "filter_hash": filter_hash, "offset": offset},
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
).encode()
|
||||
encoded = base64.urlsafe_b64encode(payload).decode().rstrip("=")
|
||||
checksum = hashlib.sha256(b"workload-projection/1.0.0:" + payload).hexdigest()
|
||||
return f"{encoded}.{checksum}"
|
||||
|
||||
|
||||
def _decode_cursor(cursor: str, generation_id: str, filter_hash: str) -> int:
|
||||
try:
|
||||
encoded, checksum = cursor.split(".", 1)
|
||||
payload = base64.urlsafe_b64decode(encoded + "=" * (-len(encoded) % 4))
|
||||
expected = hashlib.sha256(b"workload-projection/1.0.0:" + payload).hexdigest()
|
||||
value = json.loads(payload)
|
||||
if checksum != expected:
|
||||
raise ValueError("checksum")
|
||||
if value["generation_id"] != generation_id or value["filter_hash"] != filter_hash:
|
||||
raise ValueError("generation or filters")
|
||||
offset = int(value["offset"])
|
||||
if offset < 0:
|
||||
raise ValueError("offset")
|
||||
return offset
|
||||
except (KeyError, TypeError, ValueError, json.JSONDecodeError) as exc:
|
||||
raise WorkloadCursorMismatch(
|
||||
"cursor_snapshot_mismatch: restart from the first page"
|
||||
) from exc
|
||||
|
||||
|
||||
def _hash(value: Any) -> str:
|
||||
encoded = json.dumps(value, sort_keys=True, separators=(",", ":")).encode()
|
||||
return hashlib.sha256(encoded).hexdigest()
|
||||
|
||||
|
||||
def _parse_time(value: str) -> datetime:
|
||||
return datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
|
||||
|
||||
def _utc(value: datetime) -> str:
|
||||
if value.tzinfo is None:
|
||||
value = value.replace(tzinfo=timezone.utc)
|
||||
normalized = value.astimezone(timezone.utc)
|
||||
return normalized.isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
def _now() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
77
hub_core/runtime/workload_projection_routes.py
Normal file
77
hub_core/runtime/workload_projection_routes.py
Normal file
|
|
@ -0,0 +1,77 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, Request
|
||||
|
||||
from hub_core.runtime.workload_projection import (
|
||||
WorkloadCursorMismatch,
|
||||
WorkloadProjectionService,
|
||||
)
|
||||
|
||||
|
||||
def get_workload_projection_service(request: Request) -> WorkloadProjectionService:
|
||||
return request.app.state.workload_projection
|
||||
|
||||
|
||||
def create_workload_projection_router() -> APIRouter:
|
||||
router = APIRouter(prefix="/ports/projections/workloads")
|
||||
|
||||
@router.get(
|
||||
"",
|
||||
response_model=dict[str, Any],
|
||||
tags=["workload-projection"],
|
||||
openapi_extra={"x-port-id": "port.projection.query", "x-direction": "out"},
|
||||
)
|
||||
async def query_workloads(
|
||||
rapp_id: str | None = None,
|
||||
name: str | None = None,
|
||||
deployable: str | None = None,
|
||||
cursor: str | None = None,
|
||||
limit: int = Query(default=100, ge=1, le=500),
|
||||
service: WorkloadProjectionService = Depends(get_workload_projection_service),
|
||||
) -> dict[str, Any]:
|
||||
try:
|
||||
result = await service.query(
|
||||
rapp_id=rapp_id,
|
||||
name=name,
|
||||
deployable=deployable,
|
||||
cursor=cursor,
|
||||
limit=limit,
|
||||
)
|
||||
except WorkloadCursorMismatch as exc:
|
||||
raise HTTPException(status_code=409, detail=str(exc)) from exc
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
return _available(result)
|
||||
|
||||
@router.get(
|
||||
"/resolve",
|
||||
response_model=dict[str, Any],
|
||||
tags=["workload-projection"],
|
||||
openapi_extra={"x-port-id": "port.projection.query", "x-direction": "out"},
|
||||
)
|
||||
async def resolve_workload(
|
||||
rapp_id: str,
|
||||
name: str,
|
||||
deployable: str | None = None,
|
||||
service: WorkloadProjectionService = Depends(get_workload_projection_service),
|
||||
) -> dict[str, Any]:
|
||||
try:
|
||||
result = await service.resolve(
|
||||
rapp_id=rapp_id, name=name, deployable=deployable
|
||||
)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
return _available(result)
|
||||
|
||||
return router
|
||||
|
||||
|
||||
def _available(result: dict[str, Any] | None) -> dict[str, Any]:
|
||||
if result is None:
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail="no accepted workload projection is available",
|
||||
)
|
||||
return result
|
||||
Loading…
Add table
Add a link
Reference in a new issue