tegwick
a6afb08d82
Receives the State Hub rows of the 2026-10-21 node allocation review (CUST-WP-0071 T03/T04, RESOURCE-WP-0008). The release preflight now derives the API surge from the values the release deploys instead of the stale 100m STATE-WP-0091 constant, and reports the steady remainder after rollout. Ordering: apply after rail-knative reductions, before forge/telemetry increases. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Assistant: claude-code Assistant-Model: opus Assistant-Process: 8340@bnt-lap001 Assistant-Session: 2e192b3d-b3e8-4403-94d4-d55a2c8b476c
338 lines
14 KiB
Python
338 lines
14 KiB
Python
#!/usr/bin/env python3
|
|
"""Read-only scheduling-headroom preflight for State Hub Helm promotion.
|
|
|
|
STATE-WP-0091-T01. Does not start Helm, does not lower requests, and does not
|
|
claim a scheduling guarantee from an aggregate remaining-millicores sum.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
|
|
CPU_FACTORS = {"n": 0.000001, "u": 0.001, "m": 1.0, "": 1000.0}
|
|
BYTE_FACTORS = {
|
|
"": 1,
|
|
"Ki": 1024,
|
|
"Mi": 1024**2,
|
|
"Gi": 1024**3,
|
|
"Ti": 1024**4,
|
|
"K": 1000,
|
|
"M": 1000**2,
|
|
"G": 1000**3,
|
|
"T": 1000**4,
|
|
}
|
|
|
|
# Fallback surge figures, used only when no values file can be read. With
|
|
# maxSurge=1 / maxUnavailable=0 the surge pod carries the *target* release's
|
|
# API request, so the real figure is derived from the Helm values the release
|
|
# will deploy (STATE-WP-0099). The historical 100m constant under-modelled the
|
|
# 250m API request; tests keep these fallbacks equal to the shipped values.
|
|
DEFAULT_API_SURGE_CPU_M = 390
|
|
DEFAULT_MCP_SURGE_CPU_M = 10
|
|
DEFAULT_MIGRATE_CPU_M = 50
|
|
DEFAULT_API_SURGE_MEMORY = 512 * 1024**2
|
|
DEFAULT_FRESHNESS_SECONDS = 15 * 60
|
|
|
|
REPO_ROOT = Path(__file__).resolve().parent.parent
|
|
DEFAULT_VALUES_FILES = (
|
|
REPO_ROOT / "deploy/railiance/apps/charts/state-hub/values.yaml",
|
|
REPO_ROOT / "deploy/railiance/apps/helm/state-hub-values.yaml",
|
|
)
|
|
|
|
|
|
def cpu_millicores(value: str | None) -> int:
|
|
import re
|
|
|
|
match = re.fullmatch(r"([0-9]+(?:\.[0-9]+)?)(n|u|m)?", str(value or "0"))
|
|
if not match:
|
|
raise ValueError(f"unsupported CPU quantity: {value}")
|
|
return round(float(match.group(1)) * CPU_FACTORS[match.group(2) or ""])
|
|
|
|
|
|
def bytes_value(value: str | None) -> int:
|
|
import re
|
|
|
|
match = re.fullmatch(r"([0-9]+(?:\.[0-9]+)?)(Ki|Mi|Gi|Ti|K|M|G|T)?", str(value or "0"))
|
|
if not match:
|
|
raise ValueError(f"unsupported byte quantity: {value}")
|
|
return round(float(match.group(1)) * BYTE_FACTORS[match.group(2) or ""])
|
|
|
|
|
|
def pod_effective_requests(pod: dict[str, Any]) -> tuple[int, int]:
|
|
"""Pod request is max(sum(containers), max(init containers))."""
|
|
spec = pod.get("spec") or {}
|
|
containers = spec.get("containers") or []
|
|
inits = spec.get("initContainers") or []
|
|
cpu = sum(cpu_millicores((c.get("resources") or {}).get("requests", {}).get("cpu")) for c in containers)
|
|
mem = sum(bytes_value((c.get("resources") or {}).get("requests", {}).get("memory")) for c in containers)
|
|
init_cpu = max((cpu_millicores((c.get("resources") or {}).get("requests", {}).get("cpu")) for c in inits), default=0)
|
|
init_mem = max((bytes_value((c.get("resources") or {}).get("requests", {}).get("memory")) for c in inits), default=0)
|
|
return max(cpu, init_cpu), max(mem, init_mem)
|
|
|
|
|
|
def _deep_merge(base: dict[str, Any], override: dict[str, Any]) -> dict[str, Any]:
|
|
merged = dict(base)
|
|
for key, value in override.items():
|
|
if isinstance(value, dict) and isinstance(merged.get(key), dict):
|
|
merged[key] = _deep_merge(merged[key], value)
|
|
else:
|
|
merged[key] = value
|
|
return merged
|
|
|
|
|
|
def _request(section: dict[str, Any] | None, key: str) -> Any:
|
|
return (((section or {}).get("resources") or {}).get("requests") or {}).get(key)
|
|
|
|
|
|
def release_from_values(values: dict[str, Any]) -> dict[str, Any]:
|
|
"""Derive surge figures from merged Helm values (chart defaults + overrides).
|
|
|
|
Helm merges `-f` files in order; pass them already merged. Only keys that are
|
|
present are returned, so missing sections fall back to the defaults.
|
|
"""
|
|
release: dict[str, Any] = {"surge_source": "values"}
|
|
api_cpu = _request(values, "cpu")
|
|
if api_cpu is not None:
|
|
release["api_surge_cpu_m"] = cpu_millicores(api_cpu)
|
|
api_mem = _request(values, "memory")
|
|
if api_mem is not None:
|
|
release["api_surge_memory_bytes"] = bytes_value(api_mem)
|
|
mcp = values.get("mcp") or {}
|
|
if mcp.get("enabled") is False:
|
|
release["mcp_surge_cpu_m"] = 0
|
|
elif _request(mcp, "cpu") is not None:
|
|
release["mcp_surge_cpu_m"] = cpu_millicores(_request(mcp, "cpu"))
|
|
migrations = values.get("migrations") or {}
|
|
if migrations.get("enabled") is False:
|
|
release["migrate_cpu_m"] = 0
|
|
elif _request(migrations, "cpu") is not None:
|
|
release["migrate_cpu_m"] = cpu_millicores(_request(migrations, "cpu"))
|
|
return release
|
|
|
|
|
|
def load_values(paths: list[Path] | tuple[Path, ...]) -> dict[str, Any]:
|
|
import yaml
|
|
|
|
merged: dict[str, Any] = {}
|
|
for path in paths:
|
|
merged = _deep_merge(merged, yaml.safe_load(Path(path).read_text()) or {})
|
|
return merged
|
|
|
|
|
|
def evaluate(observation: dict[str, Any], release: dict[str, Any] | None = None) -> dict[str, Any]:
|
|
"""Return a preflight report. `ok` is whether Helm may start, not a guarantee."""
|
|
release = release or {}
|
|
api_cpu = int(release.get("api_surge_cpu_m", DEFAULT_API_SURGE_CPU_M))
|
|
mcp_cpu = int(release.get("mcp_surge_cpu_m", DEFAULT_MCP_SURGE_CPU_M))
|
|
migrate_cpu = int(release.get("migrate_cpu_m", DEFAULT_MIGRATE_CPU_M))
|
|
api_mem = int(release.get("api_surge_memory_bytes", DEFAULT_API_SURGE_MEMORY))
|
|
freshness_limit = int(release.get("freshness_seconds", DEFAULT_FRESHNESS_SECONDS))
|
|
atomic = bool(release.get("atomic", True))
|
|
surge_source = str(release.get("surge_source", "release" if "api_surge_cpu_m" in release else "defaults"))
|
|
|
|
reasons: list[str] = []
|
|
notes: list[str] = []
|
|
nodes = observation.get("nodes") or []
|
|
if not nodes:
|
|
reasons.append("no node capacity evidence")
|
|
remaining_cpu = 0
|
|
remaining_mem = 0
|
|
for node in nodes:
|
|
if node.get("unschedulable") or not node.get("ready", True):
|
|
reasons.append(f"node {node.get('name')} is not schedulable")
|
|
continue
|
|
remaining_cpu += int(node["allocatable_cpu_m"]) - int(node["allocated_cpu_m"])
|
|
remaining_mem += int(node["allocatable_memory_bytes"]) - int(node["allocated_memory_bytes"])
|
|
|
|
pending = observation.get("pending_unrelated") or []
|
|
pending_cpu = sum(int(p.get("cpu_m", 0)) for p in pending)
|
|
pending_mem = sum(int(p.get("memory_bytes", 0)) for p in pending)
|
|
|
|
observed_at = observation.get("observed_at")
|
|
freshness_seconds = observation.get("freshness_seconds")
|
|
if freshness_seconds is None and observed_at:
|
|
try:
|
|
stamp = datetime.fromisoformat(str(observed_at).replace("Z", "+00:00"))
|
|
freshness_seconds = int((datetime.now(timezone.utc) - stamp).total_seconds())
|
|
except ValueError:
|
|
freshness_seconds = None
|
|
reasons.append("observed_at is not parseable")
|
|
if freshness_seconds is not None and freshness_seconds > freshness_limit:
|
|
reasons.append(
|
|
f"capacity evidence is {freshness_seconds}s old (limit {freshness_limit}s)"
|
|
)
|
|
|
|
if remaining_cpu < migrate_cpu:
|
|
reasons.append(
|
|
f"migration hook needs {migrate_cpu}m CPU; remaining {remaining_cpu}m"
|
|
)
|
|
if remaining_cpu < api_cpu:
|
|
reasons.append(
|
|
f"API maxSurge=1 needs {api_cpu}m CPU; remaining {remaining_cpu}m "
|
|
f"(Insufficient CPU class; revision 61 had 65m remaining)"
|
|
)
|
|
if remaining_mem < api_mem:
|
|
reasons.append(
|
|
f"API maxSurge=1 needs {api_mem} bytes memory; remaining {remaining_mem}"
|
|
)
|
|
concurrent = api_cpu + mcp_cpu
|
|
if remaining_cpu < concurrent:
|
|
notes.append(
|
|
f"API+MCP concurrent surge would need {concurrent}m; remaining {remaining_cpu}m. "
|
|
"Helm may roll both deployments together; this is not a scheduling guarantee."
|
|
)
|
|
effective_cpu = remaining_cpu - pending_cpu
|
|
effective_mem = remaining_mem - pending_mem
|
|
if pending and effective_cpu < api_cpu:
|
|
reasons.append(
|
|
f"unrelated pending demand {pending_cpu}m leaves {effective_cpu}m "
|
|
f"against API surge {api_cpu}m"
|
|
)
|
|
if pending and effective_mem < api_mem:
|
|
reasons.append(
|
|
f"unrelated pending memory {pending_mem} leaves {effective_mem} against API surge"
|
|
)
|
|
|
|
live_api_cpu = observation.get("live_api_request_cpu_m")
|
|
remaining_after_rollout = None
|
|
if live_api_cpu is not None:
|
|
remaining_after_rollout = remaining_cpu - (api_cpu - int(live_api_cpu))
|
|
if int(live_api_cpu) != api_cpu:
|
|
notes.append(
|
|
f"release changes the API request {int(live_api_cpu)}m -> {api_cpu}m; "
|
|
f"steady remaining after rollout is {remaining_after_rollout}m"
|
|
)
|
|
if remaining_after_rollout < api_cpu:
|
|
notes.append(
|
|
f"after this rollout {remaining_after_rollout}m remains, below the {api_cpu}m "
|
|
"surge; the next release will be refused until headroom is freed"
|
|
)
|
|
|
|
if not atomic:
|
|
reasons.append("promotion path must keep helm --atomic; preflight will not start a non-atomic upgrade")
|
|
|
|
ok = not reasons
|
|
if ok and remaining_cpu - api_cpu < 50:
|
|
notes.append(
|
|
f"remaining {remaining_cpu}m is only {remaining_cpu - api_cpu}m above the "
|
|
f"{api_cpu}m API surge (revision 63 was 105m). Not factory capacity admission."
|
|
)
|
|
notes.append("Aggregate remaining millicores is not a kube-scheduler guarantee.")
|
|
notes.append("Preflight does not lower requests and does not start Helm.")
|
|
|
|
return {
|
|
"schema": "state-hub.release-headroom-preflight.v1",
|
|
"ok": ok,
|
|
"observed_at": observed_at,
|
|
"freshness_seconds": freshness_seconds,
|
|
"remaining_cpu_m": remaining_cpu,
|
|
"remaining_memory_bytes": remaining_mem,
|
|
"pending_unrelated_cpu_m": pending_cpu,
|
|
"api_surge_cpu_m": api_cpu,
|
|
"surge_source": surge_source,
|
|
"live_api_request_cpu_m": live_api_cpu,
|
|
"remaining_after_rollout_cpu_m": remaining_after_rollout,
|
|
"mcp_surge_cpu_m": mcp_cpu,
|
|
"migrate_cpu_m": migrate_cpu,
|
|
"atomic": atomic,
|
|
"reasons": reasons,
|
|
"notes": notes,
|
|
"hook_order": [
|
|
"pre-upgrade migrate job (helm.sh/hook-weight -5)",
|
|
"API RollingUpdate maxSurge=1 maxUnavailable=0",
|
|
"MCP RollingUpdate maxSurge=1 maxUnavailable=0",
|
|
],
|
|
}
|
|
|
|
|
|
def collect_live(kubectl: str = "kubectl") -> dict[str, Any]:
|
|
nodes = json.loads(subprocess.check_output([kubectl, "get", "nodes", "-o", "json"], timeout=60))
|
|
pods = json.loads(subprocess.check_output([kubectl, "get", "pods", "-A", "-o", "json"], timeout=60))
|
|
allocated: dict[str, dict[str, int]] = {}
|
|
pending_unrelated: list[dict[str, Any]] = []
|
|
now = datetime.now(timezone.utc)
|
|
for node in nodes.get("items") or []:
|
|
name = node["metadata"]["name"]
|
|
alloc = node.get("status", {}).get("allocatable") or {}
|
|
allocated[name] = {
|
|
"allocatable_cpu_m": cpu_millicores(alloc.get("cpu")),
|
|
"allocatable_memory_bytes": bytes_value(alloc.get("memory")),
|
|
"allocated_cpu_m": 0,
|
|
"allocated_memory_bytes": 0,
|
|
"ready": any(
|
|
c.get("type") == "Ready" and c.get("status") == "True"
|
|
for c in (node.get("status") or {}).get("conditions") or []
|
|
),
|
|
"unschedulable": bool((node.get("spec") or {}).get("unschedulable")),
|
|
"name": name,
|
|
}
|
|
for pod in pods.get("items") or []:
|
|
phase = (pod.get("status") or {}).get("phase")
|
|
ns = pod.get("metadata", {}).get("namespace")
|
|
name = pod.get("metadata", {}).get("name")
|
|
cpu, mem = pod_effective_requests(pod)
|
|
node_name = (pod.get("spec") or {}).get("nodeName")
|
|
if phase == "Pending" and not node_name:
|
|
if ns != "state-hub":
|
|
pending_unrelated.append({"name": f"{ns}/{name}", "cpu_m": cpu, "memory_bytes": mem})
|
|
continue
|
|
if phase in {"Succeeded", "Failed"}:
|
|
continue
|
|
if node_name in allocated:
|
|
allocated[node_name]["allocated_cpu_m"] += cpu
|
|
allocated[node_name]["allocated_memory_bytes"] += mem
|
|
live_api_cpu = None
|
|
try:
|
|
deploy = json.loads(subprocess.check_output(
|
|
[kubectl, "-n", "state-hub", "get", "deployment", "state-hub", "-o", "json"], timeout=60
|
|
))
|
|
for container in deploy["spec"]["template"]["spec"]["containers"]:
|
|
if container.get("name") == "state-hub":
|
|
live_api_cpu = cpu_millicores(_request(container, "cpu"))
|
|
except (subprocess.SubprocessError, KeyError, ValueError, OSError):
|
|
live_api_cpu = None
|
|
return {
|
|
"observed_at": now.isoformat(),
|
|
"freshness_seconds": 0,
|
|
"live_api_request_cpu_m": live_api_cpu,
|
|
"nodes": list(allocated.values()),
|
|
"pending_unrelated": pending_unrelated,
|
|
"source": "kubectl",
|
|
}
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--fixture", type=Path, help="JSON observation instead of live kubectl")
|
|
parser.add_argument("--release", type=Path, help="JSON release surge description (overrides values)")
|
|
parser.add_argument(
|
|
"--values", type=Path, action="append",
|
|
help="Helm values file, in -f order (repeatable). Default: chart values + production values",
|
|
)
|
|
parser.add_argument("--kubectl", default="kubectl")
|
|
args = parser.parse_args(argv)
|
|
if args.fixture:
|
|
observation = json.loads(args.fixture.read_text())
|
|
else:
|
|
observation = collect_live(args.kubectl)
|
|
values_files = args.values or [p for p in DEFAULT_VALUES_FILES if p.exists()]
|
|
release = release_from_values(load_values(values_files)) if values_files else {}
|
|
if args.release:
|
|
explicit = json.loads(args.release.read_text())
|
|
release.update(explicit)
|
|
if "api_surge_cpu_m" in explicit:
|
|
release["surge_source"] = "release"
|
|
report = evaluate(observation, release)
|
|
json.dump(report, sys.stdout, indent=2)
|
|
sys.stdout.write("\n")
|
|
return 0 if report["ok"] else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|