diff --git a/Makefile b/Makefile index ed66b17..52c2458 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: start install install-cli dashboard-install dashboard-check db db-tools migrate seed api dashboard dashboard-local sync check-primary primary-port-clear check check-local test test-python clean register-project register-codex-project register-mcp configure-codex bootstrap-env dev-hub edge-relay mcp-profile validate-adr add-domain rename-domain add-repo list-repos register-path register-from-classification register-from-classification-all cleanup-stale tunnels-up tunnels-status tunnels-check bridges install-hooks install-hooks-all gitea-inventory token-reconcile railiance-state-hub-render railiance-state-hub-client-dry-run railiance-state-hub-server-dry-run +.PHONY: start install install-cli dashboard-install dashboard-check db db-tools migrate seed api dashboard dashboard-local sync check-primary primary-port-clear check check-local test test-python clean register-project register-codex-project register-mcp configure-codex bootstrap-env dev-hub edge-relay mcp-profile validate-adr add-domain rename-domain add-repo list-repos register-path register-from-classification register-from-classification-all cleanup-stale tunnels-up tunnels-status tunnels-check bridges install-hooks install-hooks-all gitea-inventory token-reconcile railiance-state-hub-render railiance-state-hub-client-dry-run railiance-state-hub-server-dry-run railiance-state-hub-headroom COMPOSE = docker compose -f infra/docker-compose.yml --env-file .env PYTHON ?= python3 @@ -134,6 +134,17 @@ sync-classification-allowed: check-classification-allowed: python3 scripts/sync_classification_allowed.py --check +# STATE-WP-0091: read-only surge-headroom check. Does not start Helm. +# Live: make railiance-state-hub-headroom +# Fixtures: make railiance-state-hub-headroom HEADROOM_FIXTURE=tests/fixtures/release_headroom/rev63-105m.json +HEADROOM_FIXTURE ?= +railiance-state-hub-headroom: + @if [ -n "$(HEADROOM_FIXTURE)" ]; then \ + $(PYTHON) scripts/release_headroom_preflight.py --fixture "$(HEADROOM_FIXTURE)"; \ + else \ + $(PYTHON) scripts/release_headroom_preflight.py --kubectl "$(KUBECTL)"; \ + fi + railiance-state-hub-render: check-classification-allowed $(HELM) template $(RAILIANCE_STATE_HUB_RELEASE) $(RAILIANCE_STATE_HUB_CHART) \ --namespace $(RAILIANCE_STATE_HUB_NAMESPACE) \ diff --git a/deploy/railiance/apps/charts/state-hub/PROMOTE.md b/deploy/railiance/apps/charts/state-hub/PROMOTE.md index 0d8d213..7a59369 100644 --- a/deploy/railiance/apps/charts/state-hub/PROMOTE.md +++ b/deploy/railiance/apps/charts/state-hub/PROMOTE.md @@ -11,6 +11,18 @@ The chart's `image.repository` defaults to `forgejo.coulomb.social/coulomb/state ## Promote a build to the cluster +0. **Headroom preflight (STATE-WP-0091).** Read-only. Does not lower requests + and does not start Helm. Refuse if remaining CPU cannot cover the API + `maxSurge=1` pod (100m), the migrate hook (50m), or unrelated pending demand. + Aggregate remaining millicores is **not** a scheduling guarantee. Keep + `--atomic` so a failed pull/roll rolls back. + ``` + make railiance-state-hub-headroom + # fixture replay of revision 61/63: + make railiance-state-hub-headroom HEADROOM_FIXTURE=tests/fixtures/release_headroom/rev61-65m.json + ``` + Do not proceed if the report `ok` is false. + 1. Pick a CI-produced tag to promote — **use an immutable `main-`**, not `latest` (avoid surprise upgrades). List candidates: ``` diff --git a/docs/evidence/2026-09-14-release-headroom.md b/docs/evidence/2026-09-14-release-headroom.md new file mode 100644 index 0000000..21b604a --- /dev/null +++ b/docs/evidence/2026-09-14-release-headroom.md @@ -0,0 +1,47 @@ +# STATE-WP-0091 — release headroom and demand handoff + +Date: 2026-09-14 + +## T01 — preflight + +`scripts/release_headroom_preflight.py` is on the promotion path +(`make railiance-state-hub-headroom`, `PROMOTE.md` step 0). It is read-only. + +Proved against fixtures of the 2026-09-10 incident: + +| Case | Remaining | Result | +| --- | --- | --- | +| Revision 61 | 65m | refuse API 100m surge | +| Revision 63 | 105m | narrowly sufficient; not factory admission | +| 105m plus 50m unrelated pending | 55m effective | refuse | + +Atomic rollback stays `--atomic`. Requests are not lowered by this tool. + +Live kubectl on 2026-09-14 refused: remaining 15m, pending unrelated 25m, +migrate hook 50m unmet. The preflight is doing its job on the current node. + +## T02 — MCP demand and margin + +STATE-WP-0090 recorded 1,078 five-minute samples in a seven-day window +(partial coverage): MCP p99 2.35m, peak 30.45m. The MCP request was cut +50m → 10m; limit 500m, memory and replicas unchanged. + +That sample set is still not a week of complete coverage. It is enough to +say: + +- 10m request is above p99 and below peak; bursts still use the 500m limit. +- 105m node remainder is only 5m above the API surge. Concurrent MCP surge + (10m) or any unrelated pending pod ≥ 6m makes it insufficient. +- **Durable release margin for State Hub:** remaining CPU after current + allocations must cover API surge 100m + migrate hook 50m sequenced + separately, and must refuse when unrelated pending demand would consume + the 5m sliver. Do not treat 105m as shared-capacity admission. + +Shared allocation and provisioning stay with other owners. Receiving records: + +- `RCLUSTER-WP-0015` (railiance-cluster) — node remaining vs surge, pending pods +- `RESOURCE-WP-0007` (resource-control) — reef-railiance-k3s portfolio, not admission +- `RPF-WP-0041` (railiance-platform) — no platform provisioning change from 105m +- `CUST-WP-0071` already owns weekly fleet allocation review + +HFACT T01/T04 factory demand is not admitted by this 105m figure. diff --git a/scripts/release_headroom_preflight.py b/scripts/release_headroom_preflight.py new file mode 100644 index 0000000..d369cc5 --- /dev/null +++ b/scripts/release_headroom_preflight.py @@ -0,0 +1,239 @@ +#!/usr/bin/env python3 +"""Read-only scheduling-headroom preflight for State Hub Helm promotion. + +STATE-WP-0091-T01. Does not start Helm, does not lower requests, and does not +claim a scheduling guarantee from an aggregate remaining-millicores sum. +""" +from __future__ import annotations + +import argparse +import json +import subprocess +import sys +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + + +CPU_FACTORS = {"n": 0.000001, "u": 0.001, "m": 1.0, "": 1000.0} +BYTE_FACTORS = { + "": 1, + "Ki": 1024, + "Mi": 1024**2, + "Gi": 1024**3, + "Ti": 1024**4, + "K": 1000, + "M": 1000**2, + "G": 1000**3, + "T": 1000**4, +} + +# Historical State Hub API surge (maxSurge=1, maxUnavailable=0). +DEFAULT_API_SURGE_CPU_M = 100 +DEFAULT_MCP_SURGE_CPU_M = 10 +DEFAULT_MIGRATE_CPU_M = 50 +DEFAULT_API_SURGE_MEMORY = 512 * 1024**2 +DEFAULT_FRESHNESS_SECONDS = 15 * 60 + + +def cpu_millicores(value: str | None) -> int: + import re + + match = re.fullmatch(r"([0-9]+(?:\.[0-9]+)?)(n|u|m)?", str(value or "0")) + if not match: + raise ValueError(f"unsupported CPU quantity: {value}") + return round(float(match.group(1)) * CPU_FACTORS[match.group(2) or ""]) + + +def bytes_value(value: str | None) -> int: + import re + + match = re.fullmatch(r"([0-9]+(?:\.[0-9]+)?)(Ki|Mi|Gi|Ti|K|M|G|T)?", str(value or "0")) + if not match: + raise ValueError(f"unsupported byte quantity: {value}") + return round(float(match.group(1)) * BYTE_FACTORS[match.group(2) or ""]) + + +def pod_effective_requests(pod: dict[str, Any]) -> tuple[int, int]: + """Pod request is max(sum(containers), max(init containers)).""" + spec = pod.get("spec") or {} + containers = spec.get("containers") or [] + inits = spec.get("initContainers") or [] + cpu = sum(cpu_millicores((c.get("resources") or {}).get("requests", {}).get("cpu")) for c in containers) + mem = sum(bytes_value((c.get("resources") or {}).get("requests", {}).get("memory")) for c in containers) + init_cpu = max((cpu_millicores((c.get("resources") or {}).get("requests", {}).get("cpu")) for c in inits), default=0) + init_mem = max((bytes_value((c.get("resources") or {}).get("requests", {}).get("memory")) for c in inits), default=0) + return max(cpu, init_cpu), max(mem, init_mem) + + +def evaluate(observation: dict[str, Any], release: dict[str, Any] | None = None) -> dict[str, Any]: + """Return a preflight report. `ok` is whether Helm may start, not a guarantee.""" + release = release or {} + api_cpu = int(release.get("api_surge_cpu_m", DEFAULT_API_SURGE_CPU_M)) + mcp_cpu = int(release.get("mcp_surge_cpu_m", DEFAULT_MCP_SURGE_CPU_M)) + migrate_cpu = int(release.get("migrate_cpu_m", DEFAULT_MIGRATE_CPU_M)) + api_mem = int(release.get("api_surge_memory_bytes", DEFAULT_API_SURGE_MEMORY)) + freshness_limit = int(release.get("freshness_seconds", DEFAULT_FRESHNESS_SECONDS)) + atomic = bool(release.get("atomic", True)) + + reasons: list[str] = [] + notes: list[str] = [] + nodes = observation.get("nodes") or [] + if not nodes: + reasons.append("no node capacity evidence") + remaining_cpu = 0 + remaining_mem = 0 + for node in nodes: + if node.get("unschedulable") or not node.get("ready", True): + reasons.append(f"node {node.get('name')} is not schedulable") + continue + remaining_cpu += int(node["allocatable_cpu_m"]) - int(node["allocated_cpu_m"]) + remaining_mem += int(node["allocatable_memory_bytes"]) - int(node["allocated_memory_bytes"]) + + pending = observation.get("pending_unrelated") or [] + pending_cpu = sum(int(p.get("cpu_m", 0)) for p in pending) + pending_mem = sum(int(p.get("memory_bytes", 0)) for p in pending) + + observed_at = observation.get("observed_at") + freshness_seconds = observation.get("freshness_seconds") + if freshness_seconds is None and observed_at: + try: + stamp = datetime.fromisoformat(str(observed_at).replace("Z", "+00:00")) + freshness_seconds = int((datetime.now(timezone.utc) - stamp).total_seconds()) + except ValueError: + freshness_seconds = None + reasons.append("observed_at is not parseable") + if freshness_seconds is not None and freshness_seconds > freshness_limit: + reasons.append( + f"capacity evidence is {freshness_seconds}s old (limit {freshness_limit}s)" + ) + + if remaining_cpu < migrate_cpu: + reasons.append( + f"migration hook needs {migrate_cpu}m CPU; remaining {remaining_cpu}m" + ) + if remaining_cpu < api_cpu: + reasons.append( + f"API maxSurge=1 needs {api_cpu}m CPU; remaining {remaining_cpu}m " + f"(Insufficient CPU class; revision 61 had 65m remaining)" + ) + if remaining_mem < api_mem: + reasons.append( + f"API maxSurge=1 needs {api_mem} bytes memory; remaining {remaining_mem}" + ) + concurrent = api_cpu + mcp_cpu + if remaining_cpu < concurrent: + notes.append( + f"API+MCP concurrent surge would need {concurrent}m; remaining {remaining_cpu}m. " + "Helm may roll both deployments together; this is not a scheduling guarantee." + ) + effective_cpu = remaining_cpu - pending_cpu + effective_mem = remaining_mem - pending_mem + if pending and effective_cpu < api_cpu: + reasons.append( + f"unrelated pending demand {pending_cpu}m leaves {effective_cpu}m " + f"against API surge {api_cpu}m" + ) + if pending and effective_mem < api_mem: + reasons.append( + f"unrelated pending memory {pending_mem} leaves {effective_mem} against API surge" + ) + + if not atomic: + reasons.append("promotion path must keep helm --atomic; preflight will not start a non-atomic upgrade") + + ok = not reasons + if ok and remaining_cpu - api_cpu < 50: + notes.append( + f"remaining {remaining_cpu}m is only {remaining_cpu - api_cpu}m above the " + f"{api_cpu}m API surge (revision 63 was 105m). Not factory capacity admission." + ) + notes.append("Aggregate remaining millicores is not a kube-scheduler guarantee.") + notes.append("Preflight does not lower requests and does not start Helm.") + + return { + "schema": "state-hub.release-headroom-preflight.v1", + "ok": ok, + "observed_at": observed_at, + "freshness_seconds": freshness_seconds, + "remaining_cpu_m": remaining_cpu, + "remaining_memory_bytes": remaining_mem, + "pending_unrelated_cpu_m": pending_cpu, + "api_surge_cpu_m": api_cpu, + "mcp_surge_cpu_m": mcp_cpu, + "migrate_cpu_m": migrate_cpu, + "atomic": atomic, + "reasons": reasons, + "notes": notes, + "hook_order": [ + "pre-upgrade migrate job (helm.sh/hook-weight -5)", + "API RollingUpdate maxSurge=1 maxUnavailable=0", + "MCP RollingUpdate maxSurge=1 maxUnavailable=0", + ], + } + + +def collect_live(kubectl: str = "kubectl") -> dict[str, Any]: + nodes = json.loads(subprocess.check_output([kubectl, "get", "nodes", "-o", "json"], timeout=60)) + pods = json.loads(subprocess.check_output([kubectl, "get", "pods", "-A", "-o", "json"], timeout=60)) + allocated: dict[str, dict[str, int]] = {} + pending_unrelated: list[dict[str, Any]] = [] + now = datetime.now(timezone.utc) + for node in nodes.get("items") or []: + name = node["metadata"]["name"] + alloc = node.get("status", {}).get("allocatable") or {} + allocated[name] = { + "allocatable_cpu_m": cpu_millicores(alloc.get("cpu")), + "allocatable_memory_bytes": bytes_value(alloc.get("memory")), + "allocated_cpu_m": 0, + "allocated_memory_bytes": 0, + "ready": any( + c.get("type") == "Ready" and c.get("status") == "True" + for c in (node.get("status") or {}).get("conditions") or [] + ), + "unschedulable": bool((node.get("spec") or {}).get("unschedulable")), + "name": name, + } + for pod in pods.get("items") or []: + phase = (pod.get("status") or {}).get("phase") + ns = pod.get("metadata", {}).get("namespace") + name = pod.get("metadata", {}).get("name") + cpu, mem = pod_effective_requests(pod) + node_name = (pod.get("spec") or {}).get("nodeName") + if phase == "Pending" and not node_name: + if ns != "state-hub": + pending_unrelated.append({"name": f"{ns}/{name}", "cpu_m": cpu, "memory_bytes": mem}) + continue + if phase in {"Succeeded", "Failed"}: + continue + if node_name in allocated: + allocated[node_name]["allocated_cpu_m"] += cpu + allocated[node_name]["allocated_memory_bytes"] += mem + return { + "observed_at": now.isoformat(), + "freshness_seconds": 0, + "nodes": list(allocated.values()), + "pending_unrelated": pending_unrelated, + "source": "kubectl", + } + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--fixture", type=Path, help="JSON observation instead of live kubectl") + parser.add_argument("--release", type=Path, help="JSON release surge description") + parser.add_argument("--kubectl", default="kubectl") + args = parser.parse_args(argv) + if args.fixture: + observation = json.loads(args.fixture.read_text()) + else: + observation = collect_live(args.kubectl) + release = json.loads(args.release.read_text()) if args.release else {} + report = evaluate(observation, release) + json.dump(report, sys.stdout, indent=2) + sys.stdout.write("\n") + return 0 if report["ok"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/fixtures/release_headroom/rev61-65m.json b/tests/fixtures/release_headroom/rev61-65m.json new file mode 100644 index 0000000..6d0f237 --- /dev/null +++ b/tests/fixtures/release_headroom/rev61-65m.json @@ -0,0 +1,16 @@ +{ + "observed_at": "2026-09-10T12:00:00+00:00", + "freshness_seconds": 0, + "nodes": [ + { + "name": "railiance01", + "ready": true, + "unschedulable": false, + "allocatable_cpu_m": 4000, + "allocatable_memory_bytes": 16000000000, + "allocated_cpu_m": 3935, + "allocated_memory_bytes": 8000000000 + } + ], + "pending_unrelated": [] +} diff --git a/tests/fixtures/release_headroom/rev63-105m-concurrent-pending.json b/tests/fixtures/release_headroom/rev63-105m-concurrent-pending.json new file mode 100644 index 0000000..39860f0 --- /dev/null +++ b/tests/fixtures/release_headroom/rev63-105m-concurrent-pending.json @@ -0,0 +1,22 @@ +{ + "observed_at": "2026-09-10T13:05:00+00:00", + "freshness_seconds": 0, + "nodes": [ + { + "name": "railiance01", + "ready": true, + "unschedulable": false, + "allocatable_cpu_m": 4000, + "allocatable_memory_bytes": 16000000000, + "allocated_cpu_m": 3895, + "allocated_memory_bytes": 8000000000 + } + ], + "pending_unrelated": [ + { + "name": "other/unrelated-worker", + "cpu_m": 50, + "memory_bytes": 0 + } + ] +} diff --git a/tests/fixtures/release_headroom/rev63-105m.json b/tests/fixtures/release_headroom/rev63-105m.json new file mode 100644 index 0000000..685e083 --- /dev/null +++ b/tests/fixtures/release_headroom/rev63-105m.json @@ -0,0 +1,16 @@ +{ + "observed_at": "2026-09-10T13:00:00+00:00", + "freshness_seconds": 0, + "nodes": [ + { + "name": "railiance01", + "ready": true, + "unschedulable": false, + "allocatable_cpu_m": 4000, + "allocatable_memory_bytes": 16000000000, + "allocated_cpu_m": 3895, + "allocated_memory_bytes": 8000000000 + } + ], + "pending_unrelated": [] +} diff --git a/tests/test_release_headroom_preflight.py b/tests/test_release_headroom_preflight.py new file mode 100644 index 0000000..db09bf9 --- /dev/null +++ b/tests/test_release_headroom_preflight.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +import json +from pathlib import Path + +from scripts.release_headroom_preflight import evaluate, main, pod_effective_requests + +FIXTURES = Path(__file__).parent / "fixtures" / "release_headroom" + + +def _load(name: str) -> dict: + return json.loads((FIXTURES / name).read_text()) + + +def test_revision_61_65m_refuses_api_surge() -> None: + report = evaluate(_load("rev61-65m.json")) + assert report["ok"] is False + assert report["remaining_cpu_m"] == 65 + assert any("65m remaining" in r or "needs 100m" in r for r in report["reasons"]) + assert "not a kube-scheduler guarantee" in " ".join(report["notes"]).lower() or any( + "guarantee" in n.lower() for n in report["notes"] + ) + + +def test_revision_63_105m_is_narrowly_sufficient() -> None: + report = evaluate(_load("rev63-105m.json")) + assert report["ok"] is True + assert report["remaining_cpu_m"] == 105 + assert report["reasons"] == [] + assert any("105m" in n or "Not factory" in n for n in report["notes"]) + + +def test_unrelated_pending_demand_refuses_105m() -> None: + report = evaluate(_load("rev63-105m-concurrent-pending.json")) + assert report["ok"] is False + assert any("unrelated pending" in r for r in report["reasons"]) + + +def test_stale_observation_refuses() -> None: + observation = _load("rev63-105m.json") + observation["freshness_seconds"] = 3600 + report = evaluate(observation) + assert report["ok"] is False + assert any("old" in r for r in report["reasons"]) + + +def test_non_atomic_refuses() -> None: + report = evaluate(_load("rev63-105m.json"), {"atomic": False}) + assert report["ok"] is False + assert any("atomic" in r for r in report["reasons"]) + + +def test_init_overhead_uses_max_not_sum() -> None: + pod = { + "spec": { + "initContainers": [{"resources": {"requests": {"cpu": "80m", "memory": "64Mi"}}}], + "containers": [{"resources": {"requests": {"cpu": "100m", "memory": "512Mi"}}}], + } + } + cpu, _mem = pod_effective_requests(pod) + assert cpu == 100 + + +def test_cli_fixture_exit_codes() -> None: + assert main(["--fixture", str(FIXTURES / "rev63-105m.json")]) == 0 + assert main(["--fixture", str(FIXTURES / "rev61-65m.json")]) == 1 diff --git a/workplans/STATE-WP-0091-release-headroom.md b/workplans/STATE-WP-0091-release-headroom.md index 9fb026b..eaa5aa0 100644 --- a/workplans/STATE-WP-0091-release-headroom.md +++ b/workplans/STATE-WP-0091-release-headroom.md @@ -4,11 +4,11 @@ type: workplan title: "Check release headroom and retain a capacity allocation handoff" domain: infotech repo: state-hub -status: ready +status: finished owner: codex topic_slug: infotech created: "2026-09-10" -updated: "2026-09-10" +updated: "2026-09-14" related: [STATE-WP-0090, RCLUSTER-WP-0014, RESOURCE-WP-0003, HFACT-WP-0001] state_hub_workstream_id: "4c64d6a6-554d-5b16-a533-9be407fa6515" --- @@ -24,7 +24,7 @@ admission follows from this release. ```task id: STATE-WP-0091-T01 -status: todo +status: done priority: high state_hub_task_id: "f84c0f1b-3354-5466-a239-ef9ef85daa07" ``` @@ -41,7 +41,7 @@ case, including unrelated concurrent demand. Keep atomic rollback. ```task id: STATE-WP-0091-T02 -status: todo +status: done priority: high state_hub_task_id: "a433e2d3-6fd4-5313-9273-e3769c2cefeb" ``` @@ -58,3 +58,15 @@ allocation, procurement or scaling is authorized by this workplan. Close only after the repeatable preflight and explicit receiving work records are verified. Evidence: docs/evidence/2026-09-10-projection-convergence.json. + +**T01 done 2026-09-14.** `scripts/release_headroom_preflight.py` plus +`make railiance-state-hub-headroom`. Fixtures prove 65m refuse, 105m narrow +pass, and 105m+50m pending refuse. Helm is not started; requests are not +lowered; `--atomic` remains required. + +**T02 done 2026-09-14.** Existing 1,078 samples remain partial coverage. +Durable margin: remaining must cover 100m API surge and must refuse when +unrelated pending demand consumes the 5m sliver. Not factory admission. +Receiving records: `RCLUSTER-WP-0015`, `RESOURCE-WP-0007`, `RPF-WP-0041`. +`CUST-WP-0071` keeps weekly fleet review. Details: +`docs/evidence/2026-09-14-release-headroom.md`.