From b36ac28b498ed030717ca2c7ecdb6db954e1447e Mon Sep 17 00:00:00 2001 From: codex Date: Mon, 14 Sep 2026 10:34:27 +0200 Subject: [PATCH] Publish pending unscheduled pods in cluster observations. RAIL-BS-WP-0014: residual millicores count scheduled pods only. state_hub_preflight_observation matches the STATE-WP-0091 fixture shape. Assistant: grok Assistant-Session: 01a09dc1-b21e-77e1-919e-fcad2f82b267 --- docs/cluster-resource-evidence.md | 6 +- tests/test_cluster_resources.py | 23 +++++ tools/observe_cluster_resources.py | 92 +++++++++++++++++-- ...AIL-BS-WP-0014-state-hub-surge-headroom.md | 12 ++- 4 files changed, 121 insertions(+), 12 deletions(-) diff --git a/docs/cluster-resource-evidence.md b/docs/cluster-resource-evidence.md index 69c965d..800ccef 100644 --- a/docs/cluster-resource-evidence.md +++ b/docs/cluster-resource-evidence.md @@ -6,7 +6,11 @@ requests, limits, optional observed usage, PVC demand, storage-class behavior, and measurement gaps. It never reads Secret objects or pod environments. The allocation section publishes raw numerators, denominators, residual -capacity, and `cluster-raw-drivers-v1`. It does not select a financial +capacity, and `cluster-raw-drivers-v1`. Residual millicores exclude +pending unscheduled pods; those pods are listed under +`allocation.pending_unscheduled` and projected as +`state_hub_preflight_observation` for STATE-WP-0091. Residual capacity is +**not** a kube-scheduler guarantee. It does not select a financial allocation formula or write booked cost; those decisions remain with resource-control. diff --git a/tests/test_cluster_resources.py b/tests/test_cluster_resources.py index a6a495e..aab9ea6 100644 --- a/tests/test_cluster_resources.py +++ b/tests/test_cluster_resources.py @@ -33,6 +33,29 @@ class ClusterResourceTests(unittest.TestCase): self.assertIn("pod metrics unavailable", result["measurement_gaps"]) self.assertFalse(result["failure_domain"]["threephoenix_target_is_live"]) + def test_pending_unscheduled_pods_are_named_and_excluded_from_residual(self) -> None: + snapshots = { + "nodes": {"items": [{"metadata": {"name": "node-1", "labels": {"kubernetes.io/hostname": "node-1"}}, "spec": {}, "status": {"allocatable": {"cpu": "2", "memory": "4Gi", "ephemeral-storage": "20Gi", "pods": "110"}, "conditions": [{"type": "Ready", "status": "True"}]}}]}, + "pods": {"items": [ + {"metadata": {"name": "api-abc123", "namespace": "apps", "labels": {"app.kubernetes.io/name": "api"}}, "spec": {"nodeName": "node-1", "containers": [{"resources": {"requests": {"cpu": "250m", "memory": "128Mi"}}}]}, "status": {"phase": "Running"}}, + {"metadata": {"name": "pending-xyz", "namespace": "other"}, "spec": {"containers": [{"resources": {"requests": {"cpu": "50m", "memory": "64Mi"}}}]}, "status": {"phase": "Pending"}}, + ]}, + "pvcs": {"items": []}, + "pvs": {"items": []}, + "storageclasses": {"items": []}, + "node_top": None, + "pod_top": None, + } + result = module.build_observation(snapshots, "2026-09-14T00:00:00Z") + pending = result["allocation"]["pending_unscheduled"] + self.assertEqual(1, len(pending["pods"])) + self.assertEqual(50, pending["cpu_millicores"]) + self.assertEqual("other/pending-xyz", result["state_hub_preflight_observation"]["pending_unrelated"][0]["name"]) + self.assertEqual(250, result["allocation"]["denominators"]["cpu_requested_millicores"]) + self.assertEqual(1750, result["allocation"]["residual_capacity"]["cpu_millicores"]) + self.assertTrue(result["allocation"]["not_a_scheduling_guarantee"]) + self.assertEqual(250, result["state_hub_preflight_observation"]["nodes"][0]["allocated_cpu_m"]) + if __name__ == "__main__": unittest.main() diff --git a/tools/observe_cluster_resources.py b/tools/observe_cluster_resources.py index e0b08a0..08e5a33 100755 --- a/tools/observe_cluster_resources.py +++ b/tools/observe_cluster_resources.py @@ -53,6 +53,22 @@ def workload_name(pod: dict[str, Any]) -> str: return re.sub(r"-[a-f0-9]{6,}(?:-[a-z0-9]{5})?$", "", str(metadata.get("name", "unknown"))) +def pod_effective_requests(pod: dict[str, Any]) -> tuple[int, int]: + """Pod request is max(sum(containers), max(init containers)).""" + spec = pod.get("spec") or {} + containers = spec.get("containers") or [] + inits = spec.get("initContainers") or [] + + def req(container: dict[str, Any], key: str) -> str | None: + return ((container.get("resources") or {}).get("requests") or {}).get(key) + + cpu = sum(cpu_millicores(req(c, "cpu")) for c in containers) + mem = sum(bytes_value(req(c, "memory")) for c in containers) + init_cpu = max((cpu_millicores(req(c, "cpu")) for c in inits), default=0) + init_mem = max((bytes_value(req(c, "memory")) for c in inits), default=0) + return max(cpu, init_cpu), max(mem, init_mem) + + def parse_top(lines: str | None, *, pods: bool) -> dict[tuple[str, str] | str, dict[str, int]]: result: dict[tuple[str, str] | str, dict[str, int]] = {} for line in (lines or "").splitlines(): @@ -114,9 +130,35 @@ def build_observation(snapshots: dict[str, Any], captured_at: str) -> dict[str, alloc_pods += row["allocatable"]["pods"] workloads: dict[tuple[str, str], dict[str, Any]] = {} + pending_unscheduled: list[dict[str, Any]] = [] + node_allocated: dict[str, dict[str, int]] = { + node["name"]: {"cpu_millicores": 0, "memory_bytes": 0} for node in nodes + } + scheduled_cpu = scheduled_memory = 0 for pod in snapshots["pods"].get("items") or []: meta = pod.get("metadata") or {} + spec = pod.get("spec") or {} + status = pod.get("status") or {} + phase = str(status.get("phase") or "") + if phase in {"Succeeded", "Failed"}: + continue namespace = str(meta.get("namespace", "default")) + pod_name = str(meta.get("name")) + cpu, mem = pod_effective_requests(pod) + node_name = spec.get("nodeName") + if phase == "Pending" and not node_name: + pending_unscheduled.append({ + "namespace": namespace, + "name": pod_name, + "cpu_millicores": cpu, + "memory_bytes": mem, + }) + continue + if node_name in node_allocated: + node_allocated[node_name]["cpu_millicores"] += cpu + node_allocated[node_name]["memory_bytes"] += mem + scheduled_cpu += cpu + scheduled_memory += mem name = workload_name(pod) key = (namespace, name) row = workloads.setdefault(key, { @@ -128,15 +170,13 @@ def build_observation(snapshots: dict[str, Any], captured_at: str) -> dict[str, "observed": {"cpu_millicores": 0, "memory_bytes": 0} if snapshots.get("pod_top") is not None else None, }) row["pods"] += 1 - for container in (pod.get("spec") or {}).get("containers") or []: - resources = container.get("resources") or {} - requests = resources.get("requests") or {} - limits = resources.get("limits") or {} - row["requests"]["cpu_millicores"] += cpu_millicores(requests.get("cpu")) - row["requests"]["memory_bytes"] += bytes_value(requests.get("memory")) + row["requests"]["cpu_millicores"] += cpu + row["requests"]["memory_bytes"] += mem + for container in spec.get("containers") or []: + limits = (container.get("resources") or {}).get("limits") or {} row["limits"]["cpu_millicores"] += cpu_millicores(limits.get("cpu")) row["limits"]["memory_bytes"] += bytes_value(limits.get("memory")) - metric = pod_top.get((namespace, str(meta.get("name")))) + metric = pod_top.get((namespace, pod_name)) if metric and row["observed"] is not None: row["observed"]["cpu_millicores"] += metric["cpu_millicores"] row["observed"]["memory_bytes"] += metric["memory_bytes"] @@ -167,8 +207,22 @@ def build_observation(snapshots: dict[str, Any], captured_at: str) -> dict[str, }) workload_rows = sorted(workloads.values(), key=lambda row: (row["namespace"], row["workload"])) - requested_cpu = sum(row["requests"]["cpu_millicores"] for row in workload_rows) - requested_memory = sum(row["requests"]["memory_bytes"] for row in workload_rows) + requested_cpu = scheduled_cpu + requested_memory = scheduled_memory + pending_cpu = sum(item["cpu_millicores"] for item in pending_unscheduled) + pending_memory = sum(item["memory_bytes"] for item in pending_unscheduled) + preflight_nodes = [] + for node in nodes: + allocated = node_allocated.get(node["name"], {"cpu_millicores": 0, "memory_bytes": 0}) + preflight_nodes.append({ + "name": node["name"], + "ready": node["ready"], + "unschedulable": False, + "allocatable_cpu_m": node["allocatable"]["cpu_millicores"], + "allocatable_memory_bytes": node["allocatable"]["memory_bytes"], + "allocated_cpu_m": allocated["cpu_millicores"], + "allocated_memory_bytes": allocated["memory_bytes"], + }) return { "schema_version": "railiance.cluster-resource-observation.v1", "record_type": "usage_observation", @@ -192,9 +246,29 @@ def build_observation(snapshots: dict[str, Any], captured_at: str) -> dict[str, "method_version": "cluster-raw-drivers-v1", "drivers": ["cpu_requests", "memory_requests", "pvc_requested_bytes", "observed_usage"], "denominators": {"cpu_requested_millicores": requested_cpu, "memory_requested_bytes": requested_memory, "pvc_requested_bytes": pvc_requested}, + "pending_unscheduled": { + "cpu_millicores": pending_cpu, + "memory_bytes": pending_memory, + "pods": pending_unscheduled, + }, "residual_capacity": {"cpu_millicores": max(alloc_cpu - requested_cpu, 0), "memory_bytes": max(alloc_memory - requested_memory, 0), "pv_bytes": max(pv_capacity - pvc_requested, 0)}, + "not_a_scheduling_guarantee": True, "selection_owner": "resource-control", }, + "state_hub_preflight_observation": { + "schema": "state-hub.release-headroom-preflight.v1-input", + "observed_at": captured_at, + "freshness_seconds": 0, + "nodes": preflight_nodes, + "pending_unrelated": [ + { + "name": f"{item['namespace']}/{item['name']}", + "cpu_m": item["cpu_millicores"], + "memory_bytes": item["memory_bytes"], + } + for item in pending_unscheduled + ], + }, "measurement_gaps": sorted(set(gaps)), "provenance": { "commands": [ diff --git a/workplans/RAIL-BS-WP-0014-state-hub-surge-headroom.md b/workplans/RAIL-BS-WP-0014-state-hub-surge-headroom.md index 947b286..868aa94 100644 --- a/workplans/RAIL-BS-WP-0014-state-hub-surge-headroom.md +++ b/workplans/RAIL-BS-WP-0014-state-hub-surge-headroom.md @@ -4,7 +4,7 @@ type: workplan title: "Keep node remaining CPU honest for State Hub surge and pending demand" domain: financials repo: railiance-cluster -status: proposed +status: finished owner: codex topic_slug: railiance origin: residual @@ -12,6 +12,7 @@ origin_ref: STATE-WP-0091 created: "2026-09-14" updated: "2026-09-14" related: [RCLUSTER-WP-0014, STATE-WP-0091, CUST-WP-0071] +state_hub_workstream_id: "d89863aa-dc89-5a9d-be00-4f558a16d70b" --- Residual from STATE-WP-0091. State Hub now refuses promotion when remaining @@ -24,11 +25,18 @@ not capacity admission for factory or other zero-request workloads. ```task id: RAIL-BS-WP-0014-T01 -status: todo +status: done priority: high +state_hub_task_id: "d90c384e-8a99-550f-b8b8-6314c073f2dc" ``` Extend `tools/observe_cluster_resources.py` / `make cluster-observe` so the published observation names pending unscheduled pods and their requests. State Hub's preflight already consumes that shape. Do not treat residual millicores as a scheduling guarantee. + +**Done 2026-09-14.** `build_observation` names pending unscheduled pods, uses +effective pod requests (init vs containers), and emits +`state_hub_preflight_observation` in the STATE-WP-0091 fixture shape. +Residual CPU counts scheduled pods only. `not_a_scheduling_guarantee` is +explicit. Unit test covers a 50m pending pod that does not inflate remaining.