diff --git a/deploy/railiance/apps/helm/state-hub-values.yaml b/deploy/railiance/apps/helm/state-hub-values.yaml index 99adb99..0e3f022 100644 --- a/deploy/railiance/apps/helm/state-hub-values.yaml +++ b/deploy/railiance/apps/helm/state-hub-values.yaml @@ -39,8 +39,11 @@ resources: # for a node with <250m unallocated; ~830m of Knative/Kourier requests were # released that day (the-custodian docs/kubernetes-change-gate-decision.md). # State Hub idles near 5m but bursts past 400m under concurrent agent load. + # 2026-10-21 (STATE-WP-0099, CUST-WP-0071 node allocation review): 7d p95 + # 258m sat above the 250m request; request = p95 x1.5 = 390m. Limit unchanged. + # scripts/release_headroom_preflight.py derives the API surge from this value. requests: - cpu: 250m + cpu: 390m memory: 512Mi limits: cpu: 1000m diff --git a/deploy/railiance/platform/state-hub-db-cluster.yaml b/deploy/railiance/platform/state-hub-db-cluster.yaml index 26280c2..c2b12c6 100644 --- a/deploy/railiance/platform/state-hub-db-cluster.yaml +++ b/deploy/railiance/platform/state-hub-db-cluster.yaml @@ -20,6 +20,14 @@ spec: imageName: ghcr.io/cloudnative-pg/postgresql:16 storage: size: 10Gi + # STATE-WP-0099 (CUST-WP-0071 node allocation review, 2026-10-21): previously + # no requests. 7d CPU p95 37m / peak 133m -> max(50m floor, p95x1.5, peak x0.5) + # = 70m; memory working set p95 349Mi x1.15 = 416Mi. No limits (unchanged). + # Changing this restarts the single instance (brief hub DB outage). + resources: + requests: + cpu: 70m + memory: 416Mi bootstrap: initdb: database: state_hub diff --git a/docs/evidence/2026-10-21-resource-wave1/before.json b/docs/evidence/2026-10-21-resource-wave1/before.json new file mode 100644 index 0000000..89c8af6 --- /dev/null +++ b/docs/evidence/2026-10-21-resource-wave1/before.json @@ -0,0 +1,172 @@ +{ + "schema": "state-hub.resource-wave1-before.v1", + "workplan": "STATE-WP-0099", + "related": [ + "CUST-WP-0071", + "RESOURCE-WP-0008", + "STATE-WP-0091" + ], + "observed_at": "2026-10-21T06:39:56.971454+00:00", + "collected_by": "claude-code (read-only: kubectl get, helm history, curl /state/health)", + "node": { + "name": "239.62.205.92.host.secureserver.net", + "allocatable_cpu_m": 4000, + "allocatable_memory_mi": 15993, + "requested_cpu_m": 3605, + "requested_memory_mi": 9856, + "unrequested_cpu_m": 395 + }, + "helm": { + "release": "state-hub", + "namespace": "state-hub", + "revision": 77, + "status": "deployed", + "updated": "2026-10-17T01:53:02Z", + "app_version": "main-5dd3df9", + "rollback_target_revision": 77, + "note": "effective resources taken from the live Deployment specs below; release values were not dumped" + }, + "deployments": { + "state-hub": { + "replicas": 1, + "strategy": { + "rollingUpdate": { + "maxSurge": 1, + "maxUnavailable": 0 + }, + "type": "RollingUpdate" + }, + "containers": { + "state-hub": { + "limits": { + "cpu": "1", + "memory": "2Gi" + }, + "requests": { + "cpu": "250m", + "memory": "512Mi" + } + } + } + }, + "state-hub-mcp": { + "replicas": 1, + "strategy": { + "rollingUpdate": { + "maxSurge": 1, + "maxUnavailable": 0 + }, + "type": "RollingUpdate" + }, + "containers": { + "state-hub-mcp": { + "limits": { + "cpu": "500m", + "memory": "512Mi" + }, + "requests": { + "cpu": "10m", + "memory": "128Mi" + } + } + } + }, + "state-hub-memo-dispatch": { + "replicas": 1, + "strategy": { + "type": "Recreate" + }, + "containers": { + "dispatcher": { + "limits": { + "cpu": "100m", + "memory": "96Mi" + }, + "requests": { + "cpu": "5m", + "memory": "24Mi" + } + } + } + } + }, + "pods": { + "state-hub-6c5b9c676-8chmx": { + "phase": "Running", + "ready": true, + "restarts": 0, + "started": "2026-10-17T01:53:16Z" + }, + "state-hub-mcp-b955b86b4-x849p": { + "phase": "Running", + "ready": true, + "restarts": 0, + "started": "2026-10-17T01:53:16Z" + }, + "state-hub-memo-dispatch-5758dd4c-pk87d": { + "phase": "Running", + "ready": true, + "restarts": 1, + "started": "2026-10-04T22:10:30Z" + } + }, + "state_hub_db": { + "cluster_spec_resources": {}, + "instances": 1, + "managed_by_label": "manual", + "pod": "state-hub-db-1", + "pod_container_resources": { + "postgres": {} + }, + "pod_restarts_lifetime": 214, + "last_termination": { + "terminated": { + "containerID": "containerd://bf9293a209739e5611aeeb0c012ba10230b16bbcac8363be4475a8da2bc62a61", + "exitCode": 255, + "finishedAt": "2026-10-14T18:21:45Z", + "reason": "Unknown", + "startedAt": "2026-08-15T22:30:35Z" + } + }, + "note": "restart count is lifetime since 2026-07-06; last termination 2026-10-14 exitCode 255 reason Unknown (node-level restart), not a resource kill. Baseline for 'no new restarts'." + }, + "hub_health": { + "status": "ok", + "db": "connected", + "instance_role": "primary", + "schema_applied": "c4d5e6f7a8b9" + }, + "mcp": { + "service": "10.43.110.80:8001", + "endpoint": "10.42.0.60", + "pod_ready": true, + "note": "no workstation forward on :8001; verified by Ready pod + endpoint" + }, + "preflight_before_change": { + "schema": "state-hub.release-headroom-preflight.v1", + "ok": true, + "observed_at": "2026-10-21T06:39:56.971454+00:00", + "freshness_seconds": 0, + "remaining_cpu_m": 395, + "remaining_memory_bytes": 6435315712, + "pending_unrelated_cpu_m": 0, + "api_surge_cpu_m": 100, + "mcp_surge_cpu_m": 10, + "migrate_cpu_m": 50, + "atomic": true, + "reasons": [], + "notes": [ + "Aggregate remaining millicores is not a kube-scheduler guarantee.", + "Preflight does not lower requests and does not start Helm." + ], + "hook_order": [ + "pre-upgrade migrate job (helm.sh/hook-weight -5)", + "API RollingUpdate maxSurge=1 maxUnavailable=0", + "MCP RollingUpdate maxSurge=1 maxUnavailable=0" + ] + }, + "review_inputs": { + "railiance_apps": "2 pods 260m (API 250 + MCP 10) -> 390 proposed; p95 258.5m, peak 975m, throttle 0.0", + "state_hub_db": "0 -> 70m / 0 -> 416Mi; p95 36.7m peak 133.2m; mem p95 349Mi peak 359Mi" + } +} diff --git a/scripts/release_headroom_preflight.py b/scripts/release_headroom_preflight.py index d369cc5..b44f9e3 100644 --- a/scripts/release_headroom_preflight.py +++ b/scripts/release_headroom_preflight.py @@ -28,13 +28,23 @@ BYTE_FACTORS = { "T": 1000**4, } -# Historical State Hub API surge (maxSurge=1, maxUnavailable=0). -DEFAULT_API_SURGE_CPU_M = 100 +# Fallback surge figures, used only when no values file can be read. With +# maxSurge=1 / maxUnavailable=0 the surge pod carries the *target* release's +# API request, so the real figure is derived from the Helm values the release +# will deploy (STATE-WP-0099). The historical 100m constant under-modelled the +# 250m API request; tests keep these fallbacks equal to the shipped values. +DEFAULT_API_SURGE_CPU_M = 390 DEFAULT_MCP_SURGE_CPU_M = 10 DEFAULT_MIGRATE_CPU_M = 50 DEFAULT_API_SURGE_MEMORY = 512 * 1024**2 DEFAULT_FRESHNESS_SECONDS = 15 * 60 +REPO_ROOT = Path(__file__).resolve().parent.parent +DEFAULT_VALUES_FILES = ( + REPO_ROOT / "deploy/railiance/apps/charts/state-hub/values.yaml", + REPO_ROOT / "deploy/railiance/apps/helm/state-hub-values.yaml", +) + def cpu_millicores(value: str | None) -> int: import re @@ -66,6 +76,55 @@ def pod_effective_requests(pod: dict[str, Any]) -> tuple[int, int]: return max(cpu, init_cpu), max(mem, init_mem) +def _deep_merge(base: dict[str, Any], override: dict[str, Any]) -> dict[str, Any]: + merged = dict(base) + for key, value in override.items(): + if isinstance(value, dict) and isinstance(merged.get(key), dict): + merged[key] = _deep_merge(merged[key], value) + else: + merged[key] = value + return merged + + +def _request(section: dict[str, Any] | None, key: str) -> Any: + return (((section or {}).get("resources") or {}).get("requests") or {}).get(key) + + +def release_from_values(values: dict[str, Any]) -> dict[str, Any]: + """Derive surge figures from merged Helm values (chart defaults + overrides). + + Helm merges `-f` files in order; pass them already merged. Only keys that are + present are returned, so missing sections fall back to the defaults. + """ + release: dict[str, Any] = {"surge_source": "values"} + api_cpu = _request(values, "cpu") + if api_cpu is not None: + release["api_surge_cpu_m"] = cpu_millicores(api_cpu) + api_mem = _request(values, "memory") + if api_mem is not None: + release["api_surge_memory_bytes"] = bytes_value(api_mem) + mcp = values.get("mcp") or {} + if mcp.get("enabled") is False: + release["mcp_surge_cpu_m"] = 0 + elif _request(mcp, "cpu") is not None: + release["mcp_surge_cpu_m"] = cpu_millicores(_request(mcp, "cpu")) + migrations = values.get("migrations") or {} + if migrations.get("enabled") is False: + release["migrate_cpu_m"] = 0 + elif _request(migrations, "cpu") is not None: + release["migrate_cpu_m"] = cpu_millicores(_request(migrations, "cpu")) + return release + + +def load_values(paths: list[Path] | tuple[Path, ...]) -> dict[str, Any]: + import yaml + + merged: dict[str, Any] = {} + for path in paths: + merged = _deep_merge(merged, yaml.safe_load(Path(path).read_text()) or {}) + return merged + + def evaluate(observation: dict[str, Any], release: dict[str, Any] | None = None) -> dict[str, Any]: """Return a preflight report. `ok` is whether Helm may start, not a guarantee.""" release = release or {} @@ -75,6 +134,7 @@ def evaluate(observation: dict[str, Any], release: dict[str, Any] | None = None) api_mem = int(release.get("api_surge_memory_bytes", DEFAULT_API_SURGE_MEMORY)) freshness_limit = int(release.get("freshness_seconds", DEFAULT_FRESHNESS_SECONDS)) atomic = bool(release.get("atomic", True)) + surge_source = str(release.get("surge_source", "release" if "api_surge_cpu_m" in release else "defaults")) reasons: list[str] = [] notes: list[str] = [] @@ -139,6 +199,21 @@ def evaluate(observation: dict[str, Any], release: dict[str, Any] | None = None) f"unrelated pending memory {pending_mem} leaves {effective_mem} against API surge" ) + live_api_cpu = observation.get("live_api_request_cpu_m") + remaining_after_rollout = None + if live_api_cpu is not None: + remaining_after_rollout = remaining_cpu - (api_cpu - int(live_api_cpu)) + if int(live_api_cpu) != api_cpu: + notes.append( + f"release changes the API request {int(live_api_cpu)}m -> {api_cpu}m; " + f"steady remaining after rollout is {remaining_after_rollout}m" + ) + if remaining_after_rollout < api_cpu: + notes.append( + f"after this rollout {remaining_after_rollout}m remains, below the {api_cpu}m " + "surge; the next release will be refused until headroom is freed" + ) + if not atomic: reasons.append("promotion path must keep helm --atomic; preflight will not start a non-atomic upgrade") @@ -160,6 +235,9 @@ def evaluate(observation: dict[str, Any], release: dict[str, Any] | None = None) "remaining_memory_bytes": remaining_mem, "pending_unrelated_cpu_m": pending_cpu, "api_surge_cpu_m": api_cpu, + "surge_source": surge_source, + "live_api_request_cpu_m": live_api_cpu, + "remaining_after_rollout_cpu_m": remaining_after_rollout, "mcp_surge_cpu_m": mcp_cpu, "migrate_cpu_m": migrate_cpu, "atomic": atomic, @@ -209,9 +287,20 @@ def collect_live(kubectl: str = "kubectl") -> dict[str, Any]: if node_name in allocated: allocated[node_name]["allocated_cpu_m"] += cpu allocated[node_name]["allocated_memory_bytes"] += mem + live_api_cpu = None + try: + deploy = json.loads(subprocess.check_output( + [kubectl, "-n", "state-hub", "get", "deployment", "state-hub", "-o", "json"], timeout=60 + )) + for container in deploy["spec"]["template"]["spec"]["containers"]: + if container.get("name") == "state-hub": + live_api_cpu = cpu_millicores(_request(container, "cpu")) + except (subprocess.SubprocessError, KeyError, ValueError, OSError): + live_api_cpu = None return { "observed_at": now.isoformat(), "freshness_seconds": 0, + "live_api_request_cpu_m": live_api_cpu, "nodes": list(allocated.values()), "pending_unrelated": pending_unrelated, "source": "kubectl", @@ -221,14 +310,24 @@ def collect_live(kubectl: str = "kubectl") -> dict[str, Any]: def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--fixture", type=Path, help="JSON observation instead of live kubectl") - parser.add_argument("--release", type=Path, help="JSON release surge description") + parser.add_argument("--release", type=Path, help="JSON release surge description (overrides values)") + parser.add_argument( + "--values", type=Path, action="append", + help="Helm values file, in -f order (repeatable). Default: chart values + production values", + ) parser.add_argument("--kubectl", default="kubectl") args = parser.parse_args(argv) if args.fixture: observation = json.loads(args.fixture.read_text()) else: observation = collect_live(args.kubectl) - release = json.loads(args.release.read_text()) if args.release else {} + values_files = args.values or [p for p in DEFAULT_VALUES_FILES if p.exists()] + release = release_from_values(load_values(values_files)) if values_files else {} + if args.release: + explicit = json.loads(args.release.read_text()) + release.update(explicit) + if "api_surge_cpu_m" in explicit: + release["surge_source"] = "release" report = evaluate(observation, release) json.dump(report, sys.stdout, indent=2) sys.stdout.write("\n") diff --git a/tests/fixtures/release_headroom/2026-10-21-110m-after-wave1.json b/tests/fixtures/release_headroom/2026-10-21-110m-after-wave1.json new file mode 100644 index 0000000..3db1c1a --- /dev/null +++ b/tests/fixtures/release_headroom/2026-10-21-110m-after-wave1.json @@ -0,0 +1,17 @@ +{ + "observed_at": "2026-10-21T06:39:56+00:00", + "freshness_seconds": 0, + "nodes": [ + { + "name": "railiance01", + "ready": true, + "unschedulable": false, + "allocatable_cpu_m": 4000, + "allocatable_memory_bytes": 16000000000, + "allocated_cpu_m": 3890, + "allocated_memory_bytes": 8000000000 + } + ], + "pending_unrelated": [], + "live_api_request_cpu_m": 390 +} diff --git a/tests/fixtures/release_headroom/2026-10-21-320m-forge-first.json b/tests/fixtures/release_headroom/2026-10-21-320m-forge-first.json new file mode 100644 index 0000000..59b8487 --- /dev/null +++ b/tests/fixtures/release_headroom/2026-10-21-320m-forge-first.json @@ -0,0 +1,17 @@ +{ + "observed_at": "2026-10-21T06:39:56+00:00", + "freshness_seconds": 0, + "nodes": [ + { + "name": "railiance01", + "ready": true, + "unschedulable": false, + "allocatable_cpu_m": 4000, + "allocatable_memory_bytes": 16000000000, + "allocated_cpu_m": 3680, + "allocated_memory_bytes": 8000000000 + } + ], + "pending_unrelated": [], + "live_api_request_cpu_m": 250 +} diff --git a/tests/fixtures/release_headroom/2026-10-21-395m-live250.json b/tests/fixtures/release_headroom/2026-10-21-395m-live250.json new file mode 100644 index 0000000..7df164b --- /dev/null +++ b/tests/fixtures/release_headroom/2026-10-21-395m-live250.json @@ -0,0 +1,17 @@ +{ + "observed_at": "2026-10-21T06:39:56+00:00", + "freshness_seconds": 0, + "nodes": [ + { + "name": "railiance01", + "ready": true, + "unschedulable": false, + "allocatable_cpu_m": 4000, + "allocatable_memory_bytes": 16000000000, + "allocated_cpu_m": 3605, + "allocated_memory_bytes": 8000000000 + } + ], + "pending_unrelated": [], + "live_api_request_cpu_m": 250 +} diff --git a/tests/fixtures/release_headroom/2026-10-21-555m-after-knative.json b/tests/fixtures/release_headroom/2026-10-21-555m-after-knative.json new file mode 100644 index 0000000..99777ca --- /dev/null +++ b/tests/fixtures/release_headroom/2026-10-21-555m-after-knative.json @@ -0,0 +1,17 @@ +{ + "observed_at": "2026-10-21T06:39:56+00:00", + "freshness_seconds": 0, + "nodes": [ + { + "name": "railiance01", + "ready": true, + "unschedulable": false, + "allocatable_cpu_m": 4000, + "allocatable_memory_bytes": 16000000000, + "allocated_cpu_m": 3445, + "allocated_memory_bytes": 8000000000 + } + ], + "pending_unrelated": [], + "live_api_request_cpu_m": 250 +} diff --git a/tests/fixtures/release_headroom/historical-100m-release.json b/tests/fixtures/release_headroom/historical-100m-release.json new file mode 100644 index 0000000..998d920 --- /dev/null +++ b/tests/fixtures/release_headroom/historical-100m-release.json @@ -0,0 +1 @@ +{"api_surge_cpu_m": 100, "surge_source": "historical-100m"} diff --git a/tests/test_release_headroom_preflight.py b/tests/test_release_headroom_preflight.py index db09bf9..6583d48 100644 --- a/tests/test_release_headroom_preflight.py +++ b/tests/test_release_headroom_preflight.py @@ -3,11 +3,25 @@ from __future__ import annotations import json from pathlib import Path -from scripts.release_headroom_preflight import evaluate, main, pod_effective_requests +from scripts.release_headroom_preflight import ( + DEFAULT_API_SURGE_CPU_M, + DEFAULT_API_SURGE_MEMORY, + DEFAULT_MCP_SURGE_CPU_M, + DEFAULT_MIGRATE_CPU_M, + DEFAULT_VALUES_FILES, + evaluate, + load_values, + main, + pod_effective_requests, + release_from_values, +) FIXTURES = Path(__file__).parent / "fixtures" / "release_headroom" +HISTORICAL_100M = {"api_surge_cpu_m": 100} + + def _load(name: str) -> dict: return json.loads((FIXTURES / name).read_text()) @@ -23,7 +37,7 @@ def test_revision_61_65m_refuses_api_surge() -> None: def test_revision_63_105m_is_narrowly_sufficient() -> None: - report = evaluate(_load("rev63-105m.json")) + report = evaluate(_load("rev63-105m.json"), HISTORICAL_100M) assert report["ok"] is True assert report["remaining_cpu_m"] == 105 assert report["reasons"] == [] @@ -31,7 +45,7 @@ def test_revision_63_105m_is_narrowly_sufficient() -> None: def test_unrelated_pending_demand_refuses_105m() -> None: - report = evaluate(_load("rev63-105m-concurrent-pending.json")) + report = evaluate(_load("rev63-105m-concurrent-pending.json"), HISTORICAL_100M) assert report["ok"] is False assert any("unrelated pending" in r for r in report["reasons"]) @@ -39,13 +53,13 @@ def test_unrelated_pending_demand_refuses_105m() -> None: def test_stale_observation_refuses() -> None: observation = _load("rev63-105m.json") observation["freshness_seconds"] = 3600 - report = evaluate(observation) + report = evaluate(observation, HISTORICAL_100M) assert report["ok"] is False assert any("old" in r for r in report["reasons"]) def test_non_atomic_refuses() -> None: - report = evaluate(_load("rev63-105m.json"), {"atomic": False}) + report = evaluate(_load("rev63-105m.json"), {**HISTORICAL_100M, "atomic": False}) assert report["ok"] is False assert any("atomic" in r for r in report["reasons"]) @@ -62,5 +76,65 @@ def test_init_overhead_uses_max_not_sum() -> None: def test_cli_fixture_exit_codes() -> None: - assert main(["--fixture", str(FIXTURES / "rev63-105m.json")]) == 0 - assert main(["--fixture", str(FIXTURES / "rev61-65m.json")]) == 1 + historical = str(FIXTURES / "historical-100m-release.json") + assert main(["--fixture", str(FIXTURES / "rev63-105m.json"), "--release", historical]) == 0 + assert main(["--fixture", str(FIXTURES / "rev61-65m.json"), "--release", historical]) == 1 + # Default surge now derives from the shipped values (390m): 105m refuses. + assert main(["--fixture", str(FIXTURES / "rev63-105m.json")]) == 1 + + +# STATE-WP-0099: the surge follows the API request the release deploys. + + +def test_shipped_values_drive_surge_and_match_fallbacks() -> None: + release = release_from_values(load_values(DEFAULT_VALUES_FILES)) + assert release["api_surge_cpu_m"] == 390 + assert release["api_surge_cpu_m"] == DEFAULT_API_SURGE_CPU_M + assert release["api_surge_memory_bytes"] == DEFAULT_API_SURGE_MEMORY + assert release["mcp_surge_cpu_m"] == DEFAULT_MCP_SURGE_CPU_M + assert release["migrate_cpu_m"] == DEFAULT_MIGRATE_CPU_M + + +def test_values_override_order_and_disabled_sections() -> None: + chart = {"resources": {"requests": {"cpu": "250m", "memory": "512Mi"}}, + "mcp": {"enabled": False, "resources": {"requests": {"cpu": "50m"}}}, + "migrations": {"enabled": True, "resources": {"requests": {"cpu": "50m"}}}} + override = {"resources": {"requests": {"cpu": "0.39"}}, "migrations": {"enabled": False}} + from scripts.release_headroom_preflight import _deep_merge + + release = release_from_values(_deep_merge(chart, override)) + assert release["api_surge_cpu_m"] == 390 + assert release["api_surge_memory_bytes"] == 512 * 1024**2 + assert release["mcp_surge_cpu_m"] == 0 + assert release["migrate_cpu_m"] == 0 + + +def test_today_395m_fits_390m_surge_narrowly() -> None: + report = evaluate(_load("2026-10-21-395m-live250.json")) + assert report["ok"] is True + assert report["api_surge_cpu_m"] == 390 + assert report["remaining_after_rollout_cpu_m"] == 255 + notes = " ".join(report["notes"]) + assert "250m -> 390m" in notes + assert "only 5m above" in notes + assert "next release will be refused" in notes + + +def test_after_knative_reduction_fits_with_margin() -> None: + report = evaluate(_load("2026-10-21-555m-after-knative.json")) + assert report["ok"] is True + assert report["remaining_after_rollout_cpu_m"] == 415 + assert "next release will be refused" not in " ".join(report["notes"]) + + +def test_forge_increase_before_state_hub_refuses() -> None: + report = evaluate(_load("2026-10-21-320m-forge-first.json")) + assert report["ok"] is False + assert any("needs 390m" in r for r in report["reasons"]) + + +def test_after_full_wave1_routine_release_refuses() -> None: + report = evaluate(_load("2026-10-21-110m-after-wave1.json")) + assert report["ok"] is False + assert report["remaining_cpu_m"] == 110 + assert any("needs 390m" in r for r in report["reasons"]) diff --git a/workplans/STATE-WP-0099-node-allocation-wave1.md b/workplans/STATE-WP-0099-node-allocation-wave1.md new file mode 100644 index 0000000..52e58d8 --- /dev/null +++ b/workplans/STATE-WP-0099-node-allocation-wave1.md @@ -0,0 +1,255 @@ +--- +id: STATE-WP-0099 +type: workplan +title: "Apply the wave-1 node allocation change for the State Hub API, database and release preflight" +domain: infotech +repo: state-hub +status: active +owner: claude-code +topic_slug: infotech +created: "2026-10-21" +updated: "2026-10-21" +flavor: implementation +related: [CUST-WP-0071, RESOURCE-WP-0008, STATE-WP-0091] +--- + +This workplan receives the State Hub rows of the 2026-10-21 node allocation +review (resource-control `docs/evidence/2026-10-21-node-allocation-review.md` +and `2026-10-21-node-allocation-review/2026-10-21-node-budget.{md,json}`). +It also covers wave 1 of the CUST-WP-0071 T03/T04 rollout +(`the-custodian/docs/decisions/2026-10-21-cust-wp-0071-delivery-interview.md`). +The founder's answer "I may apply wave 1" lets the Claude session push and +apply these reviewed diffs through this repo's existing deploy path, one owner +at a time, with before/after evidence and a rollback receipt. + +## Change + +| Item | Before (live, 2026-10-21 06:39Z) | After | Source | +| --- | --- | --- | --- | +| API `state-hub` CPU request | 250m | **390m** | `deploy/railiance/apps/helm/state-hub-values.yaml` | +| API memory request / limits | 512Mi / 1000m, 2Gi | unchanged | same | +| MCP `state-hub-mcp` | 10m / 128Mi | unchanged | same | +| `databases/state-hub-db` (CNPG) requests | none | **70m / 416Mi**, no limits | `deploy/railiance/platform/state-hub-db-cluster.yaml` | +| Preflight API surge | constant 100m | **derived from the values the release deploys** (390m); fallback constant 390m | `scripts/release_headroom_preflight.py` | + +Reasons: + +- API: 7d p95 258m sits above the 250m request and the 975m peak comes from + release and migration bursts. The review sets 390m (p95 x1.5, rounded up) + as an override and leaves the limit unchanged. The budget row + `state-hub/railiance-apps` reads "260→390" for two pods. That row adds the + MCP 10m into the current figure but not into the proposed one. This change + sets the API to 390m and keeps MCP at 10m, so the workload goes 260m→400m + (+140m) and the review's node budget is 10m short. The 258.5m p95 is the + workload figure. MCP p95 is about 2m, so the API's own p95 is about 256m and + the result rounds to 390m either way. +- DB: the pod had no requests. 7d CPU p95 is 36.7m and peak 133.2m, so + max(50m DB floor, p95 x1.5 = 55m, peak x0.5 = 67m) gives 70m. The memory + working set p95 is 349Mi, x1.15 gives 416Mi. shared_buffers is the CNPG + default (no override in the Cluster spec). The 416Mi figure is a request + only and lowers nothing. +- Preflight: STATE-WP-0091 modelled a 100m API surge (revision 63). The live + API pod has requested 250m since 2026-09-21. With maxSurge=1 and + maxUnavailable=0 the surge pod carries the request of the release being + deployed. The preflight now reads that request from the chart values merged + with the production values (`--values` is repeatable in Helm `-f` order, and + `--release` JSON still overrides). It also reads the live API request and + reports `remaining_after_rollout_cpu_m`. When the release changes the request + it notes the steady remainder, and it notes when the next release would be + refused. A test keeps the fallback constants equal to the shipped values. + +## Required ordering (node has 395m unrequested) + +Before this change the node had 4000m allocatable and 3605m requested, which +leaves 395m. The API change needs a **390m surge pod** while the old 250m pod +is still running. Its net steady cost is +140m. The DB change restarts the +single instance in place, with a net cost of +70m. +Pod-effective deltas of the other wave-1 owners, taken from the review rows: +rail-knative −160m, railiance-forge +225m (gitea 100→200 counting its init +container, runner 0→125), rapp-telemetry +10m. + +| Sequence | Free before the state-hub API rollout | API surge 390m | +| --- | ---: | --- | +| state-hub first (today) | 395m | fits by only 5m. Any CronJob pod (10–25m) or pending pod breaks it. **Not acceptable** | +| rail-knative reductions first | 555m | **fits, 165m margin** | +| knative + forge + telemetry first | 320m | **refused** (preflight: needs 390m) | + +The order is therefore fixed: + +1. **rail-knative** wave-1 reductions applied and settled (pods Running, no + Pending pods). Free is then about 555m. +2. **state-hub API 250m→390m** (T03). The live preflight must report `ok: true` + with `api_surge_cpu_m: 390` immediately before Helm starts. Avoid the + 02:40–04:00 CronJob window. Free afterwards is about 415m. +3. **state-hub-db 0→70m** (T04). Free afterwards is about 345m. +4. Then rapp-telemetry (+10m) and railiance-forge (+225m). Free afterwards is + about 110m. + +T03 must run **after the wave-1 reductions and before the wave-1 increases +(railiance-forge, rapp-telemetry)**. If forge lands first, the API change +waits until wave 2 frees headroom. + +Consequence: after the whole of wave 1 the node has about 110m free, which is +below the 390m surge. **Routine State Hub releases will be refused by the +preflight until wave 2 decreases land** (the review's proposed steady total +is 2400m). That is the preflight working as intended, not a defect (T06). + +## Evidence + +- BEFORE (read-only): `docs/evidence/2026-10-21-resource-wave1/before.json`. + It holds the live Deployment and CNPG specs, pod readiness and restarts, + node allocation, Helm revision 77, hub health and the preflight as it ran + before this change (old 100m model, ok). +- Fixtures for the four ordering cases: + `tests/fixtures/release_headroom/2026-10-21-*.json`. +- Baselines for "no new restarts": API pod `state-hub-6c5b9c676-8chmx` 0, MCP + pod `state-hub-mcp-b955b86b4-x849p` 0, and DB pod `state-hub-db-1` at 214 + lifetime restarts. The last DB termination was on 2026-10-14 with exit 255, + reason Unknown, from a node restart. The DB pod is recreated by T04, so its + count resets. + +## Deploy path + +The State Hub runs in-cluster on railiance01 (k3s single node). The API and +MCP are deployed by **Helm from this repo checkout** (release `state-hub`, ns +`state-hub`, not ArgoCD). The DB Cluster is **applied manually** from +`deploy/railiance/platform/state-hub-db-cluster.yaml` +(`app.kubernetes.io/managed-by: manual`). Its live last-applied spec equals +the source spec before this change. Run from the workstation with +`KUBECONFIG=~/.kube/config-railiance01`, after pushing this commit to Forgejo. + +## Pre-check: read-only evidence and review against live + +```task +id: STATE-WP-0099-T01 +status: done +priority: high +``` + +BEFORE state captured read-only. Review figures checked against the live +specs: API 250m, MCP 10m, DB without requests, 395m free, and the preflight +still on its 100m constant. + +## Source change: values, DB Cluster, derived preflight surge, tests + +```task +id: STATE-WP-0099-T02 +status: done +priority: high +``` + +`state-hub-values.yaml` sets the API to 390m. `state-hub-db-cluster.yaml` +sets 70m/416Mi. `release_headroom_preflight.py` derives the surge from the +values. `tests/test_release_headroom_preflight.py` gains six tests. The +historical 100m cases are kept explicitly through `historical-100m-release.json`. +`uv run pytest -q tests/test_release_headroom_preflight.py` gives 13 passed. + +## Apply the API request (after rail-knative reductions, before forge/telemetry increases) + +```task +id: STATE-WP-0099-T03 +status: todo +priority: high +``` + +```bash +cd ~/state-hub && export KUBECONFIG=~/.kube/config-railiance01 +git push origin main # Forgejo; then: +kubectl get pods -A --field-selector=status.phase=Pending # must be empty +make railiance-state-hub-headroom # must print "ok": true, api_surge_cpu_m 390 +make railiance-state-hub-server-dry-run +helm upgrade state-hub deploy/railiance/apps/charts/state-hub -n state-hub \ + -f deploy/railiance/apps/helm/state-hub-values.yaml --atomic --wait --timeout 5m +``` + +The image tag is unchanged: the values pin `main-c103cbf@sha256:4cc4…`, which +is what runs live. The upgrade still runs the pre-upgrade migrate hook (50m, +no-op schema). MCP does not roll because its spec is unchanged. + +Verify: + +```bash +curl -s http://127.0.0.1:8000/state/health # status ok, db connected, instance_role primary +kubectl -n state-hub get deploy state-hub -o jsonpath='{.spec.template.spec.containers[0].resources.requests.cpu}' # 390m +kubectl -n state-hub get pods # all 1/1 Running, RESTARTS 0 +kubectl -n state-hub get endpoints state-hub-mcp # one address +ssh railiance01 'curl -s -m 3 -o /dev/null -w "%{http_code}\n" http://10.43.110.80:8001/sse' # 200 (stream cut by -m) +kubectl describe node | grep -A3 "Allocated resources" # cpu requests ≈ prior +140m +``` + +Rollback (restores 250m. A rollback surge needs 250m free, which the order +above leaves): + +```bash +helm rollback state-hub 77 --wait --timeout 5m # revision 77 = pre-change (live 2026-10-17) +``` + +Also revert the values line in git if the rollback is to be kept. + +## Apply the DB request (after T03) + +```task +id: STATE-WP-0099-T04 +status: todo +priority: high +``` + +```bash +kubectl apply --dry-run=server -f deploy/railiance/platform/state-hub-db-cluster.yaml +kubectl diff -f deploy/railiance/platform/state-hub-db-cluster.yaml # only spec.resources +kubectl apply -f deploy/railiance/platform/state-hub-db-cluster.yaml +kubectl -n databases get cluster state-hub-db -w # until "Cluster in healthy state" +``` + +There is a single instance (`primaryUpdateStrategy: unsupervised`, +`primaryUpdateMethod: restart`). CNPG recreates `state-hub-db-1`, so expect a +short window, typically under a minute, in which the hub reports DB errors. +Announce it before applying, and run it outside the CronJob window. + +Verify: + +```bash +kubectl -n databases get pod state-hub-db-1 -o jsonpath='{.spec.containers[0].resources}' # requests 70m/416Mi +kubectl -n databases get pod state-hub-db-1 # 1/1 Running, RESTARTS 0 after 15 min +curl -s http://127.0.0.1:8000/state/health # db connected, schema ok +kubectl -n state-hub get pods # API/MCP RESTARTS still 0 +``` + +Rollback: + +```bash +git show 39b51a2:deploy/railiance/platform/state-hub-db-cluster.yaml | kubectl apply -f - +``` + +The last-applied three-way merge removes `spec.resources`, and the instance +restarts again. + +## After evidence and receipt + +```task +id: STATE-WP-0099-T05 +status: todo +priority: medium +``` + +Re-run the BEFORE collection (same fields) as +`docs/evidence/2026-10-21-resource-wave1/after.json`. Add the live preflight +output, the Helm revision number, the restart counts 15 minutes after each +step, and the rollback commands with revision 77 and the base commit. Hand the +receipt to CUST-WP-0071-T04. + +## Release headroom after wave 1 + +```task +id: STATE-WP-0099-T06 +status: wait +priority: medium +depends_on: [CUST-WP-0071-T04] +blocking_reason: "After wave 1 ~110m stays free, below the 390m surge; routine State Hub releases are refused until wave-2 decreases land" +``` + +Once wave 2 has been applied, run `make railiance-state-hub-headroom` and +record whether a routine release (live 390m equal to target 390m) passes. +Do not lower the API request and do not drop the preflight to get a release +through. If an urgent release is needed earlier, Bernd decides between waiting +for wave 2 and an attended release with a brief API outage.