diff --git a/docs/evidence/2026-09-14-allocation-reconcile.json b/docs/evidence/2026-09-14-allocation-reconcile.json new file mode 100644 index 0000000..913b4dd --- /dev/null +++ b/docs/evidence/2026-09-14-allocation-reconcile.json @@ -0,0 +1,664 @@ +{ + "schema": "custodian.allocation-reconcile.v1", + "workplan_id": "CUST-WP-0071-T01", + "captured_at": "2026-09-14T09:45:22Z", + "cluster_observation_schema": "railiance.cluster-resource-observation.v1", + "count_host_and_cluster_cpus_once": true, + "host": { + "owner": "railiance-cluster", + "resource_id": "resource:hosteurope:railiance01", + "same_cpus_as_cluster": true, + "cluster_resource_id": "resource:railiance:reef-railiance:k3s" + }, + "capacity_cpu_m": 4000, + "scheduled_request_cpu_m": 3985, + "pending_unscheduled_cpu_m": 25, + "residual_cpu_m": 15, + "not_a_scheduling_guarantee": true, + "workloads": [ + { + "namespace": "knative-serving", + "workload": "knative-serving", + "pods": 4, + "cpu_request_m": 600, + "memory_request_bytes": 377487360, + "owner": "rail-kubernetes", + "service": "knative-serving", + "tenant": "unknown" + }, + { + "namespace": "core-hub", + "workload": "rapp-core-hub", + "pods": 3, + "cpu_request_m": 200, + "memory_request_bytes": 939524096, + "owner": "core-hub", + "service": "core-hub", + "tenant": "unknown" + }, + { + "namespace": "databases", + "workload": "forgejo-db", + "pods": 1, + "cpu_request_m": 200, + "memory_request_bytes": 536870912, + "owner": "rapp-postgres", + "service": "apps-pg", + "tenant": "unknown" + }, + { + "namespace": "knative-serving", + "workload": "net-kourier-controller", + "pods": 1, + "cpu_request_m": 200, + "memory_request_bytes": 209715200, + "owner": "rail-kubernetes", + "service": "knative-serving", + "tenant": "unknown" + }, + { + "namespace": "kourier-system", + "workload": "3scale-kourier-gateway", + "pods": 1, + "cpu_request_m": 200, + "memory_request_bytes": 209715200, + "owner": "rail-kubernetes", + "service": "kourier", + "tenant": "unknown" + }, + { + "namespace": "sso", + "workload": "net-kingdom-sso-mfa", + "pods": 4, + "cpu_request_m": 150, + "memory_request_bytes": 268435456, + "owner": "net-kingdom", + "service": "keycape-authelia", + "tenant": "unknown" + }, + { + "namespace": "mfa", + "workload": "net-kingdom-sso-mfa", + "pods": 1, + "cpu_request_m": 110, + "memory_request_bytes": 402653184, + "owner": "net-kingdom", + "service": "lldap-mfa", + "tenant": "unknown" + }, + { + "namespace": "reuse", + "workload": "reuse-surface", + "pods": 2, + "cpu_request_m": 110, + "memory_request_bytes": 301989888, + "owner": "reuse-surface", + "service": "reuse-surface", + "tenant": "unknown" + }, + { + "namespace": "state-hub", + "workload": "railiance-apps", + "pods": 2, + "cpu_request_m": 110, + "memory_request_bytes": 671088640, + "owner": "state-hub", + "service": "state-hub", + "tenant": "unknown" + }, + { + "namespace": "flex-auth", + "workload": "flex-auth", + "pods": 5, + "cpu_request_m": 105, + "memory_request_bytes": 167772160, + "owner": "flex-auth", + "service": "flex-auth", + "tenant": "unknown" + }, + { + "namespace": "activity-core", + "workload": "activity-core", + "pods": 10, + "cpu_request_m": 100, + "memory_request_bytes": 268435456, + "owner": "activity-core", + "service": "activity-core", + "tenant": "unknown" + }, + { + "namespace": "cnpg-system", + "workload": "cloudnative-pg", + "pods": 1, + "cpu_request_m": 100, + "memory_request_bytes": 104857600, + "owner": "rapp-postgres", + "service": "cloudnative-pg", + "tenant": "unknown" + }, + { + "namespace": "coulomb-social", + "workload": "coulomb-social", + "pods": 1, + "cpu_request_m": 100, + "memory_request_bytes": 268435456, + "owner": "coulomb-social", + "service": "coulomb-social", + "tenant": "unknown" + }, + { + "namespace": "databases", + "workload": "apps-pg", + "pods": 1, + "cpu_request_m": 100, + "memory_request_bytes": 268435456, + "owner": "rapp-postgres", + "service": "apps-pg", + "tenant": "unknown" + }, + { + "namespace": "databases", + "workload": "net-kingdom-pg", + "pods": 1, + "cpu_request_m": 100, + "memory_request_bytes": 268435456, + "owner": "rapp-postgres", + "service": "apps-pg", + "tenant": "unknown" + }, + { + "namespace": "databases", + "workload": "platform-pg", + "pods": 1, + "cpu_request_m": 100, + "memory_request_bytes": 268435456, + "owner": "rapp-postgres", + "service": "apps-pg", + "tenant": "unknown" + }, + { + "namespace": "databases", + "workload": "platform-pg-2", + "pods": 1, + "cpu_request_m": 100, + "memory_request_bytes": 268435456, + "owner": "rapp-postgres", + "service": "apps-pg", + "tenant": "unknown" + }, + { + "namespace": "forgejo", + "workload": "gitea", + "pods": 1, + "cpu_request_m": 100, + "memory_request_bytes": 134217728, + "owner": "railiance-forge", + "service": "forgejo", + "tenant": "unknown" + }, + { + "namespace": "kube-system", + "workload": "coredns-7bdb54f89", + "pods": 1, + "cpu_request_m": 100, + "memory_request_bytes": 73400320, + "owner": "railiance-cluster", + "service": "k3s-control-plane", + "tenant": "unknown" + }, + { + "namespace": "kube-system", + "workload": "metrics-server-786d997795", + "pods": 1, + "cpu_request_m": 100, + "memory_request_bytes": 73400320, + "owner": "railiance-cluster", + "service": "k3s-control-plane", + "tenant": "unknown" + }, + { + "namespace": "openbao", + "workload": "openbao", + "pods": 1, + "cpu_request_m": 100, + "memory_request_bytes": 268435456, + "owner": "railiance-platform", + "service": "openbao", + "tenant": "unknown" + }, + { + "namespace": "telemetry", + "workload": "prometheus", + "pods": 1, + "cpu_request_m": 100, + "memory_request_bytes": 536870912, + "owner": "rapp-telemetry", + "service": "prometheus-grafana", + "tenant": "unknown" + }, + { + "namespace": "user-engine", + "workload": "user-engine-pg", + "pods": 1, + "cpu_request_m": 100, + "memory_request_bytes": 268435456, + "owner": "user-engine", + "service": "user-engine", + "tenant": "unknown" + }, + { + "namespace": "telemetry", + "workload": "grafana", + "pods": 1, + "cpu_request_m": 70, + "memory_request_bytes": 201326592, + "owner": "rapp-telemetry", + "service": "prometheus-grafana", + "tenant": "unknown" + }, + { + "namespace": "vergabe-demo-company", + "workload": "vergabe-teilnahme", + "pods": 1, + "cpu_request_m": 60, + "memory_request_bytes": 268435456, + "owner": "railiance-apps", + "service": "vergabe-teilnahme", + "tenant": "unknown" + }, + { + "namespace": "audit-core", + "workload": "audit-core", + "pods": 1, + "cpu_request_m": 50, + "memory_request_bytes": 67108864, + "owner": "audit-core", + "service": "audit-core", + "tenant": "unknown" + }, + { + "namespace": "coulomb", + "workload": "ihp-railiance-probe", + "pods": 1, + "cpu_request_m": 50, + "memory_request_bytes": 134217728, + "owner": "unknown", + "service": "ihp-railiance-probe", + "tenant": "unknown" + }, + { + "namespace": "issue-core", + "workload": "issue-core", + "pods": 1, + "cpu_request_m": 50, + "memory_request_bytes": 134217728, + "owner": "issue-core", + "service": "issue-core", + "tenant": "unknown" + }, + { + "namespace": "platform-pg-drill", + "workload": "platform-pg-drill", + "pods": 1, + "cpu_request_m": 50, + "memory_request_bytes": 134217728, + "owner": "rapp-postgres", + "service": "platform-pg-drill", + "tenant": "unknown" + }, + { + "namespace": "target-revenue", + "workload": "target-revenue", + "pods": 1, + "cpu_request_m": 50, + "memory_request_bytes": 268435456, + "owner": "target-revenue", + "service": "target-revenue", + "tenant": "unknown" + }, + { + "namespace": "user-engine", + "workload": "user-engine", + "pods": 1, + "cpu_request_m": 50, + "memory_request_bytes": 67108864, + "owner": "user-engine", + "service": "user-engine", + "tenant": "unknown" + }, + { + "namespace": "canned-prompts", + "workload": "canned-prompts", + "pods": 1, + "cpu_request_m": 25, + "memory_request_bytes": 100663296, + "owner": "canned-prompts", + "service": "canned-prompts", + "tenant": "unknown" + }, + { + "namespace": "email-connect", + "workload": "email-connect", + "pods": 1, + "cpu_request_m": 25, + "memory_request_bytes": 67108864, + "owner": "email-connect", + "service": "email-connect", + "tenant": "unknown" + }, + { + "namespace": "openbao", + "workload": "rapp-openbao", + "pods": 1, + "cpu_request_m": 25, + "memory_request_bytes": 33554432, + "owner": "railiance-platform", + "service": "openbao", + "tenant": "unknown" + }, + { + "namespace": "rein-aharness", + "workload": "rein-aharness", + "pods": 1, + "cpu_request_m": 25, + "memory_request_bytes": 67108864, + "owner": "rein-aharness", + "service": "rein-aharness", + "tenant": "unknown" + }, + { + "namespace": "sbom-nexus", + "workload": "sbom-nexus", + "pods": 1, + "cpu_request_m": 25, + "memory_request_bytes": 67108864, + "owner": "sbom-nexus", + "service": "sbom-nexus", + "tenant": "unknown" + }, + { + "namespace": "telemetry", + "workload": "alertmanager", + "pods": 1, + "cpu_request_m": 25, + "memory_request_bytes": 67108864, + "owner": "rapp-telemetry", + "service": "prometheus-grafana", + "tenant": "unknown" + }, + { + "namespace": "telemetry", + "workload": "kube-state-metrics", + "pods": 1, + "cpu_request_m": 25, + "memory_request_bytes": 67108864, + "owner": "rapp-telemetry", + "service": "prometheus-grafana", + "tenant": "unknown" + }, + { + "namespace": "telemetry", + "workload": "rapp-telemetry", + "pods": 1, + "cpu_request_m": 25, + "memory_request_bytes": 134217728, + "owner": "rapp-telemetry", + "service": "prometheus-grafana", + "tenant": "unknown" + }, + { + "namespace": "tenant-engine", + "workload": "tenant-engine", + "pods": 1, + "cpu_request_m": 25, + "memory_request_bytes": 50331648, + "owner": "tenant-engine", + "service": "tenant-engine", + "tenant": "unknown" + }, + { + "namespace": "rapp-qonto-egress", + "workload": "qonto-egress-proxy", + "pods": 1, + "cpu_request_m": 20, + "memory_request_bytes": 67108864, + "owner": "rapp-qonto", + "service": "qonto-egress", + "tenant": "tenant:friendly:binky" + }, + { + "namespace": "approval-engine", + "workload": "approval-engine", + "pods": 1, + "cpu_request_m": 10, + "memory_request_bytes": 67108864, + "owner": "approval-engine", + "service": "approval-engine", + "tenant": "unknown" + }, + { + "namespace": "policy-nexus", + "workload": "policy-nexus", + "pods": 1, + "cpu_request_m": 10, + "memory_request_bytes": 33554432, + "owner": "policy-nexus", + "service": "policy-nexus", + "tenant": "unknown" + }, + { + "namespace": "informed-decision", + "workload": "informed-decision", + "pods": 1, + "cpu_request_m": 5, + "memory_request_bytes": 67108864, + "owner": "informed-decision", + "service": "informed-decision", + "tenant": "unknown" + }, + { + "namespace": "cert-manager", + "workload": "cainjector", + "pods": 1, + "cpu_request_m": 0, + "memory_request_bytes": 0, + "owner": "railiance-cluster", + "service": "cert-manager", + "tenant": "unknown" + }, + { + "namespace": "cert-manager", + "workload": "cert-manager", + "pods": 1, + "cpu_request_m": 0, + "memory_request_bytes": 0, + "owner": "railiance-cluster", + "service": "cert-manager", + "tenant": "unknown" + }, + { + "namespace": "cert-manager", + "workload": "webhook", + "pods": 1, + "cpu_request_m": 0, + "memory_request_bytes": 0, + "owner": "railiance-cluster", + "service": "cert-manager", + "tenant": "unknown" + }, + { + "namespace": "databases", + "workload": "state-hub-db", + "pods": 1, + "cpu_request_m": 0, + "memory_request_bytes": 0, + "owner": "rapp-postgres", + "service": "apps-pg", + "tenant": "unknown" + }, + { + "namespace": "external-secrets", + "workload": "external-secrets", + "pods": 1, + "cpu_request_m": 0, + "memory_request_bytes": 0, + "owner": "railiance-platform", + "service": "external-secrets", + "tenant": "unknown" + }, + { + "namespace": "external-secrets", + "workload": "external-secrets-cert-controller", + "pods": 1, + "cpu_request_m": 0, + "memory_request_bytes": 0, + "owner": "railiance-platform", + "service": "external-secrets", + "tenant": "unknown" + }, + { + "namespace": "external-secrets", + "workload": "external-secrets-webhook", + "pods": 1, + "cpu_request_m": 0, + "memory_request_bytes": 0, + "owner": "railiance-platform", + "service": "external-secrets", + "tenant": "unknown" + }, + { + "namespace": "forgejo", + "workload": "forgejo-runner", + "pods": 1, + "cpu_request_m": 0, + "memory_request_bytes": 0, + "owner": "railiance-forge", + "service": "forgejo", + "tenant": "unknown" + }, + { + "namespace": "kube-system", + "workload": "local-path-provisioner", + "pods": 1, + "cpu_request_m": 0, + "memory_request_bytes": 0, + "owner": "railiance-cluster", + "service": "k3s-control-plane", + "tenant": "unknown" + }, + { + "namespace": "kube-system", + "workload": "svclb-traefik-0c8aecaf", + "pods": 1, + "cpu_request_m": 0, + "memory_request_bytes": 0, + "owner": "railiance-cluster", + "service": "k3s-control-plane", + "tenant": "unknown" + }, + { + "namespace": "kube-system", + "workload": "traefik", + "pods": 1, + "cpu_request_m": 0, + "memory_request_bytes": 0, + "owner": "railiance-cluster", + "service": "k3s-control-plane", + "tenant": "unknown" + }, + { + "namespace": "target-revenue", + "workload": "target-revenue-pg", + "pods": 1, + "cpu_request_m": 0, + "memory_request_bytes": 0, + "owner": "target-revenue", + "service": "target-revenue", + "tenant": "unknown" + } + ], + "unknown_namespaces": [], + "zero_request_workloads": [ + "cert-manager/cainjector", + "cert-manager/cert-manager", + "cert-manager/webhook", + "databases/state-hub-db", + "external-secrets/external-secrets", + "external-secrets/external-secrets-cert-controller", + "external-secrets/external-secrets-webhook", + "forgejo/forgejo-runner", + "kube-system/local-path-provisioner", + "kube-system/svclb-traefik-0c8aecaf", + "kube-system/traefik", + "target-revenue/target-revenue-pg" + ], + "pending_unscheduled": [ + { + "namespace": "user-engine", + "name": "user-engine-backup-29822640-64rqp", + "cpu_millicores": 25, + "memory_bytes": 67108864 + } + ], + "measurement_gaps": [ + "node 239.62.205.92.host.secureserver.net lacks independent region/zone labels" + ], + "state_hub_preflight_observation": { + "schema": "state-hub.release-headroom-preflight.v1-input", + "observed_at": "2026-09-14T09:45:22Z", + "freshness_seconds": 0, + "nodes": [ + { + "name": "239.62.205.92.host.secureserver.net", + "ready": true, + "unschedulable": false, + "allocatable_cpu_m": 4000, + "allocatable_memory_bytes": 16770076672, + "allocated_cpu_m": 3985, + "allocated_memory_bytes": 9319743488 + } + ], + "pending_unrelated": [ + { + "name": "user-engine/user-engine-backup-29822640-64rqp", + "cpu_m": 25, + "memory_bytes": 67108864 + } + ] + }, + "signal_notes": [ + "Host resource:hosteurope:railiance01 and cluster resource:railiance:reef-railiance:k3s are the same 4 vCPU; do not add them.", + "Prometheus namespace_cpu:kube_pod_container_resource_requests:sum omitted Forgejo 100m on 2026-09-11; Kubernetes API is the scheduler-effective source.", + "node-exporter was disabled; host CPU usage is not a reservation and is not treated as spare capacity.", + "Zero-request pods are demand, not zero. Missing metrics are listed as gaps, not zeros.", + "STATE-WP-0091 preflight consumes state_hub_preflight_observation; residual millicores are not admission." + ], + "state_hub_preflight": { + "schema": "state-hub.release-headroom-preflight.v1", + "ok": false, + "observed_at": "2026-09-14T09:45:22Z", + "freshness_seconds": 0, + "remaining_cpu_m": 15, + "remaining_memory_bytes": 7450333184, + "pending_unrelated_cpu_m": 25, + "api_surge_cpu_m": 100, + "mcp_surge_cpu_m": 10, + "migrate_cpu_m": 50, + "atomic": true, + "reasons": [ + "migration hook needs 50m CPU; remaining 15m", + "API maxSurge=1 needs 100m CPU; remaining 15m (Insufficient CPU class; revision 61 had 65m remaining)", + "unrelated pending demand 25m leaves -10m against API surge 100m" + ], + "notes": [ + "API+MCP concurrent surge would need 110m; remaining 15m. Helm may roll both deployments together; this is not a scheduling guarantee.", + "Aggregate remaining millicores is not a kube-scheduler guarantee.", + "Preflight does not lower requests and does not start Helm." + ], + "hook_order": [ + "pre-upgrade migrate job (helm.sh/hook-weight -5)", + "API RollingUpdate maxSurge=1 maxUnavailable=0", + "MCP RollingUpdate maxSurge=1 maxUnavailable=0" + ] + } +} diff --git a/docs/evidence/2026-09-14-allocation-reconcile.md b/docs/evidence/2026-09-14-allocation-reconcile.md new file mode 100644 index 0000000..4a33c29 --- /dev/null +++ b/docs/evidence/2026-09-14-allocation-reconcile.md @@ -0,0 +1,90 @@ +# Railiance allocation reconcile — 2026-09-14T09:45:22Z + +CUST-WP-0071-T01. Scheduler-effective requests from the Kubernetes API +via `railiance-cluster` observe (RCLUSTER-WP-0014 / RAIL-BS-WP-0014). +Host and cluster are the same 4 vCPU and are counted once. + +- Capacity: 4000m +- Scheduled requests: 3985m +- Pending unscheduled: 25m +- Residual (not a guarantee): 15m +- Unknown namespaces: none +- Zero-request workloads: 12 + +| Namespace | Workload | Owner | Tenant | CPU request (m) | Pods | +|---|---|---|---|---:|---:| +| knative-serving | knative-serving | rail-kubernetes | unknown | 600 | 4 | +| core-hub | rapp-core-hub | core-hub | unknown | 200 | 3 | +| databases | forgejo-db | rapp-postgres | unknown | 200 | 1 | +| knative-serving | net-kourier-controller | rail-kubernetes | unknown | 200 | 1 | +| kourier-system | 3scale-kourier-gateway | rail-kubernetes | unknown | 200 | 1 | +| sso | net-kingdom-sso-mfa | net-kingdom | unknown | 150 | 4 | +| mfa | net-kingdom-sso-mfa | net-kingdom | unknown | 110 | 1 | +| reuse | reuse-surface | reuse-surface | unknown | 110 | 2 | +| state-hub | railiance-apps | state-hub | unknown | 110 | 2 | +| flex-auth | flex-auth | flex-auth | unknown | 105 | 5 | +| activity-core | activity-core | activity-core | unknown | 100 | 10 | +| cnpg-system | cloudnative-pg | rapp-postgres | unknown | 100 | 1 | +| coulomb-social | coulomb-social | coulomb-social | unknown | 100 | 1 | +| databases | apps-pg | rapp-postgres | unknown | 100 | 1 | +| databases | net-kingdom-pg | rapp-postgres | unknown | 100 | 1 | +| databases | platform-pg | rapp-postgres | unknown | 100 | 1 | +| databases | platform-pg-2 | rapp-postgres | unknown | 100 | 1 | +| forgejo | gitea | railiance-forge | unknown | 100 | 1 | +| kube-system | coredns-7bdb54f89 | railiance-cluster | unknown | 100 | 1 | +| kube-system | metrics-server-786d997795 | railiance-cluster | unknown | 100 | 1 | +| openbao | openbao | railiance-platform | unknown | 100 | 1 | +| telemetry | prometheus | rapp-telemetry | unknown | 100 | 1 | +| user-engine | user-engine-pg | user-engine | unknown | 100 | 1 | +| telemetry | grafana | rapp-telemetry | unknown | 70 | 1 | +| vergabe-demo-company | vergabe-teilnahme | railiance-apps | unknown | 60 | 1 | +| audit-core | audit-core | audit-core | unknown | 50 | 1 | +| coulomb | ihp-railiance-probe | unknown | unknown | 50 | 1 | +| issue-core | issue-core | issue-core | unknown | 50 | 1 | +| platform-pg-drill | platform-pg-drill | rapp-postgres | unknown | 50 | 1 | +| target-revenue | target-revenue | target-revenue | unknown | 50 | 1 | +| user-engine | user-engine | user-engine | unknown | 50 | 1 | +| canned-prompts | canned-prompts | canned-prompts | unknown | 25 | 1 | +| email-connect | email-connect | email-connect | unknown | 25 | 1 | +| openbao | rapp-openbao | railiance-platform | unknown | 25 | 1 | +| rein-aharness | rein-aharness | rein-aharness | unknown | 25 | 1 | +| sbom-nexus | sbom-nexus | sbom-nexus | unknown | 25 | 1 | +| telemetry | alertmanager | rapp-telemetry | unknown | 25 | 1 | +| telemetry | kube-state-metrics | rapp-telemetry | unknown | 25 | 1 | +| telemetry | rapp-telemetry | rapp-telemetry | unknown | 25 | 1 | +| tenant-engine | tenant-engine | tenant-engine | unknown | 25 | 1 | +| rapp-qonto-egress | qonto-egress-proxy | rapp-qonto | tenant:friendly:binky | 20 | 1 | +| approval-engine | approval-engine | approval-engine | unknown | 10 | 1 | +| policy-nexus | policy-nexus | policy-nexus | unknown | 10 | 1 | +| informed-decision | informed-decision | informed-decision | unknown | 5 | 1 | +| cert-manager | cainjector | railiance-cluster | unknown | 0 | 1 | +| cert-manager | cert-manager | railiance-cluster | unknown | 0 | 1 | +| cert-manager | webhook | railiance-cluster | unknown | 0 | 1 | +| databases | state-hub-db | rapp-postgres | unknown | 0 | 1 | +| external-secrets | external-secrets | railiance-platform | unknown | 0 | 1 | +| external-secrets | external-secrets-cert-controller | railiance-platform | unknown | 0 | 1 | +| external-secrets | external-secrets-webhook | railiance-platform | unknown | 0 | 1 | +| forgejo | forgejo-runner | railiance-forge | unknown | 0 | 1 | +| kube-system | local-path-provisioner | railiance-cluster | unknown | 0 | 1 | +| kube-system | svclb-traefik-0c8aecaf | railiance-cluster | unknown | 0 | 1 | +| kube-system | traefik | railiance-cluster | unknown | 0 | 1 | +| target-revenue | target-revenue-pg | target-revenue | unknown | 0 | 1 | + +## STATE-WP-0091 preflight + +- ok: `False` +- remaining_cpu_m: 15 +- refuse: migration hook needs 50m CPU; remaining 15m +- refuse: API maxSurge=1 needs 100m CPU; remaining 15m (Insufficient CPU class; revision 61 had 65m remaining) +- refuse: unrelated pending demand 25m leaves -10m against API surge 100m +- note: API+MCP concurrent surge would need 110m; remaining 15m. Helm may roll both deployments together; this is not a scheduling guarantee. +- note: Aggregate remaining millicores is not a kube-scheduler guarantee. +- note: Preflight does not lower requests and does not start Helm. + +## Signal notes + +- Host resource:hosteurope:railiance01 and cluster resource:railiance:reef-railiance:k3s are the same 4 vCPU; do not add them. +- Prometheus namespace_cpu:kube_pod_container_resource_requests:sum omitted Forgejo 100m on 2026-09-11; Kubernetes API is the scheduler-effective source. +- node-exporter was disabled; host CPU usage is not a reservation and is not treated as spare capacity. +- Zero-request pods are demand, not zero. Missing metrics are listed as gaps, not zeros. +- STATE-WP-0091 preflight consumes state_hub_preflight_observation; residual millicores are not admission. diff --git a/docs/evidence/namespace-ownership.yaml b/docs/evidence/namespace-ownership.yaml new file mode 100644 index 0000000..bb9c636 --- /dev/null +++ b/docs/evidence/namespace-ownership.yaml @@ -0,0 +1,42 @@ +# Namespace → owner/service mapping for CUST-WP-0071-T01. +# Unknown namespaces stay unknown; do not invent tenants. +namespaces: + knative-serving: {owner: rail-kubernetes, service: knative-serving, tenant: unknown} + kourier-system: {owner: rail-kubernetes, service: kourier, tenant: unknown} + kube-system: {owner: railiance-cluster, service: k3s-control-plane, tenant: unknown} + cert-manager: {owner: railiance-cluster, service: cert-manager, tenant: unknown} + external-secrets: {owner: railiance-platform, service: external-secrets, tenant: unknown} + databases: {owner: rapp-postgres, service: apps-pg, tenant: unknown} + cnpg-system: {owner: rapp-postgres, service: cloudnative-pg, tenant: unknown} + platform-pg-drill: {owner: rapp-postgres, service: platform-pg-drill, tenant: unknown} + telemetry: {owner: rapp-telemetry, service: prometheus-grafana, tenant: unknown} + core-hub: {owner: core-hub, service: core-hub, tenant: unknown} + sso: {owner: net-kingdom, service: keycape-authelia, tenant: unknown} + mfa: {owner: net-kingdom, service: lldap-mfa, tenant: unknown} + user-engine: {owner: user-engine, service: user-engine, tenant: unknown} + tenant-engine: {owner: tenant-engine, service: tenant-engine, tenant: unknown} + openbao: {owner: railiance-platform, service: openbao, tenant: unknown} + reuse: {owner: reuse-surface, service: reuse-surface, tenant: unknown} + state-hub: {owner: state-hub, service: state-hub, tenant: unknown} + activity-core: {owner: activity-core, service: activity-core, tenant: unknown} + coulomb-social: {owner: coulomb-social, service: coulomb-social, tenant: unknown} + flex-auth: {owner: flex-auth, service: flex-auth, tenant: unknown} + forgejo: {owner: railiance-forge, service: forgejo, tenant: unknown} + audit-core: {owner: audit-core, service: audit-core, tenant: unknown} + coulomb: {owner: unknown, service: ihp-railiance-probe, tenant: unknown} + issue-core: {owner: issue-core, service: issue-core, tenant: unknown} + canned-prompts: {owner: canned-prompts, service: canned-prompts, tenant: unknown} + email-connect: {owner: email-connect, service: email-connect, tenant: unknown} + rein-aharness: {owner: rein-aharness, service: rein-aharness, tenant: unknown} + sbom-nexus: {owner: sbom-nexus, service: sbom-nexus, tenant: unknown} + rapp-qonto-egress: {owner: rapp-qonto, service: qonto-egress, tenant: tenant:friendly:binky} + informed-decision: {owner: informed-decision, service: informed-decision, tenant: unknown} + policy-nexus: {owner: policy-nexus, service: policy-nexus, tenant: unknown} + target-revenue: {owner: target-revenue, service: target-revenue, tenant: unknown} + approval-engine: {owner: approval-engine, service: approval-engine, tenant: unknown} + vergabe-demo-company: {owner: railiance-apps, service: vergabe-teilnahme, tenant: unknown} +host: + owner: railiance-cluster + resource_id: resource:hosteurope:railiance01 + same_cpus_as_cluster: true + cluster_resource_id: resource:railiance:reef-railiance:k3s diff --git a/scripts/reconcile_railiance_allocation.py b/scripts/reconcile_railiance_allocation.py new file mode 100644 index 0000000..0e61dcd --- /dev/null +++ b/scripts/reconcile_railiance_allocation.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python3 +"""CUST-WP-0071-T01: join live cluster allocation with owner/service/tenant.""" +from __future__ import annotations + +import argparse +import importlib.util +import json +import sys +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +import yaml + +ROOT = Path(__file__).resolve().parents[1] +CLUSTER_OBS = Path.home() / "railiance-cluster" / "tools" / "observe_cluster_resources.py" +HEADROOM = Path.home() / "state-hub" / "scripts" / "release_headroom_preflight.py" +OWNERS = ROOT / "docs" / "evidence" / "namespace-ownership.yaml" + + +def load_module(path: Path, name: str): + spec = importlib.util.spec_from_file_location(name, path) + if spec is None or spec.loader is None: + raise RuntimeError(f"cannot load {path}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def join(observation: dict[str, Any], owners: dict[str, Any]) -> dict[str, Any]: + mapping = owners.get("namespaces") or {} + rows = [] + unknown = [] + zero_request = [] + for workload in observation.get("utilization", {}).get("workloads") or []: + ns = workload["namespace"] + meta = mapping.get(ns, {"owner": "unknown", "service": "unknown", "tenant": "unknown"}) + row = { + "namespace": ns, + "workload": workload["workload"], + "pods": workload["pods"], + "cpu_request_m": workload["requests"]["cpu_millicores"], + "memory_request_bytes": workload["requests"]["memory_bytes"], + "owner": meta.get("owner", "unknown"), + "service": meta.get("service", "unknown"), + "tenant": meta.get("tenant", "unknown"), + } + rows.append(row) + if ns not in mapping: + unknown.append(ns) + if workload["requests"]["cpu_millicores"] == 0: + zero_request.append(f"{ns}/{workload['workload']}") + pending = observation.get("allocation", {}).get("pending_unscheduled") or {} + alloc = observation.get("allocation") or {} + capacity = observation.get("capacity", {}).get("totals") or {} + residual = (alloc.get("residual_capacity") or {}).get("cpu_millicores", 0) + requested = (alloc.get("denominators") or {}).get("cpu_requested_millicores", 0) + return { + "schema": "custodian.allocation-reconcile.v1", + "workplan_id": "CUST-WP-0071-T01", + "captured_at": observation.get("captured_at"), + "cluster_observation_schema": observation.get("schema_version"), + "count_host_and_cluster_cpus_once": True, + "host": owners.get("host"), + "capacity_cpu_m": capacity.get("cpu_millicores"), + "scheduled_request_cpu_m": requested, + "pending_unscheduled_cpu_m": pending.get("cpu_millicores", 0), + "residual_cpu_m": residual, + "not_a_scheduling_guarantee": True, + "workloads": sorted(rows, key=lambda r: (-r["cpu_request_m"], r["namespace"], r["workload"])), + "unknown_namespaces": sorted(set(unknown)), + "zero_request_workloads": sorted(zero_request), + "pending_unscheduled": pending.get("pods") or [], + "measurement_gaps": observation.get("measurement_gaps") or [], + "state_hub_preflight_observation": observation.get("state_hub_preflight_observation"), + "signal_notes": [ + "Host resource:hosteurope:railiance01 and cluster resource:railiance:reef-railiance:k3s are the same 4 vCPU; do not add them.", + "Prometheus namespace_cpu:kube_pod_container_resource_requests:sum omitted Forgejo 100m on 2026-09-11; Kubernetes API is the scheduler-effective source.", + "node-exporter was disabled; host CPU usage is not a reservation and is not treated as spare capacity.", + "Zero-request pods are demand, not zero. Missing metrics are listed as gaps, not zeros.", + "STATE-WP-0091 preflight consumes state_hub_preflight_observation; residual millicores are not admission.", + ], + } + + +def render_markdown(report: dict[str, Any], preflight: dict[str, Any] | None) -> str: + lines = [ + f"# Railiance allocation reconcile — {report['captured_at']}", + "", + "CUST-WP-0071-T01. Scheduler-effective requests from the Kubernetes API", + "via `railiance-cluster` observe (RCLUSTER-WP-0014 / RAIL-BS-WP-0014).", + "Host and cluster are the same 4 vCPU and are counted once.", + "", + f"- Capacity: {report['capacity_cpu_m']}m", + f"- Scheduled requests: {report['scheduled_request_cpu_m']}m", + f"- Pending unscheduled: {report['pending_unscheduled_cpu_m']}m", + f"- Residual (not a guarantee): {report['residual_cpu_m']}m", + f"- Unknown namespaces: {', '.join(report['unknown_namespaces']) or 'none'}", + f"- Zero-request workloads: {len(report['zero_request_workloads'])}", + "", + "| Namespace | Workload | Owner | Tenant | CPU request (m) | Pods |", + "|---|---|---|---|---:|---:|", + ] + for row in report["workloads"]: + lines.append( + f"| {row['namespace']} | {row['workload']} | {row['owner']} | {row['tenant']} | {row['cpu_request_m']} | {row['pods']} |" + ) + lines += ["", "## STATE-WP-0091 preflight", ""] + if preflight: + lines.append(f"- ok: `{preflight.get('ok')}`") + lines.append(f"- remaining_cpu_m: {preflight.get('remaining_cpu_m')}") + for reason in preflight.get("reasons") or []: + lines.append(f"- refuse: {reason}") + for note in preflight.get("notes") or []: + lines.append(f"- note: {note}") + lines += ["", "## Signal notes", ""] + for note in report["signal_notes"]: + lines.append(f"- {note}") + lines.append("") + return "\n".join(lines) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--observation", type=Path, help="Existing cluster observation JSON") + parser.add_argument("--kubectl", default="kubectl") + parser.add_argument("--output-dir", type=Path, default=ROOT / "docs" / "evidence") + args = parser.parse_args() + cluster = load_module(CLUSTER_OBS, "cluster_resources") + if args.observation: + observation = json.loads(args.observation.read_text()) + else: + observation = cluster.collect(args.kubectl) + owners = yaml.safe_load(OWNERS.read_text()) + report = join(observation, owners) + preflight = None + if observation.get("state_hub_preflight_observation"): + headroom = load_module(HEADROOM, "release_headroom") + preflight = headroom.evaluate(observation["state_hub_preflight_observation"]) + report["state_hub_preflight"] = preflight + stamp = datetime.now(UTC).strftime("%Y-%m-%d") + out_json = args.output_dir / f"{stamp}-allocation-reconcile.json" + out_md = args.output_dir / f"{stamp}-allocation-reconcile.md" + args.output_dir.mkdir(parents=True, exist_ok=True) + out_json.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") + out_md.write_text(render_markdown(report, preflight), encoding="utf-8") + print(out_json) + print(out_md) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/test_reconcile_railiance_allocation.py b/scripts/test_reconcile_railiance_allocation.py new file mode 100644 index 0000000..9dc3fae --- /dev/null +++ b/scripts/test_reconcile_railiance_allocation.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +import unittest +from pathlib import Path + +import yaml + +import reconcile_railiance_allocation as rec + +OWNERS = yaml.safe_load((Path(__file__).resolve().parents[1] / "docs" / "evidence" / "namespace-ownership.yaml").read_text()) + + +class ReconcileTests(unittest.TestCase): + def test_unknown_and_pending_are_explicit(self) -> None: + observation = { + "schema_version": "railiance.cluster-resource-observation.v1", + "captured_at": "2026-09-14T00:00:00Z", + "capacity": {"totals": {"cpu_millicores": 4000}}, + "utilization": {"workloads": [ + {"namespace": "state-hub", "workload": "state-hub", "pods": 2, "requests": {"cpu_millicores": 110, "memory_bytes": 1}}, + {"namespace": "brand-new", "workload": "x", "pods": 1, "requests": {"cpu_millicores": 0, "memory_bytes": 0}}, + ]}, + "allocation": { + "denominators": {"cpu_requested_millicores": 110}, + "residual_capacity": {"cpu_millicores": 3890}, + "pending_unscheduled": {"cpu_millicores": 25, "pods": [{"namespace": "other", "name": "p"}]}, + }, + "measurement_gaps": ["pod metrics unavailable"], + } + report = rec.join(observation, OWNERS) + self.assertEqual(["brand-new"], report["unknown_namespaces"]) + self.assertIn("brand-new/x", report["zero_request_workloads"]) + self.assertEqual(25, report["pending_unscheduled_cpu_m"]) + self.assertEqual("state-hub", report["workloads"][0]["owner"]) + self.assertTrue(report["count_host_and_cluster_cpus_once"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/workplans/CUST-WP-0071-measured-workload-sizing-and-weekly-review.md b/workplans/CUST-WP-0071-measured-workload-sizing-and-weekly-review.md index c89647d..41c1409 100644 --- a/workplans/CUST-WP-0071-measured-workload-sizing-and-weekly-review.md +++ b/workplans/CUST-WP-0071-measured-workload-sizing-and-weekly-review.md @@ -8,7 +8,7 @@ status: ready owner: the-custodian topic_slug: custodian created: "2026-09-11" -updated: "2026-09-11" +updated: "2026-09-14" related: [STATE-WP-0091, RCLUSTER-WP-0014, RESOURCE-WP-0003, RAPP-TELEMETRY-WP-0001, RAIL-FAB-WP-0028, RAPPS-WP-0014, VERGABE-WP-0019, HFACT-WP-0001] state_hub_workstream_id: "2249bddb-7524-5add-bd5c-c4163a6ca0f3" ---