coulomb/rail-knative
5
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

RAIL-KNATIVE-WP-0003: cap Knative HPAs for one node, trim substrate CPU requests to 60m

Browse source 
Declare maxReplicas activator 20->3, webhook 5->2, 3scale-kourier-gateway
10->3 and 10m CPU requests for all six substrate containers (220m->60m),
per the resource-control node budget of 2026-10-21 (CUST-WP-0071,
RESOURCE-WP-0008). Read-only BEFORE evidence under
docs/evidence/2026-10-21-hpa-caps-before/. Not applied; railiance-cluster
overlays must follow (handoff in T02).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

Assistant: claude-code
Assistant-Model: opus
Assistant-Process: 8340@bnt-lap001
Assistant-Session: 2e192b3d-b3e8-4403-94d4-d55a2c8b476c
This commit is contained in:
tegwick 2026-10-21 08:40:35 +02:00
parent a0e4093a18
commit 76cb9c7ad9
14 changed files with 1365 additions and 30 deletions
Download patch file Download diff file Expand all files Collapse all files

2
docs/evidence/2026-10-21-hpa-caps-before/captured-at.txt Normal file
View file

@ -0,0 +1,2 @@
2026-10-21T06:38:16Z
default

711
docs/evidence/2026-10-21-hpa-caps-before/deploy-knative-serving.yaml Normal file
View file

@ -0,0 +1,711 @@
apiVersion: v1
items:
- apiVersion: apps/v1
kind: Deployment
metadata:
annotations:
deployment.kubernetes.io/revision: "2"
kubectl.kubernetes.io/last-applied-configuration: |
{"apiVersion":"apps/v1","kind":"Deployment","metadata":{"annotations":{},"labels":{"app.kubernetes.io/component":"activator","app.kubernetes.io/name":"knative-serving","app.kubernetes.io/version":"1.22.0"},"name":"activator","namespace":"knative-serving"},"spec":{"selector":{"matchLabels":{"app":"activator","role":"activator"}},"template":{"metadata":{"labels":{"app":"activator","app.kubernetes.io/component":"activator","app.kubernetes.io/name":"knative-serving","app.kubernetes.io/version":"1.22.0","role":"activator"}},"spec":{"affinity":{"podAntiAffinity":{"preferredDuringSchedulingIgnoredDuringExecution":[{"podAffinityTerm":{"labelSelector":{"matchLabels":{"app":"activator"}},"topologyKey":"kubernetes.io/hostname"},"weight":100}]}},"containers":[{"env":[{"name":"GOGC","value":"500"},{"name":"POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"POD_IP","valueFrom":{"fieldRef":{"fieldPath":"status.podIP"}}},{"name":"SYSTEM_NAMESPACE","valueFrom":{"fieldRef":{"fieldPath":"metadata.namespace"}}},{"name":"CONFIG_LOGGING_NAME","value":"config-logging"},{"name":"CONFIG_OBSERVABILITY_NAME","value":"config-observability"}],"image":"gcr.io/knative-releases/knative.dev/serving/cmd/activator@sha256:8f7e9df2642a8ba715ebc55b62abee17b0984b6b6af0c6ec00dabaadf5f8cccb","livenessProbe":{"failureThreshold":12,"httpGet":{"port":8012},"initialDelaySeconds":15,"periodSeconds":10},"name":"activator","ports":[{"containerPort":9090,"name":"metrics"},{"containerPort":8008,"name":"profiling"},{"containerPort":8012,"name":"http1"},{"containerPort":8013,"name":"h2c"}],"readinessProbe":{"failureThreshold":5,"httpGet":{"port":8012},"periodSeconds":5},"resources":{"limits":{"cpu":"1000m","memory":"600Mi"},"requests":{"cpu":"300m","memory":"60Mi"}},"securityContext":{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]},"readOnlyRootFilesystem":true,"runAsNonRoot":true,"seccompProfile":{"type":"RuntimeDefault"}}}],"serviceAccountName":"activator","terminationGracePeriodSeconds":600}}}}
creationTimestamp: "2026-07-26T16:45:42Z"
generation: 2
labels:
app.kubernetes.io/component: activator
app.kubernetes.io/name: knative-serving
app.kubernetes.io/version: 1.22.0
name: activator
namespace: knative-serving
resourceVersion: "75292980"
uid: 855e1767-8c4e-43ac-b5a7-94bf0f5a1cef
spec:
progressDeadlineSeconds: 600
replicas: 1
revisionHistoryLimit: 10
selector:
matchLabels:
app: activator
role: activator
strategy:
rollingUpdate:
maxSurge: 25%
maxUnavailable: 25%
type: RollingUpdate
template:
metadata:
labels:
app: activator
app.kubernetes.io/component: activator
app.kubernetes.io/name: knative-serving
app.kubernetes.io/version: 1.22.0
role: activator
spec:
affinity:
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- podAffinityTerm:
labelSelector:
matchLabels:
app: activator
topologyKey: kubernetes.io/hostname
weight: 100
containers:
- env:
- name: GOGC
value: "500"
- name: POD_NAME
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.name
- name: POD_IP
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: status.podIP
- name: SYSTEM_NAMESPACE
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.namespace
- name: CONFIG_LOGGING_NAME
value: config-logging
- name: CONFIG_OBSERVABILITY_NAME
value: config-observability
image: gcr.io/knative-releases/knative.dev/serving/cmd/activator@sha256:8f7e9df2642a8ba715ebc55b62abee17b0984b6b6af0c6ec00dabaadf5f8cccb
imagePullPolicy: IfNotPresent
livenessProbe:
failureThreshold: 12
httpGet:
path: /
port: 8012
scheme: HTTP
initialDelaySeconds: 15
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 1
name: activator
ports:
- containerPort: 9090
name: metrics
protocol: TCP
- containerPort: 8008
name: profiling
protocol: TCP
- containerPort: 8012
name: http1
protocol: TCP
- containerPort: 8013
name: h2c
protocol: TCP
readinessProbe:
failureThreshold: 5
httpGet:
path: /
port: 8012
scheme: HTTP
periodSeconds: 5
successThreshold: 1
timeoutSeconds: 1
resources:
limits:
cpu: "1"
memory: 600Mi
requests:
cpu: 50m
memory: 60Mi
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
readOnlyRootFilesystem: true
runAsNonRoot: true
seccompProfile:
type: RuntimeDefault
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
dnsPolicy: ClusterFirst
restartPolicy: Always
schedulerName: default-scheduler
securityContext: {}
serviceAccount: activator
serviceAccountName: activator
terminationGracePeriodSeconds: 600
status:
availableReplicas: 1
conditions:
- lastTransitionTime: "2026-07-26T16:45:42Z"
lastUpdateTime: "2026-09-21T16:29:55Z"
message: ReplicaSet "activator-6b4748664b" has successfully progressed.
reason: NewReplicaSetAvailable
status: "True"
type: Progressing
- lastTransitionTime: "2026-10-14T18:23:05Z"
lastUpdateTime: "2026-10-14T18:23:05Z"
message: Deployment has minimum availability.
reason: MinimumReplicasAvailable
status: "True"
type: Available
observedGeneration: 2
readyReplicas: 1
replicas: 1
terminatingReplicas: 0
updatedReplicas: 1
- apiVersion: apps/v1
kind: Deployment
metadata:
annotations:
deployment.kubernetes.io/revision: "2"
kubectl.kubernetes.io/last-applied-configuration: |
{"apiVersion":"apps/v1","kind":"Deployment","metadata":{"annotations":{},"labels":{"app.kubernetes.io/component":"autoscaler","app.kubernetes.io/name":"knative-serving","app.kubernetes.io/version":"1.22.0"},"name":"autoscaler","namespace":"knative-serving"},"spec":{"replicas":1,"selector":{"matchLabels":{"app":"autoscaler"}},"strategy":{"rollingUpdate":{"maxUnavailable":0},"type":"RollingUpdate"},"template":{"metadata":{"labels":{"app":"autoscaler","app.kubernetes.io/component":"autoscaler","app.kubernetes.io/name":"knative-serving","app.kubernetes.io/version":"1.22.0"}},"spec":{"affinity":{"podAntiAffinity":{"preferredDuringSchedulingIgnoredDuringExecution":[{"podAffinityTerm":{"labelSelector":{"matchLabels":{"app":"autoscaler"}},"topologyKey":"kubernetes.io/hostname"},"weight":100}]}},"containers":[{"env":[{"name":"POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"POD_IP","valueFrom":{"fieldRef":{"fieldPath":"status.podIP"}}},{"name":"SYSTEM_NAMESPACE","valueFrom":{"fieldRef":{"fieldPath":"metadata.namespace"}}},{"name":"CONFIG_LOGGING_NAME","value":"config-logging"},{"name":"CONFIG_OBSERVABILITY_NAME","value":"config-observability"}],"image":"gcr.io/knative-releases/knative.dev/serving/cmd/autoscaler@sha256:ec66d97244395b57a67923d4a8c5a95f0264805f1fdb2ee0cdb49ec0f3bd7855","livenessProbe":{"failureThreshold":6,"httpGet":{"port":8080}},"name":"autoscaler","ports":[{"containerPort":9090,"name":"metrics"},{"containerPort":8008,"name":"profiling"},{"containerPort":8080,"name":"websocket"}],"readinessProbe":{"httpGet":{"port":8080}},"resources":{"limits":{"cpu":"1000m","memory":"1000Mi"},"requests":{"cpu":"100m","memory":"100Mi"}},"securityContext":{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]},"readOnlyRootFilesystem":true,"runAsNonRoot":true,"seccompProfile":{"type":"RuntimeDefault"}}}],"serviceAccountName":"controller"}}}}
creationTimestamp: "2026-07-26T16:45:42Z"
generation: 2
labels:
app.kubernetes.io/component: autoscaler
app.kubernetes.io/name: knative-serving
app.kubernetes.io/version: 1.22.0
name: autoscaler
namespace: knative-serving
resourceVersion: "75292939"
uid: 24c54b27-5e45-41ef-8c8e-5b40c1733a5d
spec:
progressDeadlineSeconds: 600
replicas: 1
revisionHistoryLimit: 10
selector:
matchLabels:
app: autoscaler
strategy:
rollingUpdate:
maxSurge: 25%
maxUnavailable: 0
type: RollingUpdate
template:
metadata:
labels:
app: autoscaler
app.kubernetes.io/component: autoscaler
app.kubernetes.io/name: knative-serving
app.kubernetes.io/version: 1.22.0
spec:
affinity:
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- podAffinityTerm:
labelSelector:
matchLabels:
app: autoscaler
topologyKey: kubernetes.io/hostname
weight: 100
containers:
- env:
- name: POD_NAME
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.name
- name: POD_IP
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: status.podIP
- name: SYSTEM_NAMESPACE
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.namespace
- name: CONFIG_LOGGING_NAME
value: config-logging
- name: CONFIG_OBSERVABILITY_NAME
value: config-observability
image: gcr.io/knative-releases/knative.dev/serving/cmd/autoscaler@sha256:ec66d97244395b57a67923d4a8c5a95f0264805f1fdb2ee0cdb49ec0f3bd7855
imagePullPolicy: IfNotPresent
livenessProbe:
failureThreshold: 6
httpGet:
path: /
port: 8080
scheme: HTTP
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 1
name: autoscaler
ports:
- containerPort: 9090
name: metrics
protocol: TCP
- containerPort: 8008
name: profiling
protocol: TCP
- containerPort: 8080
name: websocket
protocol: TCP
readinessProbe:
failureThreshold: 3
httpGet:
path: /
port: 8080
scheme: HTTP
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 1
resources:
limits:
cpu: "1"
memory: 1000Mi
requests:
cpu: 30m
memory: 100Mi
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
readOnlyRootFilesystem: true
runAsNonRoot: true
seccompProfile:
type: RuntimeDefault
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
dnsPolicy: ClusterFirst
restartPolicy: Always
schedulerName: default-scheduler
securityContext: {}
serviceAccount: controller
serviceAccountName: controller
terminationGracePeriodSeconds: 30
status:
availableReplicas: 1
conditions:
- lastTransitionTime: "2026-07-26T16:45:42Z"
lastUpdateTime: "2026-09-21T16:30:02Z"
message: ReplicaSet "autoscaler-764bdc4bc8" has successfully progressed.
reason: NewReplicaSetAvailable
status: "True"
type: Progressing
- lastTransitionTime: "2026-10-14T18:23:04Z"
lastUpdateTime: "2026-10-14T18:23:04Z"
message: Deployment has minimum availability.
reason: MinimumReplicasAvailable
status: "True"
type: Available
observedGeneration: 2
readyReplicas: 1
replicas: 1
terminatingReplicas: 0
updatedReplicas: 1
- apiVersion: apps/v1
kind: Deployment
metadata:
annotations:
deployment.kubernetes.io/revision: "2"
kubectl.kubernetes.io/last-applied-configuration: |
{"apiVersion":"apps/v1","kind":"Deployment","metadata":{"annotations":{},"labels":{"app.kubernetes.io/component":"controller","app.kubernetes.io/name":"knative-serving","app.kubernetes.io/version":"1.22.0"},"name":"controller","namespace":"knative-serving"},"spec":{"selector":{"matchLabels":{"app":"controller"}},"template":{"metadata":{"labels":{"app":"controller","app.kubernetes.io/component":"controller","app.kubernetes.io/name":"knative-serving","app.kubernetes.io/version":"1.22.0"}},"spec":{"affinity":{"podAntiAffinity":{"preferredDuringSchedulingIgnoredDuringExecution":[{"podAffinityTerm":{"labelSelector":{"matchLabels":{"app":"controller"}},"topologyKey":"kubernetes.io/hostname"},"weight":100}]}},"containers":[{"env":[{"name":"POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"SYSTEM_NAMESPACE","valueFrom":{"fieldRef":{"fieldPath":"metadata.namespace"}}},{"name":"CONFIG_LOGGING_NAME","value":"config-logging"},{"name":"CONFIG_OBSERVABILITY_NAME","value":"config-observability"}],"image":"gcr.io/knative-releases/knative.dev/serving/cmd/controller@sha256:6833f2c83e9f356c274ddb2de40e5471ad2b1c08882c9bd11f5afa240e86d574","livenessProbe":{"failureThreshold":6,"httpGet":{"path":"/health","port":"probes","scheme":"HTTP"},"periodSeconds":5},"name":"controller","ports":[{"containerPort":9090,"name":"metrics"},{"containerPort":8008,"name":"profiling"},{"containerPort":8080,"name":"probes"}],"readinessProbe":{"failureThreshold":3,"httpGet":{"path":"/readiness","port":"probes","scheme":"HTTP"},"periodSeconds":5},"resources":{"limits":{"cpu":"1000m","memory":"1000Mi"},"requests":{"cpu":"100m","memory":"100Mi"}},"securityContext":{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]},"readOnlyRootFilesystem":true,"runAsNonRoot":true,"seccompProfile":{"type":"RuntimeDefault"}}}],"serviceAccountName":"controller"}}}}
creationTimestamp: "2026-07-26T16:45:42Z"
generation: 2
labels:
app.kubernetes.io/component: controller
app.kubernetes.io/name: knative-serving
app.kubernetes.io/version: 1.22.0
name: controller
namespace: knative-serving
resourceVersion: "75293078"
uid: a8bfc048-b465-43dd-aa0a-15c8722edd5b
spec:
progressDeadlineSeconds: 600
replicas: 1
revisionHistoryLimit: 10
selector:
matchLabels:
app: controller
strategy:
rollingUpdate:
maxSurge: 25%
maxUnavailable: 25%
type: RollingUpdate
template:
metadata:
labels:
app: controller
app.kubernetes.io/component: controller
app.kubernetes.io/name: knative-serving
app.kubernetes.io/version: 1.22.0
spec:
affinity:
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- podAffinityTerm:
labelSelector:
matchLabels:
app: controller
topologyKey: kubernetes.io/hostname
weight: 100
containers:
- env:
- name: POD_NAME
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.name
- name: SYSTEM_NAMESPACE
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.namespace
- name: CONFIG_LOGGING_NAME
value: config-logging
- name: CONFIG_OBSERVABILITY_NAME
value: config-observability
image: gcr.io/knative-releases/knative.dev/serving/cmd/controller@sha256:6833f2c83e9f356c274ddb2de40e5471ad2b1c08882c9bd11f5afa240e86d574
imagePullPolicy: IfNotPresent
livenessProbe:
failureThreshold: 6
httpGet:
path: /health
port: probes
scheme: HTTP
periodSeconds: 5
successThreshold: 1
timeoutSeconds: 1
name: controller
ports:
- containerPort: 9090
name: metrics
protocol: TCP
- containerPort: 8008
name: profiling
protocol: TCP
- containerPort: 8080
name: probes
protocol: TCP
readinessProbe:
failureThreshold: 3
httpGet:
path: /readiness
port: probes
scheme: HTTP
periodSeconds: 5
successThreshold: 1
timeoutSeconds: 1
resources:
limits:
cpu: "1"
memory: 1000Mi
requests:
cpu: 30m
memory: 100Mi
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
readOnlyRootFilesystem: true
runAsNonRoot: true
seccompProfile:
type: RuntimeDefault
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
dnsPolicy: ClusterFirst
restartPolicy: Always
schedulerName: default-scheduler
securityContext: {}
serviceAccount: controller
serviceAccountName: controller
terminationGracePeriodSeconds: 30
status:
availableReplicas: 1
conditions:
- lastTransitionTime: "2026-07-26T16:45:42Z"
lastUpdateTime: "2026-09-21T16:29:55Z"
message: ReplicaSet "controller-547767bf48" has successfully progressed.
reason: NewReplicaSetAvailable
status: "True"
type: Progressing
- lastTransitionTime: "2026-10-14T18:23:08Z"
lastUpdateTime: "2026-10-14T18:23:08Z"
message: Deployment has minimum availability.
reason: MinimumReplicasAvailable
status: "True"
type: Available
observedGeneration: 2
readyReplicas: 1
replicas: 1
terminatingReplicas: 0
updatedReplicas: 1
- apiVersion: apps/v1
kind: Deployment
metadata:
annotations:
deployment.kubernetes.io/revision: "2"
kubectl.kubernetes.io/last-applied-configuration: |
{"apiVersion":"apps/v1","kind":"Deployment","metadata":{"annotations":{},"labels":{"app.kubernetes.io/component":"webhook","app.kubernetes.io/name":"knative-serving","app.kubernetes.io/version":"1.22.0"},"name":"webhook","namespace":"knative-serving"},"spec":{"selector":{"matchLabels":{"app":"webhook","role":"webhook"}},"template":{"metadata":{"labels":{"app":"webhook","app.kubernetes.io/component":"webhook","app.kubernetes.io/name":"knative-serving","app.kubernetes.io/version":"1.22.0","role":"webhook"}},"spec":{"affinity":{"podAntiAffinity":{"preferredDuringSchedulingIgnoredDuringExecution":[{"podAffinityTerm":{"labelSelector":{"matchLabels":{"app":"webhook"}},"topologyKey":"kubernetes.io/hostname"},"weight":100}]}},"containers":[{"env":[{"name":"POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"SYSTEM_NAMESPACE","valueFrom":{"fieldRef":{"fieldPath":"metadata.namespace"}}},{"name":"CONFIG_LOGGING_NAME","value":"config-logging"},{"name":"CONFIG_OBSERVABILITY_NAME","value":"config-observability"},{"name":"WEBHOOK_NAME","value":"webhook"},{"name":"WEBHOOK_PORT","value":"8443"}],"image":"gcr.io/knative-releases/knative.dev/serving/cmd/webhook@sha256:c505888f7ee34ccadc1b9a6877d6d49691f633ea28cf8cbe03fcfa6d3afcd584","livenessProbe":{"failureThreshold":6,"httpGet":{"port":8443,"scheme":"HTTPS"},"initialDelaySeconds":20,"periodSeconds":10},"name":"webhook","ports":[{"containerPort":9090,"name":"metrics"},{"containerPort":8008,"name":"profiling"},{"containerPort":8443,"name":"https-webhook"}],"readinessProbe":{"httpGet":{"port":8443,"scheme":"HTTPS"},"periodSeconds":1},"resources":{"limits":{"cpu":"500m","memory":"500Mi"},"requests":{"cpu":"100m","memory":"100Mi"}},"securityContext":{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]},"readOnlyRootFilesystem":true,"runAsNonRoot":true,"seccompProfile":{"type":"RuntimeDefault"}}}],"serviceAccountName":"controller","terminationGracePeriodSeconds":300}}}}
creationTimestamp: "2026-07-26T16:45:42Z"
generation: 2
labels:
app.kubernetes.io/component: webhook
app.kubernetes.io/name: knative-serving
app.kubernetes.io/version: 1.22.0
name: webhook
namespace: knative-serving
resourceVersion: "75292683"
uid: 651670a3-f99d-4828-b5e0-a4b24a3428ea
spec:
progressDeadlineSeconds: 600
replicas: 1
revisionHistoryLimit: 10
selector:
matchLabels:
app: webhook
role: webhook
strategy:
rollingUpdate:
maxSurge: 25%
maxUnavailable: 25%
type: RollingUpdate
template:
metadata:
labels:
app: webhook
app.kubernetes.io/component: webhook
app.kubernetes.io/name: knative-serving
app.kubernetes.io/version: 1.22.0
role: webhook
spec:
affinity:
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- podAffinityTerm:
labelSelector:
matchLabels:
app: webhook
topologyKey: kubernetes.io/hostname
weight: 100
containers:
- env:
- name: POD_NAME
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.name
- name: SYSTEM_NAMESPACE
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.namespace
- name: CONFIG_LOGGING_NAME
value: config-logging
- name: CONFIG_OBSERVABILITY_NAME
value: config-observability
- name: WEBHOOK_NAME
value: webhook
- name: WEBHOOK_PORT
value: "8443"
image: gcr.io/knative-releases/knative.dev/serving/cmd/webhook@sha256:c505888f7ee34ccadc1b9a6877d6d49691f633ea28cf8cbe03fcfa6d3afcd584
imagePullPolicy: IfNotPresent
livenessProbe:
failureThreshold: 6
httpGet:
path: /
port: 8443
scheme: HTTPS
initialDelaySeconds: 20
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 1
name: webhook
ports:
- containerPort: 9090
name: metrics
protocol: TCP
- containerPort: 8008
name: profiling
protocol: TCP
- containerPort: 8443
name: https-webhook
protocol: TCP
readinessProbe:
failureThreshold: 3
httpGet:
path: /
port: 8443
scheme: HTTPS
periodSeconds: 1
successThreshold: 1
timeoutSeconds: 1
resources:
limits:
cpu: 500m
memory: 500Mi
requests:
cpu: 30m
memory: 100Mi
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
readOnlyRootFilesystem: true
runAsNonRoot: true
seccompProfile:
type: RuntimeDefault
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
dnsPolicy: ClusterFirst
restartPolicy: Always
schedulerName: default-scheduler
securityContext: {}
serviceAccount: controller
serviceAccountName: controller
terminationGracePeriodSeconds: 300
status:
availableReplicas: 1
conditions:
- lastTransitionTime: "2026-07-26T16:45:42Z"
lastUpdateTime: "2026-09-21T16:29:51Z"
message: ReplicaSet "webhook-8566f4d4bb" has successfully progressed.
reason: NewReplicaSetAvailable
status: "True"
type: Progressing
- lastTransitionTime: "2026-10-14T18:22:58Z"
lastUpdateTime: "2026-10-14T18:22:58Z"
message: Deployment has minimum availability.
reason: MinimumReplicasAvailable
status: "True"
type: Available
observedGeneration: 2
readyReplicas: 1
replicas: 1
terminatingReplicas: 0
updatedReplicas: 1
- apiVersion: apps/v1
kind: Deployment
metadata:
annotations:
deployment.kubernetes.io/revision: "2"
kubectl.kubernetes.io/last-applied-configuration: |
{"apiVersion":"apps/v1","kind":"Deployment","metadata":{"annotations":{},"labels":{"app.kubernetes.io/component":"net-kourier","app.kubernetes.io/name":"knative-serving","app.kubernetes.io/version":"1.22.0","networking.knative.dev/ingress-provider":"kourier"},"name":"net-kourier-controller","namespace":"knative-serving"},"spec":{"replicas":1,"selector":{"matchLabels":{"app":"net-kourier-controller"}},"strategy":{"rollingUpdate":{"maxSurge":"100%","maxUnavailable":0},"type":"RollingUpdate"},"template":{"metadata":{"annotations":{"prometheus.io/path":"/metrics","prometheus.io/port":"9090","prometheus.io/scrape":"true"},"labels":{"app":"net-kourier-controller"}},"spec":{"containers":[{"env":[{"name":"CERTS_SECRET_NAMESPACE","value":""},{"name":"CERTS_SECRET_NAME","value":""},{"name":"SYSTEM_NAMESPACE","valueFrom":{"fieldRef":{"fieldPath":"metadata.namespace"}}},{"name":"METRICS_DOMAIN","value":"knative.dev/samples"},{"name":"KOURIER_GATEWAY_NAMESPACE","value":"kourier-system"},{"name":"ENABLE_SECRET_INFORMER_FILTERING_BY_CERT_UID","value":"false"},{"name":"KUBE_API_BURST","value":"200"},{"name":"KUBE_API_QPS","value":"200"}],"image":"gcr.io/knative-releases/knative.dev/net-kourier/cmd/kourier@sha256:a2e8f568a4bf68ef6391bc3afea956d11a8698f6a22c4be50844b1da4f8bcc74","livenessProbe":{"failureThreshold":6,"grpc":{"port":18000},"periodSeconds":10},"name":"controller","ports":[{"containerPort":18000,"name":"http2-xds","protocol":"TCP"},{"containerPort":9090,"name":"metrics","protocol":"TCP"}],"readinessProbe":{"failureThreshold":3,"grpc":{"port":18000},"periodSeconds":10},"resources":{"limits":{"cpu":"1","memory":"500Mi"},"requests":{"cpu":"200m","memory":"200Mi"}},"securityContext":{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]},"readOnlyRootFilesystem":true,"runAsNonRoot":true,"seccompProfile":{"type":"RuntimeDefault"}}}],"restartPolicy":"Always","serviceAccountName":"net-kourier"}}}}
creationTimestamp: "2026-07-26T16:45:57Z"
generation: 2
labels:
app.kubernetes.io/component: net-kourier
app.kubernetes.io/name: knative-serving
app.kubernetes.io/version: 1.22.0
networking.knative.dev/ingress-provider: kourier
name: net-kourier-controller
namespace: knative-serving
resourceVersion: "75293158"
uid: 94df7e95-cf6b-457a-8d27-490a0f9f2816
spec:
progressDeadlineSeconds: 600
replicas: 1
revisionHistoryLimit: 10
selector:
matchLabels:
app: net-kourier-controller
strategy:
rollingUpdate:
maxSurge: 100%
maxUnavailable: 0
type: RollingUpdate
template:
metadata:
annotations:
prometheus.io/path: /metrics
prometheus.io/port: "9090"
prometheus.io/scrape: "true"
labels:
app: net-kourier-controller
spec:
containers:
- env:
- name: CERTS_SECRET_NAMESPACE
- name: CERTS_SECRET_NAME
- name: SYSTEM_NAMESPACE
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.namespace
- name: METRICS_DOMAIN
value: knative.dev/samples
- name: KOURIER_GATEWAY_NAMESPACE
value: kourier-system
- name: ENABLE_SECRET_INFORMER_FILTERING_BY_CERT_UID
value: "false"
- name: KUBE_API_BURST
value: "200"
- name: KUBE_API_QPS
value: "200"
image: gcr.io/knative-releases/knative.dev/net-kourier/cmd/kourier@sha256:a2e8f568a4bf68ef6391bc3afea956d11a8698f6a22c4be50844b1da4f8bcc74
imagePullPolicy: IfNotPresent
livenessProbe:
failureThreshold: 6
grpc:
port: 18000
service: ""
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 1
name: controller
ports:
- containerPort: 18000
name: http2-xds
protocol: TCP
- containerPort: 9090
name: metrics
protocol: TCP
readinessProbe:
failureThreshold: 3
grpc:
port: 18000
service: ""
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 1
resources:
limits:
cpu: "1"
memory: 500Mi
requests:
cpu: 30m
memory: 200Mi
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
readOnlyRootFilesystem: true
runAsNonRoot: true
seccompProfile:
type: RuntimeDefault
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
dnsPolicy: ClusterFirst
restartPolicy: Always
schedulerName: default-scheduler
securityContext: {}
serviceAccount: net-kourier
serviceAccountName: net-kourier
terminationGracePeriodSeconds: 30
status:
availableReplicas: 1
conditions:
- lastTransitionTime: "2026-07-26T16:45:57Z"
lastUpdateTime: "2026-09-21T16:29:41Z"
message: ReplicaSet "net-kourier-controller-5bc4c79595" has successfully progressed.
reason: NewReplicaSetAvailable
status: "True"
type: Progressing
- lastTransitionTime: "2026-10-14T18:23:11Z"
lastUpdateTime: "2026-10-14T18:23:11Z"
message: Deployment has minimum availability.
reason: MinimumReplicasAvailable
status: "True"
type: Available
observedGeneration: 2
readyReplicas: 1
replicas: 1
terminatingReplicas: 0
updatedReplicas: 1
kind: List
metadata:
resourceVersion: ""

161
docs/evidence/2026-10-21-hpa-caps-before/deploy-kourier-system.yaml Normal file
View file

@ -0,0 +1,161 @@
apiVersion: apps/v1
kind: Deployment
metadata:
annotations:
deployment.kubernetes.io/revision: "3"
kubectl.kubernetes.io/last-applied-configuration: |
{"apiVersion":"apps/v1","kind":"Deployment","metadata":{"annotations":{},"labels":{"app.kubernetes.io/component":"net-kourier","app.kubernetes.io/name":"knative-serving","app.kubernetes.io/version":"1.22.0","networking.knative.dev/ingress-provider":"kourier"},"name":"3scale-kourier-gateway","namespace":"kourier-system"},"spec":{"selector":{"matchLabels":{"app":"3scale-kourier-gateway"}},"strategy":{"rollingUpdate":{"maxSurge":"100%","maxUnavailable":0},"type":"RollingUpdate"},"template":{"metadata":{"annotations":{"networking.knative.dev/poke":"v0.26","prometheus.io/path":"/stats/prometheus","prometheus.io/port":"9000","prometheus.io/scrape":"true"},"labels":{"app":"3scale-kourier-gateway"}},"spec":{"containers":[{"args":["--base-id 1","-c /tmp/config/envoy-bootstrap.yaml","--log-level info","--drain-time-s $(DRAIN_TIME_SECONDS)","--drain-strategy immediate"],"command":["/usr/local/bin/envoy"],"env":[{"name":"DRAIN_TIME_SECONDS","value":"15"}],"image":"docker.io/envoyproxy/envoy:v1.37-latest","lifecycle":{"preStop":{"exec":{"command":["/bin/sh","-c","curl -X POST http://localhost:9901/drain_listeners?graceful; sleep $DRAIN_TIME_SECONDS"]}}},"livenessProbe":{"failureThreshold":6,"httpGet":{"httpHeaders":[{"name":"Host","value":"internalkourier"}],"path":"/ready","port":8081,"scheme":"HTTP"},"initialDelaySeconds":10,"periodSeconds":5,"timeoutSeconds":3},"name":"kourier-gateway","ports":[{"containerPort":8080,"name":"http2-external","protocol":"TCP"},{"containerPort":8081,"name":"http2-internal","protocol":"TCP"},{"containerPort":8443,"name":"https-external","protocol":"TCP"},{"containerPort":8090,"name":"http-probe","protocol":"TCP"},{"containerPort":9443,"name":"https-probe","protocol":"TCP"},{"containerPort":9000,"name":"metrics","protocol":"TCP"}],"readinessProbe":{"failureThreshold":3,"httpGet":{"httpHeaders":[{"name":"Host","value":"internalkourier"}],"path":"/ready","port":8081,"scheme":"HTTP"},"initialDelaySeconds":10,"periodSeconds":5,"timeoutSeconds":3},"resources":{"limits":{"cpu":"1","memory":"800Mi"},"requests":{"cpu":"200m","memory":"200Mi"}},"securityContext":{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]},"readOnlyRootFilesystem":false,"runAsGroup":65534,"runAsNonRoot":true,"runAsUser":65534,"seccompProfile":{"type":"RuntimeDefault"}},"volumeMounts":[{"mountPath":"/tmp/config","name":"config-volume"}]}],"restartPolicy":"Always","terminationGracePeriodSeconds":30,"volumes":[{"configMap":{"name":"kourier-bootstrap"},"name":"config-volume"}]}}}}
creationTimestamp: "2026-07-26T16:45:57Z"
generation: 3
labels:
app.kubernetes.io/component: net-kourier
app.kubernetes.io/name: knative-serving
app.kubernetes.io/version: 1.22.0
networking.knative.dev/ingress-provider: kourier
name: 3scale-kourier-gateway
namespace: kourier-system
resourceVersion: "75293597"
uid: 562459bf-ed8a-4654-8e48-db4506ea72bf
spec:
progressDeadlineSeconds: 600
replicas: 1
revisionHistoryLimit: 10
selector:
matchLabels:
app: 3scale-kourier-gateway
strategy:
rollingUpdate:
maxSurge: 100%
maxUnavailable: 0
type: RollingUpdate
template:
metadata:
annotations:
networking.knative.dev/poke: v0.26
prometheus.io/path: /stats/prometheus
prometheus.io/port: "9000"
prometheus.io/scrape: "true"
labels:
app: 3scale-kourier-gateway
spec:
containers:
- args:
- --base-id 1
- -c /tmp/config/envoy-bootstrap.yaml
- --log-level info
- --drain-time-s $(DRAIN_TIME_SECONDS)
- --drain-strategy immediate
command:
- /usr/local/bin/envoy
env:
- name: DRAIN_TIME_SECONDS
value: "15"
image: docker.io/envoyproxy/envoy@sha256:1c2b79776c6e3b38e8b0113b825e6a599f9bfc08d680c199d80bf8964856c529
imagePullPolicy: IfNotPresent
lifecycle:
preStop:
exec:
command:
- /bin/sh
- -c
- curl -X POST http://localhost:9901/drain_listeners?graceful; sleep
$DRAIN_TIME_SECONDS
livenessProbe:
failureThreshold: 6
httpGet:
httpHeaders:
- name: Host
value: internalkourier
path: /ready
port: 8081
scheme: HTTP
initialDelaySeconds: 10
periodSeconds: 5
successThreshold: 1
timeoutSeconds: 3
name: kourier-gateway
ports:
- containerPort: 8080
name: http2-external
protocol: TCP
- containerPort: 8081
name: http2-internal
protocol: TCP
- containerPort: 8443
name: https-external
protocol: TCP
- containerPort: 8090
name: http-probe
protocol: TCP
- containerPort: 9443
name: https-probe
protocol: TCP
- containerPort: 9000
name: metrics
protocol: TCP
readinessProbe:
failureThreshold: 3
httpGet:
httpHeaders:
- name: Host
value: internalkourier
path: /ready
port: 8081
scheme: HTTP
initialDelaySeconds: 10
periodSeconds: 5
successThreshold: 1
timeoutSeconds: 3
resources:
limits:
cpu: "1"
memory: 800Mi
requests:
cpu: 50m
memory: 200Mi
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
readOnlyRootFilesystem: false
runAsGroup: 65534
runAsNonRoot: true
runAsUser: 65534
seccompProfile:
type: RuntimeDefault
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /tmp/config
name: config-volume
dnsPolicy: ClusterFirst
restartPolicy: Always
schedulerName: default-scheduler
securityContext: {}
terminationGracePeriodSeconds: 30
volumes:
- configMap:
defaultMode: 420
name: kourier-bootstrap
name: config-volume
status:
availableReplicas: 1
conditions:
- lastTransitionTime: "2026-07-26T16:45:57Z"
lastUpdateTime: "2026-09-21T16:30:02Z"
message: ReplicaSet "3scale-kourier-gateway-5cf57fc9fd" has successfully progressed.
reason: NewReplicaSetAvailable
status: "True"
type: Progressing
- lastTransitionTime: "2026-10-14T18:23:33Z"
lastUpdateTime: "2026-10-14T18:23:33Z"
message: Deployment has minimum availability.
reason: MinimumReplicasAvailable
status: "True"
type: Available
observedGeneration: 3
readyReplicas: 1
replicas: 1
terminatingReplicas: 0
updatedReplicas: 1

117
docs/evidence/2026-10-21-hpa-caps-before/hpa-knative-serving.yaml Normal file
View file

@ -0,0 +1,117 @@
apiVersion: v1
items:
- apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
annotations:
kubectl.kubernetes.io/last-applied-configuration: |
{"apiVersion":"autoscaling/v2","kind":"HorizontalPodAutoscaler","metadata":{"annotations":{},"labels":{"app.kubernetes.io/component":"activator","app.kubernetes.io/name":"knative-serving","app.kubernetes.io/version":"1.22.0"},"name":"activator","namespace":"knative-serving"},"spec":{"maxReplicas":20,"metrics":[{"resource":{"name":"cpu","target":{"averageUtilization":100,"type":"Utilization"}},"type":"Resource"}],"minReplicas":1,"scaleTargetRef":{"apiVersion":"apps/v1","kind":"Deployment","name":"activator"}}}
creationTimestamp: "2026-07-26T16:45:42Z"
labels:
app.kubernetes.io/component: activator
app.kubernetes.io/name: knative-serving
app.kubernetes.io/version: 1.22.0
name: activator
namespace: knative-serving
resourceVersion: "78344170"
uid: 6dfd641f-46c8-4e2b-85b6-3a3b4b3aaa0d
spec:
maxReplicas: 20
metrics:
- resource:
name: cpu
target:
averageUtilization: 100
type: Utilization
type: Resource
minReplicas: 1
scaleTargetRef:
apiVersion: apps/v1
kind: Deployment
name: activator
status:
conditions:
- lastTransitionTime: "2026-07-26T16:45:57Z"
message: recommended size matches current size
reason: ReadyForNewScale
status: "True"
type: AbleToScale
- lastTransitionTime: "2026-10-14T18:23:29Z"
message: the HPA was able to successfully calculate a replica count from cpu
resource utilization (percentage of request)
reason: ValidMetricFound
status: "True"
type: ScalingActive
- lastTransitionTime: "2026-09-21T16:30:32Z"
message: the desired count is within the acceptable range
reason: DesiredWithinRange
status: "False"
type: ScalingLimited
currentMetrics:
- resource:
current:
averageUtilization: 2
averageValue: 1m
name: cpu
type: Resource
currentReplicas: 1
desiredReplicas: 1
- apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
annotations:
kubectl.kubernetes.io/last-applied-configuration: |
{"apiVersion":"autoscaling/v2","kind":"HorizontalPodAutoscaler","metadata":{"annotations":{},"labels":{"app.kubernetes.io/component":"webhook","app.kubernetes.io/name":"knative-serving","app.kubernetes.io/version":"1.22.0"},"name":"webhook","namespace":"knative-serving"},"spec":{"maxReplicas":5,"metrics":[{"resource":{"name":"cpu","target":{"averageUtilization":100,"type":"Utilization"}},"type":"Resource"}],"minReplicas":1,"scaleTargetRef":{"apiVersion":"apps/v1","kind":"Deployment","name":"webhook"}}}
creationTimestamp: "2026-07-26T16:45:42Z"
labels:
app.kubernetes.io/component: webhook
app.kubernetes.io/name: knative-serving
app.kubernetes.io/version: 1.22.0
name: webhook
namespace: knative-serving
resourceVersion: "78351594"
uid: 02419da2-c8d7-407f-aa99-b09ba146e3f0
spec:
maxReplicas: 5
metrics:
- resource:
name: cpu
target:
averageUtilization: 100
type: Utilization
type: Resource
minReplicas: 1
scaleTargetRef:
apiVersion: apps/v1
kind: Deployment
name: webhook
status:
conditions:
- lastTransitionTime: "2026-07-26T16:45:57Z"
message: recommended size matches current size
reason: ReadyForNewScale
status: "True"
type: AbleToScale
- lastTransitionTime: "2026-10-14T18:23:14Z"
message: the HPA was able to successfully calculate a replica count from cpu
resource utilization (percentage of request)
reason: ValidMetricFound
status: "True"
type: ScalingActive
- lastTransitionTime: "2026-07-26T16:46:12Z"
message: the desired count is within the acceptable range
reason: DesiredWithinRange
status: "False"
type: ScalingLimited
currentMetrics:
- resource:
current:
averageUtilization: 10
averageValue: 3m
name: cpu
type: Resource
currentReplicas: 1
desiredReplicas: 1
kind: List
metadata:
resourceVersion: ""

57
docs/evidence/2026-10-21-hpa-caps-before/hpa-kourier-system.yaml Normal file
View file

@ -0,0 +1,57 @@
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
annotations:
kubectl.kubernetes.io/last-applied-configuration: |
{"apiVersion":"autoscaling/v2","kind":"HorizontalPodAutoscaler","metadata":{"annotations":{},"labels":{"app.kubernetes.io/component":"net-kourier","app.kubernetes.io/name":"knative-serving","app.kubernetes.io/version":"1.22.0","networking.knative.dev/ingress-provider":"kourier"},"name":"3scale-kourier-gateway","namespace":"kourier-system"},"spec":{"maxReplicas":10,"metrics":[{"resource":{"name":"cpu","target":{"averageUtilization":100,"type":"Utilization"}},"type":"Resource"}],"minReplicas":1,"scaleTargetRef":{"apiVersion":"apps/v1","kind":"Deployment","name":"3scale-kourier-gateway"}}}
creationTimestamp: "2026-07-26T16:45:58Z"
labels:
app.kubernetes.io/component: net-kourier
app.kubernetes.io/name: knative-serving
app.kubernetes.io/version: 1.22.0
networking.knative.dev/ingress-provider: kourier
name: 3scale-kourier-gateway
namespace: kourier-system
resourceVersion: "78351099"
uid: 6caa5e4c-fa74-4d85-96cd-5d970ba2efd6
spec:
maxReplicas: 10
metrics:
- resource:
name: cpu
target:
averageUtilization: 100
type: Utilization
type: Resource
minReplicas: 1
scaleTargetRef:
apiVersion: apps/v1
kind: Deployment
name: 3scale-kourier-gateway
status:
conditions:
- lastTransitionTime: "2026-07-26T16:46:13Z"
message: recommended size matches current size
reason: ReadyForNewScale
status: "True"
type: AbleToScale
- lastTransitionTime: "2026-10-14T18:23:14Z"
message: the HPA was able to successfully calculate a replica count from cpu resource
utilization (percentage of request)
reason: ValidMetricFound
status: "True"
type: ScalingActive
- lastTransitionTime: "2026-07-26T16:46:58Z"
message: the desired count is within the acceptable range
reason: DesiredWithinRange
status: "False"
type: ScalingLimited
currentMetrics:
- resource:
current:
averageUtilization: 8
averageValue: 4m
name: cpu
type: Resource
currentReplicas: 1
desiredReplicas: 1

5
docs/evidence/2026-10-21-hpa-caps-before/pdb.txt Normal file
View file

@ -0,0 +1,5 @@
NAME MIN AVAILABLE MAX UNAVAILABLE ALLOWED DISRUPTIONS AGE
activator-pdb 80% N/A 0 86d
webhook-pdb 80% N/A 0 86d
NAME MIN AVAILABLE MAX UNAVAILABLE ALLOWED DISRUPTIONS AGE
3scale-kourier-gateway-pdb 80% N/A 0 86d

6
docs/evidence/2026-10-21-hpa-caps-before/resources.txt Normal file
View file

@ -0,0 +1,6 @@
knative-serving/activator activator req={"cpu":"50m","memory":"60Mi"} lim={"cpu":"1","memory":"600Mi"}
knative-serving/autoscaler autoscaler req={"cpu":"30m","memory":"100Mi"} lim={"cpu":"1","memory":"1000Mi"}
knative-serving/controller controller req={"cpu":"30m","memory":"100Mi"} lim={"cpu":"1","memory":"1000Mi"}
knative-serving/net-kourier-controller controller req={"cpu":"30m","memory":"200Mi"} lim={"cpu":"1","memory":"500Mi"}
knative-serving/webhook webhook req={"cpu":"30m","memory":"100Mi"} lim={"cpu":"500m","memory":"500Mi"}
kourier-system/3scale-kourier-gateway kourier-gateway req={"cpu":"50m","memory":"200Mi"} lim={"cpu":"1","memory":"800Mi"}

33
docs/evidence/2026-10-21-hpa-caps-before/summary.txt Normal file
View file

@ -0,0 +1,33 @@
NAME REFERENCE TARGETS MINPODS MAXPODS REPLICAS AGE
activator Deployment/activator cpu: 2%/100% 1 20 1 86d
webhook Deployment/webhook cpu: 10%/100% 1 5 1 86d
NAME REFERENCE TARGETS MINPODS MAXPODS REPLICAS AGE
3scale-kourier-gateway Deployment/3scale-kourier-gateway cpu: 8%/100% 1 10 1 86d
NAME CPU(cores) MEMORY(bytes)
activator-6b4748664b-kpc4n 1m 19Mi
autoscaler-764bdc4bc8-gl6x2 3m 27Mi
controller-547767bf48-5nmcf 2m 36Mi
net-kourier-controller-5bc4c79595-q5xqw 2m 42Mi
webhook-8566f4d4bb-rrrkg 3m 18Mi
NAME CPU(cores) MEMORY(bytes)
3scale-kourier-gateway-5cf57fc9fd-fplqc 4m 18Mi
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
activator-6b4748664b-kpc4n 1/1 Running 1 (6d12h ago) 29d 10.42.0.214 239.62.205.92.host.secureserver.net <none> <none>
autoscaler-764bdc4bc8-gl6x2 1/1 Running 1 (6d12h ago) 29d 10.42.0.230 239.62.205.92.host.secureserver.net <none> <none>
controller-547767bf48-5nmcf 1/1 Running 2 (6d12h ago) 29d 10.42.0.240 239.62.205.92.host.secureserver.net <none> <none>
net-kourier-controller-5bc4c79595-q5xqw 1/1 Running 3 (6d12h ago) 29d 10.42.0.187 239.62.205.92.host.secureserver.net <none> <none>
webhook-8566f4d4bb-rrrkg 1/1 Running 3 (6d12h ago) 29d 10.42.0.199 239.62.205.92.host.secureserver.net <none> <none>
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
3scale-kourier-gateway-5cf57fc9fd-fplqc 1/1 Running 2 (6d12h ago) 29d 10.42.0.245 239.62.205.92.host.secureserver.net <none> <none>
NAMESPACE NAME URL LATESTCREATED LATESTREADY READY REASON
rapp-qonto rapp-qonto http://rapp-qonto.rapp-qonto.svc.cluster.local rapp-qonto-00010 rapp-qonto-00010 True
Allocated resources:
(Total limits may be over 100 percent, i.e., overcommitted.)
Resource Requests Limits
-------- -------- ------
cpu 3605m (90%) 37500m (937%)
memory 9856Mi (61%) 34562Mi (216%)
ephemeral-storage 256Mi (0%) 1Gi (0%)
hugepages-1Gi 0 (0%) 0 (0%)
hugepages-2Mi 0 (0%) 0 (0%)
Events: <none>

38
docs/substrate-runbook.md
View file

@ -1,23 +1,33 @@
# Knative substrate runbook: declared CPU requests
# Knative substrate runbook: declared CPU requests and HPA ceilings
The Knative Serving and Kourier v1.22.0 install on railiance01 is performed by
`railiance-cluster/install/knative/install.sh` from checksum-pinned upstream
assets. rail-knative declares the CPU requests that install must carry, in
`substrate/v1.22.0/cpu-requests.patch.yaml` (strategic-merge patches) and
`substrate/v1.22.0/cpu-requests.patch.yaml` (strategic-merge patches), the HPA
ceilings in `substrate/v1.22.0/hpa-caps.patch.yaml`, and
`substrate/v1.22.0/kustomization.yaml` (overlay for the staged `core.yaml` and
`kourier.yaml`; `crds.yaml` stays a separate first apply).
| Deployment | Container | Upstream v1.22.0 | Declared |
|---|---|---|---|
| knative-serving/activator | activator | 300m | 50m |
| kourier-system/3scale-kourier-gateway | kourier-gateway | 200m | 50m |
| knative-serving/net-kourier-controller | controller | 200m | 30m |
| knative-serving/controller | controller | 100m | 30m |
| knative-serving/webhook | webhook | 100m | 30m |
| knative-serving/autoscaler | autoscaler | 100m | 30m |
| Deployment | Container | Upstream v1.22.0 | 2026-09-21 | Declared (WP-0003) |
|---|---|---|---|---|
| knative-serving/activator | activator | 300m | 50m | 10m |
| kourier-system/3scale-kourier-gateway | kourier-gateway | 200m | 50m | 10m |
| knative-serving/net-kourier-controller | controller | 200m | 30m | 10m |
| knative-serving/controller | controller | 100m | 30m | 10m |
| knative-serving/webhook | webhook | 100m | 30m | 10m |
| knative-serving/autoscaler | autoscaler | 100m | 30m | 10m |
Only CPU requests differ from upstream. Memory requests and all limits are
upstream's. These values were set live on 2026-09-21 as
| HPA | Upstream min/max | Declared min/max (WP-0003) |
|---|---|---|
| knative-serving/activator | 1/20 | 1/3 |
| knative-serving/webhook | 1/5 | 1/2 |
| kourier-system/3scale-kourier-gateway | 1/10 | 1/3 |
Only CPU requests and HPA `maxReplicas` differ from upstream. Memory requests,
all limits, `minReplicas` and the HPA CPU targets are upstream's. The
RAIL-KNATIVE-WP-0003 column (60m total, 7d p95 at most 7.2m per workload) is
declared but not live until the railiance-cluster overlays carry it and the
installer re-runs; see that workplan. The 2026-09-21 values were set live as
`ADMINISTER @ realm:kubernetes/railiance01`, activation=APPROVED by the founder,
because the node had 100% of its allocatable CPU requested and backups could
not be scheduled. Record: `the-custodian/docs/kubernetes-change-gate-decision.md`.
@ -47,7 +57,9 @@ Kourier gateway and the webhook target 100% CPU utilisation *of the request*.
Lowering a request makes the same load read as a higher percentage, so the
HPA scales out sooner (at about 50m or 30m of use, not 300m or 100m). At
2026-09-21 use they read about 2-10%. If the request is raised or lowered
again, check the HPA targets in the same change.
again, check the HPA targets in the same change. At 10m requests (WP-0003)
the 2026-10-21 use reads 10-40%, and the declared `maxReplicas` bound any
scale-out on the single node to 3+2+3 pods.
## Read-only drift check

21
substrate/v1.22.0/cpu-requests.patch.yaml
View file

@ -2,9 +2,12 @@
# Strategic-merge patches over the upstream release manifests (serving-core.yaml,
# kourier.yaml), checksum-pinned in railiance-cluster/install/knative/release-lock.env.
# Only CPU requests are set; memory requests and all limits stay upstream.
# Live on railiance01 since 2026-09-21 (ADMINISTER @ realm:kubernetes/railiance01,
# activation=APPROVED by the founder). Re-applying upstream without these patches
# restores the upstream values noted per document and re-exhausts the node.
# First set live on railiance01 2026-09-21 (220m total; ADMINISTER @
# realm:kubernetes/railiance01, activation=APPROVED by the founder). Trimmed to
# 10m each (60m total) by RAIL-KNATIVE-WP-0003 per the resource-control node
# budget of 2026-10-21 (7d p95 <= 7.2m per workload). Re-applying upstream
# without these patches restores the upstream values noted per document and
# re-exhausts the node.
# Apply after each upstream apply, e.g. for each document:
# kubectl patch deployment <name> -n <ns> --type strategic --patch-file <doc>
# or list this file under `patches:` in a kustomization over the staged assets.
@ -22,7 +25,7 @@ spec:
- name: activator
resources:
requests:
cpu: 50m
cpu: 10m
---
# upstream v1.22.0 request: 100m
apiVersion: apps/v1
@ -37,7 +40,7 @@ spec:
- name: autoscaler
resources:
requests:
cpu: 30m
cpu: 10m
---
# upstream v1.22.0 request: 100m
apiVersion: apps/v1
@ -52,7 +55,7 @@ spec:
- name: controller
resources:
requests:
cpu: 30m
cpu: 10m
---
# upstream v1.22.0 request: 100m
apiVersion: apps/v1
@ -67,7 +70,7 @@ spec:
- name: webhook
resources:
requests:
cpu: 30m
cpu: 10m
---
# upstream v1.22.0 request: 200m
apiVersion: apps/v1
@ -82,7 +85,7 @@ spec:
- name: controller
resources:
requests:
cpu: 30m
cpu: 10m
---
# upstream v1.22.0 request: 200m
apiVersion: apps/v1
@ -97,4 +100,4 @@ spec:
- name: kourier-gateway
resources:
requests:
cpu: 50m
cpu: 10m

37
substrate/v1.22.0/hpa-caps.patch.yaml Normal file
View file

@ -0,0 +1,37 @@
# Declared HPA ceilings for the Knative Serving + Kourier v1.22.0 substrate on a
# single node (railiance01). Strategic-merge patches over the upstream HPAs in
# serving-core.yaml and kourier.yaml; only maxReplicas changes. minReplicas (1)
# and the CPU target (100% of request) stay upstream.
# The HPAs scale on percent of the CPU request, so after the 10m request trim
# (cpu-requests.patch.yaml) the same load reads ~5x higher. These caps bound the
# worst-case scale-out on one node to 3+2+3 pods instead of 20+5+10.
# RAIL-KNATIVE-WP-0003; evidence: resource-control
# docs/evidence/2026-10-21-node-allocation-review.md ("Knative HPA scale-out
# headroom" 1520m -> 230m).
---
# upstream v1.22.0: minReplicas 1, maxReplicas 20
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
name: activator
namespace: knative-serving
spec:
maxReplicas: 3
---
# upstream v1.22.0: minReplicas 1, maxReplicas 5
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
name: webhook
namespace: knative-serving
spec:
maxReplicas: 2
---
# upstream v1.22.0: minReplicas 1, maxReplicas 10
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
name: 3scale-kourier-gateway
namespace: kourier-system
spec:
maxReplicas: 3

1
substrate/v1.22.0/kustomization.yaml
View file

@ -10,3 +10,4 @@ resources:
- kourier.yaml
patches:
- path: cpu-requests.patch.yaml
- path: hpa-caps.patch.yaml

38
tests/test_substrate_requests.py
View file

@ -4,14 +4,15 @@ import yaml
ROOT = Path(__file__).parents[1]
SUBSTRATE = ROOT / "substrate/v1.22.0"
# Live on railiance01 since 2026-09-21; see docs/substrate-runbook.md.
# 220m live since 2026-09-21; trimmed to 10m each by RAIL-KNATIVE-WP-0003.
# See docs/substrate-runbook.md.
DECLARED = {
("knative-serving", "activator", "activator"): "50m",
("knative-serving", "autoscaler", "autoscaler"): "30m",
("knative-serving", "controller", "controller"): "30m",
("knative-serving", "webhook", "webhook"): "30m",
("knative-serving", "net-kourier-controller", "controller"): "30m",
("kourier-system", "3scale-kourier-gateway", "kourier-gateway"): "50m",
("knative-serving", "activator", "activator"): "10m",
("knative-serving", "autoscaler", "autoscaler"): "10m",
("knative-serving", "controller", "controller"): "10m",
("knative-serving", "webhook", "webhook"): "10m",
("knative-serving", "net-kourier-controller", "controller"): "10m",
("kourier-system", "3scale-kourier-gateway", "kourier-gateway"): "10m",
}
@ -30,4 +31,25 @@ def test_patches_declare_exactly_the_cpu_requests():
def test_kustomization_patches_the_pinned_assets():
k = yaml.safe_load((SUBSTRATE / "kustomization.yaml").read_text())
assert k["resources"] == ["core.yaml", "kourier.yaml"]
assert k["patches"] == [{"path": "cpu-requests.patch.yaml"}]
assert k["patches"] == [
{"path": "cpu-requests.patch.yaml"},
{"path": "hpa-caps.patch.yaml"},
]
# Single-node ceilings, RAIL-KNATIVE-WP-0003. minReplicas stays upstream (1).
HPA_CAPS = {
("knative-serving", "activator"): 3,
("knative-serving", "webhook"): 2,
("kourier-system", "3scale-kourier-gateway"): 3,
}
def test_hpa_patches_only_cap_max_replicas():
found = {}
for doc in yaml.safe_load_all((SUBSTRATE / "hpa-caps.patch.yaml").read_text()):
assert doc["apiVersion"] == "autoscaling/v2"
assert doc["kind"] == "HorizontalPodAutoscaler"
assert set(doc["spec"]) == {"maxReplicas"}
found[(doc["metadata"]["namespace"], doc["metadata"]["name"])] = doc["spec"]["maxReplicas"]
assert found == HPA_CAPS

168
workplans/RAIL-KNATIVE-WP-0003-single-node-hpa-caps-and-request-trim.md Normal file
View file

@ -0,0 +1,168 @@
---
id: RAIL-KNATIVE-WP-0003
type: workplan
title: "Cap Knative HPA ceilings for one node and trim substrate CPU requests to 60m"
domain: financials
repo: rail-knative
status: active
owner: codex
topic_slug: railiance
created: "2026-10-21"
updated: "2026-10-21"
depends_on: []
related: [CUST-WP-0071, RESOURCE-WP-0008]
---
# RAIL-KNATIVE-WP-0003 - Single-node HPA caps and CPU request trim
Receiving workplan for the three rail-knative rows of the resource-control node
budget (CUST-WP-0071 T03/T04, RESOURCE-WP-0008).
## Evidence and rationale
- Review: `resource-control/docs/evidence/2026-10-21-node-allocation-review.md`
(Knative/Kourier item; "Knative HPA scale-out headroom" 1520m -> 230m).
- Budget rows: `resource-control/docs/evidence/2026-10-21-node-allocation-review/2026-10-21-node-budget.{md,json}`
- knative-serving/knative-serving (4 pods): 140m -> 40m, 7d p95 7.2m, peak 9.9m, throttle 0
- knative-serving/net-kourier-controller: 30m -> 10m, p95 0.8m, peak 1.6m
- kourier-system/3scale-kourier-gateway: 50m -> 10m, p95 3.6m, peak 3.8m
- Delivery decision: `the-custodian/docs/decisions/2026-10-21-cust-wp-0071-delivery-interview.md`.
- BEFORE state (read-only, 2026-10-21): `docs/evidence/2026-10-21-hpa-caps-before/`.
Live HPAs: activator 1/20 (cpu 2%/100%), webhook 1/5 (10%/100%),
3scale-kourier-gateway 1/10 (8%/100%). All six Deployments 1/1 Ready.
Node CPU requests 3605m (90%).
railiance01 is one node. The upstream HPA ceilings (20+5+10 pods) can request
up to 1520m on it, and since the HPAs scale on percent of request, every
request trim makes them scale out sooner. Capping `maxReplicas` makes the trim
safe; `minReplicas` stays 1 (upstream). Knative recommends >= 2 activator
replicas for HA across nodes; with one node that buys nothing, so min 1 stays.
Limits stay upstream (1 CPU, webhook 500m), so the request trim cannot cause
CPU throttling; only scheduling weight and HPA percent change. Memory requests
stay upstream in this workplan (the budget also proposes memory down; that is a
separate, later step).
| Object | Before (live) | After (declared) |
|---|---|---|
| HPA knative-serving/activator max | 20 | 3 |
| HPA knative-serving/webhook max | 5 | 2 |
| HPA kourier-system/3scale-kourier-gateway max | 10 | 3 |
| activator cpu request | 50m | 10m |
| autoscaler cpu request | 30m | 10m |
| controller cpu request | 30m | 10m |
| webhook cpu request | 30m | 10m |
| net-kourier-controller cpu request | 30m | 10m |
| 3scale-kourier-gateway cpu request | 50m | 10m |
| total cpu request | 220m | 60m |
## T01 - Declare caps and trimmed requests over the pinned v1.22.0 assets
```task
id: RAIL-KNATIVE-WP-0003-T01
status: done
priority: high
```
2026-10-21: `substrate/v1.22.0/cpu-requests.patch.yaml` now declares 10m for
all six containers; new `substrate/v1.22.0/hpa-caps.patch.yaml` sets only
`maxReplicas` on the three upstream HPAs; the kustomization lists both;
`tests/test_substrate_requests.py` and `docs/substrate-runbook.md` updated.
Validation: `python3 -m pytest -q -p no:cacheprovider tests` 5 passed;
`validate_contracts.py --rail declarations/rail.yaml` passed. Rendered with
`kubectl kustomize` over the checksum-verified upstream `core.yaml` and
`kourier.yaml` (staged by `railiance-cluster/install/knative/render.sh`):
exactly 9 of 72 documents differ from upstream - six Deployment CPU requests
and three HPA `maxReplicas`; memory, limits and `minReplicas` unchanged.
## T02 - Owner handoff, apply and verify on railiance01
```task
id: RAIL-KNATIVE-WP-0003-T02
status: wait
priority: high
needs_human: true
blocking_reason: "Coordinator applies wave-1 changes one owner at a time after review; railiance-cluster overlays must follow this declaration"
```
**Cross-repo coupling.** The deploy path is
`railiance-cluster/install/knative/install.sh` (render.sh + `kubectl apply`
over ssh; no ArgoCD Application). Its overlays still carry the 2026-09-21
values, and `railiance-cluster/tests/test_knative_render.py::test_matches_rail_knative_declaration`
compares them with this repository, so it fails until railiance-cluster
follows: update `install/knative/overlays/{serving-core,kourier}/cpu-requests.patch.yaml`
to 10m, add the HPA caps patch to both overlays (activator and webhook in
serving-core, 3scale-kourier-gateway in kourier), update the `DECLARED` table
in the test and the request list in `install/knative/verify.sh`, and add HPA
`maxReplicas` checks to the verifier. Until then, a re-run of the installer
reverts the live change to 220m and 20/5/10.
**Apply (coordinator, in this order: caps first, so the lower requests never
meet the upstream ceilings).** Each request patch rolls one pod (maxSurge 1);
node has ~395m request headroom, each new pod asks 10m.
```bash
ssh railiance01 'set -e
kubectl patch hpa activator -n knative-serving --type merge -p "{\"spec\":{\"maxReplicas\":3}}"
kubectl patch hpa webhook -n knative-serving --type merge -p "{\"spec\":{\"maxReplicas\":2}}"
kubectl patch hpa 3scale-kourier-gateway -n kourier-system --type merge -p "{\"spec\":{\"maxReplicas\":3}}"
for d in activator autoscaler controller webhook; do
kubectl set resources deployment/$d -n knative-serving -c $d --requests=cpu=10m
done
kubectl set resources deployment/net-kourier-controller -n knative-serving -c controller --requests=cpu=10m
kubectl set resources deployment/3scale-kourier-gateway -n kourier-system -c kourier-gateway --requests=cpu=10m
for d in activator autoscaler controller webhook net-kourier-controller; do
kubectl rollout status deployment/$d -n knative-serving --timeout=300s
done
kubectl rollout status deployment/3scale-kourier-gateway -n kourier-system --timeout=300s'
```
Owner path once railiance-cluster has followed (same result, durable):
```bash
cd ~/railiance-cluster && python3 -m pytest -q tests/test_knative_render.py \
&& install/knative/install.sh railiance01 && install/knative/verify.sh railiance01
```
**Verify (all must hold).**
```bash
ssh railiance01 'kubectl get hpa -n knative-serving; kubectl get hpa -n kourier-system
kubectl get deploy -n knative-serving; kubectl get deploy -n kourier-system
kubectl get deploy -n knative-serving -o jsonpath="{range .items[*]}{.metadata.name} {.spec.template.spec.containers[0].resources.requests.cpu}{\"\n\"}{end}"
kubectl get deploy -n kourier-system -o jsonpath="{range .items[*]}{.metadata.name} {.spec.template.spec.containers[0].resources.requests.cpu}{\"\n\"}{end}"
kubectl get ksvc -A; kubectl top pods -n knative-serving; kubectl top pods -n kourier-system'
```
- HPA max 3/2/3, min 1; six Deployments Available, pods Ready, no restarts.
- All six requests read 10m.
- `ksvc rapp-qonto` stays Ready=True and answers a request through Kourier
(scale-from-zero via the activator included) — e.g. an in-cluster
`curl -sS -o /dev/null -w '%{http_code}' http://rapp-qonto.rapp-qonto.svc.cluster.local`
from a debug pod.
- No throttling regression: resource-control 7d `throttle_ratio_7d` stays 0
for the three workloads at the next weekly allocation report; HPA replicas
stay at 1 at idle (activator ~10%, webhook ~30%, gateway ~40% of request).
**Rollback (restores the BEFORE state exactly).**
```bash
ssh railiance01 'set -e
kubectl set resources deployment/activator -n knative-serving -c activator --requests=cpu=50m
for d in autoscaler controller webhook; do
kubectl set resources deployment/$d -n knative-serving -c $d --requests=cpu=30m
done
kubectl set resources deployment/net-kourier-controller -n knative-serving -c controller --requests=cpu=30m
kubectl set resources deployment/3scale-kourier-gateway -n kourier-system -c kourier-gateway --requests=cpu=50m
kubectl patch hpa activator -n knative-serving --type merge -p "{\"spec\":{\"maxReplicas\":20}}"
kubectl patch hpa webhook -n knative-serving --type merge -p "{\"spec\":{\"maxReplicas\":5}}"
kubectl patch hpa 3scale-kourier-gateway -n kourier-system --type merge -p "{\"spec\":{\"maxReplicas\":10}}"'
```
and `git revert` this workplan's declaration commit (plus the railiance-cluster
follow-up if it landed). Full BEFORE objects: `docs/evidence/2026-10-21-hpa-caps-before/`.
Risks: a 10m request gives these pods little CPU weight under node contention
(limits still allow bursts); the caps limit burst capacity of the activator
and gateway to 3 pods, which on one node is the practical ceiling anyway; the
80% PDBs on activator, webhook and gateway already allow 0 disruptions at 1
replica (unchanged, but it blocks node drain).