diff --git a/docs/evidence/2026-10-21-hpa-caps-before/captured-at.txt b/docs/evidence/2026-10-21-hpa-caps-before/captured-at.txt new file mode 100644 index 0000000..765334b --- /dev/null +++ b/docs/evidence/2026-10-21-hpa-caps-before/captured-at.txt @@ -0,0 +1,2 @@ +2026-10-21T06:38:16Z +default diff --git a/docs/evidence/2026-10-21-hpa-caps-before/deploy-knative-serving.yaml b/docs/evidence/2026-10-21-hpa-caps-before/deploy-knative-serving.yaml new file mode 100644 index 0000000..7b1d74b --- /dev/null +++ b/docs/evidence/2026-10-21-hpa-caps-before/deploy-knative-serving.yaml @@ -0,0 +1,711 @@ +apiVersion: v1 +items: +- apiVersion: apps/v1 + kind: Deployment + metadata: + annotations: + deployment.kubernetes.io/revision: "2" + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"apps/v1","kind":"Deployment","metadata":{"annotations":{},"labels":{"app.kubernetes.io/component":"activator","app.kubernetes.io/name":"knative-serving","app.kubernetes.io/version":"1.22.0"},"name":"activator","namespace":"knative-serving"},"spec":{"selector":{"matchLabels":{"app":"activator","role":"activator"}},"template":{"metadata":{"labels":{"app":"activator","app.kubernetes.io/component":"activator","app.kubernetes.io/name":"knative-serving","app.kubernetes.io/version":"1.22.0","role":"activator"}},"spec":{"affinity":{"podAntiAffinity":{"preferredDuringSchedulingIgnoredDuringExecution":[{"podAffinityTerm":{"labelSelector":{"matchLabels":{"app":"activator"}},"topologyKey":"kubernetes.io/hostname"},"weight":100}]}},"containers":[{"env":[{"name":"GOGC","value":"500"},{"name":"POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"POD_IP","valueFrom":{"fieldRef":{"fieldPath":"status.podIP"}}},{"name":"SYSTEM_NAMESPACE","valueFrom":{"fieldRef":{"fieldPath":"metadata.namespace"}}},{"name":"CONFIG_LOGGING_NAME","value":"config-logging"},{"name":"CONFIG_OBSERVABILITY_NAME","value":"config-observability"}],"image":"gcr.io/knative-releases/knative.dev/serving/cmd/activator@sha256:8f7e9df2642a8ba715ebc55b62abee17b0984b6b6af0c6ec00dabaadf5f8cccb","livenessProbe":{"failureThreshold":12,"httpGet":{"port":8012},"initialDelaySeconds":15,"periodSeconds":10},"name":"activator","ports":[{"containerPort":9090,"name":"metrics"},{"containerPort":8008,"name":"profiling"},{"containerPort":8012,"name":"http1"},{"containerPort":8013,"name":"h2c"}],"readinessProbe":{"failureThreshold":5,"httpGet":{"port":8012},"periodSeconds":5},"resources":{"limits":{"cpu":"1000m","memory":"600Mi"},"requests":{"cpu":"300m","memory":"60Mi"}},"securityContext":{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]},"readOnlyRootFilesystem":true,"runAsNonRoot":true,"seccompProfile":{"type":"RuntimeDefault"}}}],"serviceAccountName":"activator","terminationGracePeriodSeconds":600}}}} + creationTimestamp: "2026-07-26T16:45:42Z" + generation: 2 + labels: + app.kubernetes.io/component: activator + app.kubernetes.io/name: knative-serving + app.kubernetes.io/version: 1.22.0 + name: activator + namespace: knative-serving + resourceVersion: "75292980" + uid: 855e1767-8c4e-43ac-b5a7-94bf0f5a1cef + spec: + progressDeadlineSeconds: 600 + replicas: 1 + revisionHistoryLimit: 10 + selector: + matchLabels: + app: activator + role: activator + strategy: + rollingUpdate: + maxSurge: 25% + maxUnavailable: 25% + type: RollingUpdate + template: + metadata: + labels: + app: activator + app.kubernetes.io/component: activator + app.kubernetes.io/name: knative-serving + app.kubernetes.io/version: 1.22.0 + role: activator + spec: + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchLabels: + app: activator + topologyKey: kubernetes.io/hostname + weight: 100 + containers: + - env: + - name: GOGC + value: "500" + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: POD_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: SYSTEM_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: CONFIG_LOGGING_NAME + value: config-logging + - name: CONFIG_OBSERVABILITY_NAME + value: config-observability + image: gcr.io/knative-releases/knative.dev/serving/cmd/activator@sha256:8f7e9df2642a8ba715ebc55b62abee17b0984b6b6af0c6ec00dabaadf5f8cccb + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 12 + httpGet: + path: / + port: 8012 + scheme: HTTP + initialDelaySeconds: 15 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + name: activator + ports: + - containerPort: 9090 + name: metrics + protocol: TCP + - containerPort: 8008 + name: profiling + protocol: TCP + - containerPort: 8012 + name: http1 + protocol: TCP + - containerPort: 8013 + name: h2c + protocol: TCP + readinessProbe: + failureThreshold: 5 + httpGet: + path: / + port: 8012 + scheme: HTTP + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 1 + resources: + limits: + cpu: "1" + memory: 600Mi + requests: + cpu: 50m + memory: 60Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + dnsPolicy: ClusterFirst + restartPolicy: Always + schedulerName: default-scheduler + securityContext: {} + serviceAccount: activator + serviceAccountName: activator + terminationGracePeriodSeconds: 600 + status: + availableReplicas: 1 + conditions: + - lastTransitionTime: "2026-07-26T16:45:42Z" + lastUpdateTime: "2026-09-21T16:29:55Z" + message: ReplicaSet "activator-6b4748664b" has successfully progressed. + reason: NewReplicaSetAvailable + status: "True" + type: Progressing + - lastTransitionTime: "2026-10-14T18:23:05Z" + lastUpdateTime: "2026-10-14T18:23:05Z" + message: Deployment has minimum availability. + reason: MinimumReplicasAvailable + status: "True" + type: Available + observedGeneration: 2 + readyReplicas: 1 + replicas: 1 + terminatingReplicas: 0 + updatedReplicas: 1 +- apiVersion: apps/v1 + kind: Deployment + metadata: + annotations: + deployment.kubernetes.io/revision: "2" + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"apps/v1","kind":"Deployment","metadata":{"annotations":{},"labels":{"app.kubernetes.io/component":"autoscaler","app.kubernetes.io/name":"knative-serving","app.kubernetes.io/version":"1.22.0"},"name":"autoscaler","namespace":"knative-serving"},"spec":{"replicas":1,"selector":{"matchLabels":{"app":"autoscaler"}},"strategy":{"rollingUpdate":{"maxUnavailable":0},"type":"RollingUpdate"},"template":{"metadata":{"labels":{"app":"autoscaler","app.kubernetes.io/component":"autoscaler","app.kubernetes.io/name":"knative-serving","app.kubernetes.io/version":"1.22.0"}},"spec":{"affinity":{"podAntiAffinity":{"preferredDuringSchedulingIgnoredDuringExecution":[{"podAffinityTerm":{"labelSelector":{"matchLabels":{"app":"autoscaler"}},"topologyKey":"kubernetes.io/hostname"},"weight":100}]}},"containers":[{"env":[{"name":"POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"POD_IP","valueFrom":{"fieldRef":{"fieldPath":"status.podIP"}}},{"name":"SYSTEM_NAMESPACE","valueFrom":{"fieldRef":{"fieldPath":"metadata.namespace"}}},{"name":"CONFIG_LOGGING_NAME","value":"config-logging"},{"name":"CONFIG_OBSERVABILITY_NAME","value":"config-observability"}],"image":"gcr.io/knative-releases/knative.dev/serving/cmd/autoscaler@sha256:ec66d97244395b57a67923d4a8c5a95f0264805f1fdb2ee0cdb49ec0f3bd7855","livenessProbe":{"failureThreshold":6,"httpGet":{"port":8080}},"name":"autoscaler","ports":[{"containerPort":9090,"name":"metrics"},{"containerPort":8008,"name":"profiling"},{"containerPort":8080,"name":"websocket"}],"readinessProbe":{"httpGet":{"port":8080}},"resources":{"limits":{"cpu":"1000m","memory":"1000Mi"},"requests":{"cpu":"100m","memory":"100Mi"}},"securityContext":{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]},"readOnlyRootFilesystem":true,"runAsNonRoot":true,"seccompProfile":{"type":"RuntimeDefault"}}}],"serviceAccountName":"controller"}}}} + creationTimestamp: "2026-07-26T16:45:42Z" + generation: 2 + labels: + app.kubernetes.io/component: autoscaler + app.kubernetes.io/name: knative-serving + app.kubernetes.io/version: 1.22.0 + name: autoscaler + namespace: knative-serving + resourceVersion: "75292939" + uid: 24c54b27-5e45-41ef-8c8e-5b40c1733a5d + spec: + progressDeadlineSeconds: 600 + replicas: 1 + revisionHistoryLimit: 10 + selector: + matchLabels: + app: autoscaler + strategy: + rollingUpdate: + maxSurge: 25% + maxUnavailable: 0 + type: RollingUpdate + template: + metadata: + labels: + app: autoscaler + app.kubernetes.io/component: autoscaler + app.kubernetes.io/name: knative-serving + app.kubernetes.io/version: 1.22.0 + spec: + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchLabels: + app: autoscaler + topologyKey: kubernetes.io/hostname + weight: 100 + containers: + - env: + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: POD_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: SYSTEM_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: CONFIG_LOGGING_NAME + value: config-logging + - name: CONFIG_OBSERVABILITY_NAME + value: config-observability + image: gcr.io/knative-releases/knative.dev/serving/cmd/autoscaler@sha256:ec66d97244395b57a67923d4a8c5a95f0264805f1fdb2ee0cdb49ec0f3bd7855 + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 6 + httpGet: + path: / + port: 8080 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + name: autoscaler + ports: + - containerPort: 9090 + name: metrics + protocol: TCP + - containerPort: 8008 + name: profiling + protocol: TCP + - containerPort: 8080 + name: websocket + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: / + port: 8080 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + resources: + limits: + cpu: "1" + memory: 1000Mi + requests: + cpu: 30m + memory: 100Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + dnsPolicy: ClusterFirst + restartPolicy: Always + schedulerName: default-scheduler + securityContext: {} + serviceAccount: controller + serviceAccountName: controller + terminationGracePeriodSeconds: 30 + status: + availableReplicas: 1 + conditions: + - lastTransitionTime: "2026-07-26T16:45:42Z" + lastUpdateTime: "2026-09-21T16:30:02Z" + message: ReplicaSet "autoscaler-764bdc4bc8" has successfully progressed. + reason: NewReplicaSetAvailable + status: "True" + type: Progressing + - lastTransitionTime: "2026-10-14T18:23:04Z" + lastUpdateTime: "2026-10-14T18:23:04Z" + message: Deployment has minimum availability. + reason: MinimumReplicasAvailable + status: "True" + type: Available + observedGeneration: 2 + readyReplicas: 1 + replicas: 1 + terminatingReplicas: 0 + updatedReplicas: 1 +- apiVersion: apps/v1 + kind: Deployment + metadata: + annotations: + deployment.kubernetes.io/revision: "2" + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"apps/v1","kind":"Deployment","metadata":{"annotations":{},"labels":{"app.kubernetes.io/component":"controller","app.kubernetes.io/name":"knative-serving","app.kubernetes.io/version":"1.22.0"},"name":"controller","namespace":"knative-serving"},"spec":{"selector":{"matchLabels":{"app":"controller"}},"template":{"metadata":{"labels":{"app":"controller","app.kubernetes.io/component":"controller","app.kubernetes.io/name":"knative-serving","app.kubernetes.io/version":"1.22.0"}},"spec":{"affinity":{"podAntiAffinity":{"preferredDuringSchedulingIgnoredDuringExecution":[{"podAffinityTerm":{"labelSelector":{"matchLabels":{"app":"controller"}},"topologyKey":"kubernetes.io/hostname"},"weight":100}]}},"containers":[{"env":[{"name":"POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"SYSTEM_NAMESPACE","valueFrom":{"fieldRef":{"fieldPath":"metadata.namespace"}}},{"name":"CONFIG_LOGGING_NAME","value":"config-logging"},{"name":"CONFIG_OBSERVABILITY_NAME","value":"config-observability"}],"image":"gcr.io/knative-releases/knative.dev/serving/cmd/controller@sha256:6833f2c83e9f356c274ddb2de40e5471ad2b1c08882c9bd11f5afa240e86d574","livenessProbe":{"failureThreshold":6,"httpGet":{"path":"/health","port":"probes","scheme":"HTTP"},"periodSeconds":5},"name":"controller","ports":[{"containerPort":9090,"name":"metrics"},{"containerPort":8008,"name":"profiling"},{"containerPort":8080,"name":"probes"}],"readinessProbe":{"failureThreshold":3,"httpGet":{"path":"/readiness","port":"probes","scheme":"HTTP"},"periodSeconds":5},"resources":{"limits":{"cpu":"1000m","memory":"1000Mi"},"requests":{"cpu":"100m","memory":"100Mi"}},"securityContext":{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]},"readOnlyRootFilesystem":true,"runAsNonRoot":true,"seccompProfile":{"type":"RuntimeDefault"}}}],"serviceAccountName":"controller"}}}} + creationTimestamp: "2026-07-26T16:45:42Z" + generation: 2 + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: knative-serving + app.kubernetes.io/version: 1.22.0 + name: controller + namespace: knative-serving + resourceVersion: "75293078" + uid: a8bfc048-b465-43dd-aa0a-15c8722edd5b + spec: + progressDeadlineSeconds: 600 + replicas: 1 + revisionHistoryLimit: 10 + selector: + matchLabels: + app: controller + strategy: + rollingUpdate: + maxSurge: 25% + maxUnavailable: 25% + type: RollingUpdate + template: + metadata: + labels: + app: controller + app.kubernetes.io/component: controller + app.kubernetes.io/name: knative-serving + app.kubernetes.io/version: 1.22.0 + spec: + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchLabels: + app: controller + topologyKey: kubernetes.io/hostname + weight: 100 + containers: + - env: + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: SYSTEM_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: CONFIG_LOGGING_NAME + value: config-logging + - name: CONFIG_OBSERVABILITY_NAME + value: config-observability + image: gcr.io/knative-releases/knative.dev/serving/cmd/controller@sha256:6833f2c83e9f356c274ddb2de40e5471ad2b1c08882c9bd11f5afa240e86d574 + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 6 + httpGet: + path: /health + port: probes + scheme: HTTP + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 1 + name: controller + ports: + - containerPort: 9090 + name: metrics + protocol: TCP + - containerPort: 8008 + name: profiling + protocol: TCP + - containerPort: 8080 + name: probes + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /readiness + port: probes + scheme: HTTP + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 1 + resources: + limits: + cpu: "1" + memory: 1000Mi + requests: + cpu: 30m + memory: 100Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + dnsPolicy: ClusterFirst + restartPolicy: Always + schedulerName: default-scheduler + securityContext: {} + serviceAccount: controller + serviceAccountName: controller + terminationGracePeriodSeconds: 30 + status: + availableReplicas: 1 + conditions: + - lastTransitionTime: "2026-07-26T16:45:42Z" + lastUpdateTime: "2026-09-21T16:29:55Z" + message: ReplicaSet "controller-547767bf48" has successfully progressed. + reason: NewReplicaSetAvailable + status: "True" + type: Progressing + - lastTransitionTime: "2026-10-14T18:23:08Z" + lastUpdateTime: "2026-10-14T18:23:08Z" + message: Deployment has minimum availability. + reason: MinimumReplicasAvailable + status: "True" + type: Available + observedGeneration: 2 + readyReplicas: 1 + replicas: 1 + terminatingReplicas: 0 + updatedReplicas: 1 +- apiVersion: apps/v1 + kind: Deployment + metadata: + annotations: + deployment.kubernetes.io/revision: "2" + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"apps/v1","kind":"Deployment","metadata":{"annotations":{},"labels":{"app.kubernetes.io/component":"webhook","app.kubernetes.io/name":"knative-serving","app.kubernetes.io/version":"1.22.0"},"name":"webhook","namespace":"knative-serving"},"spec":{"selector":{"matchLabels":{"app":"webhook","role":"webhook"}},"template":{"metadata":{"labels":{"app":"webhook","app.kubernetes.io/component":"webhook","app.kubernetes.io/name":"knative-serving","app.kubernetes.io/version":"1.22.0","role":"webhook"}},"spec":{"affinity":{"podAntiAffinity":{"preferredDuringSchedulingIgnoredDuringExecution":[{"podAffinityTerm":{"labelSelector":{"matchLabels":{"app":"webhook"}},"topologyKey":"kubernetes.io/hostname"},"weight":100}]}},"containers":[{"env":[{"name":"POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"SYSTEM_NAMESPACE","valueFrom":{"fieldRef":{"fieldPath":"metadata.namespace"}}},{"name":"CONFIG_LOGGING_NAME","value":"config-logging"},{"name":"CONFIG_OBSERVABILITY_NAME","value":"config-observability"},{"name":"WEBHOOK_NAME","value":"webhook"},{"name":"WEBHOOK_PORT","value":"8443"}],"image":"gcr.io/knative-releases/knative.dev/serving/cmd/webhook@sha256:c505888f7ee34ccadc1b9a6877d6d49691f633ea28cf8cbe03fcfa6d3afcd584","livenessProbe":{"failureThreshold":6,"httpGet":{"port":8443,"scheme":"HTTPS"},"initialDelaySeconds":20,"periodSeconds":10},"name":"webhook","ports":[{"containerPort":9090,"name":"metrics"},{"containerPort":8008,"name":"profiling"},{"containerPort":8443,"name":"https-webhook"}],"readinessProbe":{"httpGet":{"port":8443,"scheme":"HTTPS"},"periodSeconds":1},"resources":{"limits":{"cpu":"500m","memory":"500Mi"},"requests":{"cpu":"100m","memory":"100Mi"}},"securityContext":{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]},"readOnlyRootFilesystem":true,"runAsNonRoot":true,"seccompProfile":{"type":"RuntimeDefault"}}}],"serviceAccountName":"controller","terminationGracePeriodSeconds":300}}}} + creationTimestamp: "2026-07-26T16:45:42Z" + generation: 2 + labels: + app.kubernetes.io/component: webhook + app.kubernetes.io/name: knative-serving + app.kubernetes.io/version: 1.22.0 + name: webhook + namespace: knative-serving + resourceVersion: "75292683" + uid: 651670a3-f99d-4828-b5e0-a4b24a3428ea + spec: + progressDeadlineSeconds: 600 + replicas: 1 + revisionHistoryLimit: 10 + selector: + matchLabels: + app: webhook + role: webhook + strategy: + rollingUpdate: + maxSurge: 25% + maxUnavailable: 25% + type: RollingUpdate + template: + metadata: + labels: + app: webhook + app.kubernetes.io/component: webhook + app.kubernetes.io/name: knative-serving + app.kubernetes.io/version: 1.22.0 + role: webhook + spec: + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchLabels: + app: webhook + topologyKey: kubernetes.io/hostname + weight: 100 + containers: + - env: + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: SYSTEM_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: CONFIG_LOGGING_NAME + value: config-logging + - name: CONFIG_OBSERVABILITY_NAME + value: config-observability + - name: WEBHOOK_NAME + value: webhook + - name: WEBHOOK_PORT + value: "8443" + image: gcr.io/knative-releases/knative.dev/serving/cmd/webhook@sha256:c505888f7ee34ccadc1b9a6877d6d49691f633ea28cf8cbe03fcfa6d3afcd584 + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 6 + httpGet: + path: / + port: 8443 + scheme: HTTPS + initialDelaySeconds: 20 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + name: webhook + ports: + - containerPort: 9090 + name: metrics + protocol: TCP + - containerPort: 8008 + name: profiling + protocol: TCP + - containerPort: 8443 + name: https-webhook + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: / + port: 8443 + scheme: HTTPS + periodSeconds: 1 + successThreshold: 1 + timeoutSeconds: 1 + resources: + limits: + cpu: 500m + memory: 500Mi + requests: + cpu: 30m + memory: 100Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + dnsPolicy: ClusterFirst + restartPolicy: Always + schedulerName: default-scheduler + securityContext: {} + serviceAccount: controller + serviceAccountName: controller + terminationGracePeriodSeconds: 300 + status: + availableReplicas: 1 + conditions: + - lastTransitionTime: "2026-07-26T16:45:42Z" + lastUpdateTime: "2026-09-21T16:29:51Z" + message: ReplicaSet "webhook-8566f4d4bb" has successfully progressed. + reason: NewReplicaSetAvailable + status: "True" + type: Progressing + - lastTransitionTime: "2026-10-14T18:22:58Z" + lastUpdateTime: "2026-10-14T18:22:58Z" + message: Deployment has minimum availability. + reason: MinimumReplicasAvailable + status: "True" + type: Available + observedGeneration: 2 + readyReplicas: 1 + replicas: 1 + terminatingReplicas: 0 + updatedReplicas: 1 +- apiVersion: apps/v1 + kind: Deployment + metadata: + annotations: + deployment.kubernetes.io/revision: "2" + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"apps/v1","kind":"Deployment","metadata":{"annotations":{},"labels":{"app.kubernetes.io/component":"net-kourier","app.kubernetes.io/name":"knative-serving","app.kubernetes.io/version":"1.22.0","networking.knative.dev/ingress-provider":"kourier"},"name":"net-kourier-controller","namespace":"knative-serving"},"spec":{"replicas":1,"selector":{"matchLabels":{"app":"net-kourier-controller"}},"strategy":{"rollingUpdate":{"maxSurge":"100%","maxUnavailable":0},"type":"RollingUpdate"},"template":{"metadata":{"annotations":{"prometheus.io/path":"/metrics","prometheus.io/port":"9090","prometheus.io/scrape":"true"},"labels":{"app":"net-kourier-controller"}},"spec":{"containers":[{"env":[{"name":"CERTS_SECRET_NAMESPACE","value":""},{"name":"CERTS_SECRET_NAME","value":""},{"name":"SYSTEM_NAMESPACE","valueFrom":{"fieldRef":{"fieldPath":"metadata.namespace"}}},{"name":"METRICS_DOMAIN","value":"knative.dev/samples"},{"name":"KOURIER_GATEWAY_NAMESPACE","value":"kourier-system"},{"name":"ENABLE_SECRET_INFORMER_FILTERING_BY_CERT_UID","value":"false"},{"name":"KUBE_API_BURST","value":"200"},{"name":"KUBE_API_QPS","value":"200"}],"image":"gcr.io/knative-releases/knative.dev/net-kourier/cmd/kourier@sha256:a2e8f568a4bf68ef6391bc3afea956d11a8698f6a22c4be50844b1da4f8bcc74","livenessProbe":{"failureThreshold":6,"grpc":{"port":18000},"periodSeconds":10},"name":"controller","ports":[{"containerPort":18000,"name":"http2-xds","protocol":"TCP"},{"containerPort":9090,"name":"metrics","protocol":"TCP"}],"readinessProbe":{"failureThreshold":3,"grpc":{"port":18000},"periodSeconds":10},"resources":{"limits":{"cpu":"1","memory":"500Mi"},"requests":{"cpu":"200m","memory":"200Mi"}},"securityContext":{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]},"readOnlyRootFilesystem":true,"runAsNonRoot":true,"seccompProfile":{"type":"RuntimeDefault"}}}],"restartPolicy":"Always","serviceAccountName":"net-kourier"}}}} + creationTimestamp: "2026-07-26T16:45:57Z" + generation: 2 + labels: + app.kubernetes.io/component: net-kourier + app.kubernetes.io/name: knative-serving + app.kubernetes.io/version: 1.22.0 + networking.knative.dev/ingress-provider: kourier + name: net-kourier-controller + namespace: knative-serving + resourceVersion: "75293158" + uid: 94df7e95-cf6b-457a-8d27-490a0f9f2816 + spec: + progressDeadlineSeconds: 600 + replicas: 1 + revisionHistoryLimit: 10 + selector: + matchLabels: + app: net-kourier-controller + strategy: + rollingUpdate: + maxSurge: 100% + maxUnavailable: 0 + type: RollingUpdate + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: "9090" + prometheus.io/scrape: "true" + labels: + app: net-kourier-controller + spec: + containers: + - env: + - name: CERTS_SECRET_NAMESPACE + - name: CERTS_SECRET_NAME + - name: SYSTEM_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: METRICS_DOMAIN + value: knative.dev/samples + - name: KOURIER_GATEWAY_NAMESPACE + value: kourier-system + - name: ENABLE_SECRET_INFORMER_FILTERING_BY_CERT_UID + value: "false" + - name: KUBE_API_BURST + value: "200" + - name: KUBE_API_QPS + value: "200" + image: gcr.io/knative-releases/knative.dev/net-kourier/cmd/kourier@sha256:a2e8f568a4bf68ef6391bc3afea956d11a8698f6a22c4be50844b1da4f8bcc74 + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 6 + grpc: + port: 18000 + service: "" + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + name: controller + ports: + - containerPort: 18000 + name: http2-xds + protocol: TCP + - containerPort: 9090 + name: metrics + protocol: TCP + readinessProbe: + failureThreshold: 3 + grpc: + port: 18000 + service: "" + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + resources: + limits: + cpu: "1" + memory: 500Mi + requests: + cpu: 30m + memory: 200Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + dnsPolicy: ClusterFirst + restartPolicy: Always + schedulerName: default-scheduler + securityContext: {} + serviceAccount: net-kourier + serviceAccountName: net-kourier + terminationGracePeriodSeconds: 30 + status: + availableReplicas: 1 + conditions: + - lastTransitionTime: "2026-07-26T16:45:57Z" + lastUpdateTime: "2026-09-21T16:29:41Z" + message: ReplicaSet "net-kourier-controller-5bc4c79595" has successfully progressed. + reason: NewReplicaSetAvailable + status: "True" + type: Progressing + - lastTransitionTime: "2026-10-14T18:23:11Z" + lastUpdateTime: "2026-10-14T18:23:11Z" + message: Deployment has minimum availability. + reason: MinimumReplicasAvailable + status: "True" + type: Available + observedGeneration: 2 + readyReplicas: 1 + replicas: 1 + terminatingReplicas: 0 + updatedReplicas: 1 +kind: List +metadata: + resourceVersion: "" diff --git a/docs/evidence/2026-10-21-hpa-caps-before/deploy-kourier-system.yaml b/docs/evidence/2026-10-21-hpa-caps-before/deploy-kourier-system.yaml new file mode 100644 index 0000000..bf5058d --- /dev/null +++ b/docs/evidence/2026-10-21-hpa-caps-before/deploy-kourier-system.yaml @@ -0,0 +1,161 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + deployment.kubernetes.io/revision: "3" + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"apps/v1","kind":"Deployment","metadata":{"annotations":{},"labels":{"app.kubernetes.io/component":"net-kourier","app.kubernetes.io/name":"knative-serving","app.kubernetes.io/version":"1.22.0","networking.knative.dev/ingress-provider":"kourier"},"name":"3scale-kourier-gateway","namespace":"kourier-system"},"spec":{"selector":{"matchLabels":{"app":"3scale-kourier-gateway"}},"strategy":{"rollingUpdate":{"maxSurge":"100%","maxUnavailable":0},"type":"RollingUpdate"},"template":{"metadata":{"annotations":{"networking.knative.dev/poke":"v0.26","prometheus.io/path":"/stats/prometheus","prometheus.io/port":"9000","prometheus.io/scrape":"true"},"labels":{"app":"3scale-kourier-gateway"}},"spec":{"containers":[{"args":["--base-id 1","-c /tmp/config/envoy-bootstrap.yaml","--log-level info","--drain-time-s $(DRAIN_TIME_SECONDS)","--drain-strategy immediate"],"command":["/usr/local/bin/envoy"],"env":[{"name":"DRAIN_TIME_SECONDS","value":"15"}],"image":"docker.io/envoyproxy/envoy:v1.37-latest","lifecycle":{"preStop":{"exec":{"command":["/bin/sh","-c","curl -X POST http://localhost:9901/drain_listeners?graceful; sleep $DRAIN_TIME_SECONDS"]}}},"livenessProbe":{"failureThreshold":6,"httpGet":{"httpHeaders":[{"name":"Host","value":"internalkourier"}],"path":"/ready","port":8081,"scheme":"HTTP"},"initialDelaySeconds":10,"periodSeconds":5,"timeoutSeconds":3},"name":"kourier-gateway","ports":[{"containerPort":8080,"name":"http2-external","protocol":"TCP"},{"containerPort":8081,"name":"http2-internal","protocol":"TCP"},{"containerPort":8443,"name":"https-external","protocol":"TCP"},{"containerPort":8090,"name":"http-probe","protocol":"TCP"},{"containerPort":9443,"name":"https-probe","protocol":"TCP"},{"containerPort":9000,"name":"metrics","protocol":"TCP"}],"readinessProbe":{"failureThreshold":3,"httpGet":{"httpHeaders":[{"name":"Host","value":"internalkourier"}],"path":"/ready","port":8081,"scheme":"HTTP"},"initialDelaySeconds":10,"periodSeconds":5,"timeoutSeconds":3},"resources":{"limits":{"cpu":"1","memory":"800Mi"},"requests":{"cpu":"200m","memory":"200Mi"}},"securityContext":{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]},"readOnlyRootFilesystem":false,"runAsGroup":65534,"runAsNonRoot":true,"runAsUser":65534,"seccompProfile":{"type":"RuntimeDefault"}},"volumeMounts":[{"mountPath":"/tmp/config","name":"config-volume"}]}],"restartPolicy":"Always","terminationGracePeriodSeconds":30,"volumes":[{"configMap":{"name":"kourier-bootstrap"},"name":"config-volume"}]}}}} + creationTimestamp: "2026-07-26T16:45:57Z" + generation: 3 + labels: + app.kubernetes.io/component: net-kourier + app.kubernetes.io/name: knative-serving + app.kubernetes.io/version: 1.22.0 + networking.knative.dev/ingress-provider: kourier + name: 3scale-kourier-gateway + namespace: kourier-system + resourceVersion: "75293597" + uid: 562459bf-ed8a-4654-8e48-db4506ea72bf +spec: + progressDeadlineSeconds: 600 + replicas: 1 + revisionHistoryLimit: 10 + selector: + matchLabels: + app: 3scale-kourier-gateway + strategy: + rollingUpdate: + maxSurge: 100% + maxUnavailable: 0 + type: RollingUpdate + template: + metadata: + annotations: + networking.knative.dev/poke: v0.26 + prometheus.io/path: /stats/prometheus + prometheus.io/port: "9000" + prometheus.io/scrape: "true" + labels: + app: 3scale-kourier-gateway + spec: + containers: + - args: + - --base-id 1 + - -c /tmp/config/envoy-bootstrap.yaml + - --log-level info + - --drain-time-s $(DRAIN_TIME_SECONDS) + - --drain-strategy immediate + command: + - /usr/local/bin/envoy + env: + - name: DRAIN_TIME_SECONDS + value: "15" + image: docker.io/envoyproxy/envoy@sha256:1c2b79776c6e3b38e8b0113b825e6a599f9bfc08d680c199d80bf8964856c529 + imagePullPolicy: IfNotPresent + lifecycle: + preStop: + exec: + command: + - /bin/sh + - -c + - curl -X POST http://localhost:9901/drain_listeners?graceful; sleep + $DRAIN_TIME_SECONDS + livenessProbe: + failureThreshold: 6 + httpGet: + httpHeaders: + - name: Host + value: internalkourier + path: /ready + port: 8081 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 3 + name: kourier-gateway + ports: + - containerPort: 8080 + name: http2-external + protocol: TCP + - containerPort: 8081 + name: http2-internal + protocol: TCP + - containerPort: 8443 + name: https-external + protocol: TCP + - containerPort: 8090 + name: http-probe + protocol: TCP + - containerPort: 9443 + name: https-probe + protocol: TCP + - containerPort: 9000 + name: metrics + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + httpHeaders: + - name: Host + value: internalkourier + path: /ready + port: 8081 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 3 + resources: + limits: + cpu: "1" + memory: 800Mi + requests: + cpu: 50m + memory: 200Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: false + runAsGroup: 65534 + runAsNonRoot: true + runAsUser: 65534 + seccompProfile: + type: RuntimeDefault + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /tmp/config + name: config-volume + dnsPolicy: ClusterFirst + restartPolicy: Always + schedulerName: default-scheduler + securityContext: {} + terminationGracePeriodSeconds: 30 + volumes: + - configMap: + defaultMode: 420 + name: kourier-bootstrap + name: config-volume +status: + availableReplicas: 1 + conditions: + - lastTransitionTime: "2026-07-26T16:45:57Z" + lastUpdateTime: "2026-09-21T16:30:02Z" + message: ReplicaSet "3scale-kourier-gateway-5cf57fc9fd" has successfully progressed. + reason: NewReplicaSetAvailable + status: "True" + type: Progressing + - lastTransitionTime: "2026-10-14T18:23:33Z" + lastUpdateTime: "2026-10-14T18:23:33Z" + message: Deployment has minimum availability. + reason: MinimumReplicasAvailable + status: "True" + type: Available + observedGeneration: 3 + readyReplicas: 1 + replicas: 1 + terminatingReplicas: 0 + updatedReplicas: 1 diff --git a/docs/evidence/2026-10-21-hpa-caps-before/hpa-knative-serving.yaml b/docs/evidence/2026-10-21-hpa-caps-before/hpa-knative-serving.yaml new file mode 100644 index 0000000..b20ae3c --- /dev/null +++ b/docs/evidence/2026-10-21-hpa-caps-before/hpa-knative-serving.yaml @@ -0,0 +1,117 @@ +apiVersion: v1 +items: +- apiVersion: autoscaling/v2 + kind: HorizontalPodAutoscaler + metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"autoscaling/v2","kind":"HorizontalPodAutoscaler","metadata":{"annotations":{},"labels":{"app.kubernetes.io/component":"activator","app.kubernetes.io/name":"knative-serving","app.kubernetes.io/version":"1.22.0"},"name":"activator","namespace":"knative-serving"},"spec":{"maxReplicas":20,"metrics":[{"resource":{"name":"cpu","target":{"averageUtilization":100,"type":"Utilization"}},"type":"Resource"}],"minReplicas":1,"scaleTargetRef":{"apiVersion":"apps/v1","kind":"Deployment","name":"activator"}}} + creationTimestamp: "2026-07-26T16:45:42Z" + labels: + app.kubernetes.io/component: activator + app.kubernetes.io/name: knative-serving + app.kubernetes.io/version: 1.22.0 + name: activator + namespace: knative-serving + resourceVersion: "78344170" + uid: 6dfd641f-46c8-4e2b-85b6-3a3b4b3aaa0d + spec: + maxReplicas: 20 + metrics: + - resource: + name: cpu + target: + averageUtilization: 100 + type: Utilization + type: Resource + minReplicas: 1 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: activator + status: + conditions: + - lastTransitionTime: "2026-07-26T16:45:57Z" + message: recommended size matches current size + reason: ReadyForNewScale + status: "True" + type: AbleToScale + - lastTransitionTime: "2026-10-14T18:23:29Z" + message: the HPA was able to successfully calculate a replica count from cpu + resource utilization (percentage of request) + reason: ValidMetricFound + status: "True" + type: ScalingActive + - lastTransitionTime: "2026-09-21T16:30:32Z" + message: the desired count is within the acceptable range + reason: DesiredWithinRange + status: "False" + type: ScalingLimited + currentMetrics: + - resource: + current: + averageUtilization: 2 + averageValue: 1m + name: cpu + type: Resource + currentReplicas: 1 + desiredReplicas: 1 +- apiVersion: autoscaling/v2 + kind: HorizontalPodAutoscaler + metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"autoscaling/v2","kind":"HorizontalPodAutoscaler","metadata":{"annotations":{},"labels":{"app.kubernetes.io/component":"webhook","app.kubernetes.io/name":"knative-serving","app.kubernetes.io/version":"1.22.0"},"name":"webhook","namespace":"knative-serving"},"spec":{"maxReplicas":5,"metrics":[{"resource":{"name":"cpu","target":{"averageUtilization":100,"type":"Utilization"}},"type":"Resource"}],"minReplicas":1,"scaleTargetRef":{"apiVersion":"apps/v1","kind":"Deployment","name":"webhook"}}} + creationTimestamp: "2026-07-26T16:45:42Z" + labels: + app.kubernetes.io/component: webhook + app.kubernetes.io/name: knative-serving + app.kubernetes.io/version: 1.22.0 + name: webhook + namespace: knative-serving + resourceVersion: "78351594" + uid: 02419da2-c8d7-407f-aa99-b09ba146e3f0 + spec: + maxReplicas: 5 + metrics: + - resource: + name: cpu + target: + averageUtilization: 100 + type: Utilization + type: Resource + minReplicas: 1 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: webhook + status: + conditions: + - lastTransitionTime: "2026-07-26T16:45:57Z" + message: recommended size matches current size + reason: ReadyForNewScale + status: "True" + type: AbleToScale + - lastTransitionTime: "2026-10-14T18:23:14Z" + message: the HPA was able to successfully calculate a replica count from cpu + resource utilization (percentage of request) + reason: ValidMetricFound + status: "True" + type: ScalingActive + - lastTransitionTime: "2026-07-26T16:46:12Z" + message: the desired count is within the acceptable range + reason: DesiredWithinRange + status: "False" + type: ScalingLimited + currentMetrics: + - resource: + current: + averageUtilization: 10 + averageValue: 3m + name: cpu + type: Resource + currentReplicas: 1 + desiredReplicas: 1 +kind: List +metadata: + resourceVersion: "" diff --git a/docs/evidence/2026-10-21-hpa-caps-before/hpa-kourier-system.yaml b/docs/evidence/2026-10-21-hpa-caps-before/hpa-kourier-system.yaml new file mode 100644 index 0000000..f913c70 --- /dev/null +++ b/docs/evidence/2026-10-21-hpa-caps-before/hpa-kourier-system.yaml @@ -0,0 +1,57 @@ +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"autoscaling/v2","kind":"HorizontalPodAutoscaler","metadata":{"annotations":{},"labels":{"app.kubernetes.io/component":"net-kourier","app.kubernetes.io/name":"knative-serving","app.kubernetes.io/version":"1.22.0","networking.knative.dev/ingress-provider":"kourier"},"name":"3scale-kourier-gateway","namespace":"kourier-system"},"spec":{"maxReplicas":10,"metrics":[{"resource":{"name":"cpu","target":{"averageUtilization":100,"type":"Utilization"}},"type":"Resource"}],"minReplicas":1,"scaleTargetRef":{"apiVersion":"apps/v1","kind":"Deployment","name":"3scale-kourier-gateway"}}} + creationTimestamp: "2026-07-26T16:45:58Z" + labels: + app.kubernetes.io/component: net-kourier + app.kubernetes.io/name: knative-serving + app.kubernetes.io/version: 1.22.0 + networking.knative.dev/ingress-provider: kourier + name: 3scale-kourier-gateway + namespace: kourier-system + resourceVersion: "78351099" + uid: 6caa5e4c-fa74-4d85-96cd-5d970ba2efd6 +spec: + maxReplicas: 10 + metrics: + - resource: + name: cpu + target: + averageUtilization: 100 + type: Utilization + type: Resource + minReplicas: 1 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: 3scale-kourier-gateway +status: + conditions: + - lastTransitionTime: "2026-07-26T16:46:13Z" + message: recommended size matches current size + reason: ReadyForNewScale + status: "True" + type: AbleToScale + - lastTransitionTime: "2026-10-14T18:23:14Z" + message: the HPA was able to successfully calculate a replica count from cpu resource + utilization (percentage of request) + reason: ValidMetricFound + status: "True" + type: ScalingActive + - lastTransitionTime: "2026-07-26T16:46:58Z" + message: the desired count is within the acceptable range + reason: DesiredWithinRange + status: "False" + type: ScalingLimited + currentMetrics: + - resource: + current: + averageUtilization: 8 + averageValue: 4m + name: cpu + type: Resource + currentReplicas: 1 + desiredReplicas: 1 diff --git a/docs/evidence/2026-10-21-hpa-caps-before/pdb.txt b/docs/evidence/2026-10-21-hpa-caps-before/pdb.txt new file mode 100644 index 0000000..0848951 --- /dev/null +++ b/docs/evidence/2026-10-21-hpa-caps-before/pdb.txt @@ -0,0 +1,5 @@ +NAME MIN AVAILABLE MAX UNAVAILABLE ALLOWED DISRUPTIONS AGE +activator-pdb 80% N/A 0 86d +webhook-pdb 80% N/A 0 86d +NAME MIN AVAILABLE MAX UNAVAILABLE ALLOWED DISRUPTIONS AGE +3scale-kourier-gateway-pdb 80% N/A 0 86d diff --git a/docs/evidence/2026-10-21-hpa-caps-before/resources.txt b/docs/evidence/2026-10-21-hpa-caps-before/resources.txt new file mode 100644 index 0000000..40730f5 --- /dev/null +++ b/docs/evidence/2026-10-21-hpa-caps-before/resources.txt @@ -0,0 +1,6 @@ +knative-serving/activator activator req={"cpu":"50m","memory":"60Mi"} lim={"cpu":"1","memory":"600Mi"} +knative-serving/autoscaler autoscaler req={"cpu":"30m","memory":"100Mi"} lim={"cpu":"1","memory":"1000Mi"} +knative-serving/controller controller req={"cpu":"30m","memory":"100Mi"} lim={"cpu":"1","memory":"1000Mi"} +knative-serving/net-kourier-controller controller req={"cpu":"30m","memory":"200Mi"} lim={"cpu":"1","memory":"500Mi"} +knative-serving/webhook webhook req={"cpu":"30m","memory":"100Mi"} lim={"cpu":"500m","memory":"500Mi"} +kourier-system/3scale-kourier-gateway kourier-gateway req={"cpu":"50m","memory":"200Mi"} lim={"cpu":"1","memory":"800Mi"} diff --git a/docs/evidence/2026-10-21-hpa-caps-before/summary.txt b/docs/evidence/2026-10-21-hpa-caps-before/summary.txt new file mode 100644 index 0000000..d009e2c --- /dev/null +++ b/docs/evidence/2026-10-21-hpa-caps-before/summary.txt @@ -0,0 +1,33 @@ +NAME REFERENCE TARGETS MINPODS MAXPODS REPLICAS AGE +activator Deployment/activator cpu: 2%/100% 1 20 1 86d +webhook Deployment/webhook cpu: 10%/100% 1 5 1 86d +NAME REFERENCE TARGETS MINPODS MAXPODS REPLICAS AGE +3scale-kourier-gateway Deployment/3scale-kourier-gateway cpu: 8%/100% 1 10 1 86d +NAME CPU(cores) MEMORY(bytes) +activator-6b4748664b-kpc4n 1m 19Mi +autoscaler-764bdc4bc8-gl6x2 3m 27Mi +controller-547767bf48-5nmcf 2m 36Mi +net-kourier-controller-5bc4c79595-q5xqw 2m 42Mi +webhook-8566f4d4bb-rrrkg 3m 18Mi +NAME CPU(cores) MEMORY(bytes) +3scale-kourier-gateway-5cf57fc9fd-fplqc 4m 18Mi +NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES +activator-6b4748664b-kpc4n 1/1 Running 1 (6d12h ago) 29d 10.42.0.214 239.62.205.92.host.secureserver.net +autoscaler-764bdc4bc8-gl6x2 1/1 Running 1 (6d12h ago) 29d 10.42.0.230 239.62.205.92.host.secureserver.net +controller-547767bf48-5nmcf 1/1 Running 2 (6d12h ago) 29d 10.42.0.240 239.62.205.92.host.secureserver.net +net-kourier-controller-5bc4c79595-q5xqw 1/1 Running 3 (6d12h ago) 29d 10.42.0.187 239.62.205.92.host.secureserver.net +webhook-8566f4d4bb-rrrkg 1/1 Running 3 (6d12h ago) 29d 10.42.0.199 239.62.205.92.host.secureserver.net +NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES +3scale-kourier-gateway-5cf57fc9fd-fplqc 1/1 Running 2 (6d12h ago) 29d 10.42.0.245 239.62.205.92.host.secureserver.net +NAMESPACE NAME URL LATESTCREATED LATESTREADY READY REASON +rapp-qonto rapp-qonto http://rapp-qonto.rapp-qonto.svc.cluster.local rapp-qonto-00010 rapp-qonto-00010 True +Allocated resources: + (Total limits may be over 100 percent, i.e., overcommitted.) + Resource Requests Limits + -------- -------- ------ + cpu 3605m (90%) 37500m (937%) + memory 9856Mi (61%) 34562Mi (216%) + ephemeral-storage 256Mi (0%) 1Gi (0%) + hugepages-1Gi 0 (0%) 0 (0%) + hugepages-2Mi 0 (0%) 0 (0%) +Events: diff --git a/docs/substrate-runbook.md b/docs/substrate-runbook.md index a3eb8ee..ae6610f 100644 --- a/docs/substrate-runbook.md +++ b/docs/substrate-runbook.md @@ -1,23 +1,33 @@ -# Knative substrate runbook: declared CPU requests +# Knative substrate runbook: declared CPU requests and HPA ceilings The Knative Serving and Kourier v1.22.0 install on railiance01 is performed by `railiance-cluster/install/knative/install.sh` from checksum-pinned upstream assets. rail-knative declares the CPU requests that install must carry, in -`substrate/v1.22.0/cpu-requests.patch.yaml` (strategic-merge patches) and +`substrate/v1.22.0/cpu-requests.patch.yaml` (strategic-merge patches), the HPA +ceilings in `substrate/v1.22.0/hpa-caps.patch.yaml`, and `substrate/v1.22.0/kustomization.yaml` (overlay for the staged `core.yaml` and `kourier.yaml`; `crds.yaml` stays a separate first apply). -| Deployment | Container | Upstream v1.22.0 | Declared | -|---|---|---|---| -| knative-serving/activator | activator | 300m | 50m | -| kourier-system/3scale-kourier-gateway | kourier-gateway | 200m | 50m | -| knative-serving/net-kourier-controller | controller | 200m | 30m | -| knative-serving/controller | controller | 100m | 30m | -| knative-serving/webhook | webhook | 100m | 30m | -| knative-serving/autoscaler | autoscaler | 100m | 30m | +| Deployment | Container | Upstream v1.22.0 | 2026-09-21 | Declared (WP-0003) | +|---|---|---|---|---| +| knative-serving/activator | activator | 300m | 50m | 10m | +| kourier-system/3scale-kourier-gateway | kourier-gateway | 200m | 50m | 10m | +| knative-serving/net-kourier-controller | controller | 200m | 30m | 10m | +| knative-serving/controller | controller | 100m | 30m | 10m | +| knative-serving/webhook | webhook | 100m | 30m | 10m | +| knative-serving/autoscaler | autoscaler | 100m | 30m | 10m | -Only CPU requests differ from upstream. Memory requests and all limits are -upstream's. These values were set live on 2026-09-21 as +| HPA | Upstream min/max | Declared min/max (WP-0003) | +|---|---|---| +| knative-serving/activator | 1/20 | 1/3 | +| knative-serving/webhook | 1/5 | 1/2 | +| kourier-system/3scale-kourier-gateway | 1/10 | 1/3 | + +Only CPU requests and HPA `maxReplicas` differ from upstream. Memory requests, +all limits, `minReplicas` and the HPA CPU targets are upstream's. The +RAIL-KNATIVE-WP-0003 column (60m total, 7d p95 at most 7.2m per workload) is +declared but not live until the railiance-cluster overlays carry it and the +installer re-runs; see that workplan. The 2026-09-21 values were set live as `ADMINISTER @ realm:kubernetes/railiance01`, activation=APPROVED by the founder, because the node had 100% of its allocatable CPU requested and backups could not be scheduled. Record: `the-custodian/docs/kubernetes-change-gate-decision.md`. @@ -47,7 +57,9 @@ Kourier gateway and the webhook target 100% CPU utilisation *of the request*. Lowering a request makes the same load read as a higher percentage, so the HPA scales out sooner (at about 50m or 30m of use, not 300m or 100m). At 2026-09-21 use they read about 2-10%. If the request is raised or lowered -again, check the HPA targets in the same change. +again, check the HPA targets in the same change. At 10m requests (WP-0003) +the 2026-10-21 use reads 10-40%, and the declared `maxReplicas` bound any +scale-out on the single node to 3+2+3 pods. ## Read-only drift check diff --git a/substrate/v1.22.0/cpu-requests.patch.yaml b/substrate/v1.22.0/cpu-requests.patch.yaml index 83a8295..36cd1e6 100644 --- a/substrate/v1.22.0/cpu-requests.patch.yaml +++ b/substrate/v1.22.0/cpu-requests.patch.yaml @@ -2,9 +2,12 @@ # Strategic-merge patches over the upstream release manifests (serving-core.yaml, # kourier.yaml), checksum-pinned in railiance-cluster/install/knative/release-lock.env. # Only CPU requests are set; memory requests and all limits stay upstream. -# Live on railiance01 since 2026-09-21 (ADMINISTER @ realm:kubernetes/railiance01, -# activation=APPROVED by the founder). Re-applying upstream without these patches -# restores the upstream values noted per document and re-exhausts the node. +# First set live on railiance01 2026-09-21 (220m total; ADMINISTER @ +# realm:kubernetes/railiance01, activation=APPROVED by the founder). Trimmed to +# 10m each (60m total) by RAIL-KNATIVE-WP-0003 per the resource-control node +# budget of 2026-10-21 (7d p95 <= 7.2m per workload). Re-applying upstream +# without these patches restores the upstream values noted per document and +# re-exhausts the node. # Apply after each upstream apply, e.g. for each document: # kubectl patch deployment -n --type strategic --patch-file # or list this file under `patches:` in a kustomization over the staged assets. @@ -22,7 +25,7 @@ spec: - name: activator resources: requests: - cpu: 50m + cpu: 10m --- # upstream v1.22.0 request: 100m apiVersion: apps/v1 @@ -37,7 +40,7 @@ spec: - name: autoscaler resources: requests: - cpu: 30m + cpu: 10m --- # upstream v1.22.0 request: 100m apiVersion: apps/v1 @@ -52,7 +55,7 @@ spec: - name: controller resources: requests: - cpu: 30m + cpu: 10m --- # upstream v1.22.0 request: 100m apiVersion: apps/v1 @@ -67,7 +70,7 @@ spec: - name: webhook resources: requests: - cpu: 30m + cpu: 10m --- # upstream v1.22.0 request: 200m apiVersion: apps/v1 @@ -82,7 +85,7 @@ spec: - name: controller resources: requests: - cpu: 30m + cpu: 10m --- # upstream v1.22.0 request: 200m apiVersion: apps/v1 @@ -97,4 +100,4 @@ spec: - name: kourier-gateway resources: requests: - cpu: 50m + cpu: 10m diff --git a/substrate/v1.22.0/hpa-caps.patch.yaml b/substrate/v1.22.0/hpa-caps.patch.yaml new file mode 100644 index 0000000..9e67c05 --- /dev/null +++ b/substrate/v1.22.0/hpa-caps.patch.yaml @@ -0,0 +1,37 @@ +# Declared HPA ceilings for the Knative Serving + Kourier v1.22.0 substrate on a +# single node (railiance01). Strategic-merge patches over the upstream HPAs in +# serving-core.yaml and kourier.yaml; only maxReplicas changes. minReplicas (1) +# and the CPU target (100% of request) stay upstream. +# The HPAs scale on percent of the CPU request, so after the 10m request trim +# (cpu-requests.patch.yaml) the same load reads ~5x higher. These caps bound the +# worst-case scale-out on one node to 3+2+3 pods instead of 20+5+10. +# RAIL-KNATIVE-WP-0003; evidence: resource-control +# docs/evidence/2026-10-21-node-allocation-review.md ("Knative HPA scale-out +# headroom" 1520m -> 230m). +--- +# upstream v1.22.0: minReplicas 1, maxReplicas 20 +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: activator + namespace: knative-serving +spec: + maxReplicas: 3 +--- +# upstream v1.22.0: minReplicas 1, maxReplicas 5 +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: webhook + namespace: knative-serving +spec: + maxReplicas: 2 +--- +# upstream v1.22.0: minReplicas 1, maxReplicas 10 +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: 3scale-kourier-gateway + namespace: kourier-system +spec: + maxReplicas: 3 diff --git a/substrate/v1.22.0/kustomization.yaml b/substrate/v1.22.0/kustomization.yaml index 44ffd58..d1db1fc 100644 --- a/substrate/v1.22.0/kustomization.yaml +++ b/substrate/v1.22.0/kustomization.yaml @@ -10,3 +10,4 @@ resources: - kourier.yaml patches: - path: cpu-requests.patch.yaml + - path: hpa-caps.patch.yaml diff --git a/tests/test_substrate_requests.py b/tests/test_substrate_requests.py index cc128a6..42dedb1 100644 --- a/tests/test_substrate_requests.py +++ b/tests/test_substrate_requests.py @@ -4,14 +4,15 @@ import yaml ROOT = Path(__file__).parents[1] SUBSTRATE = ROOT / "substrate/v1.22.0" -# Live on railiance01 since 2026-09-21; see docs/substrate-runbook.md. +# 220m live since 2026-09-21; trimmed to 10m each by RAIL-KNATIVE-WP-0003. +# See docs/substrate-runbook.md. DECLARED = { - ("knative-serving", "activator", "activator"): "50m", - ("knative-serving", "autoscaler", "autoscaler"): "30m", - ("knative-serving", "controller", "controller"): "30m", - ("knative-serving", "webhook", "webhook"): "30m", - ("knative-serving", "net-kourier-controller", "controller"): "30m", - ("kourier-system", "3scale-kourier-gateway", "kourier-gateway"): "50m", + ("knative-serving", "activator", "activator"): "10m", + ("knative-serving", "autoscaler", "autoscaler"): "10m", + ("knative-serving", "controller", "controller"): "10m", + ("knative-serving", "webhook", "webhook"): "10m", + ("knative-serving", "net-kourier-controller", "controller"): "10m", + ("kourier-system", "3scale-kourier-gateway", "kourier-gateway"): "10m", } @@ -30,4 +31,25 @@ def test_patches_declare_exactly_the_cpu_requests(): def test_kustomization_patches_the_pinned_assets(): k = yaml.safe_load((SUBSTRATE / "kustomization.yaml").read_text()) assert k["resources"] == ["core.yaml", "kourier.yaml"] - assert k["patches"] == [{"path": "cpu-requests.patch.yaml"}] + assert k["patches"] == [ + {"path": "cpu-requests.patch.yaml"}, + {"path": "hpa-caps.patch.yaml"}, + ] + + +# Single-node ceilings, RAIL-KNATIVE-WP-0003. minReplicas stays upstream (1). +HPA_CAPS = { + ("knative-serving", "activator"): 3, + ("knative-serving", "webhook"): 2, + ("kourier-system", "3scale-kourier-gateway"): 3, +} + + +def test_hpa_patches_only_cap_max_replicas(): + found = {} + for doc in yaml.safe_load_all((SUBSTRATE / "hpa-caps.patch.yaml").read_text()): + assert doc["apiVersion"] == "autoscaling/v2" + assert doc["kind"] == "HorizontalPodAutoscaler" + assert set(doc["spec"]) == {"maxReplicas"} + found[(doc["metadata"]["namespace"], doc["metadata"]["name"])] = doc["spec"]["maxReplicas"] + assert found == HPA_CAPS diff --git a/workplans/RAIL-KNATIVE-WP-0003-single-node-hpa-caps-and-request-trim.md b/workplans/RAIL-KNATIVE-WP-0003-single-node-hpa-caps-and-request-trim.md new file mode 100644 index 0000000..4e62085 --- /dev/null +++ b/workplans/RAIL-KNATIVE-WP-0003-single-node-hpa-caps-and-request-trim.md @@ -0,0 +1,168 @@ +--- +id: RAIL-KNATIVE-WP-0003 +type: workplan +title: "Cap Knative HPA ceilings for one node and trim substrate CPU requests to 60m" +domain: financials +repo: rail-knative +status: active +owner: codex +topic_slug: railiance +created: "2026-10-21" +updated: "2026-10-21" +depends_on: [] +related: [CUST-WP-0071, RESOURCE-WP-0008] +--- + +# RAIL-KNATIVE-WP-0003 - Single-node HPA caps and CPU request trim + +Receiving workplan for the three rail-knative rows of the resource-control node +budget (CUST-WP-0071 T03/T04, RESOURCE-WP-0008). + +## Evidence and rationale + +- Review: `resource-control/docs/evidence/2026-10-21-node-allocation-review.md` + (Knative/Kourier item; "Knative HPA scale-out headroom" 1520m -> 230m). +- Budget rows: `resource-control/docs/evidence/2026-10-21-node-allocation-review/2026-10-21-node-budget.{md,json}` + - knative-serving/knative-serving (4 pods): 140m -> 40m, 7d p95 7.2m, peak 9.9m, throttle 0 + - knative-serving/net-kourier-controller: 30m -> 10m, p95 0.8m, peak 1.6m + - kourier-system/3scale-kourier-gateway: 50m -> 10m, p95 3.6m, peak 3.8m +- Delivery decision: `the-custodian/docs/decisions/2026-10-21-cust-wp-0071-delivery-interview.md`. +- BEFORE state (read-only, 2026-10-21): `docs/evidence/2026-10-21-hpa-caps-before/`. + Live HPAs: activator 1/20 (cpu 2%/100%), webhook 1/5 (10%/100%), + 3scale-kourier-gateway 1/10 (8%/100%). All six Deployments 1/1 Ready. + Node CPU requests 3605m (90%). + +railiance01 is one node. The upstream HPA ceilings (20+5+10 pods) can request +up to 1520m on it, and since the HPAs scale on percent of request, every +request trim makes them scale out sooner. Capping `maxReplicas` makes the trim +safe; `minReplicas` stays 1 (upstream). Knative recommends >= 2 activator +replicas for HA across nodes; with one node that buys nothing, so min 1 stays. +Limits stay upstream (1 CPU, webhook 500m), so the request trim cannot cause +CPU throttling; only scheduling weight and HPA percent change. Memory requests +stay upstream in this workplan (the budget also proposes memory down; that is a +separate, later step). + +| Object | Before (live) | After (declared) | +|---|---|---| +| HPA knative-serving/activator max | 20 | 3 | +| HPA knative-serving/webhook max | 5 | 2 | +| HPA kourier-system/3scale-kourier-gateway max | 10 | 3 | +| activator cpu request | 50m | 10m | +| autoscaler cpu request | 30m | 10m | +| controller cpu request | 30m | 10m | +| webhook cpu request | 30m | 10m | +| net-kourier-controller cpu request | 30m | 10m | +| 3scale-kourier-gateway cpu request | 50m | 10m | +| total cpu request | 220m | 60m | + +## T01 - Declare caps and trimmed requests over the pinned v1.22.0 assets + +```task +id: RAIL-KNATIVE-WP-0003-T01 +status: done +priority: high +``` + +2026-10-21: `substrate/v1.22.0/cpu-requests.patch.yaml` now declares 10m for +all six containers; new `substrate/v1.22.0/hpa-caps.patch.yaml` sets only +`maxReplicas` on the three upstream HPAs; the kustomization lists both; +`tests/test_substrate_requests.py` and `docs/substrate-runbook.md` updated. +Validation: `python3 -m pytest -q -p no:cacheprovider tests` 5 passed; +`validate_contracts.py --rail declarations/rail.yaml` passed. Rendered with +`kubectl kustomize` over the checksum-verified upstream `core.yaml` and +`kourier.yaml` (staged by `railiance-cluster/install/knative/render.sh`): +exactly 9 of 72 documents differ from upstream - six Deployment CPU requests +and three HPA `maxReplicas`; memory, limits and `minReplicas` unchanged. + +## T02 - Owner handoff, apply and verify on railiance01 + +```task +id: RAIL-KNATIVE-WP-0003-T02 +status: wait +priority: high +needs_human: true +blocking_reason: "Coordinator applies wave-1 changes one owner at a time after review; railiance-cluster overlays must follow this declaration" +``` + +**Cross-repo coupling.** The deploy path is +`railiance-cluster/install/knative/install.sh` (render.sh + `kubectl apply` +over ssh; no ArgoCD Application). Its overlays still carry the 2026-09-21 +values, and `railiance-cluster/tests/test_knative_render.py::test_matches_rail_knative_declaration` +compares them with this repository, so it fails until railiance-cluster +follows: update `install/knative/overlays/{serving-core,kourier}/cpu-requests.patch.yaml` +to 10m, add the HPA caps patch to both overlays (activator and webhook in +serving-core, 3scale-kourier-gateway in kourier), update the `DECLARED` table +in the test and the request list in `install/knative/verify.sh`, and add HPA +`maxReplicas` checks to the verifier. Until then, a re-run of the installer +reverts the live change to 220m and 20/5/10. + +**Apply (coordinator, in this order: caps first, so the lower requests never +meet the upstream ceilings).** Each request patch rolls one pod (maxSurge 1); +node has ~395m request headroom, each new pod asks 10m. + +```bash +ssh railiance01 'set -e +kubectl patch hpa activator -n knative-serving --type merge -p "{\"spec\":{\"maxReplicas\":3}}" +kubectl patch hpa webhook -n knative-serving --type merge -p "{\"spec\":{\"maxReplicas\":2}}" +kubectl patch hpa 3scale-kourier-gateway -n kourier-system --type merge -p "{\"spec\":{\"maxReplicas\":3}}" +for d in activator autoscaler controller webhook; do + kubectl set resources deployment/$d -n knative-serving -c $d --requests=cpu=10m +done +kubectl set resources deployment/net-kourier-controller -n knative-serving -c controller --requests=cpu=10m +kubectl set resources deployment/3scale-kourier-gateway -n kourier-system -c kourier-gateway --requests=cpu=10m +for d in activator autoscaler controller webhook net-kourier-controller; do + kubectl rollout status deployment/$d -n knative-serving --timeout=300s +done +kubectl rollout status deployment/3scale-kourier-gateway -n kourier-system --timeout=300s' +``` + +Owner path once railiance-cluster has followed (same result, durable): + +```bash +cd ~/railiance-cluster && python3 -m pytest -q tests/test_knative_render.py \ + && install/knative/install.sh railiance01 && install/knative/verify.sh railiance01 +``` + +**Verify (all must hold).** + +```bash +ssh railiance01 'kubectl get hpa -n knative-serving; kubectl get hpa -n kourier-system +kubectl get deploy -n knative-serving; kubectl get deploy -n kourier-system +kubectl get deploy -n knative-serving -o jsonpath="{range .items[*]}{.metadata.name} {.spec.template.spec.containers[0].resources.requests.cpu}{\"\n\"}{end}" +kubectl get deploy -n kourier-system -o jsonpath="{range .items[*]}{.metadata.name} {.spec.template.spec.containers[0].resources.requests.cpu}{\"\n\"}{end}" +kubectl get ksvc -A; kubectl top pods -n knative-serving; kubectl top pods -n kourier-system' +``` + +- HPA max 3/2/3, min 1; six Deployments Available, pods Ready, no restarts. +- All six requests read 10m. +- `ksvc rapp-qonto` stays Ready=True and answers a request through Kourier + (scale-from-zero via the activator included) — e.g. an in-cluster + `curl -sS -o /dev/null -w '%{http_code}' http://rapp-qonto.rapp-qonto.svc.cluster.local` + from a debug pod. +- No throttling regression: resource-control 7d `throttle_ratio_7d` stays 0 + for the three workloads at the next weekly allocation report; HPA replicas + stay at 1 at idle (activator ~10%, webhook ~30%, gateway ~40% of request). + +**Rollback (restores the BEFORE state exactly).** + +```bash +ssh railiance01 'set -e +kubectl set resources deployment/activator -n knative-serving -c activator --requests=cpu=50m +for d in autoscaler controller webhook; do + kubectl set resources deployment/$d -n knative-serving -c $d --requests=cpu=30m +done +kubectl set resources deployment/net-kourier-controller -n knative-serving -c controller --requests=cpu=30m +kubectl set resources deployment/3scale-kourier-gateway -n kourier-system -c kourier-gateway --requests=cpu=50m +kubectl patch hpa activator -n knative-serving --type merge -p "{\"spec\":{\"maxReplicas\":20}}" +kubectl patch hpa webhook -n knative-serving --type merge -p "{\"spec\":{\"maxReplicas\":5}}" +kubectl patch hpa 3scale-kourier-gateway -n kourier-system --type merge -p "{\"spec\":{\"maxReplicas\":10}}"' +``` + +and `git revert` this workplan's declaration commit (plus the railiance-cluster +follow-up if it landed). Full BEFORE objects: `docs/evidence/2026-10-21-hpa-caps-before/`. + +Risks: a 10m request gives these pods little CPU weight under node contention +(limits still allow bursts); the caps limit burst capacity of the activator +and gateway to 3 pods, which on one node is the practical ceiling anyway; the +80% PDBs on activator, webhook and gateway already allow 0 disruptions at 1 +replica (unchanged, but it blocks node drain).