diff --git a/multidimensional-pod-autoscaler/README.md b/multidimensional-pod-autoscaler/README.md new file mode 100644 index 000000000000..647429f4f5ae --- /dev/null +++ b/multidimensional-pod-autoscaler/README.md @@ -0,0 +1,138 @@ +# Multi-dimensional Pod Autoscaler (MPA) + +## Intro + +Multi-dimensional Pod Autoscaler (MPA) combines Kubernetes [HPA](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/) +and [VPA](../vertical-pod-autoscaler/README.md) so that scaling actions can be considered and actuated together in a holistic manner. +MPA separates the scaling recommendation and actuation completely so that any multi-dimensional autoscaling algorithms can be used as the "recommender". +The default recommender is a simple combination of HPA and VPA algorithm which + +1) sets the requests automatically based on usage and +2) sets the number of replicas based on a target metric. + +Like VPA, MPA is configured with a [Custom Resource Definition object](https://kubernetes.io/docs/concepts/api-extension/custom-resources/). +The custom resource is called [MultidimPodAutoscaler](./pkg/apis/autoscaling.k8s.io/v1alpha1/types.go). +It specifies which pods should be vertically and horizontally autoscaled as well as if/how the resource recommendations are applied. + +To enable multi-dimensional pod autoscaling on your cluster please follow the installation procedure described below. + +## Prerequisites + +* `kubectl` should be connected to the cluster you want to install MPA. +* The metrics server must be deployed in your cluster. Read more about [Metrics Server](https://github.com/kubernetes-incubator/metrics-server). +* If you are using a GKE Kubernetes cluster, you will need to grant your current Google + identity `cluster-admin` role. Otherwise, you won't be authorized to grant extra + privileges to the MPA system components. + ```console + $ gcloud info | grep Account # get current google identity + Account: [myname@example.org] + + $ kubectl create clusterrolebinding myname-cluster-admin-binding --clusterrole=cluster-admin --user=myname@example.org + Clusterrolebinding "myname-cluster-admin-binding" created + ``` +* You should make sure your API server supports Mutating Webhooks. + Its `--admission-control` flag should have `MutatingAdmissionWebhook` as one of + the values on the list and its `--runtime-config` flag should include + `admissionregistration.k8s.io/v1beta1=true`. + To change those flags, ssh to your API Server instance, edit + `/etc/kubernetes/manifests/kube-apiserver.manifest` and restart kubelet to pick + up the changes: ```sudo systemctl restart kubelet.service``` +* Please upgrade `openssl` to version 1.1.1 or higher (needs to support `-addext` option). + +## Installation + +To install MPA, please download the source code of MPA +and run the following command inside the `multidimensional-pod-autoscaler` directory: + +``` +./hack/mpa-up.sh +``` + +Note: the script currently reads environment variables: `$REGISTRY` and `$TAG`. +Make sure they're unset unless you want to use a non-default version of MPA. + +The script issues multiple `kubectl` commands to the +cluster that insert the configuration and start all needed pods +in the `kube-system` namespace. It also generates +and uploads a secret (a CA cert) used by MPA Admission Controller when communicating +with the API server. + +## Tearing Down + +To remove MPA installation: + +``` +./deploy/mpa-down.sh +``` + +## Quick Start + +After [installation](#installation) the system is ready to recommend and set +resource requests for your pods. +In order to use it, you need to insert a *MultidimPodAutoscaler* resource for +each controller that you want to have automatically computed resource requirements. +This will be most commonly a **Deployment**. +There are four modes in which *MPAs* operate, same as [VPA modes](https://github.com/kubernetes/autoscaler/tree/master/vertical-pod-autoscaler#quick-start). + +#### Example Deployment + +A simple way to check if Multidimensional Pod Autoscaler is fully operational in your +cluster is to create a sample deployment: + +``` +kubectl create -f examples/hamster.yaml +``` + +The above command creates a deployment with two pods, each running a single container +that requests 100 millicores and tries to utilize slightly above 200 millicores. + +#### Example MPA Config + +``` +--- +apiVersion: "autoscaling.k8s.io/v1alpha1" +kind: MultidimPodAutoscaler +metadata: + name: hamster-mpa + namespace: default +spec: + # recommenders field can be unset when using the default recommender. + # recommenders: + # - name: 'hamster-recommender' + scaleTargetRef: + apiVersion: "apps/v1" + kind: Deployment + name: hamster + resourcePolicy: + containerPolicies: + - containerName: '*' + minAllowed: + cpu: 100m + memory: 50Mi + maxAllowed: + cpu: 1 + memory: 500Mi + controlledResources: ["cpu", "memory"] + constraints: + minReplicas: 1 + maxReplicas: 6 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 30 +``` + +Create an MPA: + +``` +kubectl create -f examples/hamster-mpa.yaml +``` + +To see MPA config and current recommended resource requests run: + +``` +kubectl describe mpa +``` diff --git a/multidimensional-pod-autoscaler/common/kubeconfig.go b/multidimensional-pod-autoscaler/common/kubeconfig.go new file mode 100644 index 000000000000..65c72418d678 --- /dev/null +++ b/multidimensional-pod-autoscaler/common/kubeconfig.go @@ -0,0 +1,47 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package common + +import ( + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/klog/v2" +) + +// CreateKubeConfigOrDie builds and returns a kubeconfig from file or in-cluster configuration. +func CreateKubeConfigOrDie(kubeconfig string, kubeApiQps float32, kubeApiBurst int) *rest.Config { + var config *rest.Config + var err error + if len(kubeconfig) > 0 { + klog.V(1).InfoS("Using kubeconfig", "file", kubeconfig) + // use the current context in kubeconfig + config, err = clientcmd.BuildConfigFromFlags("", kubeconfig) + if err != nil { + klog.Fatalf("Failed to build kubeconfig from file: %v", err) + } + } else { + config, err = rest.InClusterConfig() + if err != nil { + klog.Fatalf("Failed to create config: %v", err) + } + } + + config.QPS = kubeApiQps + config.Burst = kubeApiBurst + + return config +} diff --git a/multidimensional-pod-autoscaler/common/version.go b/multidimensional-pod-autoscaler/common/version.go new file mode 100644 index 000000000000..9b7cba420741 --- /dev/null +++ b/multidimensional-pod-autoscaler/common/version.go @@ -0,0 +1,20 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package common + +// MultidimPodAutoscalerVersion is the version of MPA. +const MultidimPodAutoscalerVersion = "0.0.1" diff --git a/multidimensional-pod-autoscaler/deploy/admission-controller-deployment.yaml b/multidimensional-pod-autoscaler/deploy/admission-controller-deployment.yaml new file mode 100644 index 000000000000..7e86153e073b --- /dev/null +++ b/multidimensional-pod-autoscaler/deploy/admission-controller-deployment.yaml @@ -0,0 +1,60 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mpa-admission-controller + namespace: kube-system +spec: + replicas: 1 + selector: + matchLabels: + app: mpa-admission-controller + template: + metadata: + labels: + app: mpa-admission-controller + spec: + serviceAccountName: mpa-admission-controller + securityContext: + runAsNonRoot: true + runAsUser: 65534 # nobody + containers: + - name: admission-controller + image: registry.k8s.io/autoscaling/mpa-admission-controller:latest + imagePullPolicy: IfNotPresent + env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumeMounts: + - name: tls-certs + mountPath: "/etc/tls-certs" + readOnly: true + resources: + limits: + cpu: 200m + memory: 500Mi + requests: + cpu: 50m + memory: 200Mi + ports: + - containerPort: 8000 + - name: prometheus + containerPort: 8944 + volumes: + - name: tls-certs + secret: + secretName: mpa-tls-certs +--- +apiVersion: v1 +kind: Service +metadata: + name: mpa-webhook + namespace: kube-system +spec: + ports: + - port: 443 + targetPort: 8000 + selector: + app: mpa-admission-controller diff --git a/multidimensional-pod-autoscaler/deploy/mpa-rbac.yaml b/multidimensional-pod-autoscaler/deploy/mpa-rbac.yaml new file mode 100644 index 000000000000..c7d01c074a24 --- /dev/null +++ b/multidimensional-pod-autoscaler/deploy/mpa-rbac.yaml @@ -0,0 +1,393 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: system:metrics-reader +rules: + - apiGroups: + - "metrics.k8s.io" + resources: + - pods + verbs: + - get + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: system:mpa-actor +rules: + - apiGroups: + - "" + resources: + - pods + - nodes + - limitranges + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - events + verbs: + - get + - list + - watch + - create + - patch + - apiGroups: + - "poc.autoscaling.k8s.io" + resources: + - multidimpodautoscalers + verbs: + - get + - list + - watch + - patch + - apiGroups: + - "autoscaling.k8s.io" + resources: + - multidimpodautoscalers + verbs: + - get + - list + - watch + - patch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: system:mpa-status-actor +rules: + - apiGroups: + - "autoscaling.k8s.io" + resources: + - multidimpodautoscalers/status + verbs: + - get + - patch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: system:mpa-checkpoint-actor +rules: + - apiGroups: + - "poc.autoscaling.k8s.io" + resources: + - multidimpodautoscalercheckpoints + verbs: + - get + - list + - watch + - create + - patch + - delete + - apiGroups: + - "autoscaling.k8s.io" + resources: + - multidimpodautoscalercheckpoints + verbs: + - get + - list + - watch + - create + - patch + - delete + - apiGroups: + - "" + resources: + - namespaces + verbs: + - get + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: system:evictioner +rules: + - apiGroups: + - "apps" + - "extensions" + resources: + - replicasets + verbs: + - get + - apiGroups: + - "" + resources: + - pods/eviction + verbs: + - create +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: system:metrics-reader +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:metrics-reader +subjects: + - kind: ServiceAccount + name: mpa-recommender + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: system:mpa-actor +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:mpa-actor +subjects: + - kind: ServiceAccount + name: mpa-recommender + namespace: kube-system + - kind: ServiceAccount + name: mpa-updater + namespace: kube-system + - kind: ServiceAccount + name: mpa-admission-controller + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: system:mpa-status-actor +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:mpa-status-actor +subjects: + - kind: ServiceAccount + name: mpa-recommender + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: system:mpa-checkpoint-actor +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:mpa-checkpoint-actor +subjects: + - kind: ServiceAccount + name: mpa-recommender + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: system:mpa-target-reader +rules: + - apiGroups: + - '*' + resources: + - '*/scale' + verbs: + - get + - watch + - apiGroups: + - "" + resources: + - replicationcontrollers + verbs: + - get + - list + - watch + - apiGroups: + - apps + resources: + - daemonsets + - deployments + - replicasets + - statefulsets + verbs: + - get + - list + - watch + - apiGroups: + - batch + resources: + - jobs + - cronjobs + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: system:mpa-target-reader-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:mpa-target-reader +subjects: + - kind: ServiceAccount + name: mpa-recommender + namespace: kube-system + - kind: ServiceAccount + name: mpa-admission-controller + namespace: kube-system + - kind: ServiceAccount + name: mpa-updater + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: system:mpa-evictionter-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:evictioner +subjects: + - kind: ServiceAccount + name: mpa-updater + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: system:controller:mpa-horizontal-pod-autoscaler-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:controller:horizontal-pod-autoscaler +subjects: + - kind: ServiceAccount + name: mpa-updater + namespace: kube-system +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: mpa-admission-controller + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: system:mpa-admission-controller +rules: + - apiGroups: + - "" + resources: + - pods + - configmaps + - nodes + - limitranges + verbs: + - get + - list + - watch + - apiGroups: + - "admissionregistration.k8s.io" + resources: + - mutatingwebhookconfigurations + verbs: + - create + - delete + - get + - list + - apiGroups: + - "poc.autoscaling.k8s.io" + resources: + - multidimpodautoscalers + verbs: + - get + - list + - watch + - apiGroups: + - "autoscaling.k8s.io" + resources: + - multidimpodautoscalers + verbs: + - get + - list + - watch + - apiGroups: + - "coordination.k8s.io" + resources: + - leases + verbs: + - create + - update + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: system:mpa-status-reader +rules: + - apiGroups: + - "coordination.k8s.io" + resources: + - leases + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: system:mpa-status-reader-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:mpa-status-reader +subjects: + - kind: ServiceAccount + name: mpa-updater + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: system:mpa-admission-controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:mpa-admission-controller +subjects: + - kind: ServiceAccount + name: mpa-admission-controller + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: system:mpa-status-reader +rules: + - apiGroups: + - "coordination.k8s.io" + resources: + - leases + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: system:mpa-status-reader-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:mpa-status-reader +subjects: + - kind: ServiceAccount + name: mpa-updater + namespace: kube-system diff --git a/multidimensional-pod-autoscaler/deploy/mpa-v1alpha1-crd-gen.yaml b/multidimensional-pod-autoscaler/deploy/mpa-v1alpha1-crd-gen.yaml new file mode 100644 index 000000000000..0752820e7eb2 --- /dev/null +++ b/multidimensional-pod-autoscaler/deploy/mpa-v1alpha1-crd-gen.yaml @@ -0,0 +1,1534 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + api-approved.kubernetes.io: https://github.com/kubernetes/kubernetes/pull/63797 + controller-gen.kubebuilder.io/version: v0.16.5 + name: multidimpodautoscalercheckpoints.autoscaling.k8s.io +spec: + group: autoscaling.k8s.io + names: + kind: MultidimPodAutoscalerCheckpoint + listKind: MultidimPodAutoscalerCheckpointList + plural: multidimpodautoscalercheckpoints + shortNames: + - mpacheckpoint + singular: multidimpodautoscalercheckpoint + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: |- + MultidimPodAutoscalerCheckpoint is the checkpoint of the internal state of VPA that + is used for recovery after recommender's restart. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + Specification of the checkpoint. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status. + properties: + containerName: + description: Name of the checkpointed container. + type: string + mpaObjectName: + description: Name of the MPA object that stored MultidimPodAutoscalerCheckpoint + object. + type: string + type: object + status: + description: Data of the checkpoint. + properties: + cpuHistogram: + description: Checkpoint of histogram for consumption of CPU. + properties: + bucketWeights: + description: Map from bucket index to bucket weight. + type: object + x-kubernetes-preserve-unknown-fields: true + referenceTimestamp: + description: Reference timestamp for samples collected within + this histogram. + format: date-time + nullable: true + type: string + totalWeight: + description: Sum of samples to be used as denominator for weights + from BucketWeights. + type: number + type: object + firstSampleStart: + description: Timestamp of the fist sample from the histograms. + format: date-time + nullable: true + type: string + lastSampleStart: + description: Timestamp of the last sample from the histograms. + format: date-time + nullable: true + type: string + lastUpdateTime: + description: The time when the status was last refreshed. + format: date-time + nullable: true + type: string + memoryHistogram: + description: Checkpoint of histogram for consumption of memory. + properties: + bucketWeights: + description: Map from bucket index to bucket weight. + type: object + x-kubernetes-preserve-unknown-fields: true + referenceTimestamp: + description: Reference timestamp for samples collected within + this histogram. + format: date-time + nullable: true + type: string + totalWeight: + description: Sum of samples to be used as denominator for weights + from BucketWeights. + type: number + type: object + totalSamplesCount: + description: Total number of samples in the histograms. + type: integer + version: + description: Version of the format of the stored data. + type: string + type: object + type: object + served: true + storage: true +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + api-approved.kubernetes.io: https://github.com/kubernetes/kubernetes/pull/63797 + controller-gen.kubebuilder.io/version: v0.16.5 + name: multidimpodautoscalers.autoscaling.k8s.io +spec: + group: autoscaling.k8s.io + names: + kind: MultidimPodAutoscaler + listKind: MultidimPodAutoscalerList + plural: multidimpodautoscalers + shortNames: + - mpa + singular: multidimpodautoscaler + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.updatePolicy.updateMode + name: Mode + type: string + - jsonPath: .status.recommendation.containerRecommendations[0].target.cpu + name: CPU + type: string + - jsonPath: .status.recommendation.containerRecommendations[0].target.memory + name: Mem + type: string + - jsonPath: .status.conditions[?(@.type=='RecommendationProvided')].status + name: Provided + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: |- + MultidimPodAutoscaler is the configuration for a multidimensional pod autoscaler, + which automatically manages pod resources and number of replicas based on historical and + real-time resource utilization as well as workload performance. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + Specification of the behavior of the autoscaler. + More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#spec-and-status. + properties: + constraints: + description: Describes the constraints for horizontal and vertical + scaling. + properties: + container: + description: Per-container resource policies. + items: + description: VerticalScalingConstraints describes the constraints + for vertical scaling. + properties: + mode: + description: Whether autoscaler is enabled for the container. + The default is "Auto". + enum: + - Auto + - "Off" + type: string + name: + description: Name of the container. + type: string + requests: + description: Describes the vertical scaling limits. + properties: + maxAllowed: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Specifies the maximum amount of resources that will be recommended + for the container. The default is no maximum. + type: object + minAllowed: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Specifies the minimal amount of resources that will be recommended + for the container. The default is no minimum. + type: object + type: object + type: object + type: array + containerControlledResources: + description: |- + Defines controlled resources. + If not specified, the default of [cpu, memory] will be used. + items: + description: ResourceName is the name identifying various resources + in a ResourceList. + type: string + type: array + global: + description: HorizontalScalingConstraints describes the constraints + for horizontal scaling. + properties: + behavior: + description: |- + Behavior configures the scaling behavior of the target in both Up and Down direction + (scaleUp and scaleDown fields respectively). + properties: + scaleDown: + description: |- + scaleDown is scaling policy for scaling Down. + If not set, the default value is to allow to scale down to minReplicas pods, with a + 300 second stabilization window (i.e., the highest recommendation for + the last 300sec is used). + properties: + policies: + description: |- + policies is a list of potential scaling polices which can be used during scaling. + At least one policy must be specified, otherwise the HPAScalingRules will be discarded as invalid + items: + description: HPAScalingPolicy is a single policy + which must hold true for a specified past interval. + properties: + periodSeconds: + description: |- + periodSeconds specifies the window of time for which the policy should hold true. + PeriodSeconds must be greater than zero and less than or equal to 1800 (30 min). + format: int32 + type: integer + type: + description: type is used to specify the scaling + policy. + type: string + value: + description: |- + value contains the amount of change which is permitted by the policy. + It must be greater than zero + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + x-kubernetes-list-type: atomic + selectPolicy: + description: |- + selectPolicy is used to specify which policy should be used. + If not set, the default value Max is used. + type: string + stabilizationWindowSeconds: + description: |- + stabilizationWindowSeconds is the number of seconds for which past recommendations should be + considered while scaling up or scaling down. + StabilizationWindowSeconds must be greater than or equal to zero and less than or equal to 3600 (one hour). + If not set, use the default values: + - For scale up: 0 (i.e. no stabilization is done). + - For scale down: 300 (i.e. the stabilization window is 300 seconds long). + format: int32 + type: integer + type: object + scaleUp: + description: |- + scaleUp is scaling policy for scaling Up. + If not set, the default value is the higher of: + * increase no more than 4 pods per 60 seconds + * double the number of pods per 60 seconds + No stabilization is used. + properties: + policies: + description: |- + policies is a list of potential scaling polices which can be used during scaling. + At least one policy must be specified, otherwise the HPAScalingRules will be discarded as invalid + items: + description: HPAScalingPolicy is a single policy + which must hold true for a specified past interval. + properties: + periodSeconds: + description: |- + periodSeconds specifies the window of time for which the policy should hold true. + PeriodSeconds must be greater than zero and less than or equal to 1800 (30 min). + format: int32 + type: integer + type: + description: type is used to specify the scaling + policy. + type: string + value: + description: |- + value contains the amount of change which is permitted by the policy. + It must be greater than zero + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + x-kubernetes-list-type: atomic + selectPolicy: + description: |- + selectPolicy is used to specify which policy should be used. + If not set, the default value Max is used. + type: string + stabilizationWindowSeconds: + description: |- + stabilizationWindowSeconds is the number of seconds for which past recommendations should be + considered while scaling up or scaling down. + StabilizationWindowSeconds must be greater than or equal to zero and less than or equal to 3600 (one hour). + If not set, use the default values: + - For scale up: 0 (i.e. no stabilization is done). + - For scale down: 300 (i.e. the stabilization window is 300 seconds long). + format: int32 + type: integer + type: object + type: object + maxReplicas: + description: |- + Upper limit for the number of pods that can be set by the autoscaler; cannot be smaller than + MinReplicas. + format: int32 + type: integer + minReplicas: + description: Lower limit for the number of pods that can be + set by the autoscaler, default 1. + format: int32 + type: integer + required: + - maxReplicas + type: object + type: object + goals: + description: Describes the goals for autoscaling + properties: + metrics: + description: |- + Contains the specifications about the metric type and target in terms of resource + utilization or workload performance. See the individual metric source types for + more information about how each type of metric must respond. + items: + description: |- + MetricSpec specifies how to scale based on a single metric + (only `type` and one other matching field should be set at once). + properties: + containerResource: + description: |- + containerResource refers to a resource metric (such as those specified in + requests and limits) known to Kubernetes describing a single container in + each pod of the current scale target (e.g. CPU or memory). Such metrics are + built in to Kubernetes, and have special scaling options on top of those + available to normal per-pod metrics using the "pods" source. + properties: + container: + description: container is the name of the container + in the pods of the scaling target + type: string + name: + description: name is the name of the resource in question. + type: string + target: + description: target specifies the target value for the + given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric + type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the metric + (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - container + - name + - target + type: object + external: + description: |- + external refers to a global metric that is not associated + with any Kubernetes object. It allows autoscaling based on information + coming from components running outside of cluster + (for example length of queue in cloud messaging service, or + QPS from loadbalancer running outside of cluster). + properties: + metric: + description: metric identifies the target metric by + name and selector + properties: + name: + description: name is the name of the given metric + type: string + selector: + description: |- + selector is the string-encoded form of a standard kubernetes label selector for the given metric + When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping. + When unset, just the metricName will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value for the + given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric + type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the metric + (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - metric + - target + type: object + object: + description: |- + object refers to a metric describing a single kubernetes object + (for example, hits-per-second on an Ingress object). + properties: + describedObject: + description: describedObject specifies the descriptions + of a object,such as kind,name apiVersion + properties: + apiVersion: + description: apiVersion is the API version of the + referent + type: string + kind: + description: 'kind is the kind of the referent; + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'name is the name of the referent; + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + required: + - kind + - name + type: object + metric: + description: metric identifies the target metric by + name and selector + properties: + name: + description: name is the name of the given metric + type: string + selector: + description: |- + selector is the string-encoded form of a standard kubernetes label selector for the given metric + When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping. + When unset, just the metricName will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value for the + given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric + type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the metric + (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - describedObject + - metric + - target + type: object + pods: + description: |- + pods refers to a metric describing each pod in the current scale target + (for example, transactions-processed-per-second). The values will be + averaged together before being compared to the target value. + properties: + metric: + description: metric identifies the target metric by + name and selector + properties: + name: + description: name is the name of the given metric + type: string + selector: + description: |- + selector is the string-encoded form of a standard kubernetes label selector for the given metric + When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping. + When unset, just the metricName will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value for the + given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric + type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the metric + (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - metric + - target + type: object + resource: + description: |- + resource refers to a resource metric (such as those specified in + requests and limits) known to Kubernetes describing each pod in the + current scale target (e.g. CPU or memory). Such metrics are built in to + Kubernetes, and have special scaling options on top of those available + to normal per-pod metrics using the "pods" source. + properties: + name: + description: name is the name of the resource in question. + type: string + target: + description: target specifies the target value for the + given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric + type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the metric + (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - name + - target + type: object + type: + description: |- + type is the type of metric source. It should be one of "ContainerResource", "External", + "Object", "Pods" or "Resource", each mapping to a matching field in the object. + type: string + required: + - type + type: object + type: array + x-kubernetes-list-type: atomic + type: object + policy: + description: |- + Describes the rules on how changes are applied to the pods. + If not specified, all fields in the `PodUpdatePolicy` are set to their default values. + properties: + updateMode: + description: |- + Controls when autoscaler applies changes to the pod resources. + The default is 'Auto'. + enum: + - "Off" + - Initial + - Recreate + - Auto + type: string + type: object + recommenders: + description: |- + Recommender responsible for generating recommendation for the set of pods and the deployment. + List should be empty (then the default recommender will be used) or contain exactly one + recommender. + items: + description: |- + MultidimPodAutoscalerRecommenderSelector points to a specific Multidimensional Pod Autoscaler + recommender. + In the future it might pass parameters to the recommender. + properties: + name: + description: Name of the recommender responsible for generating + recommendation for this object. + type: string + required: + - name + type: object + type: array + resourcePolicy: + description: |- + Controls how the VPA autoscaler computes recommended resources. + The resource policy is also used to set constraints on the recommendations for individual + containers. If not specified, the autoscaler computes recommended resources for all + containers in the pod, without additional constraints. + properties: + containerPolicies: + description: Per-container resource policies. + items: + description: |- + ContainerResourcePolicy controls how autoscaler computes the recommended + resources for a specific container. + properties: + containerName: + description: |- + Name of the container or DefaultContainerResourcePolicy, in which + case the policy is used by the containers that don't have their own + policy specified. + type: string + controlledResources: + description: |- + Specifies the type of recommendations that will be computed + (and possibly applied) by VPA. + If not specified, the default of [ResourceCPU, ResourceMemory] will be used. + items: + description: ResourceName is the name identifying various + resources in a ResourceList. + type: string + type: array + controlledValues: + description: |- + Specifies which resource values should be controlled. + The default is "RequestsAndLimits". + enum: + - RequestsAndLimits + - RequestsOnly + type: string + maxAllowed: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Specifies the maximum amount of resources that will be recommended + for the container. The default is no maximum. + type: object + minAllowed: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Specifies the minimal amount of resources that will be recommended + for the container. The default is no minimum. + type: object + mode: + description: Whether autoscaler is enabled for the container. + The default is "Auto". + enum: + - Auto + - "Off" + type: string + type: object + type: array + type: object + scaleTargetRef: + description: |- + ScaleTargetRef points to the controller managing the set of pods for the autoscaler to + control, e.g., Deployment, StatefulSet. MultidimPodAutoscaler can be targeted at controller + implementing scale subresource (the pod set is retrieved from the controller's ScaleStatus + or some well known controllers (e.g., for DaemonSet the pod set is read from the + controller's spec). If MultidimPodAutoscaler cannot use specified target it will report + the ConfigUnsupported condition. + properties: + apiVersion: + description: apiVersion is the API version of the referent + type: string + kind: + description: 'kind is the kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'name is the name of the referent; More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + required: + - scaleTargetRef + type: object + status: + description: Current information about the autoscaler. + properties: + conditions: + description: |- + Conditions is the set of conditions required for this autoscaler to scale its target, and + indicates whether or not those conditions are met. + items: + description: MultidimPodAutoscalerCondition describes the state + of a MultidimPodAutoscaler at a certain point. + properties: + lastTransitionTime: + description: lastTransitionTime is the last time the condition + transitioned from one status to another + format: date-time + type: string + message: + description: message is a human-readable explanation containing + details about the transition + type: string + reason: + description: reason is the reason for the condition's last transition. + type: string + status: + description: status is the status of the condition (True, False, + Unknown) + type: string + type: + description: type describes the current condition + type: string + required: + - status + - type + type: object + type: array + currentMetrics: + description: The last read state of the metrics used by this autoscaler. + items: + description: MetricStatus describes the last-read state of a single + metric. + properties: + containerResource: + description: |- + container resource refers to a resource metric (such as those specified in + requests and limits) known to Kubernetes describing a single container in each pod in the + current scale target (e.g. CPU or memory). Such metrics are built in to + Kubernetes, and have special scaling options on top of those available + to normal per-pod metrics using the "pods" source. + properties: + container: + description: container is the name of the container in the + pods of the scaling target + type: string + current: + description: current contains the current value for the + given metric + properties: + averageUtilization: + description: |- + currentAverageUtilization is the current value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the current value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + value: + anyOf: + - type: integer + - type: string + description: value is the current value of the metric + (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + name: + description: name is the name of the resource in question. + type: string + required: + - container + - current + - name + type: object + external: + description: |- + external refers to a global metric that is not associated + with any Kubernetes object. It allows autoscaling based on information + coming from components running outside of cluster + (for example length of queue in cloud messaging service, or + QPS from loadbalancer running outside of cluster). + properties: + current: + description: current contains the current value for the + given metric + properties: + averageUtilization: + description: |- + currentAverageUtilization is the current value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the current value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + value: + anyOf: + - type: integer + - type: string + description: value is the current value of the metric + (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + metric: + description: metric identifies the target metric by name + and selector + properties: + name: + description: name is the name of the given metric + type: string + selector: + description: |- + selector is the string-encoded form of a standard kubernetes label selector for the given metric + When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping. + When unset, just the metricName will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + required: + - current + - metric + type: object + object: + description: |- + object refers to a metric describing a single kubernetes object + (for example, hits-per-second on an Ingress object). + properties: + current: + description: current contains the current value for the + given metric + properties: + averageUtilization: + description: |- + currentAverageUtilization is the current value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the current value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + value: + anyOf: + - type: integer + - type: string + description: value is the current value of the metric + (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + describedObject: + description: DescribedObject specifies the descriptions + of a object,such as kind,name apiVersion + properties: + apiVersion: + description: apiVersion is the API version of the referent + type: string + kind: + description: 'kind is the kind of the referent; More + info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'name is the name of the referent; More + info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + required: + - kind + - name + type: object + metric: + description: metric identifies the target metric by name + and selector + properties: + name: + description: name is the name of the given metric + type: string + selector: + description: |- + selector is the string-encoded form of a standard kubernetes label selector for the given metric + When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping. + When unset, just the metricName will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + required: + - current + - describedObject + - metric + type: object + pods: + description: |- + pods refers to a metric describing each pod in the current scale target + (for example, transactions-processed-per-second). The values will be + averaged together before being compared to the target value. + properties: + current: + description: current contains the current value for the + given metric + properties: + averageUtilization: + description: |- + currentAverageUtilization is the current value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the current value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + value: + anyOf: + - type: integer + - type: string + description: value is the current value of the metric + (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + metric: + description: metric identifies the target metric by name + and selector + properties: + name: + description: name is the name of the given metric + type: string + selector: + description: |- + selector is the string-encoded form of a standard kubernetes label selector for the given metric + When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping. + When unset, just the metricName will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + required: + - current + - metric + type: object + resource: + description: |- + resource refers to a resource metric (such as those specified in + requests and limits) known to Kubernetes describing each pod in the + current scale target (e.g. CPU or memory). Such metrics are built in to + Kubernetes, and have special scaling options on top of those available + to normal per-pod metrics using the "pods" source. + properties: + current: + description: current contains the current value for the + given metric + properties: + averageUtilization: + description: |- + currentAverageUtilization is the current value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the current value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + value: + anyOf: + - type: integer + - type: string + description: value is the current value of the metric + (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + name: + description: name is the name of the resource in question. + type: string + required: + - current + - name + type: object + type: + description: |- + type is the type of metric source. It will be one of "ContainerResource", "External", + "Object", "Pods" or "Resource", each corresponds to a matching field in the object. + type: string + required: + - type + type: object + type: array + x-kubernetes-list-type: atomic + currentReplicas: + description: Current number of replicas of pods managed by this autoscaler. + format: int32 + type: integer + desiredReplicas: + description: Desired number of replicas of pods managed by this autoscaler. + format: int32 + type: integer + lastScaleTime: + description: |- + Last time the MultidimPodAutoscaler scaled the number of pods and resizes containers; + Used by the autoscaler to control how often scaling operations are performed. + format: date-time + type: string + recommendation: + description: |- + The most recently computed amount of resources for each controlled pod recommended by the + autoscaler. + properties: + containerRecommendations: + description: Resources recommended by the autoscaler for each + container. + items: + description: |- + RecommendedContainerResources is the recommendation of resources computed by + autoscaler for a specific container. Respects the container resource policy + if present in the spec. In particular the recommendation is not produced for + containers with `ContainerScalingMode` set to 'Off'. + properties: + containerName: + description: Name of the container. + type: string + lowerBound: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Minimum recommended amount of resources. Observes ContainerResourcePolicy. + This amount is not guaranteed to be sufficient for the application to operate in a stable way, however + running with less resources is likely to have significant impact on performance/availability. + type: object + target: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Recommended amount of resources. Observes ContainerResourcePolicy. + type: object + uncappedTarget: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + The most recent recommended resources target computed by the autoscaler + for the controlled pods, based only on actual resource usage, not taking + into account the ContainerResourcePolicy. + May differ from the Recommendation if the actual resource usage causes + the target to violate the ContainerResourcePolicy (lower than MinAllowed + or higher that MaxAllowed). + Used only as status indication, will not affect actual resource assignment. + type: object + upperBound: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Maximum recommended amount of resources. Observes ContainerResourcePolicy. + Any resources allocated beyond this value are likely wasted. This value may be larger than the maximum + amount of application is actually capable of consuming. + type: object + required: + - target + type: object + type: array + type: object + required: + - currentReplicas + - desiredReplicas + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/multidimensional-pod-autoscaler/deploy/mpa-webhook.yaml b/multidimensional-pod-autoscaler/deploy/mpa-webhook.yaml new file mode 100644 index 000000000000..5c765b2bb513 --- /dev/null +++ b/multidimensional-pod-autoscaler/deploy/mpa-webhook.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: mpa-webhook + namespace: kube-system +spec: + ports: + - port: 443 + targetPort: 8000 + selector: + app: mpa-admission-controller diff --git a/multidimensional-pod-autoscaler/deploy/recommender-deployment.yaml b/multidimensional-pod-autoscaler/deploy/recommender-deployment.yaml new file mode 100644 index 000000000000..49ce99d3f978 --- /dev/null +++ b/multidimensional-pod-autoscaler/deploy/recommender-deployment.yaml @@ -0,0 +1,40 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: mpa-recommender + namespace: kube-system +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mpa-recommender + namespace: kube-system +spec: + replicas: 1 + selector: + matchLabels: + app: mpa-recommender + template: + metadata: + labels: + app: mpa-recommender + spec: + serviceAccountName: mpa-recommender + securityContext: + runAsNonRoot: true + runAsUser: 65534 # nobody + containers: + - name: recommender + image: registry.k8s.io/autoscaling/mpa-recommender:latest + imagePullPolicy: IfNotPresent + resources: + limits: + cpu: 200m + memory: 1000Mi + requests: + cpu: 50m + memory: 500Mi + ports: + - name: prometheus + containerPort: 8942 diff --git a/multidimensional-pod-autoscaler/deploy/updater-deployment.yaml b/multidimensional-pod-autoscaler/deploy/updater-deployment.yaml new file mode 100644 index 000000000000..3bcf9cf6e1db --- /dev/null +++ b/multidimensional-pod-autoscaler/deploy/updater-deployment.yaml @@ -0,0 +1,45 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: mpa-updater + namespace: kube-system +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mpa-updater + namespace: kube-system +spec: + replicas: 1 + selector: + matchLabels: + app: mpa-updater + template: + metadata: + labels: + app: mpa-updater + spec: + serviceAccountName: mpa-updater + securityContext: + runAsNonRoot: true + runAsUser: 65534 # nobody + containers: + - name: updater + image: registry.k8s.io/autoscaling/mpa-updater:latest + imagePullPolicy: IfNotPresent + env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + resources: + limits: + cpu: 200m + memory: 1000Mi + requests: + cpu: 50m + memory: 500Mi + ports: + - name: prometheus + containerPort: 8943 diff --git a/multidimensional-pod-autoscaler/e2e/go.mod b/multidimensional-pod-autoscaler/e2e/go.mod new file mode 100644 index 000000000000..2b5eb93403f6 --- /dev/null +++ b/multidimensional-pod-autoscaler/e2e/go.mod @@ -0,0 +1,195 @@ +// for updating kubernetes dependencies, use `autoscaler/multidimensional-pod-autoscaler/hack/update-kubernetes-deps-in-e2e.sh` +// for any other update, use standard `go mod` commands + +module k8s.io/autoscaler/multidimensional-pod-autoscaler/e2e + +go 1.23.0 + +toolchain go1.23.3 + +require ( + github.com/onsi/ginkgo/v2 v2.22.0 + github.com/onsi/gomega v1.36.1 + k8s.io/api v0.32.0 + k8s.io/apimachinery v0.32.0 + k8s.io/autoscaler/multidimensional-pod-autoscaler v0.0.1 + k8s.io/autoscaler/vertical-pod-autoscaler v1.2.1 + k8s.io/client-go v0.32.0 + k8s.io/component-base v0.32.0 + k8s.io/klog/v2 v2.130.1 + k8s.io/kubernetes v1.32.0 + k8s.io/pod-security-admission v0.32.0 + k8s.io/utils v0.0.0-20241210054802-24370beab758 +) + +require ( + cel.dev/expr v0.18.0 // indirect + github.com/JeffAshton/win_pdh v0.0.0-20161109143554-76bb4ee9f0ab // indirect + github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/Microsoft/hnslib v0.0.8 // indirect + github.com/NYTimes/gziphandler v1.1.1 // indirect + github.com/antlr4-go/antlr/v4 v4.13.0 // indirect + github.com/armon/circbuf v0.0.0-20190214190532-5111143e8da2 // indirect + github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/blang/semver/v4 v4.0.0 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/container-storage-interface/spec v1.9.0 // indirect + github.com/containerd/containerd/api v1.8.0 // indirect + github.com/containerd/errdefs v0.1.0 // indirect + github.com/containerd/log v0.1.0 // indirect + github.com/containerd/ttrpc v1.2.5 // indirect + github.com/coreos/go-semver v0.3.1 // indirect + github.com/coreos/go-systemd/v22 v22.5.0 // indirect + github.com/cyphar/filepath-securejoin v0.3.4 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/distribution/reference v0.6.0 // indirect + github.com/docker/go-units v0.5.0 // indirect + github.com/emicklei/go-restful/v3 v3.11.0 // indirect + github.com/euank/go-kmsg-parser v2.0.0+incompatible // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/fsnotify/fsnotify v1.8.0 // indirect + github.com/fxamacker/cbor/v2 v2.7.0 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.23.0 // indirect + github.com/go-task/slim-sprig/v3 v3.0.0 // indirect + github.com/godbus/dbus/v5 v5.1.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/google/btree v1.0.1 // indirect + github.com/google/cadvisor v0.51.0 // indirect + github.com/google/cel-go v0.22.0 // indirect + github.com/google/gnostic-models v0.6.8 // indirect + github.com/google/go-cmp v0.6.0 // indirect + github.com/google/gofuzz v1.2.0 // indirect + github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/gorilla/websocket v1.5.0 // indirect + github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/karrick/godirwalk v1.17.0 // indirect + github.com/klauspost/compress v1.17.9 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible // indirect + github.com/moby/spdystream v0.5.0 // indirect + github.com/moby/sys/mountinfo v0.7.2 // indirect + github.com/moby/sys/userns v0.1.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.0 // indirect + github.com/opencontainers/runc v1.2.1 // indirect + github.com/opencontainers/runtime-spec v1.2.0 // indirect + github.com/opencontainers/selinux v1.11.1 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/prometheus/client_golang v1.20.5 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.61.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect + github.com/spf13/cobra v1.8.1 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/stoewer/go-strcase v1.3.0 // indirect + github.com/stretchr/objx v0.5.2 // indirect + github.com/stretchr/testify v1.10.0 // indirect + github.com/x448/float16 v0.8.4 // indirect + go.etcd.io/etcd/api/v3 v3.5.16 // indirect + go.etcd.io/etcd/client/pkg/v3 v3.5.16 // indirect + go.etcd.io/etcd/client/v3 v3.5.16 // indirect + go.opentelemetry.io/contrib/instrumentation/github.com/emicklei/go-restful/otelrestful v0.42.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.53.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 // indirect + go.opentelemetry.io/otel v1.28.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0 // indirect + go.opentelemetry.io/otel/metric v1.28.0 // indirect + go.opentelemetry.io/otel/sdk v1.28.0 // indirect + go.opentelemetry.io/otel/trace v1.28.0 // indirect + go.opentelemetry.io/proto/otlp v1.3.1 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.27.0 // indirect + golang.org/x/crypto v0.30.0 // indirect + golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect + golang.org/x/net v0.32.0 // indirect + golang.org/x/oauth2 v0.24.0 // indirect + golang.org/x/sync v0.10.0 // indirect + golang.org/x/sys v0.28.0 // indirect + golang.org/x/term v0.27.0 // indirect + golang.org/x/text v0.21.0 // indirect + golang.org/x/time v0.8.0 // indirect + golang.org/x/tools v0.28.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20241118233622-e639e219e697 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20241118233622-e639e219e697 // indirect + google.golang.org/grpc v1.68.0 // indirect + google.golang.org/protobuf v1.35.2 // indirect + gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/apiextensions-apiserver v0.0.0 // indirect + k8s.io/apiserver v0.32.0 // indirect + k8s.io/cloud-provider v0.32.0 // indirect + k8s.io/component-helpers v0.32.0 // indirect + k8s.io/controller-manager v0.32.0 // indirect + k8s.io/cri-api v0.32.0 // indirect + k8s.io/cri-client v0.32.0 // indirect + k8s.io/csi-translation-lib v0.26.1 // indirect + k8s.io/dynamic-resource-allocation v0.0.0 // indirect + k8s.io/kms v0.32.0 // indirect + k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f // indirect + k8s.io/kube-scheduler v0.0.0 // indirect + k8s.io/kubectl v0.0.0 // indirect + k8s.io/kubelet v0.32.0 // indirect + k8s.io/mount-utils v0.25.0 // indirect + sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.0 // indirect + sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.4.2 // indirect + sigs.k8s.io/yaml v1.4.0 // indirect +) + +replace ( + k8s.io/api => k8s.io/api v0.32.0 + k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.32.0 + k8s.io/apimachinery => k8s.io/apimachinery v0.32.0 + k8s.io/apiserver => k8s.io/apiserver v0.32.0 + k8s.io/autoscaler => ../../ + k8s.io/autoscaler/multidimensional-pod-autoscaler => ../ + k8s.io/autoscaler/vertical-pod-autoscaler => ../../vertical-pod-autoscaler + k8s.io/cli-runtime => k8s.io/cli-runtime v0.32.0 + k8s.io/client-go => k8s.io/client-go v0.32.0 + k8s.io/cloud-provider => k8s.io/cloud-provider v0.32.0 + k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.32.0 + k8s.io/code-generator => k8s.io/code-generator v0.32.0 + k8s.io/component-base => k8s.io/component-base v0.32.0 + k8s.io/component-helpers => k8s.io/component-helpers v0.32.0 + k8s.io/controller-manager => k8s.io/controller-manager v0.32.0 + k8s.io/cri-api => k8s.io/cri-api v0.32.0 + k8s.io/cri-client => k8s.io/cri-client v0.32.0 + k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.32.0 + k8s.io/dynamic-resource-allocation => k8s.io/dynamic-resource-allocation v0.32.0 + k8s.io/endpointslice => k8s.io/endpointslice v0.32.0 + k8s.io/kms => k8s.io/kms v0.32.0 + k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.32.0 + k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.32.0 + k8s.io/kube-proxy => k8s.io/kube-proxy v0.32.0 + k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.32.0 + k8s.io/kubectl => k8s.io/kubectl v0.32.0 + k8s.io/kubelet => k8s.io/kubelet v0.32.0 + k8s.io/metrics => k8s.io/metrics v0.32.0 + k8s.io/mount-utils => k8s.io/mount-utils v0.32.0 + k8s.io/pod-security-admission => k8s.io/pod-security-admission v0.32.0 + k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.32.0 + k8s.io/sample-cli-plugin => k8s.io/sample-cli-plugin v0.32.0 + k8s.io/sample-controller => k8s.io/sample-controller v0.32.0 +) diff --git a/multidimensional-pod-autoscaler/e2e/go.sum b/multidimensional-pod-autoscaler/e2e/go.sum new file mode 100644 index 000000000000..14ad25d21151 --- /dev/null +++ b/multidimensional-pod-autoscaler/e2e/go.sum @@ -0,0 +1,396 @@ +cel.dev/expr v0.18.0 h1:CJ6drgk+Hf96lkLikr4rFf19WrU0BOWEihyZnI2TAzo= +cel.dev/expr v0.18.0/go.mod h1:MrpN08Q+lEBs+bGYdLxxHkZoUSsCp0nSKTs0nTymJgw= +github.com/JeffAshton/win_pdh v0.0.0-20161109143554-76bb4ee9f0ab h1:UKkYhof1njT1/xq4SEg5z+VpTgjmNeHwPGRQl7takDI= +github.com/JeffAshton/win_pdh v0.0.0-20161109143554-76bb4ee9f0ab/go.mod h1:3VYc5hodBMJ5+l/7J4xAyMeuM2PNuepvHlGs8yilUCA= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/Microsoft/hnslib v0.0.8 h1:EBrIiRB7i/UYIXEC2yw22dn+RLzOmsc5S0bw2xf0Qus= +github.com/Microsoft/hnslib v0.0.8/go.mod h1:EYveQJlhKh2obmEIRB3uKN6dBd9pj1frPsrTGFppKuk= +github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I= +github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= +github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI= +github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= +github.com/armon/circbuf v0.0.0-20190214190532-5111143e8da2 h1:7Ip0wMmLHLRJdrloDxZfhMm0xrLXZS8+COSu2bXmEQs= +github.com/armon/circbuf v0.0.0-20190214190532-5111143e8da2/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA= +github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/container-storage-interface/spec v1.9.0 h1:zKtX4STsq31Knz3gciCYCi1SXtO2HJDecIjDVboYavY= +github.com/container-storage-interface/spec v1.9.0/go.mod h1:ZfDu+3ZRyeVqxZM0Ds19MVLkN2d1XJ5MAfi1L3VjlT0= +github.com/containerd/containerd/api v1.8.0 h1:hVTNJKR8fMc/2Tiw60ZRijntNMd1U+JVMyTRdsD2bS0= +github.com/containerd/containerd/api v1.8.0/go.mod h1:dFv4lt6S20wTu/hMcP4350RL87qPWLVa/OHOwmmdnYc= +github.com/containerd/errdefs v0.1.0 h1:m0wCRBiu1WJT/Fr+iOoQHMQS/eP5myQ8lCv4Dz5ZURM= +github.com/containerd/errdefs v0.1.0/go.mod h1:YgWiiHtLmSeBrvpw+UfPijzbLaB77mEG1WwJTDETIV0= +github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= +github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= +github.com/containerd/ttrpc v1.2.5 h1:IFckT1EFQoFBMG4c3sMdT8EP3/aKfumK1msY+Ze4oLU= +github.com/containerd/ttrpc v1.2.5/go.mod h1:YCXHsb32f+Sq5/72xHubdiJRQY9inL4a4ZQrAbN1q9o= +github.com/containerd/typeurl/v2 v2.2.0 h1:6NBDbQzr7I5LHgp34xAXYF5DOTQDn05X58lsPEmzLso= +github.com/containerd/typeurl/v2 v2.2.0/go.mod h1:8XOOxnyatxSWuG8OfsZXVnAF4iZfedjS/8UHSPJnX4g= +github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= +github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= +github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= +github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/cyphar/filepath-securejoin v0.3.4 h1:VBWugsJh2ZxJmLFSM06/0qzQyiQX2Qs0ViKrUAcqdZ8= +github.com/cyphar/filepath-securejoin v0.3.4/go.mod h1:8s/MCNJREmFK0H02MF6Ihv1nakJe4L/w3WZLHNkvlYM= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/docker/docker v26.1.4+incompatible h1:vuTpXDuoga+Z38m1OZHzl7NKisKWaWlhjQk7IDPSLsU= +github.com/docker/docker v26.1.4+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= +github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= +github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/euank/go-kmsg-parser v2.0.0+incompatible h1:cHD53+PLQuuQyLZeriD1V/esuG4MuU0Pjs5y6iknohY= +github.com/euank/go-kmsg-parser v2.0.0+incompatible/go.mod h1:MhmAMZ8V4CYH4ybgdRwPr2TU5ThnS43puaKEMpja1uw= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= +github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= +github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= +github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= +github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= +github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= +github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= +github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= +github.com/google/cadvisor v0.51.0 h1:BspqSPdZoLKrnvuZNOvM/KiJ/A+RdixwagN20n+2H8k= +github.com/google/cadvisor v0.51.0/go.mod h1:czGE/c/P/i0QFpVNKTFrIEzord9Y10YfpwuaSWXELc0= +github.com/google/cel-go v0.22.0 h1:b3FJZxpiv1vTMo2/5RDUqAHPxkT8mmMfJIrq1llbf7g= +github.com/google/cel-go v0.22.0/go.mod h1:BuznPXXfQDpXKWQ9sPW3TzlAJN5zzFe+i9tIs0yC4s8= +github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= +github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad h1:a6HEuzUHeKH6hwfN/ZoQgRgVIWFJljSWa/zetS2WTvg= +github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= +github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= +github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0 h1:TmHmbvxPmaegwhDubVz0lICL0J5Ka2vwTzhoePEXsGE= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0/go.mod h1:qztMSjm835F2bXf+5HKAPIS5qsmQDqZna/PgVt4rWtI= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jonboulle/clockwork v0.4.0 h1:p4Cf1aMWXnXAUh8lVfewRBx1zaTSYKrKMF2g3ST4RZ4= +github.com/jonboulle/clockwork v0.4.0/go.mod h1:xgRqUGwRcjKCO1vbZUEtSLrqKoPSsUpK7fnezOII0kc= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/karrick/godirwalk v1.17.0 h1:b4kY7nqDdioR/6qnbHQyDvmA17u5G1cZ6J+CZXwSWoI= +github.com/karrick/godirwalk v1.17.0/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/lithammer/dedent v1.1.0 h1:VNzHMVCBNG1j0fh3OrsFRkVUwStdDArbgBWoPAffktY= +github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible h1:aKW/4cBs+yK6gpqU3K/oIwk9Q/XICqd3zOX/UFuvqmk= +github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= +github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= +github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/moby/spdystream v0.5.0 h1:7r0J1Si3QO/kjRitvSLVVFUjxMEb/YLj6S9FF62JBCU= +github.com/moby/spdystream v0.5.0/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI= +github.com/moby/sys/mountinfo v0.7.2 h1:1shs6aH5s4o5H2zQLn796ADW1wMrIwHsyJ2v9KouLrg= +github.com/moby/sys/mountinfo v0.7.2/go.mod h1:1YOa8w8Ih7uW0wALDUgT1dTTSBrZ+HiBLGws92L2RU4= +github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= +github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/onsi/ginkgo/v2 v2.22.0 h1:Yed107/8DjTr0lKCNt7Dn8yQ6ybuDRQoMGrNFKzMfHg= +github.com/onsi/ginkgo/v2 v2.22.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= +github.com/onsi/gomega v1.36.1 h1:bJDPBO7ibjxcbHMgSCoo4Yj18UWbKDlLwX1x9sybDcw= +github.com/onsi/gomega v1.36.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= +github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= +github.com/opencontainers/runc v1.2.1 h1:mQkmeFSUxqFaVmvIn1VQPeQIKpHFya5R07aJw0DKQa8= +github.com/opencontainers/runc v1.2.1/go.mod h1:/PXzF0h531HTMsYQnmxXkBD7YaGShm/2zcRB79dksUc= +github.com/opencontainers/runtime-spec v1.2.0 h1:z97+pHb3uELt/yiAWD691HNHQIF07bE7dzrbT927iTk= +github.com/opencontainers/runtime-spec v1.2.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/selinux v1.11.1 h1:nHFvthhM0qY8/m+vfhJylliSshm8G1jJ2jDMcgULaH8= +github.com/opencontainers/selinux v1.11.1/go.mod h1:E5dMC3VPuVvVHDYmi78qvhJp8+M586T4DlDRYpFkyec= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= +github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.61.0 h1:3gv/GThfX0cV2lpO7gkTUwZru38mxevy90Bj8YFSRQQ= +github.com/prometheus/common v0.61.0/go.mod h1:zr29OCN/2BsJRaFwG8QOBr41D6kkchKbpeNH7pAjb/s= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= +github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= +github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= +github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stoewer/go-strcase v1.3.0 h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AVEzs= +github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7otjonDflCTK0BCfls4SPy3NcCVb5dqqmbRknE= +github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75/go.mod h1:KO6IkyS8Y3j8OdNO85qEYBsRPuteD+YciPomcXdrMnk= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510 h1:S2dVYn90KE98chqDkyE9Z4N61UnQd+KOfgp5Iu53llk= +github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.etcd.io/bbolt v1.3.11 h1:yGEzV1wPz2yVCLsD8ZAiGHhHVlczyC9d1rP43/VCRJ0= +go.etcd.io/bbolt v1.3.11/go.mod h1:dksAq7YMXoljX0xu6VF5DMZGbhYYoLUalEiSySYAS4I= +go.etcd.io/etcd/api/v3 v3.5.16 h1:WvmyJVbjWqK4R1E+B12RRHz3bRGy9XVfh++MgbN+6n0= +go.etcd.io/etcd/api/v3 v3.5.16/go.mod h1:1P4SlIP/VwkDmGo3OlOD7faPeP8KDIFhqvciH5EfN28= +go.etcd.io/etcd/client/pkg/v3 v3.5.16 h1:ZgY48uH6UvB+/7R9Yf4x574uCO3jIx0TRDyetSfId3Q= +go.etcd.io/etcd/client/pkg/v3 v3.5.16/go.mod h1:V8acl8pcEK0Y2g19YlOV9m9ssUe6MgiDSobSoaBAM0E= +go.etcd.io/etcd/client/v2 v2.305.16 h1:kQrn9o5czVNaukf2A2At43cE9ZtWauOtf9vRZuiKXow= +go.etcd.io/etcd/client/v2 v2.305.16/go.mod h1:h9YxWCzcdvZENbfzBTFCnoNumr2ax3F19sKMqHFmXHE= +go.etcd.io/etcd/client/v3 v3.5.16 h1:sSmVYOAHeC9doqi0gv7v86oY/BTld0SEFGaxsU9eRhE= +go.etcd.io/etcd/client/v3 v3.5.16/go.mod h1:X+rExSGkyqxvu276cr2OwPLBaeqFu1cIl4vmRjAD/50= +go.etcd.io/etcd/pkg/v3 v3.5.16 h1:cnavs5WSPWeK4TYwPYfmcr3Joz9BH+TZ6qoUtz6/+mc= +go.etcd.io/etcd/pkg/v3 v3.5.16/go.mod h1:+lutCZHG5MBBFI/U4eYT5yL7sJfnexsoM20Y0t2uNuY= +go.etcd.io/etcd/raft/v3 v3.5.16 h1:zBXA3ZUpYs1AwiLGPafYAKKl/CORn/uaxYDwlNwndAk= +go.etcd.io/etcd/raft/v3 v3.5.16/go.mod h1:P4UP14AxofMJ/54boWilabqqWoW9eLodl6I5GdGzazI= +go.etcd.io/etcd/server/v3 v3.5.16 h1:d0/SAdJ3vVsZvF8IFVb1k8zqMZ+heGcNfft71ul9GWE= +go.etcd.io/etcd/server/v3 v3.5.16/go.mod h1:ynhyZZpdDp1Gq49jkUg5mfkDWZwXnn3eIqCqtJnrD/s= +go.opentelemetry.io/contrib/instrumentation/github.com/emicklei/go-restful/otelrestful v0.42.0 h1:Z6SbqeRZAl2OczfkFOqLx1BeYBDYehNjEnqluD7581Y= +go.opentelemetry.io/contrib/instrumentation/github.com/emicklei/go-restful/otelrestful v0.42.0/go.mod h1:XiglO+8SPMqM3Mqh5/rtxR1VHc63o8tb38QrU6tm4mU= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.53.0 h1:9G6E0TXzGFVfTnawRzrPl83iHOAV7L8NJiR8RSGYV1g= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.53.0/go.mod h1:azvtTADFQJA8mX80jIH/akaE7h+dbm/sVuaHqN13w74= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 h1:4K4tsIXefpVJtvA/8srF4V4y0akAoPHkIslgAkjixJA= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0/go.mod h1:jjdQuTGVsXV4vSs+CJ2qYDeDPf9yIJV23qlIzBm73Vg= +go.opentelemetry.io/contrib/propagators/b3 v1.17.0 h1:ImOVvHnku8jijXqkwCSyYKRDt2YrnGXD4BbhcpfbfJo= +go.opentelemetry.io/contrib/propagators/b3 v1.17.0/go.mod h1:IkfUfMpKWmynvvE0264trz0sf32NRTZL4nuAN9AbWRc= +go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo= +go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 h1:3Q/xZUyC1BBkualc9ROb4G8qkH90LXEIICcs5zv1OYY= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0/go.mod h1:s75jGIWA9OfCMzF0xr+ZgfrB5FEbbV7UuYo32ahUiFI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0 h1:qFffATk0X+HD+f1Z8lswGiOQYKHRlzfmdJm0wEaVrFA= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0/go.mod h1:MOiCmryaYtc+V0Ei+Tx9o5S1ZjA7kzLucuVuyzBZloQ= +go.opentelemetry.io/otel/metric v1.28.0 h1:f0HGvSl1KRAU1DLgLGFjrwVyismPlnuU6JD6bOeuA5Q= +go.opentelemetry.io/otel/metric v1.28.0/go.mod h1:Fb1eVBFZmLVTMb6PPohq3TO9IIhUisDsbJoL/+uQW4s= +go.opentelemetry.io/otel/sdk v1.28.0 h1:b9d7hIry8yZsgtbmM0DKyPWMMUMlK9NEKuIG4aBqWyE= +go.opentelemetry.io/otel/sdk v1.28.0/go.mod h1:oYj7ClPUA7Iw3m+r7GeEjz0qckQRJK2B8zjcZEfu7Pg= +go.opentelemetry.io/otel/trace v1.28.0 h1:GhQ9cUuQGmNDd5BTCP2dAvv75RdMxEfTmYejp+lkx9g= +go.opentelemetry.io/otel/trace v1.28.0/go.mod h1:jPyXzNPg6da9+38HEwElrQiHlVMTnVfM3/yv2OlIHaI= +go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0= +go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.30.0 h1:RwoQn3GkWiMkzlX562cLB7OxWvjH1L8xutO2WoJcRoY= +golang.org/x/crypto v0.30.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= +golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/oauth2 v0.24.0 h1:KTBBxWqUa0ykRPLtV69rRto9TLXcqYkeswu48x/gvNE= +golang.org/x/oauth2 v0.24.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= +golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/time v0.8.0 h1:9i3RxcPv3PZnitoVGMPDKZSq1xW1gK1Xy3ArNOGZfEg= +golang.org/x/time v0.8.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.28.0 h1:WuB6qZ4RPCQo5aP3WdKZS7i595EdWqWR8vqJTlwTVK8= +golang.org/x/tools v0.28.0/go.mod h1:dcIOrVd3mfQKTgrDVQHqCPMWy6lnhfhtX3hLXYVLfRw= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80 h1:KAeGQVN3M9nD0/bQXnr/ClcEMJ968gUXJQ9pwfSynuQ= +google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80/go.mod h1:cc8bqMqtv9gMOr0zHg2Vzff5ULhhL2IXP4sbcn32Dro= +google.golang.org/genproto/googleapis/api v0.0.0-20241118233622-e639e219e697 h1:pgr/4QbFyktUv9CtQ/Fq4gzEE6/Xs7iCXbktaGzLHbQ= +google.golang.org/genproto/googleapis/api v0.0.0-20241118233622-e639e219e697/go.mod h1:+D9ySVjN8nY8YCVjc5O7PZDIdZporIDY3KaGfJunh88= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241118233622-e639e219e697 h1:LWZqQOEjDyONlF1H6afSWpAL/znlREo2tHfLoe+8LMA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241118233622-e639e219e697/go.mod h1:5uTbfoYQed2U9p3KIj2/Zzm02PYhndfdmML0qC3q3FU= +google.golang.org/grpc v1.68.0 h1:aHQeeJbo8zAkAa3pRzrVjZlbz6uSfeOXlJNQM0RAbz0= +google.golang.org/grpc v1.68.0/go.mod h1:fmSPC5AsjSBCK54MyHRx48kpOti1/jRfOlwEWywNjWA= +google.golang.org/protobuf v1.35.2 h1:8Ar7bF+apOIoThw1EdZl0p1oWvMqTHmpA2fRTyZO8io= +google.golang.org/protobuf v1.35.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= +gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= +gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/api v0.32.0 h1:OL9JpbvAU5ny9ga2fb24X8H6xQlVp+aJMFlgtQjR9CE= +k8s.io/api v0.32.0/go.mod h1:4LEwHZEf6Q/cG96F3dqR965sYOfmPM7rq81BLgsE0p0= +k8s.io/apiextensions-apiserver v0.32.0 h1:S0Xlqt51qzzqjKPxfgX1xh4HBZE+p8KKBq+k2SWNOE0= +k8s.io/apiextensions-apiserver v0.32.0/go.mod h1:86hblMvN5yxMvZrZFX2OhIHAuFIMJIZ19bTvzkP+Fmw= +k8s.io/apimachinery v0.32.0 h1:cFSE7N3rmEEtv4ei5X6DaJPHHX0C+upp+v5lVPiEwpg= +k8s.io/apimachinery v0.32.0/go.mod h1:GpHVgxoKlTxClKcteaeuF1Ul/lDVb74KpZcxcmLDElE= +k8s.io/apiserver v0.32.0 h1:VJ89ZvQZ8p1sLeiWdRJpRD6oLozNZD2+qVSLi+ft5Qs= +k8s.io/apiserver v0.32.0/go.mod h1:HFh+dM1/BE/Hm4bS4nTXHVfN6Z6tFIZPi649n83b4Ag= +k8s.io/client-go v0.32.0 h1:DimtMcnN/JIKZcrSrstiwvvZvLjG0aSxy8PxN8IChp8= +k8s.io/client-go v0.32.0/go.mod h1:boDWvdM1Drk4NJj/VddSLnx59X3OPgwrOo0vGbtq9+8= +k8s.io/cloud-provider v0.32.0 h1:QXYJGmwME2q2rprymbmw2GroMChQYc/MWN6l/I4Kgp8= +k8s.io/cloud-provider v0.32.0/go.mod h1:cz3gVodkhgwi2ugj/JUPglIruLSdDaThxawuDyCHfr8= +k8s.io/component-base v0.32.0 h1:d6cWHZkCiiep41ObYQS6IcgzOUQUNpywm39KVYaUqzU= +k8s.io/component-base v0.32.0/go.mod h1:JLG2W5TUxUu5uDyKiH2R/7NnxJo1HlPoRIIbVLkK5eM= +k8s.io/component-helpers v0.32.0 h1:pQEEBmRt3pDJJX98cQvZshDgJFeKRM4YtYkMmfOlczw= +k8s.io/component-helpers v0.32.0/go.mod h1:9RuClQatbClcokXOcDWSzFKQm1huIf0FzQlPRpizlMc= +k8s.io/controller-manager v0.32.0 h1:tpQl1rvH4huFB6Avl1nhowZHtZoCNWqn6OYdZPl7Ybc= +k8s.io/controller-manager v0.32.0/go.mod h1:JRuYnYCkKj3NgBTy+KNQKIUm/lJRoDAvGbfdEmk9LhY= +k8s.io/cri-api v0.32.0 h1:pzXJfyG7Tm4acrEt5HPqAq3r4cN5guLeapAN/NM2b70= +k8s.io/cri-api v0.32.0/go.mod h1:DCzMuTh2padoinefWME0G678Mc3QFbLMF2vEweGzBAI= +k8s.io/cri-client v0.32.0 h1:K6aTYDyS2AS8O4h79eI5r26562xstdybprtaaszjn+E= +k8s.io/cri-client v0.32.0/go.mod h1:FB8qZNj8KrwIFfVIR2zBGb+l6KUhrp+k8YFsVp3D+kw= +k8s.io/csi-translation-lib v0.32.0 h1:RAn9RGgYXHJQtDSb6qQ7zvq6QObOejzmsXDARI+f4OQ= +k8s.io/csi-translation-lib v0.32.0/go.mod h1:TjCJzkTNstdOESAXNnEImrYOMIEzP14aqM7H+vkehqw= +k8s.io/dynamic-resource-allocation v0.32.0 h1:0ZLSCKzlLZLVwKHxg6vafpd2U8b7jPMO3k8bbMFodis= +k8s.io/dynamic-resource-allocation v0.32.0/go.mod h1:MfoAUi0vCJtchNirAVk7c3IYfGGB3n+zbZ9GuyX4eeo= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kms v0.32.0 h1:jwOfunHIrcdYl5FRcA+uUKKtg6qiqoPCwmS2T3XTYL4= +k8s.io/kms v0.32.0/go.mod h1:Bk2evz/Yvk0oVrvm4MvZbgq8BD34Ksxs2SRHn4/UiOM= +k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f h1:GA7//TjRY9yWGy1poLzYYJJ4JRdzg3+O6e8I+e+8T5Y= +k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f/go.mod h1:R/HEjbvWI0qdfb8viZUeVZm0X6IZnxAydC7YU42CMw4= +k8s.io/kube-scheduler v0.32.0 h1:FCsF/3TPvR51ptx/gLUrqcoKqAMhQKrydYCJzPz9VGM= +k8s.io/kube-scheduler v0.32.0/go.mod h1:yof3vmyx70TWoQ6XZruYEGIUT/r0H/ELGdnWiqPF5EE= +k8s.io/kubectl v0.32.0 h1:rpxl+ng9qeG79YA4Em9tLSfX0G8W0vfaiPVrc/WR7Xw= +k8s.io/kubectl v0.32.0/go.mod h1:qIjSX+QgPQUgdy8ps6eKsYNF+YmFOAO3WygfucIqFiE= +k8s.io/kubelet v0.32.0 h1:uLyiKlz195Wo4an/K2tyge8o3QHx0ZkhVN3pevvp59A= +k8s.io/kubelet v0.32.0/go.mod h1:lAwuVZT/Hm7EdLn0jW2D+WdrJoorjJL2rVSdhOFnegw= +k8s.io/kubernetes v1.32.0 h1:4BDBWSolqPrv8GC3YfZw0CJvh5kA1TPnoX0FxDVd+qc= +k8s.io/kubernetes v1.32.0/go.mod h1:tiIKO63GcdPRBHW2WiUFm3C0eoLczl3f7qi56Dm1W8I= +k8s.io/mount-utils v0.32.0 h1:KOQAhPzJICATXnc6XCkWoexKbkOexRnMCUW8APFfwg4= +k8s.io/mount-utils v0.32.0/go.mod h1:Kun5c2svjAPx0nnvJKYQWhfeNW+O0EpzHgRhDcYoSY0= +k8s.io/pod-security-admission v0.32.0 h1:I+Og0uZIiMpIgTgPrTbW4jlwRI5RWazi8y+jrx1v10w= +k8s.io/pod-security-admission v0.32.0/go.mod h1:RvrcY0+5UAoCIJ7BscgDF3nbmXprgfnjTW+byCyXDvA= +k8s.io/utils v0.0.0-20241210054802-24370beab758 h1:sdbE21q2nlQtFh65saZY+rRM6x6aJJI8IUa1AmH/qa0= +k8s.io/utils v0.0.0-20241210054802-24370beab758/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.0 h1:CPT0ExVicCzcpeN4baWEV2ko2Z/AsiZgEdwgcfwLgMo= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.0/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= +sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 h1:/Rv+M11QRah1itp8VhT6HoVx1Ray9eB4DBr+K+/sCJ8= +sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3/go.mod h1:18nIHnGi6636UCz6m8i4DhaJ65T6EruyzmoQqI2BVDo= +sigs.k8s.io/structured-merge-diff/v4 v4.4.2 h1:MdmvkGuXi/8io6ixD5wud3vOLwc1rj0aNqRlpuvjmwA= +sigs.k8s.io/structured-merge-diff/v4 v4.4.2/go.mod h1:N8f93tFZh9U6vpxwRArLiikrE5/2tiu1w1AGfACIGE4= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/multidimensional-pod-autoscaler/e2e/utils/certs.go b/multidimensional-pod-autoscaler/e2e/utils/certs.go new file mode 100644 index 000000000000..d57be1269104 --- /dev/null +++ b/multidimensional-pod-autoscaler/e2e/utils/certs.go @@ -0,0 +1,99 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// COPY OF https://github.com/kubernetes/kubernetes/blob/master/test/e2e/apimachinery/certs.go + +package utils + +import ( + "crypto/x509" + "io/ioutil" + "os" + + "k8s.io/client-go/util/cert" + "k8s.io/client-go/util/keyutil" + "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/utils" +) + +type certContext struct { + cert []byte + key []byte + signingCert []byte +} + +// SetupWebhookCert sets up the server cert. For example, user, apiservers and admission webhooks +// can use the cert to prove their identity to the kube-apiserver. +func SetupWebhookCert(namespaceName string) *certContext { + certDir, err := ioutil.TempDir("", "test-e2e-server-cert") + if err != nil { + framework.Failf("Failed to create a temp dir for cert generation %v", err) + } + defer os.RemoveAll(certDir) + signingKey, err := utils.NewPrivateKey() + if err != nil { + framework.Failf("Failed to create CA private key %v", err) + } + signingCert, err := cert.NewSelfSignedCACert(cert.Config{CommonName: "e2e-server-cert-ca"}, signingKey) + if err != nil { + framework.Failf("Failed to create CA cert for apiserver %v", err) + } + caCertFile, err := ioutil.TempFile(certDir, "ca.crt") + if err != nil { + framework.Failf("Failed to create a temp file for ca cert generation %v", err) + } + if err := ioutil.WriteFile(caCertFile.Name(), utils.EncodeCertPEM(signingCert), 0644); err != nil { + framework.Failf("Failed to write CA cert %v", err) + } + key, err := utils.NewPrivateKey() + if err != nil { + framework.Failf("Failed to create private key for %v", err) + } + signedCert, err := utils.NewSignedCert( + &cert.Config{ + CommonName: WebhookServiceName + "." + namespaceName + ".svc", + AltNames: cert.AltNames{DNSNames: []string{WebhookServiceName + "." + namespaceName + ".svc"}}, + Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + }, + key, signingCert, signingKey, + ) + if err != nil { + framework.Failf("Failed to create cert%v", err) + } + certFile, err := ioutil.TempFile(certDir, "server.crt") + if err != nil { + framework.Failf("Failed to create a temp file for cert generation %v", err) + } + keyFile, err := ioutil.TempFile(certDir, "server.key") + if err != nil { + framework.Failf("Failed to create a temp file for key generation %v", err) + } + if err = ioutil.WriteFile(certFile.Name(), utils.EncodeCertPEM(signedCert), 0600); err != nil { + framework.Failf("Failed to write cert file %v", err) + } + privateKeyPEM, err := keyutil.MarshalPrivateKeyToPEM(key) + if err != nil { + framework.Failf("Failed to marshal key %v", err) + } + if err = ioutil.WriteFile(keyFile.Name(), privateKeyPEM, 0644); err != nil { + framework.Failf("Failed to write key file %v", err) + } + return &certContext{ + cert: utils.EncodeCertPEM(signedCert), + key: privateKeyPEM, + signingCert: utils.EncodeCertPEM(signingCert), + } +} diff --git a/multidimensional-pod-autoscaler/e2e/utils/webhook.go b/multidimensional-pod-autoscaler/e2e/utils/webhook.go new file mode 100644 index 000000000000..3abe00fb59dc --- /dev/null +++ b/multidimensional-pod-autoscaler/e2e/utils/webhook.go @@ -0,0 +1,388 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// PARTIAL COPY OF https://github.com/kubernetes/kubernetes/blob/master/test/e2e/apimachinery/webhook.go + +package utils + +import ( + "context" + "fmt" + "strings" + "time" + + ginkgo "github.com/onsi/ginkgo/v2" + admissionregistrationv1 "k8s.io/api/admissionregistration/v1" + appsv1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/apimachinery/pkg/util/uuid" + "k8s.io/apimachinery/pkg/util/wait" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/kubernetes/test/e2e/framework" + e2edeploy "k8s.io/kubernetes/test/e2e/framework/deployment" + "k8s.io/utils/pointer" +) + +const ( + // WebhookServiceName is the webhook service name. + WebhookServiceName = "e2e-test-webhook" + + roleBindingName = "webhook-auth-reader" + secretName = "sample-webhook-secret" + deploymentName = "sample-webhook-deployment" +) + +func strPtr(s string) *string { return &s } + +// LabelNamespace applies unique label to the namespace. +func LabelNamespace(f *framework.Framework, namespace string) { + client := f.ClientSet + + // Add a unique label to the namespace + ns, err := client.CoreV1().Namespaces().Get(context.TODO(), namespace, metav1.GetOptions{}) + framework.ExpectNoError(err, "error getting namespace %s", namespace) + if ns.Labels == nil { + ns.Labels = map[string]string{} + } + ns.Labels[f.UniqueName] = "true" + _, err = client.CoreV1().Namespaces().Update(context.TODO(), ns, metav1.UpdateOptions{}) + framework.ExpectNoError(err, "error labeling namespace %s", namespace) +} + +// CreateWebhookConfigurationReadyNamespace creates a separate namespace for webhook configuration ready markers to +// prevent cross-talk with webhook configurations being tested. +func CreateWebhookConfigurationReadyNamespace(f *framework.Framework) { + ns, err := f.ClientSet.CoreV1().Namespaces().Create(context.TODO(), &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: f.Namespace.Name + "-markers", + Labels: map[string]string{f.UniqueName + "-markers": "true"}, + }, + }, metav1.CreateOptions{}) + framework.ExpectNoError(err, "creating namespace for webhook configuration ready markers") + f.AddNamespacesToDelete(ns) +} + +// RegisterMutatingWebhookForPod creates mutation webhook configuration +// and applies it to the cluster. +func RegisterMutatingWebhookForPod(f *framework.Framework, configName string, certContext *certContext, servicePort int32) func() { + client := f.ClientSet + ginkgo.By("Registering the mutating pod webhook via the AdmissionRegistration API") + + namespace := f.Namespace.Name + sideEffectsNone := admissionregistrationv1.SideEffectClassNone + + _, err := createMutatingWebhookConfiguration(f, &admissionregistrationv1.MutatingWebhookConfiguration{ + ObjectMeta: metav1.ObjectMeta{ + Name: configName, + }, + Webhooks: []admissionregistrationv1.MutatingWebhook{ + { + Name: "adding-init-container.k8s.io", + Rules: []admissionregistrationv1.RuleWithOperations{{ + Operations: []admissionregistrationv1.OperationType{admissionregistrationv1.Create}, + Rule: admissionregistrationv1.Rule{ + APIGroups: []string{""}, + APIVersions: []string{"v1"}, + Resources: []string{"pods"}, + }, + }}, + ClientConfig: admissionregistrationv1.WebhookClientConfig{ + Service: &admissionregistrationv1.ServiceReference{ + Namespace: namespace, + Name: WebhookServiceName, + Path: strPtr("/mutating-pods-sidecar"), + Port: pointer.Int32Ptr(servicePort), + }, + CABundle: certContext.signingCert, + }, + SideEffects: &sideEffectsNone, + AdmissionReviewVersions: []string{"v1", "v1beta1"}, + // Scope the webhook to just this namespace + NamespaceSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{f.UniqueName: "true"}, + }, + }, + // Register a webhook that can be probed by marker requests to detect when the configuration is ready. + newMutatingIsReadyWebhookFixture(f, certContext, servicePort), + }, + }) + framework.ExpectNoError(err, "registering mutating webhook config %s with namespace %s", configName, namespace) + + err = waitWebhookConfigurationReady(f) + framework.ExpectNoError(err, "waiting for webhook configuration to be ready") + + return func() { + client.AdmissionregistrationV1().MutatingWebhookConfigurations().Delete(context.TODO(), configName, metav1.DeleteOptions{}) + } +} + +// createMutatingWebhookConfiguration ensures the webhook config scopes object or namespace selection +// to avoid interfering with other tests, then creates the config. +func createMutatingWebhookConfiguration(f *framework.Framework, config *admissionregistrationv1.MutatingWebhookConfiguration) (*admissionregistrationv1.MutatingWebhookConfiguration, error) { + for _, webhook := range config.Webhooks { + if webhook.NamespaceSelector != nil && webhook.NamespaceSelector.MatchLabels[f.UniqueName] == "true" { + continue + } + if webhook.ObjectSelector != nil && webhook.ObjectSelector.MatchLabels[f.UniqueName] == "true" { + continue + } + framework.Failf(`webhook %s in config %s has no namespace or object selector with %s="true", and can interfere with other tests`, webhook.Name, config.Name, f.UniqueName) + } + return f.ClientSet.AdmissionregistrationV1().MutatingWebhookConfigurations().Create(context.TODO(), config, metav1.CreateOptions{}) +} + +// newMutatingIsReadyWebhookFixture creates a mutating webhook that can be added to a webhook configuration and then probed +// with "marker" requests via waitWebhookConfigurationReady to wait for a webhook configuration to be ready. +func newMutatingIsReadyWebhookFixture(f *framework.Framework, certContext *certContext, servicePort int32) admissionregistrationv1.MutatingWebhook { + sideEffectsNone := admissionregistrationv1.SideEffectClassNone + failOpen := admissionregistrationv1.Ignore + return admissionregistrationv1.MutatingWebhook{ + Name: "mutating-is-webhook-configuration-ready.k8s.io", + Rules: []admissionregistrationv1.RuleWithOperations{{ + Operations: []admissionregistrationv1.OperationType{admissionregistrationv1.Create}, + Rule: admissionregistrationv1.Rule{ + APIGroups: []string{""}, + APIVersions: []string{"v1"}, + Resources: []string{"configmaps"}, + }, + }}, + ClientConfig: admissionregistrationv1.WebhookClientConfig{ + Service: &admissionregistrationv1.ServiceReference{ + Namespace: f.Namespace.Name, + Name: WebhookServiceName, + Path: strPtr("/always-deny"), + Port: pointer.Int32Ptr(servicePort), + }, + CABundle: certContext.signingCert, + }, + // network failures while the service network routing is being set up should be ignored by the marker + FailurePolicy: &failOpen, + SideEffects: &sideEffectsNone, + AdmissionReviewVersions: []string{"v1", "v1beta1"}, + // Scope the webhook to just the markers namespace + NamespaceSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{f.UniqueName + "-markers": "true"}, + }, + // appease createMutatingWebhookConfiguration isolation requirements + ObjectSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{f.UniqueName: "true"}, + }, + } +} + +// waitWebhookConfigurationReady sends "marker" requests until a webhook configuration is ready. +// A webhook created with newValidatingIsReadyWebhookFixture or newMutatingIsReadyWebhookFixture should first be added to +// the webhook configuration. +func waitWebhookConfigurationReady(f *framework.Framework) error { + cmClient := f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name + "-markers") + return wait.PollUntilContextTimeout(context.Background(), 100*time.Millisecond, 30*time.Second, true, func(ctx context.Context) (done bool, err error) { + marker := &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: string(uuid.NewUUID()), + Labels: map[string]string{ + f.UniqueName: "true", + }, + }, + } + _, err = cmClient.Create(ctx, marker, metav1.CreateOptions{}) + if err != nil { + // The always-deny webhook does not provide a reason, so check for the error string we expect + if strings.Contains(err.Error(), "denied") { + return true, nil + } + return false, err + } + // best effort cleanup of markers that are no longer needed + _ = cmClient.Delete(ctx, marker.GetName(), metav1.DeleteOptions{}) + framework.Logf("Waiting for webhook configuration to be ready...") + return false, nil + }) +} + +// CreateAuthReaderRoleBinding creates the role binding to allow the webhook read +// the extension-apiserver-authentication configmap. +func CreateAuthReaderRoleBinding(f *framework.Framework, namespace string) { + ginkgo.By("Create role binding to let webhook read extension-apiserver-authentication") + client := f.ClientSet + _, err := client.RbacV1().RoleBindings("kube-system").Create(context.TODO(), &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: roleBindingName, + Annotations: map[string]string{ + rbacv1.AutoUpdateAnnotationKey: "true", + }, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "", + Kind: "Role", + Name: "extension-apiserver-authentication-reader", + }, + // Webhook uses the default service account. + Subjects: []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: "default", + Namespace: namespace, + }, + }, + }, metav1.CreateOptions{}) + if err != nil && apierrors.IsAlreadyExists(err) { + framework.Logf("role binding %s already exists", roleBindingName) + } else { + framework.ExpectNoError(err, "creating role binding %s:webhook to access configMap", namespace) + } +} + +// DeployWebhookAndService creates a webhook with a corresponding service. +func DeployWebhookAndService(f *framework.Framework, image string, certContext *certContext, servicePort int32, + containerPort int32, params ...string) { + ginkgo.By("Deploying the webhook pod") + client := f.ClientSet + + // Creating the secret that contains the webhook's cert. + secret := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: secretName, + }, + Type: v1.SecretTypeOpaque, + Data: map[string][]byte{ + "tls.crt": certContext.cert, + "tls.key": certContext.key, + }, + } + namespace := f.Namespace.Name + _, err := client.CoreV1().Secrets(namespace).Create(context.TODO(), secret, metav1.CreateOptions{}) + framework.ExpectNoError(err, "creating secret %q in namespace %q", secretName, namespace) + + // Create the deployment of the webhook + podLabels := map[string]string{"app": "sample-webhook", "webhook": "true"} + replicas := int32(1) + zero := int64(0) + mounts := []v1.VolumeMount{ + { + Name: "webhook-certs", + ReadOnly: true, + MountPath: "/webhook.local.config/certificates", + }, + } + volumes := []v1.Volume{ + { + Name: "webhook-certs", + VolumeSource: v1.VolumeSource{ + Secret: &v1.SecretVolumeSource{SecretName: secretName}, + }, + }, + } + containers := []v1.Container{ + { + Name: "sample-webhook", + VolumeMounts: mounts, + Args: append([]string{ + "webhook", + "--tls-cert-file=/webhook.local.config/certificates/tls.crt", + "--tls-private-key-file=/webhook.local.config/certificates/tls.key", + "--alsologtostderr", + "-v=4", + // Use a non-default port for containers. + fmt.Sprintf("--port=%d", containerPort), + }, params...), + ReadinessProbe: &v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + HTTPGet: &v1.HTTPGetAction{ + Scheme: v1.URISchemeHTTPS, + Port: intstr.FromInt(int(containerPort)), + Path: "/readyz", + }, + }, + PeriodSeconds: 1, + SuccessThreshold: 1, + FailureThreshold: 30, + }, + Image: image, + Ports: []v1.ContainerPort{{ContainerPort: containerPort}}, + }, + } + d := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: deploymentName, + Labels: podLabels, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: &replicas, + Selector: &metav1.LabelSelector{ + MatchLabels: podLabels, + }, + Strategy: appsv1.DeploymentStrategy{ + Type: appsv1.RollingUpdateDeploymentStrategyType, + }, + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: podLabels, + }, + Spec: v1.PodSpec{ + TerminationGracePeriodSeconds: &zero, + Containers: containers, + Volumes: volumes, + }, + }, + }, + } + deployment, err := client.AppsV1().Deployments(namespace).Create(context.TODO(), d, metav1.CreateOptions{}) + framework.ExpectNoError(err, "creating deployment %s in namespace %s", deploymentName, namespace) + ginkgo.By("Wait for the deployment to be ready") + err = e2edeploy.WaitForDeploymentRevisionAndImage(client, namespace, deploymentName, "1", image) + framework.ExpectNoError(err, "waiting for the deployment of image %s in %s in %s to complete", image, deploymentName, namespace) + err = e2edeploy.WaitForDeploymentComplete(client, deployment) + framework.ExpectNoError(err, "waiting for the deployment status valid", image, deploymentName, namespace) + + ginkgo.By("Deploying the webhook service") + + serviceLabels := map[string]string{"webhook": "true"} + service := &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: WebhookServiceName, + Labels: map[string]string{"test": "webhook"}, + }, + Spec: v1.ServiceSpec{ + Selector: serviceLabels, + Ports: []v1.ServicePort{ + { + Protocol: "TCP", + Port: servicePort, + TargetPort: intstr.FromInt(int(containerPort)), + }, + }, + }, + } + _, err = client.CoreV1().Services(namespace).Create(context.TODO(), service, metav1.CreateOptions{}) + framework.ExpectNoError(err, "creating service %s in namespace %s", WebhookServiceName, namespace) + + ginkgo.By("Verifying the service has paired with the endpoint") + err = framework.WaitForServiceEndpointsNum(context.TODO(), client, namespace, WebhookServiceName, 1, 1*time.Second, 30*time.Second) + framework.ExpectNoError(err, "waiting for service %s/%s have %d endpoint", namespace, WebhookServiceName, 1) +} + +// CleanWebhookTest cleans after a webhook test. +func CleanWebhookTest(client clientset.Interface, namespaceName string) { + _ = client.CoreV1().Services(namespaceName).Delete(context.TODO(), WebhookServiceName, metav1.DeleteOptions{}) + _ = client.AppsV1().Deployments(namespaceName).Delete(context.TODO(), deploymentName, metav1.DeleteOptions{}) + _ = client.CoreV1().Secrets(namespaceName).Delete(context.TODO(), WebhookServiceName, metav1.DeleteOptions{}) + _ = client.RbacV1().RoleBindings("kube-system").Delete(context.TODO(), roleBindingName, metav1.DeleteOptions{}) +} diff --git a/multidimensional-pod-autoscaler/e2e/v1alpha1/actuation.go b/multidimensional-pod-autoscaler/e2e/v1alpha1/actuation.go new file mode 100644 index 000000000000..474f393925fb --- /dev/null +++ b/multidimensional-pod-autoscaler/e2e/v1alpha1/actuation.go @@ -0,0 +1,841 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package autoscaling + +import ( + "context" + "fmt" + "time" + + "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/utils/test" + + appsv1 "k8s.io/api/apps/v1" + autoscaling "k8s.io/api/autoscaling/v1" + apiv1 "k8s.io/api/core/v1" + policyv1 "k8s.io/api/policy/v1" + apierrs "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/autoscaler/multidimensional-pod-autoscaler/e2e/utils" + "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/utils/annotations" + vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/kubernetes/test/e2e/framework" + framework_deployment "k8s.io/kubernetes/test/e2e/framework/deployment" + framework_job "k8s.io/kubernetes/test/e2e/framework/job" + framework_rc "k8s.io/kubernetes/test/e2e/framework/rc" + framework_rs "k8s.io/kubernetes/test/e2e/framework/replicaset" + framework_ss "k8s.io/kubernetes/test/e2e/framework/statefulset" + testutils "k8s.io/kubernetes/test/utils" + podsecurity "k8s.io/pod-security-admission/api" + + ginkgo "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" +) + +var _ = ActuationSuiteE2eDescribe("Actuation", func() { + f := framework.NewDefaultFramework("multidimensional-pod-autoscaling") + f.NamespacePodSecurityEnforceLevel = podsecurity.LevelBaseline + + ginkgo.It("stops when pods get pending", func() { + + ginkgo.By("Setting up a hamster deployment") + d := SetupHamsterDeployment(f, "100m", "100Mi", defaultHamsterReplicas) + + ginkgo.By("Setting up a MPA CRD with ridiculous request") + containerName := GetHamsterContainerNameByIndex(0) + mpaCRD := test.MultidimPodAutoscaler(). + WithName("hamster-mpa"). + WithNamespace(f.Namespace.Name). + WithScaleTargetRef(hamsterTargetRef). + WithContainer(containerName). + AppendRecommendation( + test.Recommendation(). + WithContainer(containerName). + WithTarget("9999", ""). // Request 9999 CPUs to make POD pending + WithLowerBound("9999", ""). + WithUpperBound("9999", ""). + GetContainerResources()). + Get() + + InstallMPA(f, mpaCRD) + + ginkgo.By("Waiting for pods to be restarted and stuck pending") + err := assertPodsPendingForDuration(f.ClientSet, d, 1, 2*time.Minute) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + }) + + ginkgo.It("never applies recommendations when update mode is Off", func() { + ginkgo.By("Setting up a hamster deployment") + d := SetupHamsterDeployment(f, "100m", "100Mi", defaultHamsterReplicas) + cpuRequest := getCPURequest(d.Spec.Template.Spec) + podList, err := GetHamsterPods(f) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + podSet := MakePodSet(podList) + + ginkgo.By("Setting up a MPA CRD in mode Off") + containerName := GetHamsterContainerNameByIndex(0) + mpaCRD := test.MultidimPodAutoscaler(). + WithName("hamster-mpa"). + WithNamespace(f.Namespace.Name). + WithScaleTargetRef(hamsterTargetRef). + WithUpdateMode(vpa_types.UpdateModeOff). + WithContainer(containerName). + AppendRecommendation( + test.Recommendation(). + WithContainer(containerName). + WithTarget("200m", ""). + WithLowerBound("200m", ""). + WithUpperBound("200m", ""). + GetContainerResources()). + Get() + + InstallMPA(f, mpaCRD) + + ginkgo.By(fmt.Sprintf("Waiting for pods to be evicted, hoping it won't happen, sleep for %s", MpaEvictionTimeout.String())) + CheckNoPodsEvicted(f, podSet) + ginkgo.By("Forcefully killing one pod") + killPod(f, podList) + + ginkgo.By("Checking the requests were not modified") + updatedPodList, err := GetHamsterPods(f) + for _, pod := range updatedPodList.Items { + gomega.Expect(getCPURequest(pod.Spec)).To(gomega.Equal(cpuRequest)) + } + }) + + ginkgo.It("applies recommendations only on restart when update mode is Initial", func() { + ginkgo.By("Setting up a hamster deployment") + SetupHamsterDeployment(f, "100m", "100Mi", defaultHamsterReplicas) + podList, err := GetHamsterPods(f) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + podSet := MakePodSet(podList) + + ginkgo.By("Setting up a MPA CRD in mode Initial") + containerName := GetHamsterContainerNameByIndex(0) + mpaCRD := test.MultidimPodAutoscaler(). + WithName("hamster-mpa"). + WithNamespace(f.Namespace.Name). + WithScaleTargetRef(hamsterTargetRef). + WithUpdateMode(vpa_types.UpdateModeInitial). + WithContainer(containerName). + AppendRecommendation( + test.Recommendation(). + WithContainer(containerName). + WithTarget("200m", ""). + WithLowerBound("200m", ""). + WithUpperBound("200m", ""). + GetContainerResources()). + Get() + + InstallMPA(f, mpaCRD) + updatedCPURequest := ParseQuantityOrDie("200m") + + ginkgo.By(fmt.Sprintf("Waiting for pods to be evicted, hoping it won't happen, sleep for %s", MpaEvictionTimeout.String())) + CheckNoPodsEvicted(f, podSet) + ginkgo.By("Forcefully killing one pod") + killPod(f, podList) + + ginkgo.By("Checking that request was modified after forceful restart") + updatedPodList, err := GetHamsterPods(f) + foundUpdated := 0 + for _, pod := range updatedPodList.Items { + podRequest := getCPURequest(pod.Spec) + framework.Logf("podReq: %v", podRequest) + if podRequest.Cmp(updatedCPURequest) == 0 { + foundUpdated += 1 + } + } + gomega.Expect(foundUpdated).To(gomega.Equal(1)) + }) + + perControllerTests := []struct { + apiVersion string + kind string + name string + }{ + { + apiVersion: "apps/v1", + kind: "Deployment", + name: "hamster-deployment", + }, + { + apiVersion: "v1", + kind: "ReplicationController", + name: "hamster-rc", + }, + { + apiVersion: "batch/v1", + kind: "Job", + name: "hamster-job", + }, + { + apiVersion: "batch/v1", + kind: "CronJob", + name: "hamster-cronjob", + }, + { + apiVersion: "apps/v1", + kind: "ReplicaSet", + name: "hamster-rs", + }, + { + apiVersion: "apps/v1", + kind: "StatefulSet", + name: "hamster-stateful", + }, + } + for _, tc := range perControllerTests { + ginkgo.It("evicts pods in a multiple-replica "+tc.kind, func() { + testEvictsReplicatedPods(f, &autoscaling.CrossVersionObjectReference{ + APIVersion: tc.apiVersion, + Kind: tc.kind, + Name: tc.name, + }) + }) + ginkgo.It("by default does not evict pods in a 1-Pod "+tc.kind, func() { + testDoesNotEvictSingletonPodByDefault(f, &autoscaling.CrossVersionObjectReference{ + APIVersion: tc.apiVersion, + Kind: tc.kind, + Name: tc.name, + }) + }) + ginkgo.It("when configured, evicts pods in a 1-Pod "+tc.kind, func() { + testEvictsSingletonPodWhenConfigured(f, &autoscaling.CrossVersionObjectReference{ + APIVersion: tc.apiVersion, + Kind: tc.kind, + Name: tc.name, + }) + }) + } + + ginkgo.It("observes pod disruption budget", func() { + + ginkgo.By("Setting up a hamster deployment") + c := f.ClientSet + ns := f.Namespace.Name + + SetupHamsterDeployment(f, "10m", "10Mi", 10) + podList, err := GetHamsterPods(f) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + podSet := MakePodSet(podList) + + ginkgo.By("Setting up prohibitive PDB for hamster deployment") + pdb := setupPDB(f, "hamster-pdb", 0 /* maxUnavailable */) + + ginkgo.By("Setting up a MPA CRD") + containerName := GetHamsterContainerNameByIndex(0) + mpaCRD := test.MultidimPodAutoscaler(). + WithName("hamster-mpa"). + WithNamespace(f.Namespace.Name). + WithScaleTargetRef(hamsterTargetRef). + WithContainer(containerName). + AppendRecommendation( + test.Recommendation(). + WithContainer(containerName). + WithTarget("25m", ""). + WithLowerBound("25m", ""). + WithUpperBound("25m", ""). + GetContainerResources()). + Get() + + InstallMPA(f, mpaCRD) + + ginkgo.By(fmt.Sprintf("Waiting for pods to be evicted, hoping it won't happen, sleep for %s", MpaEvictionTimeout.String())) + CheckNoPodsEvicted(f, podSet) + + ginkgo.By("Updating the PDB to allow for multiple pods to be evicted") + // We will check that 7 replicas are evicted in 3 minutes, which translates + // to 3 updater loops. This gives us relatively good confidence that updater + // evicts more than one pod in a loop if PDB allows it. + permissiveMaxUnavailable := 7 + // Creating new PDB and removing old one, since PDBs are immutable at the moment + setupPDB(f, "hamster-pdb-2", permissiveMaxUnavailable) + err = c.PolicyV1().PodDisruptionBudgets(ns).Delete(context.TODO(), pdb.Name, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + ginkgo.By(fmt.Sprintf("Waiting for pods to be evicted, sleep for %s", MpaEvictionTimeout.String())) + time.Sleep(MpaEvictionTimeout) + ginkgo.By("Checking enough pods were evicted.") + currentPodList, err := GetHamsterPods(f) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + evictedCount := GetEvictedPodsCount(MakePodSet(currentPodList), podSet) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(evictedCount >= permissiveMaxUnavailable).To(gomega.BeTrue()) + }) + + ginkgo.It("observes container max in LimitRange", func() { + ginkgo.By("Setting up a hamster deployment") + d := NewHamsterDeploymentWithResourcesAndLimits(f, + ParseQuantityOrDie("100m") /*cpu request*/, ParseQuantityOrDie("200Mi"), /*memory request*/ + ParseQuantityOrDie("300m") /*cpu limit*/, ParseQuantityOrDie("400Mi") /*memory limit*/) + podList := startDeploymentPods(f, d) + + ginkgo.By("Setting up a MPA CRD") + containerName := GetHamsterContainerNameByIndex(0) + mpaCRD := test.MultidimPodAutoscaler(). + WithName("hamster-mpa"). + WithNamespace(f.Namespace.Name). + WithScaleTargetRef(hamsterTargetRef). + WithContainer(containerName). + AppendRecommendation( + test.Recommendation(). + WithContainer(containerName). + WithTarget("200m", ""). + WithLowerBound("200m", ""). + WithUpperBound("200m", ""). + GetContainerResources()). + Get() + + InstallMPA(f, mpaCRD) + + // Max CPU limit is 300m and ratio is 3., so max request is 100m, while + // recommendation is 200m + // Max memory limit is 1T and ratio is 2., so max request is 0.5T + InstallLimitRangeWithMax(f, "300m", "1T", apiv1.LimitTypeContainer) + + ginkgo.By(fmt.Sprintf("Waiting for pods to be evicted, hoping it won't happen, sleep for %s", MpaEvictionTimeout.String())) + CheckNoPodsEvicted(f, MakePodSet(podList)) + }) + + ginkgo.It("observes container min in LimitRange", func() { + ginkgo.By("Setting up a hamster deployment") + d := NewHamsterDeploymentWithResourcesAndLimits(f, + ParseQuantityOrDie("100m") /*cpu request*/, ParseQuantityOrDie("200Mi"), /*memory request*/ + ParseQuantityOrDie("300m") /*cpu limit*/, ParseQuantityOrDie("400Mi") /*memory limit*/) + podList := startDeploymentPods(f, d) + + ginkgo.By("Setting up a MPA CRD") + containerName := GetHamsterContainerNameByIndex(0) + mpaCRD := test.MultidimPodAutoscaler(). + WithName("hamster-mpa"). + WithNamespace(f.Namespace.Name). + WithScaleTargetRef(hamsterTargetRef). + WithContainer(containerName). + AppendRecommendation( + test.Recommendation(). + WithContainer(containerName). + WithTarget("50m", ""). + WithLowerBound("50m", ""). + WithUpperBound("50m", ""). + GetContainerResources()). + Get() + + InstallMPA(f, mpaCRD) + + // Min CPU from limit range is 100m and ratio is 3. Min applies both to limit and request so min + // request is 100m request and 300m limit + // Min memory limit is 0 and ratio is 2., so min request is 0 + InstallLimitRangeWithMin(f, "100m", "0", apiv1.LimitTypeContainer) + + ginkgo.By(fmt.Sprintf("Waiting for pods to be evicted, hoping it won't happen, sleep for %s", MpaEvictionTimeout.String())) + CheckNoPodsEvicted(f, MakePodSet(podList)) + }) + + ginkgo.It("observes pod max in LimitRange", func() { + ginkgo.By("Setting up a hamster deployment") + d := NewHamsterDeploymentWithResourcesAndLimits(f, + ParseQuantityOrDie("100m") /*cpu request*/, ParseQuantityOrDie("200Mi"), /*memory request*/ + ParseQuantityOrDie("300m") /*cpu limit*/, ParseQuantityOrDie("400Mi") /*memory limit*/) + d.Spec.Template.Spec.Containers = append(d.Spec.Template.Spec.Containers, d.Spec.Template.Spec.Containers[0]) + d.Spec.Template.Spec.Containers[1].Name = "hamster2" + podList := startDeploymentPods(f, d) + + ginkgo.By("Setting up a MPA CRD") + container1Name := GetHamsterContainerNameByIndex(0) + container2Name := GetHamsterContainerNameByIndex(1) + mpaCRD := test.MultidimPodAutoscaler(). + WithName("hamster-mpa"). + WithNamespace(f.Namespace.Name). + WithScaleTargetRef(hamsterTargetRef). + WithContainer(container1Name). + AppendRecommendation( + test.Recommendation(). + WithContainer(container1Name). + WithTarget("200m", ""). + WithLowerBound("200m", ""). + WithUpperBound("200m", ""). + GetContainerResources()). + WithContainer(container2Name). + AppendRecommendation( + test.Recommendation(). + WithContainer(container2Name). + WithTarget("200m", ""). + WithLowerBound("200m", ""). + WithUpperBound("200m", ""). + GetContainerResources()). + Get() + + InstallMPA(f, mpaCRD) + + // Max CPU limit is 600m per pod, 300m per container and ratio is 3., so max request is 100m, + // while recommendation is 200m + // Max memory limit is 2T per pod, 1T per container and ratio is 2., so max request is 0.5T + InstallLimitRangeWithMax(f, "600m", "2T", apiv1.LimitTypePod) + + ginkgo.By(fmt.Sprintf("Waiting for pods to be evicted, hoping it won't happen, sleep for %s", MpaEvictionTimeout.String())) + CheckNoPodsEvicted(f, MakePodSet(podList)) + }) + + ginkgo.It("observes pod min in LimitRange", func() { + ginkgo.By("Setting up a hamster deployment") + d := NewHamsterDeploymentWithResourcesAndLimits(f, + ParseQuantityOrDie("100m") /*cpu request*/, ParseQuantityOrDie("200Mi"), /*memory request*/ + ParseQuantityOrDie("300m") /*cpu limit*/, ParseQuantityOrDie("400Mi") /*memory limit*/) + d.Spec.Template.Spec.Containers = append(d.Spec.Template.Spec.Containers, d.Spec.Template.Spec.Containers[0]) + container2Name := "hamster2" + d.Spec.Template.Spec.Containers[1].Name = container2Name + podList := startDeploymentPods(f, d) + + ginkgo.By("Setting up a MPA CRD") + container1Name := GetHamsterContainerNameByIndex(0) + mpaCRD := test.MultidimPodAutoscaler(). + WithName("hamster-mpa"). + WithNamespace(f.Namespace.Name). + WithScaleTargetRef(hamsterTargetRef). + WithContainer(container1Name). + AppendRecommendation( + test.Recommendation(). + WithContainer(container1Name). + WithTarget("50m", ""). + WithLowerBound("50m", ""). + WithUpperBound("50m", ""). + GetContainerResources()). + WithContainer(container2Name). + AppendRecommendation( + test.Recommendation(). + WithContainer(container2Name). + WithTarget("50m", ""). + WithLowerBound("50m", ""). + WithUpperBound("50m", ""). + GetContainerResources()). + Get() + + InstallMPA(f, mpaCRD) + + // Min CPU from limit range is 200m per pod, 100m per container and ratio is 3. Min applies both + // to limit and request so min request is 100m request and 300m limit + // Min memory limit is 0 and ratio is 2., so min request is 0 + InstallLimitRangeWithMin(f, "200m", "0", apiv1.LimitTypePod) + + ginkgo.By(fmt.Sprintf("Waiting for pods to be evicted, hoping it won't happen, sleep for %s", MpaEvictionTimeout.String())) + CheckNoPodsEvicted(f, MakePodSet(podList)) + }) + + ginkgo.It("does not act on injected sidecars", func() { + const ( + agnhostImage = "registry.k8s.io/e2e-test-images/agnhost:2.40" + sidecarParam = "--sidecar-image=registry.k8s.io/pause:3.1" + servicePort = int32(8443) + containerPort = int32(8444) + ) + + ginkgo.By("Setting up Webhook for sidecar injection") + + client := f.ClientSet + namespaceName := f.Namespace.Name + defer utils.CleanWebhookTest(client, namespaceName) + + // Make sure the namespace created for the test is labeled to be selected by the webhooks. + utils.LabelNamespace(f, f.Namespace.Name) + utils.CreateWebhookConfigurationReadyNamespace(f) + + ginkgo.By("Setting up server cert") + context := utils.SetupWebhookCert(namespaceName) + utils.CreateAuthReaderRoleBinding(f, namespaceName) + + utils.DeployWebhookAndService(f, agnhostImage, context, servicePort, containerPort, sidecarParam) + + // Webhook must be placed after mpa webhook. Webhooks are registered alphabetically. + // Use name that starts with "z". + webhookCleanup := utils.RegisterMutatingWebhookForPod(f, "z-sidecar-injection-webhook", context, servicePort) + defer webhookCleanup() + + ginkgo.By("Setting up a hamster mpa") + container1Name := GetHamsterContainerNameByIndex(0) + container2Name := GetHamsterContainerNameByIndex(1) + mpaCRD := test.MultidimPodAutoscaler(). + WithName("hamster-mpa"). + WithNamespace(f.Namespace.Name). + WithScaleTargetRef(hamsterTargetRef). + WithContainer(container1Name). + AppendRecommendation( + test.Recommendation(). + WithContainer(container1Name). + WithTarget("100m", ""). + WithLowerBound("100m", ""). + WithUpperBound("100m", ""). + GetContainerResources()). + WithContainer(container2Name). + AppendRecommendation( + test.Recommendation(). + WithContainer(container2Name). + WithTarget("5000m", ""). + WithLowerBound("5000m", ""). + WithUpperBound("5000m", ""). + GetContainerResources()). + Get() + + InstallMPA(f, mpaCRD) + + ginkgo.By("Setting up a hamster deployment") + + d := NewHamsterDeploymentWithResources(f, ParseQuantityOrDie("100m"), ParseQuantityOrDie("100Mi")) + podList := startDeploymentPods(f, d) + for _, pod := range podList.Items { + observedContainers, ok := pod.GetAnnotations()[annotations.MpaObservedContainersLabel] + gomega.Expect(ok).To(gomega.Equal(true)) + containers, err := annotations.ParseMpaObservedContainersValue(observedContainers) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(containers).To(gomega.HaveLen(1)) + gomega.Expect(pod.Spec.Containers).To(gomega.HaveLen(2)) + } + + podSet := MakePodSet(podList) + ginkgo.By(fmt.Sprintf("Waiting for pods to be evicted, hoping it won't happen, sleep for %s", MpaEvictionTimeout.String())) + CheckNoPodsEvicted(f, podSet) + }) +}) + +func getCPURequest(podSpec apiv1.PodSpec) resource.Quantity { + return podSpec.Containers[0].Resources.Requests[apiv1.ResourceCPU] +} + +func killPod(f *framework.Framework, podList *apiv1.PodList) { + f.ClientSet.CoreV1().Pods(f.Namespace.Name).Delete(context.TODO(), podList.Items[0].Name, metav1.DeleteOptions{}) + err := WaitForPodsRestarted(f, podList) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) +} + +// assertPodsPendingForDuration checks that at most pendingPodsNum pods are pending for pendingDuration +func assertPodsPendingForDuration(c clientset.Interface, deployment *appsv1.Deployment, pendingPodsNum int, pendingDuration time.Duration) error { + + pendingPods := make(map[string]time.Time) + + err := wait.PollUntilContextTimeout(context.Background(), pollInterval, pollTimeout, true, func(ctx context.Context) (done bool, err error) { + currentPodList, err := framework_deployment.GetPodsForDeployment(ctx, c, deployment) + if err != nil { + return false, err + } + + missingPods := make(map[string]bool) + for podName := range pendingPods { + missingPods[podName] = true + } + + now := time.Now() + for _, pod := range currentPodList.Items { + delete(missingPods, pod.Name) + switch pod.Status.Phase { + case apiv1.PodPending: + _, ok := pendingPods[pod.Name] + if !ok { + pendingPods[pod.Name] = now + } + default: + delete(pendingPods, pod.Name) + } + } + + for missingPod := range missingPods { + delete(pendingPods, missingPod) + } + + if len(pendingPods) < pendingPodsNum { + return false, nil + } + + if len(pendingPods) > pendingPodsNum { + return false, fmt.Errorf("%v pending pods seen - expecting %v", len(pendingPods), pendingPodsNum) + } + + for p, t := range pendingPods { + fmt.Println("task", now, p, t, now.Sub(t), pendingDuration) + if now.Sub(t) < pendingDuration { + return false, nil + } + } + + return true, nil + }) + + if err != nil { + return fmt.Errorf("assertion failed for pending pods in %v: %v", deployment.Name, err) + } + return nil +} + +func testEvictsReplicatedPods(f *framework.Framework, controller *autoscaling.CrossVersionObjectReference) { + ginkgo.By(fmt.Sprintf("Setting up a hamster %v", controller.Kind)) + setupHamsterController(f, controller.Kind, "100m", "100Mi", defaultHamsterReplicas) + podList, err := GetHamsterPods(f) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + ginkgo.By("Setting up a MPA CRD") + containerName := GetHamsterContainerNameByIndex(0) + mpaCRD := test.MultidimPodAutoscaler(). + WithName("hamster-mpa"). + WithNamespace(f.Namespace.Name). + WithScaleTargetRef(controller). + WithContainer(containerName). + AppendRecommendation( + test.Recommendation(). + WithContainer(containerName). + WithTarget("200m", ""). + WithLowerBound("200m", ""). + WithUpperBound("200m", ""). + GetContainerResources()). + Get() + + InstallMPA(f, mpaCRD) + + ginkgo.By("Waiting for pods to be evicted") + err = WaitForPodsEvicted(f, podList) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) +} + +func testDoesNotEvictSingletonPodByDefault(f *framework.Framework, controller *autoscaling.CrossVersionObjectReference) { + ginkgo.By(fmt.Sprintf("Setting up a hamster %v", controller.Kind)) + setupHamsterController(f, controller.Kind, "100m", "100Mi", 1 /*replicas*/) + podList, err := GetHamsterPods(f) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + ginkgo.By("Setting up a MPA CRD") + containerName := GetHamsterContainerNameByIndex(0) + mpaCRD := test.MultidimPodAutoscaler(). + WithName("hamster-mpa"). + WithNamespace(f.Namespace.Name). + WithScaleTargetRef(controller). + WithContainer(containerName). + AppendRecommendation( + test.Recommendation(). + WithContainer(containerName). + WithTarget("200m", ""). + WithLowerBound("200m", ""). + WithUpperBound("200m", ""). + GetContainerResources()). + Get() + + InstallMPA(f, mpaCRD) + + // No eviction is expected with the default settings of MPA object + ginkgo.By(fmt.Sprintf("Waiting for pods to be evicted, hoping it won't happen, sleep for %s", MpaEvictionTimeout.String())) + CheckNoPodsEvicted(f, MakePodSet(podList)) +} + +func testEvictsSingletonPodWhenConfigured(f *framework.Framework, controller *autoscaling.CrossVersionObjectReference) { + ginkgo.By(fmt.Sprintf("Setting up a hamster %v", controller.Kind)) + setupHamsterController(f, controller.Kind, "100m", "100Mi", 1 /*replicas*/) + podList, err := GetHamsterPods(f) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + // Prepare the MPA to allow single-Pod eviction. + ginkgo.By("Setting up a MPA CRD") + containerName := GetHamsterContainerNameByIndex(0) + mpaCRD := test.MultidimPodAutoscaler(). + WithName("hamster-mpa"). + WithNamespace(f.Namespace.Name). + WithScaleTargetRef(controller). + // WithMinReplicas(pointer.Int32(1)). + WithContainer(containerName). + AppendRecommendation( + test.Recommendation(). + WithContainer(containerName). + WithTarget("200m", ""). + WithLowerBound("200m", ""). + WithUpperBound("200m", ""). + GetContainerResources()). + Get() + + InstallMPA(f, mpaCRD) + + ginkgo.By("Waiting for pods to be evicted") + err = WaitForPodsEvicted(f, podList) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) +} + +func setupHamsterController(f *framework.Framework, controllerKind, cpu, memory string, replicas int32) *apiv1.PodList { + switch controllerKind { + case "Deployment": + SetupHamsterDeployment(f, cpu, memory, replicas) + case "ReplicationController": + setupHamsterReplicationController(f, cpu, memory, replicas) + case "Job": + setupHamsterJob(f, cpu, memory, replicas) + case "CronJob": + SetupHamsterCronJob(f, "*/2 * * * *", cpu, memory, replicas) + case "ReplicaSet": + setupHamsterRS(f, cpu, memory, replicas) + case "StatefulSet": + setupHamsterStateful(f, cpu, memory, replicas) + default: + framework.Failf("Unknown controller kind: %v", controllerKind) + return nil + } + pods, err := GetHamsterPods(f) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + return pods +} + +func setupHamsterReplicationController(f *framework.Framework, cpu, memory string, replicas int32) { + hamsterContainer := SetupHamsterContainer(cpu, memory) + rc := framework_rc.ByNameContainer("hamster-rc", replicas, hamsterLabels, hamsterContainer, nil) + + rc.Namespace = f.Namespace.Name + err := testutils.CreateRCWithRetries(f.ClientSet, f.Namespace.Name, rc) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = waitForRCPodsRunning(f, rc) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) +} + +func waitForRCPodsRunning(f *framework.Framework, rc *apiv1.ReplicationController) error { + return wait.PollUntilContextTimeout(context.Background(), pollInterval, pollTimeout, true, func(ctx context.Context) (done bool, err error) { + podList, err := GetHamsterPods(f) + if err != nil { + framework.Logf("Error listing pods, retrying: %v", err) + return false, nil + } + podsRunning := int32(0) + for _, pod := range podList.Items { + if pod.Status.Phase == apiv1.PodRunning { + podsRunning += 1 + } + } + return podsRunning == *rc.Spec.Replicas, nil + }) +} + +func setupHamsterJob(f *framework.Framework, cpu, memory string, replicas int32) { + job := framework_job.NewTestJob("notTerminate", "hamster-job", apiv1.RestartPolicyOnFailure, + replicas, replicas, nil, 10) + job.Spec.Template.Spec.Containers[0] = SetupHamsterContainer(cpu, memory) + for label, value := range hamsterLabels { + job.Spec.Template.Labels[label] = value + } + _, err := framework_job.CreateJob(context.TODO(), f.ClientSet, f.Namespace.Name, job) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = framework_job.WaitForJobPodsRunning(context.TODO(), f.ClientSet, f.Namespace.Name, job.Name, replicas) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) +} + +func setupHamsterRS(f *framework.Framework, cpu, memory string, replicas int32) { + rs := newReplicaSet("hamster-rs", f.Namespace.Name, replicas, hamsterLabels, "", "") + rs.Spec.Template.Spec.Containers[0] = SetupHamsterContainer(cpu, memory) + err := createReplicaSetWithRetries(f.ClientSet, f.Namespace.Name, rs) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = framework_rs.WaitForReadyReplicaSet(context.TODO(), f.ClientSet, f.Namespace.Name, rs.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) +} + +func setupHamsterStateful(f *framework.Framework, cpu, memory string, replicas int32) { + stateful := framework_ss.NewStatefulSet("hamster-stateful", f.Namespace.Name, + "hamster-service", replicas, nil, nil, hamsterLabels) + + stateful.Spec.Template.Spec.Containers[0] = SetupHamsterContainer(cpu, memory) + err := createStatefulSetSetWithRetries(f.ClientSet, f.Namespace.Name, stateful) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + framework_ss.WaitForRunningAndReady(context.TODO(), f.ClientSet, *stateful.Spec.Replicas, stateful) +} + +func setupPDB(f *framework.Framework, name string, maxUnavailable int) *policyv1.PodDisruptionBudget { + maxUnavailableIntstr := intstr.FromInt(maxUnavailable) + pdb := &policyv1.PodDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Spec: policyv1.PodDisruptionBudgetSpec{ + MaxUnavailable: &maxUnavailableIntstr, + Selector: &metav1.LabelSelector{ + MatchLabels: hamsterLabels, + }, + }, + } + _, err := f.ClientSet.PolicyV1().PodDisruptionBudgets(f.Namespace.Name).Create(context.TODO(), pdb, metav1.CreateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + return pdb +} + +func getCurrentPodSetForDeployment(c clientset.Interface, d *appsv1.Deployment) PodSet { + podList, err := framework_deployment.GetPodsForDeployment(context.TODO(), c, d) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + return MakePodSet(podList) +} + +func createReplicaSetWithRetries(c clientset.Interface, namespace string, obj *appsv1.ReplicaSet) error { + if obj == nil { + return fmt.Errorf("object provided to create is empty") + } + createFunc := func() (bool, error) { + _, err := c.AppsV1().ReplicaSets(namespace).Create(context.TODO(), obj, metav1.CreateOptions{}) + if err == nil || apierrs.IsAlreadyExists(err) { + return true, nil + } + return false, fmt.Errorf("failed to create object with non-retriable error: %v", err) + } + return testutils.RetryWithExponentialBackOff(createFunc) +} + +func createStatefulSetSetWithRetries(c clientset.Interface, namespace string, obj *appsv1.StatefulSet) error { + if obj == nil { + return fmt.Errorf("object provided to create is empty") + } + createFunc := func() (bool, error) { + _, err := c.AppsV1().StatefulSets(namespace).Create(context.TODO(), obj, metav1.CreateOptions{}) + if err == nil || apierrs.IsAlreadyExists(err) { + return true, nil + } + return false, fmt.Errorf("failed to create object with non-retriable error: %v", err) + } + return testutils.RetryWithExponentialBackOff(createFunc) +} + +// newReplicaSet returns a new ReplicaSet. +func newReplicaSet(name, namespace string, replicas int32, podLabels map[string]string, imageName, image string) *appsv1.ReplicaSet { + return &appsv1.ReplicaSet{ + TypeMeta: metav1.TypeMeta{ + Kind: "ReplicaSet", + APIVersion: "apps/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: name, + }, + Spec: appsv1.ReplicaSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: podLabels, + }, + Replicas: &replicas, + Template: apiv1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: podLabels, + }, + Spec: apiv1.PodSpec{ + Containers: []apiv1.Container{ + { + Name: imageName, + Image: image, + SecurityContext: &apiv1.SecurityContext{}, + }, + }, + }, + }, + }, + } +} diff --git a/multidimensional-pod-autoscaler/e2e/v1alpha1/admission_controller.go b/multidimensional-pod-autoscaler/e2e/v1alpha1/admission_controller.go new file mode 100644 index 000000000000..32538f76c86f --- /dev/null +++ b/multidimensional-pod-autoscaler/e2e/v1alpha1/admission_controller.go @@ -0,0 +1,963 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package autoscaling + +import ( + "context" + "fmt" + "io" + "strings" + "time" + + appsv1 "k8s.io/api/apps/v1" + autoscalingv1 "k8s.io/api/autoscaling/v1" + apiv1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/utils/test" + vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" + "k8s.io/kubernetes/test/e2e/framework" + framework_deployment "k8s.io/kubernetes/test/e2e/framework/deployment" + podsecurity "k8s.io/pod-security-admission/api" + + ginkgo "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" +) + +var _ = AdmissionControllerE2eDescribe("Admission-controller", func() { + f := framework.NewDefaultFramework("multidimensional-pod-autoscaling") + f.NamespacePodSecurityEnforceLevel = podsecurity.LevelBaseline + + ginkgo.It("starts pods with new recommended request", func() { + d := NewHamsterDeploymentWithResources(f, ParseQuantityOrDie("100m") /*cpu*/, ParseQuantityOrDie("100Mi") /*memory*/) + + ginkgo.By("Setting up a MPA CRD") + containerName := GetHamsterContainerNameByIndex(0) + mpaCRD := test.MultidimPodAutoscaler(). + WithName("hamster-mpa"). + WithNamespace(f.Namespace.Name). + WithScaleTargetRef(hamsterTargetRef). + WithContainer(containerName). + AppendRecommendation( + test.Recommendation(). + WithContainer(containerName). + WithTarget("250m", "200Mi"). + WithLowerBound("250m", "200Mi"). + WithUpperBound("250m", "200Mi"). + GetContainerResources()). + Get() + + InstallMPA(f, mpaCRD) + + ginkgo.By("Setting up a hamster deployment") + podList := startDeploymentPods(f, d) + + // Originally Pods had 100m CPU, 100Mi of memory, but admission controller + // should change it to recommended 250m CPU and 200Mi of memory. + for _, pod := range podList.Items { + gomega.Expect(pod.Spec.Containers[0].Resources.Requests[apiv1.ResourceCPU]).To(gomega.Equal(ParseQuantityOrDie("250m"))) + gomega.Expect(pod.Spec.Containers[0].Resources.Requests[apiv1.ResourceMemory]).To(gomega.Equal(ParseQuantityOrDie("200Mi"))) + } + }) + + ginkgo.It("starts pods with new recommended request when recommendation includes an extra container", func() { + d := NewHamsterDeploymentWithResources(f, ParseQuantityOrDie("100m") /*cpu*/, ParseQuantityOrDie("100Mi") /*memory*/) + + ginkgo.By("Setting up a MPA CRD") + removedContainerName := "removed" + container1Name := GetHamsterContainerNameByIndex(0) + mpaCRD := test.MultidimPodAutoscaler(). + WithName("hamster-mpa"). + WithNamespace(f.Namespace.Name). + WithScaleTargetRef(hamsterTargetRef). + WithContainer(removedContainerName). + AppendRecommendation( + test.Recommendation(). + WithContainer(removedContainerName). + WithTarget("500m", "500Mi"). + WithLowerBound("500m", "500Mi"). + WithUpperBound("500m", "500Mi"). + GetContainerResources()). + WithContainer(container1Name). + AppendRecommendation( + test.Recommendation(). + WithContainer(container1Name). + WithTarget("250m", "200Mi"). + WithLowerBound("250m", "200Mi"). + WithUpperBound("250m", "200Mi"). + GetContainerResources()). + Get() + + InstallMPA(f, mpaCRD) + + ginkgo.By("Setting up a hamster deployment") + podList := startDeploymentPods(f, d) + + // Originally Pods had 100m CPU, 100Mi of memory, but admission controller + // should change it to recommended 250m CPU and 200Mi of memory. + for _, pod := range podList.Items { + gomega.Expect(pod.Spec.Containers[0].Resources.Requests[apiv1.ResourceCPU]).To(gomega.Equal(ParseQuantityOrDie("250m"))) + gomega.Expect(pod.Spec.Containers[0].Resources.Requests[apiv1.ResourceMemory]).To(gomega.Equal(ParseQuantityOrDie("200Mi"))) + } + }) + + ginkgo.It("starts pods with old recommended request when recommendation has only a container that doesn't match", func() { + d := NewHamsterDeploymentWithResources(f, ParseQuantityOrDie("100m") /*cpu*/, ParseQuantityOrDie("100Mi") /*memory*/) + + ginkgo.By("Setting up a MPA CRD") + removedContainerName := "removed" + mpaCRD := test.MultidimPodAutoscaler(). + WithName("hamster-mpa"). + WithNamespace(f.Namespace.Name). + WithScaleTargetRef(hamsterTargetRef). + WithContainer(removedContainerName). + AppendRecommendation( + test.Recommendation(). + WithContainer(removedContainerName). + WithTarget("250m", "200Mi"). + WithLowerBound("250m", "200Mi"). + WithUpperBound("250m", "200Mi"). + GetContainerResources()). + Get() + + InstallMPA(f, mpaCRD) + + ginkgo.By("Setting up a hamster deployment") + podList := startDeploymentPods(f, d) + + // Originally Pods had 100m CPU, 100Mi of memory, but admission controller + // should change it to recommended 250m CPU and 200Mi of memory. + for _, pod := range podList.Items { + gomega.Expect(pod.Spec.Containers[0].Resources.Requests[apiv1.ResourceCPU]).To(gomega.Equal(ParseQuantityOrDie("100m"))) + gomega.Expect(pod.Spec.Containers[0].Resources.Requests[apiv1.ResourceMemory]).To(gomega.Equal(ParseQuantityOrDie("100Mi"))) + } + }) + + ginkgo.It("starts pod with recommendation when one container has a recommendation and one other one doesn't", func() { + d := NewNHamstersDeployment(f, 2) + d.Spec.Template.Spec.Containers[0].Resources.Requests = apiv1.ResourceList{ + apiv1.ResourceCPU: ParseQuantityOrDie("100m"), + apiv1.ResourceMemory: ParseQuantityOrDie("100Mi"), + } + d.Spec.Template.Spec.Containers[1].Resources.Requests = apiv1.ResourceList{ + apiv1.ResourceCPU: ParseQuantityOrDie("100m"), + apiv1.ResourceMemory: ParseQuantityOrDie("100Mi"), + } + framework.Logf("Created hamster deployment %v", d) + ginkgo.By("Setting up a MPA CRD") + containerName := GetHamsterContainerNameByIndex(0) + mpaCRD := test.MultidimPodAutoscaler(). + WithName("hamster-mpa"). + WithNamespace(f.Namespace.Name). + WithScaleTargetRef(hamsterTargetRef). + WithContainer(containerName). + AppendRecommendation( + test.Recommendation(). + WithContainer(containerName). + WithTarget("250m", "200Mi"). + WithLowerBound("250m", "200Mi"). + WithUpperBound("250m", "200Mi"). + GetContainerResources()). + Get() + + InstallMPA(f, mpaCRD) + + ginkgo.By("Setting up a hamster deployment") + podList := startDeploymentPods(f, d) + + // Originally Pods had 100m CPU, 100Mi of memory, but admission controller + // should change it to recommended 250m CPU and 200Mi of memory. + for _, pod := range podList.Items { + gomega.Expect(pod.Spec.Containers[0].Resources.Requests[apiv1.ResourceCPU]).To(gomega.Equal(ParseQuantityOrDie("250m"))) + gomega.Expect(pod.Spec.Containers[0].Resources.Requests[apiv1.ResourceMemory]).To(gomega.Equal(ParseQuantityOrDie("200Mi"))) + gomega.Expect(pod.Spec.Containers[1].Resources.Requests[apiv1.ResourceCPU]).To(gomega.Equal(ParseQuantityOrDie("100m"))) + gomega.Expect(pod.Spec.Containers[1].Resources.Requests[apiv1.ResourceMemory]).To(gomega.Equal(ParseQuantityOrDie("100Mi"))) + } + }) + + ginkgo.It("starts pods with default request when recommendation includes an extra container when a limit range applies", func() { + d := NewHamsterDeploymentWithResources(f, ParseQuantityOrDie("100m") /*cpu*/, ParseQuantityOrDie("100Mi") /*memory*/) + InstallLimitRangeWithMax(f, "300m", "1Gi", apiv1.LimitTypeContainer) + + ginkgo.By("Setting up a MPA CRD") + container1Name := GetHamsterContainerNameByIndex(0) + container2Name := GetHamsterContainerNameByIndex(1) + mpaCRD := test.MultidimPodAutoscaler(). + WithName("hamster-mpa"). + WithNamespace(f.Namespace.Name). + WithScaleTargetRef(hamsterTargetRef). + WithContainer(container1Name). + AppendRecommendation( + test.Recommendation(). + WithContainer(container1Name). + WithTarget("500m", "500Mi"). + WithLowerBound("500m", "500Mi"). + WithUpperBound("500m", "500Mi"). + GetContainerResources()). + WithContainer(container2Name). + AppendRecommendation( + test.Recommendation(). + WithContainer(container2Name). + WithTarget("250m", "200Mi"). + WithLowerBound("250m", "200Mi"). + WithUpperBound("250m", "200Mi"). + GetContainerResources()). + Get() + + InstallMPA(f, mpaCRD) + + ginkgo.By("Setting up a hamster deployment") + podList := startDeploymentPods(f, d) + + // Originally Pods had 100m CPU, 100Mi of memory, but admission controller + // should change it to recommended 250m CPU and 200Mi of memory. + for _, pod := range podList.Items { + // This is a bug; MPA should behave here like it does without a limit range + // Like in "starts pods with new recommended request when recommendation includes an extra container" + gomega.Expect(pod.Spec.Containers[0].Resources.Requests[apiv1.ResourceCPU]).To(gomega.Equal(ParseQuantityOrDie("100m"))) + gomega.Expect(pod.Spec.Containers[0].Resources.Requests[apiv1.ResourceMemory]).To(gomega.Equal(ParseQuantityOrDie("100Mi"))) + } + }) + + ginkgo.It("starts pods with old recommended request when recommendation has only a container that doesn't match when a limit range applies", func() { + d := NewHamsterDeploymentWithResources(f, ParseQuantityOrDie("100m") /*cpu*/, ParseQuantityOrDie("100Mi") /*memory*/) + InstallLimitRangeWithMax(f, "300m", "1Gi", apiv1.LimitTypeContainer) + + ginkgo.By("Setting up a MPA CRD") + containerName := GetHamsterContainerNameByIndex(0) + mpaCRD := test.MultidimPodAutoscaler(). + WithName("hamster-mpa"). + WithNamespace(f.Namespace.Name). + WithScaleTargetRef(hamsterTargetRef). + WithContainer(containerName). + AppendRecommendation( + test.Recommendation(). + WithContainer(containerName). + WithTarget("250m", "200Mi"). + WithLowerBound("250m", "200Mi"). + WithUpperBound("250m", "200Mi"). + GetContainerResources()). + Get() + + InstallMPA(f, mpaCRD) + + ginkgo.By("Setting up a hamster deployment") + podList := startDeploymentPods(f, d) + + // Originally Pods had 100m CPU, 100Mi of memory, but admission controller + // should change it to recommended 250m CPU and 200Mi of memory. + for _, pod := range podList.Items { + gomega.Expect(pod.Spec.Containers[0].Resources.Requests[apiv1.ResourceCPU]).To(gomega.Equal(ParseQuantityOrDie("100m"))) + gomega.Expect(pod.Spec.Containers[0].Resources.Requests[apiv1.ResourceMemory]).To(gomega.Equal(ParseQuantityOrDie("100Mi"))) + } + }) + + ginkgo.It("starts pod with default request when one container has a recommendation and one other one doesn't when a limit range applies", func() { + d := NewNHamstersDeployment(f, 2) + InstallLimitRangeWithMax(f, "400m", "1Gi", apiv1.LimitTypePod) + + d.Spec.Template.Spec.Containers[0].Resources.Requests = apiv1.ResourceList{ + apiv1.ResourceCPU: ParseQuantityOrDie("100m"), + apiv1.ResourceMemory: ParseQuantityOrDie("100Mi"), + } + d.Spec.Template.Spec.Containers[0].Resources.Limits = apiv1.ResourceList{ + apiv1.ResourceCPU: ParseQuantityOrDie("100m"), + apiv1.ResourceMemory: ParseQuantityOrDie("100Mi"), + } + d.Spec.Template.Spec.Containers[1].Resources.Requests = apiv1.ResourceList{ + apiv1.ResourceCPU: ParseQuantityOrDie("400m"), + apiv1.ResourceMemory: ParseQuantityOrDie("600Mi"), + } + d.Spec.Template.Spec.Containers[1].Resources.Limits = apiv1.ResourceList{ + apiv1.ResourceCPU: ParseQuantityOrDie("400m"), + apiv1.ResourceMemory: ParseQuantityOrDie("600Mi"), + } + framework.Logf("Created hamster deployment %v", d) + ginkgo.By("Setting up a MPA CRD") + containerName := GetHamsterContainerNameByIndex(0) + mpaCRD := test.MultidimPodAutoscaler(). + WithName("hamster-mpa"). + WithNamespace(f.Namespace.Name). + WithScaleTargetRef(hamsterTargetRef). + WithContainer(containerName). + AppendRecommendation( + test.Recommendation(). + WithContainer(containerName). + WithTarget("400m", "600Mi"). + WithLowerBound("400m", "600Mi"). + WithUpperBound("400m", "600Mi"). + GetContainerResources()). + Get() + + InstallMPA(f, mpaCRD) + + ginkgo.By("Setting up a hamster deployment") + podList := startDeploymentPods(f, d) + + // Originally both containers in each Pod had 400m CPU (one from + // recommendation the other one from request), 600Mi of memory (similarly), + // but admission controller should change it to recommended 200m CPU + // (1/2 of max in limit range) and 512Mi of memory (similarly). + for _, pod := range podList.Items { + gomega.Expect(pod.Spec.Containers[0].Resources.Requests[apiv1.ResourceCPU]).To(gomega.Equal(ParseQuantityOrDie("200m"))) + gomega.Expect(pod.Spec.Containers[0].Resources.Requests[apiv1.ResourceMemory]).To(gomega.Equal(ParseQuantityOrDie("512Mi"))) + gomega.Expect(pod.Spec.Containers[1].Resources.Requests[apiv1.ResourceCPU]).To(gomega.Equal(ParseQuantityOrDie("200m"))) + gomega.Expect(pod.Spec.Containers[1].Resources.Requests[apiv1.ResourceMemory]).To(gomega.Equal(ParseQuantityOrDie("512Mi"))) + } + }) + + ginkgo.It("doesn't block patches", func() { + d := NewHamsterDeploymentWithResources(f, ParseQuantityOrDie("100m") /*cpu*/, ParseQuantityOrDie("100Mi") /*memory*/) + + ginkgo.By("Setting up a MPA CRD") + containerName := GetHamsterContainerNameByIndex(0) + mpaCRD := test.MultidimPodAutoscaler(). + WithName("hamster-mpa"). + WithNamespace(f.Namespace.Name). + WithScaleTargetRef(hamsterTargetRef). + WithContainer(containerName). + AppendRecommendation( + test.Recommendation(). + WithContainer(containerName). + WithTarget("250m", "200Mi"). + WithLowerBound("250m", "200Mi"). + WithUpperBound("250m", "200Mi"). + GetContainerResources()). + Get() + + InstallMPA(f, mpaCRD) + + ginkgo.By("Setting up a hamster deployment") + podList := startDeploymentPods(f, d) + + ginkgo.By("Verifying hamster deployment") + for i, pod := range podList.Items { + podInfo := fmt.Sprintf("pod at index %d", i) + cpuDescription := fmt.Sprintf("%s: originally Pods had 100m CPU, admission controller should change it to recommended 250m CPU", podInfo) + gomega.Expect(pod.Spec.Containers[0].Resources.Requests[apiv1.ResourceCPU]).To(gomega.Equal(ParseQuantityOrDie("250m")), cpuDescription) + memDescription := fmt.Sprintf("%s: originally Pods had 100Mi of memory, admission controller should change it to recommended 200Mi memory", podInfo) + gomega.Expect(pod.Spec.Containers[0].Resources.Requests[apiv1.ResourceMemory]).To(gomega.Equal(ParseQuantityOrDie("200Mi")), memDescription) + } + + ginkgo.By("Modifying recommendation.") + PatchMpaRecommendation(f, mpaCRD, &vpa_types.RecommendedPodResources{ + ContainerRecommendations: []vpa_types.RecommendedContainerResources{{ + ContainerName: "hamster", + Target: apiv1.ResourceList{ + apiv1.ResourceCPU: ParseQuantityOrDie("100m"), + apiv1.ResourceMemory: ParseQuantityOrDie("100Mi"), + }, + }}, + }) + + podName := podList.Items[0].Name + ginkgo.By(fmt.Sprintf("Modifying pod %v.", podName)) + AnnotatePod(f, podName, "someAnnotation", "someValue") + }) + + ginkgo.It("keeps limits equal to request", func() { + d := NewHamsterDeploymentWithGuaranteedResources(f, ParseQuantityOrDie("100m") /*cpu*/, ParseQuantityOrDie("100Mi") /*memory*/) + + ginkgo.By("Setting up a MPA CRD") + containerName := GetHamsterContainerNameByIndex(0) + mpaCRD := test.MultidimPodAutoscaler(). + WithName("hamster-mpa"). + WithNamespace(f.Namespace.Name). + WithScaleTargetRef(hamsterTargetRef). + WithContainer(containerName). + AppendRecommendation( + test.Recommendation(). + WithContainer(containerName). + WithTarget("250m", "200Mi"). + WithLowerBound("250m", "200Mi"). + WithUpperBound("250m", "200Mi"). + GetContainerResources()). + Get() + + InstallMPA(f, mpaCRD) + + ginkgo.By("Setting up a hamster deployment") + podList := startDeploymentPods(f, d) + + // Originally Pods had 100m CPU, 100Mi of memory, but admission controller + // should change it to 250m CPU and 200Mi of memory. Limits and requests should stay equal. + for _, pod := range podList.Items { + gomega.Expect(pod.Spec.Containers[0].Resources.Requests[apiv1.ResourceCPU]).To(gomega.Equal(ParseQuantityOrDie("250m"))) + gomega.Expect(pod.Spec.Containers[0].Resources.Requests[apiv1.ResourceMemory]).To(gomega.Equal(ParseQuantityOrDie("200Mi"))) + gomega.Expect(pod.Spec.Containers[0].Resources.Limits[apiv1.ResourceCPU]).To(gomega.Equal(ParseQuantityOrDie("250m"))) + gomega.Expect(pod.Spec.Containers[0].Resources.Limits[apiv1.ResourceMemory]).To(gomega.Equal(ParseQuantityOrDie("200Mi"))) + } + }) + + ginkgo.It("keeps limits to request ratio constant", func() { + d := NewHamsterDeploymentWithResourcesAndLimits(f, + ParseQuantityOrDie("100m") /*cpu request*/, ParseQuantityOrDie("100Mi"), /*memory request*/ + ParseQuantityOrDie("150m") /*cpu limit*/, ParseQuantityOrDie("200Mi") /*memory limit*/) + + ginkgo.By("Setting up a MPA CRD") + containerName := GetHamsterContainerNameByIndex(0) + mpaCRD := test.MultidimPodAutoscaler(). + WithName("hamster-mpa"). + WithNamespace(f.Namespace.Name). + WithScaleTargetRef(hamsterTargetRef). + WithContainer(containerName). + AppendRecommendation( + test.Recommendation(). + WithContainer(containerName). + WithTarget("250m", "200Mi"). + WithLowerBound("250m", "200Mi"). + WithUpperBound("250m", "200Mi"). + GetContainerResources()). + Get() + + InstallMPA(f, mpaCRD) + + ginkgo.By("Setting up a hamster deployment") + podList := startDeploymentPods(f, d) + + // Originally Pods had 100m CPU, 100Mi of memory, but admission controller + // should change it to 250m CPU and 200Mi of memory. Limits to request ratio should stay unchanged. + for _, pod := range podList.Items { + gomega.Expect(*pod.Spec.Containers[0].Resources.Requests.Cpu()).To(gomega.Equal(ParseQuantityOrDie("250m"))) + gomega.Expect(*pod.Spec.Containers[0].Resources.Requests.Memory()).To(gomega.Equal(ParseQuantityOrDie("200Mi"))) + gomega.Expect(float64(pod.Spec.Containers[0].Resources.Limits.Cpu().MilliValue()) / float64(pod.Spec.Containers[0].Resources.Requests.Cpu().MilliValue())).To(gomega.BeNumerically("~", 1.5)) + gomega.Expect(float64(pod.Spec.Containers[0].Resources.Limits.Memory().Value()) / float64(pod.Spec.Containers[0].Resources.Requests.Memory().Value())).To(gomega.BeNumerically("~", 2.)) + } + }) + + ginkgo.It("keeps limits unchanged when container controlled values is requests only", func() { + d := NewHamsterDeploymentWithResourcesAndLimits(f, + ParseQuantityOrDie("100m") /*cpu request*/, ParseQuantityOrDie("100Mi"), /*memory request*/ + ParseQuantityOrDie("500m") /*cpu limit*/, ParseQuantityOrDie("500Mi") /*memory limit*/) + + ginkgo.By("Setting up a MPA CRD") + containerName := GetHamsterContainerNameByIndex(0) + mpaCRD := test.MultidimPodAutoscaler(). + WithName("hamster-mpa"). + WithNamespace(f.Namespace.Name). + WithScaleTargetRef(hamsterTargetRef). + WithContainer(containerName). + WithControlledValues(containerName, vpa_types.ContainerControlledValuesRequestsOnly). + AppendRecommendation( + test.Recommendation(). + WithContainer(containerName). + WithTarget("250m", "200Mi"). + WithLowerBound("250m", "200Mi"). + WithUpperBound("250m", "200Mi"). + GetContainerResources()). + Get() + + InstallMPA(f, mpaCRD) + + ginkgo.By("Setting up a hamster deployment") + podList := startDeploymentPods(f, d) + + // Originally Pods had 100m CPU, 100Mi of memory, but admission controller + // should change it to 250m CPU and 200Mi of memory. Limits should stay unchanged. + for _, pod := range podList.Items { + gomega.Expect(*pod.Spec.Containers[0].Resources.Requests.Cpu()).To(gomega.Equal(ParseQuantityOrDie("250m"))) + gomega.Expect(*pod.Spec.Containers[0].Resources.Requests.Memory()).To(gomega.Equal(ParseQuantityOrDie("200Mi"))) + gomega.Expect(*pod.Spec.Containers[0].Resources.Limits.Cpu()).To(gomega.Equal(ParseQuantityOrDie("500m"))) + gomega.Expect(*pod.Spec.Containers[0].Resources.Limits.Memory()).To(gomega.Equal(ParseQuantityOrDie("500Mi"))) + } + }) + + ginkgo.It("caps request according to container max limit set in LimitRange", func() { + startCpuRequest := ParseQuantityOrDie("100m") + startCpuLimit := ParseQuantityOrDie("150m") + startMemRequest := ParseQuantityOrDie("100Mi") + startMemLimit := ParseQuantityOrDie("200Mi") + memRecommendation := ParseQuantityOrDie("200Mi") + + d := NewHamsterDeploymentWithResourcesAndLimits(f, startCpuRequest, startMemRequest, startCpuLimit, startMemLimit) + + ginkgo.By("Setting up a MPA CRD") + containerName := GetHamsterContainerNameByIndex(0) + mpaCRD := test.MultidimPodAutoscaler(). + WithName("hamster-mpa"). + WithNamespace(f.Namespace.Name). + WithScaleTargetRef(hamsterTargetRef). + WithContainer(containerName). + AppendRecommendation( + test.Recommendation(). + WithContainer(containerName). + WithTarget("250m", "200Mi"). + WithLowerBound("250m", "200Mi"). + WithUpperBound("250m", "200Mi"). + GetContainerResources()). + Get() + + InstallMPA(f, mpaCRD) + + // Max CPU limit is 300m and ratio is 1.5, so max request is 200m, while + // recommendation is 250m + // Max memory limit is 1Gi and ratio is 2., so max request is 0.5Gi + maxCpu := ParseQuantityOrDie("300m") + InstallLimitRangeWithMax(f, maxCpu.String(), "1Gi", apiv1.LimitTypeContainer) + + ginkgo.By("Setting up a hamster deployment") + podList := startDeploymentPods(f, d) + + ginkgo.By("Verifying hamster deployment") + for i, pod := range podList.Items { + podInfo := fmt.Sprintf("pod %s at index %d", pod.Name, i) + + cpuRequestMsg := fmt.Sprintf("%s: CPU request didn't increase to the recommendation capped to max limit in LimitRange", podInfo) + gomega.Expect(*pod.Spec.Containers[0].Resources.Requests.Cpu()).To(gomega.Equal(ParseQuantityOrDie("200m")), cpuRequestMsg) + + cpuLimitMsg := fmt.Sprintf("%s: CPU limit above max in LimitRange", podInfo) + gomega.Expect(pod.Spec.Containers[0].Resources.Limits.Cpu().MilliValue()).To(gomega.BeNumerically("<=", maxCpu.MilliValue()), cpuLimitMsg) + + cpuRatioMsg := fmt.Sprintf("%s: CPU limit / request ratio isn't approximately equal to the original ratio", podInfo) + cpuRatio := float64(pod.Spec.Containers[0].Resources.Limits.Cpu().MilliValue()) / float64(pod.Spec.Containers[0].Resources.Requests.Cpu().MilliValue()) + gomega.Expect(cpuRatio).To(gomega.BeNumerically("~", 1.5), cpuRatioMsg) + + memRequestMsg := fmt.Sprintf("%s: memory request didn't increase to the recommendation capped to max limit in LimitRange", podInfo) + gomega.Expect(pod.Spec.Containers[0].Resources.Requests.Memory().Value()).To(gomega.Equal(memRecommendation.Value()), memRequestMsg) + + memLimitMsg := fmt.Sprintf("%s: memory limit above max limit in LimitRange", podInfo) + gomega.Expect(pod.Spec.Containers[0].Resources.Limits.Memory().Value()).To(gomega.BeNumerically("<=", 1024*1024*1024), memLimitMsg) + + memRatioMsg := fmt.Sprintf("%s: memory limit / request ratio isn't approximately equal to the original ratio", podInfo) + memRatio := float64(pod.Spec.Containers[0].Resources.Limits.Memory().Value()) / float64(pod.Spec.Containers[0].Resources.Requests.Memory().Value()) + gomega.Expect(memRatio).To(gomega.BeNumerically("~", 2.), memRatioMsg) + } + }) + + ginkgo.It("raises request according to container min limit set in LimitRange", func() { + d := NewHamsterDeploymentWithResourcesAndLimits(f, + ParseQuantityOrDie("100m") /*cpu request*/, ParseQuantityOrDie("200Mi"), /*memory request*/ + ParseQuantityOrDie("150m") /*cpu limit*/, ParseQuantityOrDie("400Mi") /*memory limit*/) + + ginkgo.By("Setting up a MPA CRD") + containerName := GetHamsterContainerNameByIndex(0) + mpaCRD := test.MultidimPodAutoscaler(). + WithName("hamster-mpa"). + WithNamespace(f.Namespace.Name). + WithScaleTargetRef(hamsterTargetRef). + WithContainer(containerName). + AppendRecommendation( + test.Recommendation(). + WithContainer(containerName). + WithTarget("250m", "100Mi"). + WithLowerBound("250m", "100Mi"). + WithUpperBound("250m", "100Mi"). + GetContainerResources()). + Get() + + InstallMPA(f, mpaCRD) + + // Min CPU from limit range is 50m and ratio is 1.5. Min applies to both limit and request so min + // request is 50m and min limit is 75 + // Min memory limit is 250Mi and it applies to both limit and request. Recommendation is 100Mi. + // It should be scaled up to 250Mi. + InstallLimitRangeWithMin(f, "50m", "250Mi", apiv1.LimitTypeContainer) + + ginkgo.By("Setting up a hamster deployment") + podList := startDeploymentPods(f, d) + + // Originally Pods had 100m CPU, 200Mi of memory, but admission controller + // should change it to 250m CPU and 125Mi of memory, since this is the lowest + // request that limitrange allows. + // Limit to request ratio should stay unchanged. + for _, pod := range podList.Items { + gomega.Expect(*pod.Spec.Containers[0].Resources.Requests.Cpu()).To(gomega.Equal(ParseQuantityOrDie("250m"))) + gomega.Expect(*pod.Spec.Containers[0].Resources.Requests.Memory()).To(gomega.Equal(ParseQuantityOrDie("250Mi"))) + gomega.Expect(pod.Spec.Containers[0].Resources.Limits.Cpu().MilliValue()).To(gomega.BeNumerically(">=", 75)) + gomega.Expect(pod.Spec.Containers[0].Resources.Limits.Memory().Value()).To(gomega.BeNumerically(">=", 250*1024*1024)) + gomega.Expect(float64(pod.Spec.Containers[0].Resources.Limits.Cpu().MilliValue()) / float64(pod.Spec.Containers[0].Resources.Requests.Cpu().MilliValue())).To(gomega.BeNumerically("~", 1.5)) + gomega.Expect(float64(pod.Spec.Containers[0].Resources.Limits.Memory().Value()) / float64(pod.Spec.Containers[0].Resources.Requests.Memory().Value())).To(gomega.BeNumerically("~", 2.)) + } + }) + + ginkgo.It("caps request according to pod max limit set in LimitRange", func() { + d := NewHamsterDeploymentWithResourcesAndLimits(f, + ParseQuantityOrDie("100m") /*cpu request*/, ParseQuantityOrDie("100Mi"), /*memory request*/ + ParseQuantityOrDie("150m") /*cpu limit*/, ParseQuantityOrDie("200Mi") /*memory limit*/) + d.Spec.Template.Spec.Containers = append(d.Spec.Template.Spec.Containers, d.Spec.Template.Spec.Containers[0]) + container2Name := "hamster2" + d.Spec.Template.Spec.Containers[1].Name = container2Name + + ginkgo.By("Setting up a MPA CRD") + container1Name := GetHamsterContainerNameByIndex(0) + mpaCRD := test.MultidimPodAutoscaler(). + WithName("hamster-mpa"). + WithNamespace(f.Namespace.Name). + WithScaleTargetRef(hamsterTargetRef). + WithContainer(container1Name). + AppendRecommendation( + test.Recommendation(). + WithContainer(container1Name). + WithTarget("250m", "200Mi"). + WithLowerBound("250m", "200Mi"). + WithUpperBound("250m", "200Mi"). + GetContainerResources()). + WithContainer(container2Name). + AppendRecommendation( + test.Recommendation(). + WithContainer(container2Name). + WithTarget("250m", "200Mi"). + WithLowerBound("250m", "200Mi"). + WithUpperBound("250m", "200Mi"). + GetContainerResources()). + Get() + + InstallMPA(f, mpaCRD) + + // Max CPU limit is 600m for pod, 300 per container and ratio is 1.5, so max request is 200m, + // while recommendation is 250m + // Max memory limit is 1Gi and ratio is 2., so max request is 0.5Gi + InstallLimitRangeWithMax(f, "600m", "1Gi", apiv1.LimitTypePod) + + ginkgo.By("Setting up a hamster deployment") + podList := startDeploymentPods(f, d) + + // Originally Pods had 100m CPU, 100Mi of memory, but admission controller + // should change it to 200m CPU (as this is the recommendation + // capped according to max limit in LimitRange) and 200Mi of memory, + // which is uncapped. Limit to request ratio should stay unchanged. + for _, pod := range podList.Items { + gomega.Expect(*pod.Spec.Containers[0].Resources.Requests.Cpu()).To(gomega.Equal(ParseQuantityOrDie("200m"))) + gomega.Expect(*pod.Spec.Containers[0].Resources.Requests.Memory()).To(gomega.Equal(ParseQuantityOrDie("200Mi"))) + gomega.Expect(pod.Spec.Containers[0].Resources.Limits.Cpu().MilliValue()).To(gomega.BeNumerically("<=", 300)) + gomega.Expect(pod.Spec.Containers[0].Resources.Limits.Memory().Value()).To(gomega.BeNumerically("<=", 1024*1024*1024)) + gomega.Expect(float64(pod.Spec.Containers[0].Resources.Limits.Cpu().MilliValue()) / float64(pod.Spec.Containers[0].Resources.Requests.Cpu().MilliValue())).To(gomega.BeNumerically("~", 1.5)) + gomega.Expect(float64(pod.Spec.Containers[0].Resources.Limits.Memory().Value()) / float64(pod.Spec.Containers[0].Resources.Requests.Memory().Value())).To(gomega.BeNumerically("~", 2.)) + } + }) + + ginkgo.It("raises request according to pod min limit set in LimitRange", func() { + d := NewHamsterDeploymentWithResourcesAndLimits(f, + ParseQuantityOrDie("100m") /*cpu request*/, ParseQuantityOrDie("200Mi"), /*memory request*/ + ParseQuantityOrDie("150m") /*cpu limit*/, ParseQuantityOrDie("400Mi") /*memory limit*/) + d.Spec.Template.Spec.Containers = append(d.Spec.Template.Spec.Containers, d.Spec.Template.Spec.Containers[0]) + container2Name := "hamster2" + d.Spec.Template.Spec.Containers[1].Name = container2Name + + ginkgo.By("Setting up a MPA CRD") + container1Name := GetHamsterContainerNameByIndex(0) + mpaCRD := test.MultidimPodAutoscaler(). + WithName("hamster-mpa"). + WithNamespace(f.Namespace.Name). + WithScaleTargetRef(hamsterTargetRef). + WithContainer(container1Name). + AppendRecommendation( + test.Recommendation(). + WithContainer(container1Name). + WithTarget("120m", "100Mi"). + WithLowerBound("120m", "100Mi"). + WithUpperBound("120m", "100Mi"). + GetContainerResources()). + WithContainer(container2Name). + AppendRecommendation( + test.Recommendation(). + WithContainer(container2Name). + WithTarget("120m", "100Mi"). + WithLowerBound("120m", "100Mi"). + WithUpperBound("120m", "100Mi"). + GetContainerResources()). + Get() + + InstallMPA(f, mpaCRD) + + // Min CPU from limit range is 100m, 50m per pod and ratio is 1.5. Min applies to both limit and + // request so min request is 50m and min limit is 75 + // Min memory limit is 500Mi per pod, 250 per container and it applies to both limit and request. + // Recommendation is 100Mi it should be scaled up to 250Mi. + InstallLimitRangeWithMin(f, "100m", "500Mi", apiv1.LimitTypePod) + + ginkgo.By("Setting up a hamster deployment") + podList := startDeploymentPods(f, d) + + // Originally Pods had 100m CPU, 200Mi of memory, but admission controller + // should change it to 250m CPU and 125Mi of memory, since this is the lowest + // request that limitrange allows. + // Limit to request ratio should stay unchanged. + for _, pod := range podList.Items { + gomega.Expect(*pod.Spec.Containers[0].Resources.Requests.Cpu()).To(gomega.Equal(ParseQuantityOrDie("120m"))) + gomega.Expect(*pod.Spec.Containers[0].Resources.Requests.Memory()).To(gomega.Equal(ParseQuantityOrDie("250Mi"))) + gomega.Expect(pod.Spec.Containers[0].Resources.Limits.Cpu().MilliValue()).To(gomega.BeNumerically(">=", 75)) + gomega.Expect(pod.Spec.Containers[0].Resources.Limits.Memory().Value()).To(gomega.BeNumerically(">=", 250*1024*1024)) + gomega.Expect(float64(pod.Spec.Containers[0].Resources.Limits.Cpu().MilliValue()) / float64(pod.Spec.Containers[0].Resources.Requests.Cpu().MilliValue())).To(gomega.BeNumerically("~", 1.5)) + gomega.Expect(float64(pod.Spec.Containers[0].Resources.Limits.Memory().Value()) / float64(pod.Spec.Containers[0].Resources.Requests.Memory().Value())).To(gomega.BeNumerically("~", 2.)) + } + }) + + ginkgo.It("caps request to max set in MPA", func() { + d := NewHamsterDeploymentWithResources(f, ParseQuantityOrDie("100m") /*cpu*/, ParseQuantityOrDie("100Mi") /*memory*/) + + ginkgo.By("Setting up a MPA CRD") + containerName := GetHamsterContainerNameByIndex(0) + mpaCRD := test.MultidimPodAutoscaler(). + WithName("hamster-mpa"). + WithNamespace(f.Namespace.Name). + WithScaleTargetRef(hamsterTargetRef). + WithContainer(containerName). + AppendRecommendation( + test.Recommendation(). + WithContainer(containerName). + WithTarget("250m", "200Mi"). + WithLowerBound("250m", "200Mi"). + WithUpperBound("250m", "200Mi"). + GetContainerResources()). + WithMaxAllowed(containerName, "233m", "150Mi"). + Get() + + InstallMPA(f, mpaCRD) + + ginkgo.By("Setting up a hamster deployment") + podList := startDeploymentPods(f, d) + + // Originally Pods had 100m CPU, 100Mi of memory, but admission controller + // should change it to 233m CPU and 150Mi of memory (as this is the recommendation + // capped to max specified in MPA) + for _, pod := range podList.Items { + gomega.Expect(pod.Spec.Containers[0].Resources.Requests[apiv1.ResourceCPU]).To(gomega.Equal(ParseQuantityOrDie("233m"))) + gomega.Expect(pod.Spec.Containers[0].Resources.Requests[apiv1.ResourceMemory]).To(gomega.Equal(ParseQuantityOrDie("150Mi"))) + } + }) + + ginkgo.It("raises request to min set in MPA", func() { + d := NewHamsterDeploymentWithResources(f, ParseQuantityOrDie("100m") /*cpu*/, ParseQuantityOrDie("100Mi") /*memory*/) + + ginkgo.By("Setting up a MPA CRD") + containerName := GetHamsterContainerNameByIndex(0) + mpaCRD := test.MultidimPodAutoscaler(). + WithName("hamster-mpa"). + WithNamespace(f.Namespace.Name). + WithScaleTargetRef(hamsterTargetRef). + WithContainer(containerName). + AppendRecommendation( + test.Recommendation(). + WithContainer(containerName). + WithTarget("50m", "60Mi"). + WithLowerBound("50m", "60Mi"). + WithUpperBound("50m", "60Mi"). + GetContainerResources()). + WithMinAllowed(containerName, "90m", "80Mi"). + Get() + + InstallMPA(f, mpaCRD) + + ginkgo.By("Setting up a hamster deployment") + podList := startDeploymentPods(f, d) + + // Originally Pods had 100m CPU, 100Mi of memory, but admission controller + // should change it to recommended 90m CPU and 800Mi of memory (as this the + // recommendation raised to min specified in MPA) + for _, pod := range podList.Items { + gomega.Expect(pod.Spec.Containers[0].Resources.Requests[apiv1.ResourceCPU]).To(gomega.Equal(ParseQuantityOrDie("90m"))) + gomega.Expect(pod.Spec.Containers[0].Resources.Requests[apiv1.ResourceMemory]).To(gomega.Equal(ParseQuantityOrDie("80Mi"))) + } + }) + + ginkgo.It("leaves users request when no recommendation", func() { + d := NewHamsterDeploymentWithResources(f, ParseQuantityOrDie("100m") /*cpu*/, ParseQuantityOrDie("100Mi") /*memory*/) + + ginkgo.By("Setting up a MPA CRD") + containerName := GetHamsterContainerNameByIndex(0) + mpaCRD := test.MultidimPodAutoscaler(). + WithName("hamster-mpa"). + WithNamespace(f.Namespace.Name). + WithScaleTargetRef(hamsterTargetRef). + WithContainer(containerName). + Get() + + InstallMPA(f, mpaCRD) + + ginkgo.By("Setting up a hamster deployment") + podList := startDeploymentPods(f, d) + + // MPA has no recommendation, so user's request is passed through + for _, pod := range podList.Items { + gomega.Expect(pod.Spec.Containers[0].Resources.Requests[apiv1.ResourceCPU]).To(gomega.Equal(ParseQuantityOrDie("100m"))) + gomega.Expect(pod.Spec.Containers[0].Resources.Requests[apiv1.ResourceMemory]).To(gomega.Equal(ParseQuantityOrDie("100Mi"))) + } + }) + + ginkgo.It("passes empty request when no recommendation and no user-specified request", func() { + d := NewHamsterDeployment(f) + + ginkgo.By("Setting up a MPA CRD") + containerName := GetHamsterContainerNameByIndex(0) + mpaCRD := test.MultidimPodAutoscaler(). + WithName("hamster-mpa"). + WithNamespace(f.Namespace.Name). + WithScaleTargetRef(hamsterTargetRef). + WithContainer(containerName). + Get() + + InstallMPA(f, mpaCRD) + + ginkgo.By("Setting up a hamster deployment") + podList := startDeploymentPods(f, d) + + // MPA has no recommendation, deployment has no request specified + for _, pod := range podList.Items { + gomega.Expect(pod.Spec.Containers[0].Resources.Requests).To(gomega.BeEmpty()) + } + }) + + ginkgo.It("accepts valid and rejects invalid VPA object", func() { + ginkgo.By("Setting up valid VPA object") + validVPA := []byte(`{ + "kind": "VerticalPodAutoscaler", + "apiVersion": "autoscaling.k8s.io/v1", + "metadata": {"name": "hamster-mpa-valid"}, + "spec": { + "targetRef": { + "apiVersion": "apps/v1", + "kind": "Deployment", + "name":"hamster" + }, + "resourcePolicy": { + "containerPolicies": [{"containerName": "*", "minAllowed":{"cpu":"50m"}}] + } + } + }`) + err := InstallRawMPA(f, validVPA) + gomega.Expect(err).NotTo(gomega.HaveOccurred(), "Valid VPA object rejected") + + ginkgo.By("Setting up invalid VPA object") + // The invalid object differs by name and minAllowed - there is an invalid "requests" field. + invalidVPA := []byte(`{ + "kind": "VerticalPodAutoscaler", + "apiVersion": "autoscaling.k8s.io/v1", + "metadata": {"name": "hamster-mpa-invalid"}, + "spec": { + "targetRef": { + "apiVersion": "apps/v1", + "kind": "Deployment", + "name":"hamster" + }, + "resourcePolicy": { + "containerPolicies": [{"containerName": "*", "minAllowed":{"requests":{"cpu":"50m"}}}] + } + } + }`) + err2 := InstallRawMPA(f, invalidVPA) + gomega.Expect(err2).To(gomega.HaveOccurred(), "Invalid MPA object accepted") + gomega.Expect(err2.Error()).To(gomega.MatchRegexp(`.*admission webhook .*mpa.* denied the request: .*`)) + }) + + ginkgo.It("reloads the webhook certificate", func(ctx ginkgo.SpecContext) { + ginkgo.By("Retrieving alternative certificate") + c := f.ClientSet + e2eCertsSecret, err := c.CoreV1().Secrets(metav1.NamespaceSystem).Get(ctx, "mpa-e2e-certs", metav1.GetOptions{}) + gomega.Expect(err).To(gomega.Succeed(), "Failed to get mpa-e2e-certs secret") + actualCertsSecret, err := c.CoreV1().Secrets(metav1.NamespaceSystem).Get(ctx, "mpa-tls-certs", metav1.GetOptions{}) + gomega.Expect(err).To(gomega.Succeed(), "Failed to get mpa-tls-certs secret") + actualCertsSecret.Data["serverKey.pem"] = e2eCertsSecret.Data["e2eKey.pem"] + actualCertsSecret.Data["serverCert.pem"] = e2eCertsSecret.Data["e2eCert.pem"] + _, err = c.CoreV1().Secrets(metav1.NamespaceSystem).Update(ctx, actualCertsSecret, metav1.UpdateOptions{}) + gomega.Expect(err).To(gomega.Succeed(), "Failed to update mpa-tls-certs secret with e2e rotation certs") + + ginkgo.By("Waiting for certificate reload") + pods, err := c.CoreV1().Pods(metav1.NamespaceSystem).List(ctx, metav1.ListOptions{}) + gomega.Expect(err).To(gomega.Succeed()) + + var admissionController apiv1.Pod + for _, p := range pods.Items { + if strings.HasPrefix(p.Name, "mpa-admission-controller") { + admissionController = p + } + } + gomega.Expect(admissionController.Name).ToNot(gomega.BeEmpty()) + + gomega.Eventually(func(g gomega.Gomega) string { + reader, err := c.CoreV1().Pods(metav1.NamespaceSystem).GetLogs(admissionController.Name, &apiv1.PodLogOptions{}).Stream(ctx) + g.Expect(err).To(gomega.Succeed()) + logs, err := io.ReadAll(reader) + g.Expect(err).To(gomega.Succeed()) + return string(logs) + }).Should(gomega.ContainSubstring("New certificate found, reloading")) + + ginkgo.By("Setting up invalid VPA object") + // there is an invalid "requests" field. + invalidVPA := []byte(`{ + "kind": "VerticalPodAutoscaler", + "apiVersion": "autoscaling.k8s.io/v1", + "metadata": {"name": "cert-vpa-invalid"}, + "spec": { + "targetRef": { + "apiVersion": "apps/v1", + "kind": "Deployment", + "name":"hamster" + }, + "resourcePolicy": { + "containerPolicies": [{"containerName": "*", "minAllowed":{"requests":{"cpu":"50m"}}}] + } + } + }`) + err = InstallRawMPA(f, invalidVPA) + gomega.Expect(err).To(gomega.HaveOccurred(), "Invalid MPA object accepted") + gomega.Expect(err.Error()).To(gomega.MatchRegexp(`.*admission webhook .*mpa.* denied the request: .*`), "Admission controller did not inspect the object") + }) + +}) + +func startDeploymentPods(f *framework.Framework, deployment *appsv1.Deployment) *apiv1.PodList { + // Apiserver watch can lag depending on cached object count and apiserver resource usage. + // We assume that watch can lag up to 5 seconds. + const apiserverWatchLag = 5 * time.Second + // In admission controller e2e tests a recommendation is created before deployment. + // Creating deployment with size greater than 0 would create a race between information + // about pods and information about deployment getting to the admission controller. + // Any pods that get processed by AC before it receives information about the deployment + // don't receive recommendation. + // To avoid this create deployment with size 0, then scale it up to the desired size. + desiredPodCount := *deployment.Spec.Replicas + zero := int32(0) + deployment.Spec.Replicas = &zero + c, ns := f.ClientSet, f.Namespace.Name + deployment, err := c.AppsV1().Deployments(ns).Create(context.TODO(), deployment, metav1.CreateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred(), "when creating deployment with size 0") + + err = framework_deployment.WaitForDeploymentComplete(c, deployment) + gomega.Expect(err).NotTo(gomega.HaveOccurred(), "when waiting for empty deployment to create") + // If admission controller receives pod before controller it will not apply recommendation and test will fail. + // Wait after creating deployment to ensure MPA knows about it, then scale up. + // Normally watch lag is not a problem in terms of correctness: + // - Mode "Auto": created pod without assigned resources will be handled by the eviction loop. + // - Mode "Initial": calculating recommendations takes more than potential ectd lag. + // - Mode "Off": pods are not handled by the admission controller. + // In e2e admission controller tests we want to focus on scenarios without considering watch lag. + // TODO(#2631): Remove sleep when issue is fixed. + time.Sleep(apiserverWatchLag) + + scale := autoscalingv1.Scale{ + ObjectMeta: metav1.ObjectMeta{ + Name: deployment.ObjectMeta.Name, + Namespace: deployment.ObjectMeta.Namespace, + }, + Spec: autoscalingv1.ScaleSpec{ + Replicas: desiredPodCount, + }, + } + afterScale, err := c.AppsV1().Deployments(ns).UpdateScale(context.TODO(), deployment.Name, &scale, metav1.UpdateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(afterScale.Spec.Replicas).To(gomega.Equal(desiredPodCount), fmt.Sprintf("expected %d replicas after scaling", desiredPodCount)) + + // After scaling deployment we need to retrieve current version with updated replicas count. + deployment, err = c.AppsV1().Deployments(ns).Get(context.TODO(), deployment.Name, metav1.GetOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred(), "when getting scaled deployment") + err = framework_deployment.WaitForDeploymentComplete(c, deployment) + gomega.Expect(err).NotTo(gomega.HaveOccurred(), "when waiting for deployment to resize") + + podList, err := framework_deployment.GetPodsForDeployment(context.TODO(), c, deployment) + gomega.Expect(err).NotTo(gomega.HaveOccurred(), "when listing pods after deployment resize") + return podList +} diff --git a/multidimensional-pod-autoscaler/e2e/v1alpha1/autoscaling_utils.go b/multidimensional-pod-autoscaler/e2e/v1alpha1/autoscaling_utils.go new file mode 100644 index 000000000000..ee675ca77094 --- /dev/null +++ b/multidimensional-pod-autoscaler/e2e/v1alpha1/autoscaling_utils.go @@ -0,0 +1,469 @@ +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// This is a cut down fork of k8s.io/kubernetes/test/e2e/common/autoscaling_utils.go + +package autoscaling + +import ( + "context" + "fmt" + "strconv" + "strings" + "sync" + "time" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/apimachinery/pkg/util/wait" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/kubernetes/test/e2e/framework" + e2edebug "k8s.io/kubernetes/test/e2e/framework/debug" + e2ekubectl "k8s.io/kubernetes/test/e2e/framework/kubectl" + e2erc "k8s.io/kubernetes/test/e2e/framework/rc" + "k8s.io/kubernetes/test/e2e/framework/resource" + e2eservice "k8s.io/kubernetes/test/e2e/framework/service" + testutils "k8s.io/kubernetes/test/utils" + + ginkgo "github.com/onsi/ginkgo/v2" + + scaleclient "k8s.io/client-go/scale" + imageutils "k8s.io/kubernetes/test/utils/image" +) + +const ( + dynamicConsumptionTimeInSeconds = 30 + dynamicRequestSizeInMillicores = 20 + dynamicRequestSizeInMegabytes = 100 + dynamicRequestSizeCustomMetric = 10 + port = 80 + targetPort = 8080 + timeoutRC = 120 * time.Second + startServiceTimeout = time.Minute + startServiceInterval = 5 * time.Second + rcIsNil = "ERROR: replicationController = nil" + deploymentIsNil = "ERROR: deployment = nil" + rsIsNil = "ERROR: replicaset = nil" + invalidKind = "ERROR: invalid workload kind for resource consumer" + customMetricName = "QPS" + serviceInitializationTimeout = 2 * time.Minute + serviceInitializationInterval = 15 * time.Second + stressImage = "registry.k8s.io/e2e-test-images/agnhost:2.53" +) + +var ( + resourceConsumerImage = imageutils.GetE2EImage(imageutils.ResourceConsumer) +) + +var ( + // KindRC is the GVK for ReplicationController + KindRC = schema.GroupVersionKind{Version: "v1", Kind: "ReplicationController"} + // KindDeployment is the GVK for Deployment + KindDeployment = schema.GroupVersionKind{Group: "apps", Version: "v1beta2", Kind: "Deployment"} + // KindReplicaSet is the GVK for ReplicaSet + KindReplicaSet = schema.GroupVersionKind{Group: "apps", Version: "v1beta2", Kind: "ReplicaSet"} +) + +/* +ResourceConsumer is a tool for testing. It helps create specified usage of CPU or memory (Warning: memory not supported) +typical use case: +rc.ConsumeCPU(600) +// ... check your assumption here +rc.ConsumeCPU(300) +// ... check your assumption here +*/ +type ResourceConsumer struct { + name string + controllerName string + kind schema.GroupVersionKind + nsName string + clientSet clientset.Interface + scaleClient scaleclient.ScalesGetter + cpu chan int + mem chan int + customMetric chan int + stopCPU chan int + stopMem chan int + stopCustomMetric chan int + stopWaitGroup sync.WaitGroup + consumptionTimeInSeconds int + sleepTime time.Duration + requestSizeInMillicores int + requestSizeInMegabytes int + requestSizeCustomMetric int +} + +// NewDynamicResourceConsumer is a wrapper to create a new dynamic ResourceConsumer +func NewDynamicResourceConsumer(name, nsName string, kind schema.GroupVersionKind, replicas, initCPUTotal, initMemoryTotal, initCustomMetric int, cpuLimit, memLimit int64, clientset clientset.Interface, scaleClient scaleclient.ScalesGetter) *ResourceConsumer { + return newResourceConsumer(name, nsName, kind, replicas, initCPUTotal, initMemoryTotal, initCustomMetric, dynamicConsumptionTimeInSeconds, + dynamicRequestSizeInMillicores, dynamicRequestSizeInMegabytes, dynamicRequestSizeCustomMetric, cpuLimit, memLimit, clientset, scaleClient, nil, nil) +} + +/* +NewResourceConsumer creates new ResourceConsumer +initCPUTotal argument is in millicores +initMemoryTotal argument is in megabytes +memLimit argument is in megabytes, memLimit is a maximum amount of memory that can be consumed by a single pod +cpuLimit argument is in millicores, cpuLimit is a maximum amount of cpu that can be consumed by a single pod +*/ +func newResourceConsumer(name, nsName string, kind schema.GroupVersionKind, replicas, initCPUTotal, initMemoryTotal, initCustomMetric, consumptionTimeInSeconds, requestSizeInMillicores, + requestSizeInMegabytes int, requestSizeCustomMetric int, cpuLimit, memLimit int64, clientset clientset.Interface, scaleClient scaleclient.ScalesGetter, podAnnotations, serviceAnnotations map[string]string) *ResourceConsumer { + if podAnnotations == nil { + podAnnotations = make(map[string]string) + } + if serviceAnnotations == nil { + serviceAnnotations = make(map[string]string) + } + runServiceAndWorkloadForResourceConsumer(clientset, nsName, name, kind, replicas, cpuLimit, memLimit, podAnnotations, serviceAnnotations) + rc := &ResourceConsumer{ + name: name, + controllerName: name + "-ctrl", + kind: kind, + nsName: nsName, + clientSet: clientset, + scaleClient: scaleClient, + cpu: make(chan int), + mem: make(chan int), + customMetric: make(chan int), + stopCPU: make(chan int), + stopMem: make(chan int), + stopCustomMetric: make(chan int), + consumptionTimeInSeconds: consumptionTimeInSeconds, + sleepTime: time.Duration(consumptionTimeInSeconds) * time.Second, + requestSizeInMillicores: requestSizeInMillicores, + requestSizeInMegabytes: requestSizeInMegabytes, + requestSizeCustomMetric: requestSizeCustomMetric, + } + + go rc.makeConsumeCPURequests() + rc.ConsumeCPU(initCPUTotal) + + go rc.makeConsumeMemRequests() + rc.ConsumeMem(initMemoryTotal) + go rc.makeConsumeCustomMetric() + rc.ConsumeCustomMetric(initCustomMetric) + return rc +} + +// ConsumeCPU consumes given number of CPU +func (rc *ResourceConsumer) ConsumeCPU(millicores int) { + framework.Logf("RC %s: consume %v millicores in total", rc.name, millicores) + rc.cpu <- millicores +} + +// ConsumeMem consumes given number of Mem +func (rc *ResourceConsumer) ConsumeMem(megabytes int) { + framework.Logf("RC %s: consume %v MB in total", rc.name, megabytes) + rc.mem <- megabytes +} + +// ConsumeCustomMetric consumes given number of custom metric +func (rc *ResourceConsumer) ConsumeCustomMetric(amount int) { + framework.Logf("RC %s: consume custom metric %v in total", rc.name, amount) + rc.customMetric <- amount +} + +func (rc *ResourceConsumer) makeConsumeCPURequests() { + defer ginkgo.GinkgoRecover() + rc.stopWaitGroup.Add(1) + defer rc.stopWaitGroup.Done() + sleepTime := time.Duration(0) + millicores := 0 + for { + select { + case millicores = <-rc.cpu: + framework.Logf("RC %s: setting consumption to %v millicores in total", rc.name, millicores) + case <-time.After(sleepTime): + framework.Logf("RC %s: sending request to consume %d millicores", rc.name, millicores) + rc.sendConsumeCPURequest(millicores) + sleepTime = rc.sleepTime + case <-rc.stopCPU: + framework.Logf("RC %s: stopping CPU consumer", rc.name) + return + } + } +} + +func (rc *ResourceConsumer) makeConsumeMemRequests() { + defer ginkgo.GinkgoRecover() + rc.stopWaitGroup.Add(1) + defer rc.stopWaitGroup.Done() + sleepTime := time.Duration(0) + megabytes := 0 + for { + select { + case megabytes = <-rc.mem: + framework.Logf("RC %s: setting consumption to %v MB in total", rc.name, megabytes) + case <-time.After(sleepTime): + framework.Logf("RC %s: sending request to consume %d MB", rc.name, megabytes) + rc.sendConsumeMemRequest(megabytes) + sleepTime = rc.sleepTime + case <-rc.stopMem: + framework.Logf("RC %s: stopping mem consumer", rc.name) + return + } + } +} + +func (rc *ResourceConsumer) makeConsumeCustomMetric() { + defer ginkgo.GinkgoRecover() + rc.stopWaitGroup.Add(1) + defer rc.stopWaitGroup.Done() + sleepTime := time.Duration(0) + delta := 0 + for { + select { + case delta = <-rc.customMetric: + framework.Logf("RC %s: setting bump of metric %s to %d in total", rc.name, customMetricName, delta) + case <-time.After(sleepTime): + framework.Logf("RC %s: sending request to consume %d of custom metric %s", rc.name, delta, customMetricName) + rc.sendConsumeCustomMetric(delta) + sleepTime = rc.sleepTime + case <-rc.stopCustomMetric: + framework.Logf("RC %s: stopping metric consumer", rc.name) + return + } + } +} + +func (rc *ResourceConsumer) sendConsumeCPURequest(millicores int) { + ctx, cancel := context.WithTimeout(context.Background(), framework.SingleCallTimeout) + defer cancel() + + err := wait.PollUntilContextTimeout(ctx, serviceInitializationInterval, serviceInitializationTimeout, true, func(ctx context.Context) (done bool, err error) { + proxyRequest, err := e2eservice.GetServicesProxyRequest(rc.clientSet, rc.clientSet.CoreV1().RESTClient().Post()) + framework.ExpectNoError(err) + req := proxyRequest.Namespace(rc.nsName). + Name(rc.controllerName). + Suffix("ConsumeCPU"). + Param("millicores", strconv.Itoa(millicores)). + Param("durationSec", strconv.Itoa(rc.consumptionTimeInSeconds)). + Param("requestSizeMillicores", strconv.Itoa(rc.requestSizeInMillicores)) + framework.Logf("ConsumeCPU URL: %v", *req.URL()) + _, err = req.DoRaw(ctx) + if err != nil { + framework.Logf("ConsumeCPU failure: %v", err) + return false, nil + } + return true, nil + }) + + framework.ExpectNoError(err) +} + +// sendConsumeMemRequest sends POST request for memory consumption +func (rc *ResourceConsumer) sendConsumeMemRequest(megabytes int) { + ctx, cancel := context.WithTimeout(context.Background(), framework.SingleCallTimeout) + defer cancel() + + err := wait.PollUntilContextTimeout(ctx, serviceInitializationInterval, serviceInitializationTimeout, true, func(ctx context.Context) (done bool, err error) { + proxyRequest, err := e2eservice.GetServicesProxyRequest(rc.clientSet, rc.clientSet.CoreV1().RESTClient().Post()) + framework.ExpectNoError(err) + req := proxyRequest.Namespace(rc.nsName). + Name(rc.controllerName). + Suffix("ConsumeMem"). + Param("megabytes", strconv.Itoa(megabytes)). + Param("durationSec", strconv.Itoa(rc.consumptionTimeInSeconds)). + Param("requestSizeMegabytes", strconv.Itoa(rc.requestSizeInMegabytes)) + framework.Logf("ConsumeMem URL: %v", *req.URL()) + _, err = req.DoRaw(ctx) + if err != nil { + framework.Logf("ConsumeMem failure: %v", err) + return false, nil + } + return true, nil + }) + + framework.ExpectNoError(err) +} + +// sendConsumeCustomMetric sends POST request for custom metric consumption +func (rc *ResourceConsumer) sendConsumeCustomMetric(delta int) { + ctx, cancel := context.WithTimeout(context.Background(), framework.SingleCallTimeout) + defer cancel() + + err := wait.PollUntilContextTimeout(ctx, serviceInitializationInterval, serviceInitializationTimeout, true, func(ctx context.Context) (done bool, err error) { + proxyRequest, err := e2eservice.GetServicesProxyRequest(rc.clientSet, rc.clientSet.CoreV1().RESTClient().Post()) + framework.ExpectNoError(err) + req := proxyRequest.Namespace(rc.nsName). + Name(rc.controllerName). + Suffix("BumpMetric"). + Param("metric", customMetricName). + Param("delta", strconv.Itoa(delta)). + Param("durationSec", strconv.Itoa(rc.consumptionTimeInSeconds)). + Param("requestSizeMetrics", strconv.Itoa(rc.requestSizeCustomMetric)) + framework.Logf("ConsumeCustomMetric URL: %v", *req.URL()) + _, err = req.DoRaw(ctx) + if err != nil { + framework.Logf("ConsumeCustomMetric failure: %v", err) + return false, nil + } + return true, nil + }) + framework.ExpectNoError(err) +} + +// CleanUp clean up the background goroutines responsible for consuming resources. +func (rc *ResourceConsumer) CleanUp() { + ginkgo.By(fmt.Sprintf("Removing consuming RC %s", rc.name)) + close(rc.stopCPU) + close(rc.stopMem) + close(rc.stopCustomMetric) + rc.stopWaitGroup.Wait() + // Wait some time to ensure all child goroutines are finished. + time.Sleep(10 * time.Second) + kind := rc.kind.GroupKind() + framework.ExpectNoError(resource.DeleteResourceAndWaitForGC(context.TODO(), rc.clientSet, kind, rc.nsName, rc.name)) + framework.ExpectNoError(rc.clientSet.CoreV1().Services(rc.nsName).Delete(context.TODO(), rc.name, metav1.DeleteOptions{})) + framework.ExpectNoError(resource.DeleteResourceAndWaitForGC(context.TODO(), rc.clientSet, schema.GroupKind{Kind: "ReplicationController"}, rc.nsName, rc.controllerName)) + framework.ExpectNoError(rc.clientSet.CoreV1().Services(rc.nsName).Delete(context.TODO(), rc.controllerName, metav1.DeleteOptions{})) +} + +func runServiceAndWorkloadForResourceConsumer(c clientset.Interface, ns, name string, kind schema.GroupVersionKind, replicas int, cpuRequestMillis, memRequestMb int64, podAnnotations, serviceAnnotations map[string]string) { + ginkgo.By(fmt.Sprintf("Running consuming RC %s via %s with %v replicas", name, kind, replicas)) + _, err := c.CoreV1().Services(ns).Create(context.TODO(), &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Annotations: serviceAnnotations, + }, + Spec: v1.ServiceSpec{ + Ports: []v1.ServicePort{{ + Port: port, + TargetPort: intstr.FromInt(targetPort), + }}, + + Selector: map[string]string{ + "name": name, + }, + }, + }, metav1.CreateOptions{}) + framework.ExpectNoError(err) + + rcConfig := testutils.RCConfig{ + Client: c, + Image: resourceConsumerImage, + Name: name, + Namespace: ns, + Timeout: timeoutRC, + Replicas: replicas, + CpuRequest: cpuRequestMillis, + MemRequest: memRequestMb * 1024 * 1024, // MemRequest is in bytes + Annotations: podAnnotations, + } + + switch kind { + case KindRC: + framework.ExpectNoError(e2erc.RunRC(context.TODO(), rcConfig)) + case KindDeployment: + dpConfig := testutils.DeploymentConfig{ + RCConfig: rcConfig, + } + ginkgo.By(fmt.Sprintf("creating deployment %s in namespace %s", dpConfig.Name, dpConfig.Namespace)) + dpConfig.NodeDumpFunc = e2edebug.DumpNodeDebugInfo + dpConfig.ContainerDumpFunc = e2ekubectl.LogFailedContainers + framework.ExpectNoError(testutils.RunDeployment(context.TODO(), dpConfig)) + case KindReplicaSet: + rsConfig := testutils.ReplicaSetConfig{ + RCConfig: rcConfig, + } + ginkgo.By(fmt.Sprintf("creating replicaset %s in namespace %s", rsConfig.Name, rsConfig.Namespace)) + framework.ExpectNoError(runReplicaSet(rsConfig)) + default: + framework.Failf(invalidKind) + } + + ginkgo.By(fmt.Sprintf("Running controller")) + controllerName := name + "-ctrl" + _, err = c.CoreV1().Services(ns).Create(context.TODO(), &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: controllerName, + }, + Spec: v1.ServiceSpec{ + Ports: []v1.ServicePort{{ + Port: port, + TargetPort: intstr.FromInt(targetPort), + }}, + + Selector: map[string]string{ + "name": controllerName, + }, + }, + }, metav1.CreateOptions{}) + framework.ExpectNoError(err) + + dnsClusterFirst := v1.DNSClusterFirst + controllerRcConfig := testutils.RCConfig{ + Client: c, + Image: imageutils.GetE2EImage(imageutils.Agnhost), + Name: controllerName, + Namespace: ns, + Timeout: timeoutRC, + Replicas: 1, + Command: []string{"/agnhost", "resource-consumer-controller", "--consumer-service-name=" + name, "--consumer-service-namespace=" + ns, "--consumer-port=80"}, + DNSPolicy: &dnsClusterFirst, + } + framework.ExpectNoError(e2erc.RunRC(context.TODO(), controllerRcConfig)) + + // Wait for endpoints to propagate for the controller service. + framework.ExpectNoError(framework.WaitForServiceEndpointsNum( + context.TODO(), c, ns, controllerName, 1, startServiceInterval, startServiceTimeout)) +} + +// runReplicaSet launches (and verifies correctness) of a replicaset. +func runReplicaSet(config testutils.ReplicaSetConfig) error { + ginkgo.By(fmt.Sprintf("creating replicaset %s in namespace %s", config.Name, config.Namespace)) + config.NodeDumpFunc = e2edebug.DumpNodeDebugInfo + config.ContainerDumpFunc = e2ekubectl.LogFailedContainers + return testutils.RunReplicaSet(context.TODO(), config) +} + +func runOomingReplicationController(c clientset.Interface, ns, name string, replicas int) { + ginkgo.By(fmt.Sprintf("Running OOMing RC %s with %v replicas", name, replicas)) + + rcConfig := testutils.RCConfig{ + Client: c, + Image: stressImage, + // request exactly 1025 MiB, in a single chunk (1 MiB above the limit) + Command: []string{"/agnhost", "stress", "--mem-total", "1074790400", "--mem-alloc-size", "1074790400"}, + Name: name, + Namespace: ns, + Timeout: timeoutRC, + Replicas: replicas, + Annotations: make(map[string]string), + MemRequest: 1024 * 1024 * 1024, + MemLimit: 1024 * 1024 * 1024, + } + + dpConfig := testutils.DeploymentConfig{ + RCConfig: rcConfig, + } + ginkgo.By(fmt.Sprintf("Creating deployment %s in namespace %s", dpConfig.Name, dpConfig.Namespace)) + dpConfig.NodeDumpFunc = e2edebug.DumpNodeDebugInfo + dpConfig.ContainerDumpFunc = e2ekubectl.LogFailedContainers + // Allow containers to fail (they should be OOM-killed). + failures := 999 + dpConfig.MaxContainerFailures = &failures + // Decrease the timeout since the containers are note expected to actually get up. + dpConfig.Timeout = 10 * time.Second + dpConfig.PollInterval = 5 * time.Second + err := testutils.RunDeployment(context.TODO(), dpConfig) + // Only ignore an error about Pods not starting properly - they're not expected to. + if err != nil && !strings.Contains(err.Error(), "pods started out of") { + framework.ExpectNoError(err) + } +} diff --git a/multidimensional-pod-autoscaler/e2e/v1alpha1/common.go b/multidimensional-pod-autoscaler/e2e/v1alpha1/common.go new file mode 100644 index 000000000000..e59dc5f7a775 --- /dev/null +++ b/multidimensional-pod-autoscaler/e2e/v1alpha1/common.go @@ -0,0 +1,558 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package autoscaling + +import ( + "context" + "encoding/json" + "fmt" + "time" + + ginkgo "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + autoscaling "k8s.io/api/autoscaling/v1" + batchv1 "k8s.io/api/batch/v1" + apiv1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" + mpa_types "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1alpha1" + mpa_clientset "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/client/clientset/versioned" + vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/kubernetes/test/e2e/framework" + framework_deployment "k8s.io/kubernetes/test/e2e/framework/deployment" +) + +const ( + recommenderComponent = "recommender" + updateComponent = "updater" + admissionControllerComponent = "admission-controller" + fullMpaSuite = "full-mpa" + actuationSuite = "actuation" + pollInterval = 10 * time.Second + pollTimeout = 15 * time.Minute + cronJobsWaitTimeout = 15 * time.Minute + // MpaEvictionTimeout is a timeout for MPA to restart a pod if there are no + // mechanisms blocking it (for example PDB). + MpaEvictionTimeout = 3 * time.Minute + + defaultHamsterReplicas = int32(3) + defaultHamsterBackoffLimit = int32(10) +) + +var hamsterTargetRef = &autoscaling.CrossVersionObjectReference{ + APIVersion: "apps/v1", + Kind: "Deployment", + Name: "hamster-deployment", +} + +var hamsterLabels = map[string]string{"app": "hamster"} + +// SIGDescribe adds sig-autoscaling tag to test description. +func SIGDescribe(text string, body func()) bool { + return ginkgo.Describe(fmt.Sprintf("[sig-autoscaling] %v", text), body) +} + +// E2eDescribe describes a MPA e2e test. +func E2eDescribe(scenario, name string, body func()) bool { + return SIGDescribe(fmt.Sprintf("[MPA] [%s] [v1] %s", scenario, name), body) +} + +// RecommenderE2eDescribe describes a MPA recommender e2e test. +func RecommenderE2eDescribe(name string, body func()) bool { + return E2eDescribe(recommenderComponent, name, body) +} + +// UpdaterE2eDescribe describes a MPA updater e2e test. +func UpdaterE2eDescribe(name string, body func()) bool { + return E2eDescribe(updateComponent, name, body) +} + +// AdmissionControllerE2eDescribe describes a MPA admission controller e2e test. +func AdmissionControllerE2eDescribe(name string, body func()) bool { + return E2eDescribe(admissionControllerComponent, name, body) +} + +// FullMpaE2eDescribe describes a MPA full stack e2e test. +func FullMpaE2eDescribe(name string, body func()) bool { + return E2eDescribe(fullMpaSuite, name, body) +} + +// ActuationSuiteE2eDescribe describes a MPA actuation e2e test. +func ActuationSuiteE2eDescribe(name string, body func()) bool { + return E2eDescribe(actuationSuite, name, body) +} + +// GetHamsterContainerNameByIndex returns name of i-th hamster container. +func GetHamsterContainerNameByIndex(i int) string { + switch { + case i < 0: + panic("negative index") + case i == 0: + return "hamster" + default: + return fmt.Sprintf("hamster%d", i+1) + } +} + +// SetupHamsterDeployment creates and installs a simple hamster deployment +// for e2e test purposes, then makes sure the deployment is running. +func SetupHamsterDeployment(f *framework.Framework, cpu, memory string, replicas int32) *appsv1.Deployment { + cpuQuantity := ParseQuantityOrDie(cpu) + memoryQuantity := ParseQuantityOrDie(memory) + + d := NewHamsterDeploymentWithResources(f, cpuQuantity, memoryQuantity) + d.Spec.Replicas = &replicas + d, err := f.ClientSet.AppsV1().Deployments(f.Namespace.Name).Create(context.TODO(), d, metav1.CreateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred(), "unexpected error when starting deployment creation") + err = framework_deployment.WaitForDeploymentComplete(f.ClientSet, d) + gomega.Expect(err).NotTo(gomega.HaveOccurred(), "unexpected error waiting for deployment creation to finish") + return d +} + +// NewHamsterDeployment creates a simple hamster deployment for e2e test purposes. +func NewHamsterDeployment(f *framework.Framework) *appsv1.Deployment { + return NewNHamstersDeployment(f, 1) +} + +// NewNHamstersDeployment creates a simple hamster deployment with n containers +// for e2e test purposes. +func NewNHamstersDeployment(f *framework.Framework, n int) *appsv1.Deployment { + if n < 1 { + panic("container count should be greater than 0") + } + d := framework_deployment.NewDeployment( + "hamster-deployment", /*deploymentName*/ + defaultHamsterReplicas, /*replicas*/ + hamsterLabels, /*podLabels*/ + GetHamsterContainerNameByIndex(0), /*imageName*/ + "registry.k8s.io/ubuntu-slim:0.1", /*image*/ + appsv1.RollingUpdateDeploymentStrategyType, /*strategyType*/ + ) + d.ObjectMeta.Namespace = f.Namespace.Name + d.Spec.Template.Spec.Containers[0].Command = []string{"/bin/sh"} + d.Spec.Template.Spec.Containers[0].Args = []string{"-c", "/usr/bin/yes >/dev/null"} + for i := 1; i < n; i++ { + d.Spec.Template.Spec.Containers = append(d.Spec.Template.Spec.Containers, d.Spec.Template.Spec.Containers[0]) + d.Spec.Template.Spec.Containers[i].Name = GetHamsterContainerNameByIndex(i) + } + return d +} + +// NewHamsterDeploymentWithResources creates a simple hamster deployment with specific +// resource requests for e2e test purposes. +func NewHamsterDeploymentWithResources(f *framework.Framework, cpuQuantity, memoryQuantity resource.Quantity) *appsv1.Deployment { + d := NewHamsterDeployment(f) + d.Spec.Template.Spec.Containers[0].Resources.Requests = apiv1.ResourceList{ + apiv1.ResourceCPU: cpuQuantity, + apiv1.ResourceMemory: memoryQuantity, + } + return d +} + +// NewHamsterDeploymentWithGuaranteedResources creates a simple hamster deployment with specific +// resource requests for e2e test purposes. Since the container in the pod specifies resource limits +// but not resource requests K8s will set requests equal to limits and the pod will have guaranteed +// QoS class. +func NewHamsterDeploymentWithGuaranteedResources(f *framework.Framework, cpuQuantity, memoryQuantity resource.Quantity) *appsv1.Deployment { + d := NewHamsterDeployment(f) + d.Spec.Template.Spec.Containers[0].Resources.Limits = apiv1.ResourceList{ + apiv1.ResourceCPU: cpuQuantity, + apiv1.ResourceMemory: memoryQuantity, + } + return d +} + +// NewHamsterDeploymentWithResourcesAndLimits creates a simple hamster deployment with specific +// resource requests and limits for e2e test purposes. +func NewHamsterDeploymentWithResourcesAndLimits(f *framework.Framework, cpuQuantityRequest, memoryQuantityRequest, cpuQuantityLimit, memoryQuantityLimit resource.Quantity) *appsv1.Deployment { + d := NewHamsterDeploymentWithResources(f, cpuQuantityRequest, memoryQuantityRequest) + d.Spec.Template.Spec.Containers[0].Resources.Limits = apiv1.ResourceList{ + apiv1.ResourceCPU: cpuQuantityLimit, + apiv1.ResourceMemory: memoryQuantityLimit, + } + return d +} + +func getPodSelectorExcludingDonePodsOrDie() string { + stringSelector := "status.phase!=" + string(apiv1.PodSucceeded) + + ",status.phase!=" + string(apiv1.PodFailed) + selector := fields.ParseSelectorOrDie(stringSelector) + return selector.String() +} + +// GetHamsterPods returns running hamster pods (matched by hamsterLabels) +func GetHamsterPods(f *framework.Framework) (*apiv1.PodList, error) { + label := labels.SelectorFromSet(labels.Set(hamsterLabels)) + options := metav1.ListOptions{LabelSelector: label.String(), FieldSelector: getPodSelectorExcludingDonePodsOrDie()} + return f.ClientSet.CoreV1().Pods(f.Namespace.Name).List(context.TODO(), options) +} + +// NewTestCronJob returns a CronJob for test purposes. +func NewTestCronJob(name, schedule string, replicas int32) *batchv1.CronJob { + backoffLimit := defaultHamsterBackoffLimit + sj := &batchv1.CronJob{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "CronJob", + }, + Spec: batchv1.CronJobSpec{ + Schedule: schedule, + ConcurrencyPolicy: batchv1.AllowConcurrent, + JobTemplate: batchv1.JobTemplateSpec{ + Spec: batchv1.JobSpec{ + Parallelism: &replicas, + Completions: &replicas, + BackoffLimit: &backoffLimit, + Template: apiv1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"job": name}, + }, + Spec: apiv1.PodSpec{ + RestartPolicy: apiv1.RestartPolicyOnFailure, + }, + }, + }, + }, + }, + } + + return sj +} + +func waitForActiveJobs(c clientset.Interface, ns, cronJobName string, active int) error { + return wait.PollUntilContextTimeout(context.Background(), framework.Poll, cronJobsWaitTimeout, true, func(ctx context.Context) (done bool, err error) { + curr, err := getCronJob(c, ns, cronJobName) + if err != nil { + return false, err + } + return len(curr.Status.Active) >= active, nil + }) +} + +func createCronJob(c clientset.Interface, ns string, cronJob *batchv1.CronJob) (*batchv1.CronJob, error) { + return c.BatchV1().CronJobs(ns).Create(context.TODO(), cronJob, metav1.CreateOptions{}) +} + +func getCronJob(c clientset.Interface, ns, name string) (*batchv1.CronJob, error) { + return c.BatchV1().CronJobs(ns).Get(context.TODO(), name, metav1.GetOptions{}) +} + +// SetupHamsterCronJob creates and sets up a new CronJob +func SetupHamsterCronJob(f *framework.Framework, schedule, cpu, memory string, replicas int32) { + cronJob := NewTestCronJob("hamster-cronjob", schedule, replicas) + cronJob.Spec.JobTemplate.Spec.Template.Spec.Containers = []apiv1.Container{SetupHamsterContainer(cpu, memory)} + for label, value := range hamsterLabels { + cronJob.Spec.JobTemplate.Spec.Template.Labels[label] = value + } + cronJob, err := createCronJob(f.ClientSet, f.Namespace.Name, cronJob) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = waitForActiveJobs(f.ClientSet, f.Namespace.Name, cronJob.Name, 1) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) +} + +// SetupHamsterContainer returns container with given amount of cpu and memory +func SetupHamsterContainer(cpu, memory string) apiv1.Container { + cpuQuantity := ParseQuantityOrDie(cpu) + memoryQuantity := ParseQuantityOrDie(memory) + + return apiv1.Container{ + Name: "hamster", + Image: "registry.k8s.io/ubuntu-slim:0.1", + Resources: apiv1.ResourceRequirements{ + Requests: apiv1.ResourceList{ + apiv1.ResourceCPU: cpuQuantity, + apiv1.ResourceMemory: memoryQuantity, + }, + }, + Command: []string{"/bin/sh"}, + Args: []string{"-c", "while true; do sleep 10 ; done"}, + } +} + +type patchRecord struct { + Op string `json:"op,inline"` + Path string `json:"path,inline"` + Value interface{} `json:"value"` +} + +func getMpaClientSet(f *framework.Framework) mpa_clientset.Interface { + config, err := framework.LoadConfig() + gomega.Expect(err).NotTo(gomega.HaveOccurred(), "unexpected error loading framework") + return mpa_clientset.NewForConfigOrDie(config) +} + +// InstallMPA installs a MPA object in the test cluster. +func InstallMPA(f *framework.Framework, mpa *mpa_types.MultidimPodAutoscaler) { + mpaClientSet := getMpaClientSet(f) + _, err := mpaClientSet.AutoscalingV1alpha1().MultidimPodAutoscalers(f.Namespace.Name).Create(context.TODO(), mpa, metav1.CreateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred(), "unexpected error creating MPA") + // apiserver ignore status in mpa create, so need to update status + if !isStatusEmpty(&mpa.Status) { + if mpa.Status.Recommendation != nil { + PatchMpaRecommendation(f, mpa, mpa.Status.Recommendation) + } + } +} + +func isStatusEmpty(status *mpa_types.MultidimPodAutoscalerStatus) bool { + if status == nil { + return true + } + + if len(status.Conditions) == 0 && status.Recommendation == nil { + return true + } + return false +} + +// InstallRawMPA installs a MPA object passed in as raw json in the test cluster. +func InstallRawMPA(f *framework.Framework, obj interface{}) error { + mpaClientSet := getMpaClientSet(f) + err := mpaClientSet.AutoscalingV1alpha1().RESTClient().Post(). + Namespace(f.Namespace.Name). + Resource("multidimpodautoscalers"). + Body(obj). + Do(context.TODO()) + return err.Error() +} + +// PatchMpaRecommendation installs a new recommendation for MPA object. +func PatchMpaRecommendation(f *framework.Framework, mpa *mpa_types.MultidimPodAutoscaler, + recommendation *vpa_types.RecommendedPodResources) { + newStatus := mpa.Status.DeepCopy() + newStatus.Recommendation = recommendation + bytes, err := json.Marshal([]patchRecord{{ + Op: "replace", + Path: "/status", + Value: *newStatus, + }}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + _, err = getMpaClientSet(f).AutoscalingV1alpha1().MultidimPodAutoscalers(f.Namespace.Name).Patch(context.TODO(), mpa.Name, types.JSONPatchType, bytes, metav1.PatchOptions{}, "status") + gomega.Expect(err).NotTo(gomega.HaveOccurred(), "Failed to patch MPA.") +} + +// AnnotatePod adds annotation for an existing pod. +func AnnotatePod(f *framework.Framework, podName, annotationName, annotationValue string) { + bytes, err := json.Marshal([]patchRecord{{ + Op: "add", + Path: fmt.Sprintf("/metadata/annotations/%v", annotationName), + Value: annotationValue, + }}) + pod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Patch(context.TODO(), podName, types.JSONPatchType, bytes, metav1.PatchOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred(), "Failed to patch pod.") + gomega.Expect(pod.Annotations[annotationName]).To(gomega.Equal(annotationValue)) +} + +// ParseQuantityOrDie parses quantity from string and dies with an error if +// unparsable. +func ParseQuantityOrDie(text string) resource.Quantity { + quantity, err := resource.ParseQuantity(text) + gomega.Expect(err).NotTo(gomega.HaveOccurred(), "unexpected error parsing quantity: %s", text) + return quantity +} + +// PodSet is a simplified representation of PodList mapping names to UIDs. +type PodSet map[string]types.UID + +// MakePodSet converts PodList to podset for easier comparison of pod collections. +func MakePodSet(pods *apiv1.PodList) PodSet { + result := make(PodSet) + if pods == nil { + return result + } + for _, p := range pods.Items { + result[p.Name] = p.UID + } + return result +} + +// WaitForPodsRestarted waits until some pods from the list are restarted. +func WaitForPodsRestarted(f *framework.Framework, podList *apiv1.PodList) error { + initialPodSet := MakePodSet(podList) + + return wait.PollUntilContextTimeout(context.Background(), pollInterval, pollTimeout, true, func(ctx context.Context) (done bool, err error) { + currentPodList, err := GetHamsterPods(f) + if err != nil { + return false, err + } + currentPodSet := MakePodSet(currentPodList) + return WerePodsSuccessfullyRestarted(currentPodSet, initialPodSet), nil + }) +} + +// WaitForPodsEvicted waits until some pods from the list are evicted. +func WaitForPodsEvicted(f *framework.Framework, podList *apiv1.PodList) error { + initialPodSet := MakePodSet(podList) + + return wait.PollUntilContextTimeout(context.Background(), pollInterval, pollTimeout, true, func(ctx context.Context) (done bool, err error) { + currentPodList, err := GetHamsterPods(f) + if err != nil { + return false, err + } + currentPodSet := MakePodSet(currentPodList) + return GetEvictedPodsCount(currentPodSet, initialPodSet) > 0, nil + }) +} + +// WerePodsSuccessfullyRestarted returns true if some pods from initialPodSet have been +// successfully restarted comparing to currentPodSet (pods were evicted and +// are running). +func WerePodsSuccessfullyRestarted(currentPodSet PodSet, initialPodSet PodSet) bool { + if len(currentPodSet) < len(initialPodSet) { + // If we have less pods running than in the beginning, there is a restart + // in progress - a pod was evicted but not yet recreated. + framework.Logf("Restart in progress") + return false + } + evictedCount := GetEvictedPodsCount(currentPodSet, initialPodSet) + framework.Logf("%v of initial pods were already evicted", evictedCount) + return evictedCount > 0 +} + +// GetEvictedPodsCount returns the count of pods from initialPodSet that have +// been evicted comparing to currentPodSet. +func GetEvictedPodsCount(currentPodSet PodSet, initialPodSet PodSet) int { + diffs := 0 + for name, initialUID := range initialPodSet { + currentUID, inCurrent := currentPodSet[name] + if !inCurrent { + diffs += 1 + } else if initialUID != currentUID { + diffs += 1 + } + } + return diffs +} + +// CheckNoPodsEvicted waits for long enough period for MPA to start evicting +// pods and checks that no pods were restarted. +func CheckNoPodsEvicted(f *framework.Framework, initialPodSet PodSet) { + time.Sleep(MpaEvictionTimeout) + currentPodList, err := GetHamsterPods(f) + gomega.Expect(err).NotTo(gomega.HaveOccurred(), "unexpected error when listing hamster pods to check number of pod evictions") + restarted := GetEvictedPodsCount(MakePodSet(currentPodList), initialPodSet) + gomega.Expect(restarted).To(gomega.Equal(0), "there should be no pod evictions") +} + +// WaitForMPAMatch pools MPA object until match function returns true. Returns +// polled mpa object. On timeout returns error. +func WaitForMPAMatch(c mpa_clientset.Interface, mpa *mpa_types.MultidimPodAutoscaler, match func(mpa *mpa_types.MultidimPodAutoscaler) bool) (*mpa_types.MultidimPodAutoscaler, error) { + var polledMpa *mpa_types.MultidimPodAutoscaler + err := wait.PollUntilContextTimeout(context.Background(), pollInterval, pollTimeout, true, func(ctx context.Context) (done bool, err error) { + polledMpa, err = c.AutoscalingV1alpha1().MultidimPodAutoscalers(mpa.Namespace).Get(context.TODO(), mpa.Name, metav1.GetOptions{}) + if err != nil { + return false, err + } + + if match(polledMpa) { + return true, nil + } + + return false, nil + }) + + if err != nil { + return nil, fmt.Errorf("error waiting for recommendation present in %v: %v", mpa.Name, err) + } + return polledMpa, nil +} + +// WaitForRecommendationPresent pools MPA object until recommendations are not empty. Returns +// polled mpa object. On timeout returns error. +func WaitForRecommendationPresent(c mpa_clientset.Interface, mpa *mpa_types.MultidimPodAutoscaler) (*mpa_types.MultidimPodAutoscaler, error) { + return WaitForMPAMatch(c, mpa, func(mpa *mpa_types.MultidimPodAutoscaler) bool { + return mpa.Status.Recommendation != nil && len(mpa.Status.Recommendation.ContainerRecommendations) != 0 + }) +} + +// WaitForUncappedCPURecommendationAbove pools MPA object until uncapped recommendation is above specified value. +// Returns polled MPA object. On timeout returns error. +func WaitForUncappedCPURecommendationAbove(c mpa_clientset.Interface, mpa *mpa_types.MultidimPodAutoscaler, minMilliCPU int64) (*mpa_types.MultidimPodAutoscaler, error) { + return WaitForMPAMatch(c, mpa, func(mpa *mpa_types.MultidimPodAutoscaler) bool { + if mpa.Status.Recommendation == nil || len(mpa.Status.Recommendation.ContainerRecommendations) == 0 { + return false + } + uncappedCpu := mpa.Status.Recommendation.ContainerRecommendations[0].UncappedTarget[apiv1.ResourceCPU] + return uncappedCpu.MilliValue() > minMilliCPU + }) +} + +func installLimitRange(f *framework.Framework, minCpuLimit, minMemoryLimit, maxCpuLimit, maxMemoryLimit *resource.Quantity, lrType apiv1.LimitType) { + lr := &apiv1.LimitRange{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: f.Namespace.Name, + Name: "hamster-lr", + }, + Spec: apiv1.LimitRangeSpec{ + Limits: []apiv1.LimitRangeItem{}, + }, + } + + if maxMemoryLimit != nil || maxCpuLimit != nil { + lrItem := apiv1.LimitRangeItem{ + Type: lrType, + Max: apiv1.ResourceList{}, + } + if maxCpuLimit != nil { + lrItem.Max[apiv1.ResourceCPU] = *maxCpuLimit + } + if maxMemoryLimit != nil { + lrItem.Max[apiv1.ResourceMemory] = *maxMemoryLimit + } + lr.Spec.Limits = append(lr.Spec.Limits, lrItem) + } + + if minMemoryLimit != nil || minCpuLimit != nil { + lrItem := apiv1.LimitRangeItem{ + Type: lrType, + Min: apiv1.ResourceList{}, + } + if minCpuLimit != nil { + lrItem.Min[apiv1.ResourceCPU] = *minCpuLimit + } + if minMemoryLimit != nil { + lrItem.Min[apiv1.ResourceMemory] = *minMemoryLimit + } + lr.Spec.Limits = append(lr.Spec.Limits, lrItem) + } + _, err := f.ClientSet.CoreV1().LimitRanges(f.Namespace.Name).Create(context.TODO(), lr, metav1.CreateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred(), "unexpected error when creating limit range") +} + +// InstallLimitRangeWithMax installs a LimitRange with a maximum limit for CPU and memory. +func InstallLimitRangeWithMax(f *framework.Framework, maxCpuLimit, maxMemoryLimit string, lrType apiv1.LimitType) { + ginkgo.By(fmt.Sprintf("Setting up LimitRange with max limits - CPU: %v, memory: %v", maxCpuLimit, maxMemoryLimit)) + maxCpuLimitQuantity := ParseQuantityOrDie(maxCpuLimit) + maxMemoryLimitQuantity := ParseQuantityOrDie(maxMemoryLimit) + installLimitRange(f, nil, nil, &maxCpuLimitQuantity, &maxMemoryLimitQuantity, lrType) +} + +// InstallLimitRangeWithMin installs a LimitRange with a minimum limit for CPU and memory. +func InstallLimitRangeWithMin(f *framework.Framework, minCpuLimit, minMemoryLimit string, lrType apiv1.LimitType) { + ginkgo.By(fmt.Sprintf("Setting up LimitRange with min limits - CPU: %v, memory: %v", minCpuLimit, minMemoryLimit)) + minCpuLimitQuantity := ParseQuantityOrDie(minCpuLimit) + minMemoryLimitQuantity := ParseQuantityOrDie(minMemoryLimit) + installLimitRange(f, &minCpuLimitQuantity, &minMemoryLimitQuantity, nil, nil, lrType) +} diff --git a/multidimensional-pod-autoscaler/e2e/v1alpha1/e2e.go b/multidimensional-pod-autoscaler/e2e/v1alpha1/e2e.go new file mode 100644 index 000000000000..b6124fcb2144 --- /dev/null +++ b/multidimensional-pod-autoscaler/e2e/v1alpha1/e2e.go @@ -0,0 +1,345 @@ +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package autoscaling + +// This file is a cut down fork of k8s/test/e2e/e2e.go + +import ( + "context" + "fmt" + "io/ioutil" + "path" + "testing" + "time" + + klog "k8s.io/klog/v2" + + ginkgo "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtimeutils "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/component-base/logs" + "k8s.io/component-base/version" + "k8s.io/kubernetes/test/e2e/framework" + e2edebug "k8s.io/kubernetes/test/e2e/framework/debug" + e2ekubectl "k8s.io/kubernetes/test/e2e/framework/kubectl" + "k8s.io/kubernetes/test/e2e/framework/manifest" + e2emetrics "k8s.io/kubernetes/test/e2e/framework/metrics" + + e2enode "k8s.io/kubernetes/test/e2e/framework/node" + e2epod "k8s.io/kubernetes/test/e2e/framework/pod" + testutils "k8s.io/kubernetes/test/utils" + utilnet "k8s.io/utils/net" + + clientset "k8s.io/client-go/kubernetes" + // ensure auth plugins are loaded + _ "k8s.io/client-go/plugin/pkg/client/auth" + + // ensure that cloud providers are loaded + _ "k8s.io/kubernetes/test/e2e/framework/providers/gce" +) + +const ( + // namespaceCleanupTimeout is how long to wait for the namespace to be deleted. + // If there are any orphaned namespaces to clean up, this test is running + // on a long lived cluster. A long wait here is preferably to spurious test + // failures caused by leaked resources from a previous test run. + namespaceCleanupTimeout = 15 * time.Minute +) + +var _ = ginkgo.SynchronizedBeforeSuite(func() []byte { + setupSuite() + return nil +}, func(data []byte) { + // Run on all Ginkgo nodes + setupSuitePerGinkgoNode() +}) + +var _ = ginkgo.SynchronizedAfterSuite(func() { + CleanupSuite() +}, func() { + AfterSuiteActions() +}) + +// RunE2ETests checks configuration parameters (specified through flags) and then runs +// E2E tests using the Ginkgo runner. +// If a "report directory" is specified, one or more JUnit test reports will be +// generated in this directory, and cluster logs will also be saved. +// This function is called on each Ginkgo node in parallel mode. +func RunE2ETests(t *testing.T) { + runtimeutils.ReallyCrash = true + logs.InitLogs() + defer logs.FlushLogs() + + gomega.RegisterFailHandler(framework.Fail) + suiteConfig, _ := ginkgo.GinkgoConfiguration() + // Disable skipped tests unless they are explicitly requested. + if len(suiteConfig.FocusStrings) == 0 && len(suiteConfig.SkipStrings) == 0 { + suiteConfig.SkipStrings = []string{`\[Flaky\]|\[Feature:.+\]`} + } + ginkgo.RunSpecs(t, "Kubernetes e2e suite") +} + +// Run a test container to try and contact the Kubernetes api-server from a pod, wait for it +// to flip to Ready, log its output and delete it. +func runKubernetesServiceTestContainer(c clientset.Interface, ns string) { + path := "test/images/clusterapi-tester/pod.yaml" + framework.Logf("Parsing pod from %v", path) + p, err := manifest.PodFromManifest(path) + if err != nil { + framework.Logf("Failed to parse clusterapi-tester from manifest %v: %v", path, err) + return + } + p.Namespace = ns + if _, err := c.CoreV1().Pods(ns).Create(context.TODO(), p, metav1.CreateOptions{}); err != nil { + framework.Logf("Failed to create %v: %v", p.Name, err) + return + } + defer func() { + if err := c.CoreV1().Pods(ns).Delete(context.TODO(), p.Name, metav1.DeleteOptions{}); err != nil { + framework.Logf("Failed to delete pod %v: %v", p.Name, err) + } + }() + timeout := 5 * time.Minute + if err := e2epod.WaitForPodCondition(context.TODO(), c, ns, p.Name, "clusterapi-tester", timeout, testutils.PodRunningReady); err != nil { + framework.Logf("Pod %v took longer than %v to enter running/ready: %v", p.Name, timeout, err) + return + } + logs, err := e2epod.GetPodLogs(context.TODO(), c, ns, p.Name, p.Spec.Containers[0].Name) + if err != nil { + framework.Logf("Failed to retrieve logs from %v: %v", p.Name, err) + } else { + framework.Logf("Output of clusterapi-tester:\n%v", logs) + } +} + +// getDefaultClusterIPFamily obtains the default IP family of the cluster +// using the Cluster IP address of the kubernetes service created in the default namespace +// This unequivocally identifies the default IP family because services are single family +// TODO: dual-stack may support multiple families per service +// but we can detect if a cluster is dual stack because pods have two addresses (one per family) +func getDefaultClusterIPFamily(c clientset.Interface) string { + // Get the ClusterIP of the kubernetes service created in the default namespace + svc, err := c.CoreV1().Services(metav1.NamespaceDefault).Get(context.TODO(), "kubernetes", metav1.GetOptions{}) + if err != nil { + framework.Failf("Failed to get kubernetes service ClusterIP: %v", err) + } + + if utilnet.IsIPv6String(svc.Spec.ClusterIP) { + return "ipv6" + } + return "ipv4" +} + +// waitForDaemonSets for all daemonsets in the given namespace to be ready +// (defined as all but 'allowedNotReadyNodes' pods associated with that +// daemonset are ready). +func waitForDaemonSets(c clientset.Interface, ns string, allowedNotReadyNodes int32, timeout time.Duration) error { + start := time.Now() + framework.Logf("Waiting up to %v for all daemonsets in namespace '%s' to start", + timeout, ns) + + return wait.PollUntilContextTimeout(context.Background(), framework.Poll, timeout, true, func(ctx context.Context) (done bool, err error) { + dsList, err := c.AppsV1().DaemonSets(ns).List(ctx, metav1.ListOptions{}) + if err != nil { + framework.Logf("Error getting daemonsets in namespace: '%s': %v", ns, err) + return false, err + } + var notReadyDaemonSets []string + for _, ds := range dsList.Items { + framework.Logf("%d / %d pods ready in namespace '%s' in daemonset '%s' (%d seconds elapsed)", ds.Status.NumberReady, ds.Status.DesiredNumberScheduled, ns, ds.ObjectMeta.Name, int(time.Since(start).Seconds())) + if ds.Status.DesiredNumberScheduled-ds.Status.NumberReady > allowedNotReadyNodes { + notReadyDaemonSets = append(notReadyDaemonSets, ds.ObjectMeta.Name) + } + } + + if len(notReadyDaemonSets) > 0 { + framework.Logf("there are not ready daemonsets: %v", notReadyDaemonSets) + return false, nil + } + + return true, nil + }) +} + +// setupSuite is the boilerplate that can be used to setup ginkgo test suites, on the SynchronizedBeforeSuite step. +// There are certain operations we only want to run once per overall test invocation +// (such as deleting old namespaces, or verifying that all system pods are running. +// Because of the way Ginkgo runs tests in parallel, we must use SynchronizedBeforeSuite +// to ensure that these operations only run on the first parallel Ginkgo node. +// +// This function takes two parameters: one function which runs on only the first Ginkgo node, +// returning an opaque byte array, and then a second function which runs on all Ginkgo nodes, +// accepting the byte array. +func setupSuite() { + // Run only on Ginkgo node 1 + c, err := framework.LoadClientset() + if err != nil { + klog.Fatal("Error loading client: ", err) + } + + // Delete any namespaces except those created by the system. This ensures no + // lingering resources are left over from a previous test run. + if framework.TestContext.CleanStart { + deleted, err := framework.DeleteNamespaces(context.TODO(), c, nil, /* deleteFilter */ + []string{ + metav1.NamespaceSystem, + metav1.NamespaceDefault, + metav1.NamespacePublic, + v1.NamespaceNodeLease, + }) + if err != nil { + framework.Failf("Error deleting orphaned namespaces: %v", err) + } + framework.Logf("Waiting for deletion of the following namespaces: %v", deleted) + if err := framework.WaitForNamespacesDeleted(context.TODO(), c, deleted, namespaceCleanupTimeout); err != nil { + framework.Failf("Failed to delete orphaned namespaces %v: %v", deleted, err) + } + } + + // In large clusters we may get to this point but still have a bunch + // of nodes without Routes created. Since this would make a node + // unschedulable, we need to wait until all of them are schedulable. + framework.ExpectNoError(e2enode.WaitForAllNodesSchedulable(context.TODO(), c, framework.NewTimeoutContext().NodeSchedulable)) + + // If NumNodes is not specified then auto-detect how many are scheduleable and not tainted + if framework.TestContext.CloudConfig.NumNodes == framework.DefaultNumNodes { + nodes, err := e2enode.GetReadySchedulableNodes(context.TODO(), c) + framework.ExpectNoError(err) + framework.TestContext.CloudConfig.NumNodes = len(nodes.Items) + } + + timeoutCtx := framework.NewTimeoutContext() + // Ensure all pods are running and ready before starting tests (otherwise, + // cluster infrastructure pods that are being pulled or started can block + // test pods from running, and tests that ensure all pods are running and + // ready will fail). + podStartupTimeout := timeoutCtx.SystemPodsStartup + // TODO: In large clusters, we often observe a non-starting pods due to + // #41007. To avoid those pods preventing the whole test runs (and just + // wasting the whole run), we allow for some not-ready pods (with the + // number equal to the number of allowed not-ready nodes). + if err := e2epod.WaitForPodsRunningReady(context.TODO(), c, metav1.NamespaceSystem, framework.TestContext.MinStartupPods, podStartupTimeout); err != nil { + e2edebug.DumpAllNamespaceInfo(context.TODO(), c, metav1.NamespaceSystem) + e2ekubectl.LogFailedContainers(context.TODO(), c, metav1.NamespaceSystem, framework.Logf) + runKubernetesServiceTestContainer(c, metav1.NamespaceDefault) + framework.Failf("Error waiting for all pods to be running and ready: %v", err) + } + + if err := waitForDaemonSets(c, metav1.NamespaceSystem, int32(framework.TestContext.AllowedNotReadyNodes), timeoutCtx.SystemDaemonsetStartup); err != nil { + framework.Logf("WARNING: Waiting for all daemonsets to be ready failed: %v", err) + } + + // Log the version of the server and this client. + framework.Logf("e2e test version: %s", version.Get().GitVersion) + + dc := c.DiscoveryClient + + serverVersion, serverErr := dc.ServerVersion() + if serverErr != nil { + framework.Logf("Unexpected server error retrieving version: %v", serverErr) + } + if serverVersion != nil { + framework.Logf("kube-apiserver version: %s", serverVersion.GitVersion) + } + + if framework.TestContext.NodeKiller.Enabled { + nodeKiller := e2enode.NewNodeKiller(framework.TestContext.NodeKiller, c, framework.TestContext.Provider) + go nodeKiller.Run(framework.TestContext.NodeKiller.NodeKillerStopCtx) + } +} + +// setupSuitePerGinkgoNode is the boilerplate that can be used to setup ginkgo test suites, on the SynchronizedBeforeSuite step. +// There are certain operations we only want to run once per overall test invocation on each Ginkgo node +// such as making some global variables accessible to all parallel executions +// Because of the way Ginkgo runs tests in parallel, we must use SynchronizedBeforeSuite +// Ref: https://onsi.github.io/ginkgo/#parallel-specs +func setupSuitePerGinkgoNode() { + // Obtain the default IP family of the cluster + // Some e2e test are designed to work on IPv4 only, this global variable + // allows to adapt those tests to work on both IPv4 and IPv6 + // TODO: dual-stack + // the dual stack clusters can be ipv4-ipv6 or ipv6-ipv4, order matters, + // and services use the primary IP family by default + c, err := framework.LoadClientset() + if err != nil { + klog.Fatal("Error loading client: ", err) + } + framework.TestContext.IPFamily = getDefaultClusterIPFamily(c) + framework.Logf("Cluster IP family: %s", framework.TestContext.IPFamily) +} + +// CleanupSuite is the boilerplate that can be used after tests on ginkgo were run, on the SynchronizedAfterSuite step. +// Similar to SynchronizedBeforeSuite, we want to run some operations only once (such as collecting cluster logs). +// Here, the order of functions is reversed; first, the function which runs everywhere, +// and then the function that only runs on the first Ginkgo node. +func CleanupSuite() { + // Run on all Ginkgo nodes + framework.Logf("Running AfterSuite actions on all nodes") +} + +// AfterSuiteActions are actions that are run on ginkgo's SynchronizedAfterSuite +func AfterSuiteActions() { + // Run only Ginkgo on node 1 + framework.Logf("Running AfterSuite actions on node 1") + if framework.TestContext.ReportDir != "" { + framework.CoreDump(framework.TestContext.ReportDir) + } + if framework.TestContext.GatherSuiteMetricsAfterTest { + if err := gatherTestSuiteMetrics(); err != nil { + framework.Logf("Error gathering metrics: %v", err) + } + } + if framework.TestContext.NodeKiller.Enabled { + framework.TestContext.NodeKiller.NodeKillerStop() + } +} + +func gatherTestSuiteMetrics() error { + framework.Logf("Gathering metrics") + c, err := framework.LoadClientset() + if err != nil { + return fmt.Errorf("error loading client: %v", err) + } + + // Grab metrics for apiserver, scheduler, controller-manager, kubelet (for non-kubemark case) and cluster autoscaler (optionally). + grabber, err := e2emetrics.NewMetricsGrabber(context.TODO(), c, nil, nil, !framework.ProviderIs("kubemark"), true, true, true, framework.TestContext.IncludeClusterAutoscalerMetrics, false) + if err != nil { + return fmt.Errorf("failed to create MetricsGrabber: %v", err) + } + + received, err := grabber.Grab(context.TODO()) + if err != nil { + return fmt.Errorf("failed to grab metrics: %v", err) + } + + metricsForE2E := (*e2emetrics.ComponentCollection)(&received) + metricsJSON := metricsForE2E.PrintJSON() + if framework.TestContext.ReportDir != "" { + filePath := path.Join(framework.TestContext.ReportDir, "MetricsForE2ESuite_"+time.Now().Format(time.RFC3339)+".json") + if err := ioutil.WriteFile(filePath, []byte(metricsJSON), 0644); err != nil { + return fmt.Errorf("error writing to %q: %v", filePath, err) + } + } else { + framework.Logf("\n\nTest Suite Metrics:\n%s\n", metricsJSON) + } + + return nil +} diff --git a/multidimensional-pod-autoscaler/e2e/v1alpha1/e2e_test.go b/multidimensional-pod-autoscaler/e2e/v1alpha1/e2e_test.go new file mode 100644 index 000000000000..98819575c43a --- /dev/null +++ b/multidimensional-pod-autoscaler/e2e/v1alpha1/e2e_test.go @@ -0,0 +1,77 @@ +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package autoscaling + +import ( + "flag" + "fmt" + "math/rand" + "os" + "testing" + "time" + + // Never, ever remove the line with "/ginkgo". Without it, + // the ginkgo test runner will not detect that this + // directory contains a Ginkgo test suite. + // See https://github.com/kubernetes/kubernetes/issues/74827 + // "github.com/onsi/ginkgo" + + "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/framework/config" + "k8s.io/kubernetes/test/e2e/framework/testfiles" + "k8s.io/kubernetes/test/utils/image" +) + +var viperConfig = flag.String("viper-config", "", "The name of a viper config file (https://github.com/spf13/viper#what-is-viper). All e2e command line parameters can also be configured in such a file. May contain a path and may or may not contain the file suffix. The default is to look for an optional file with `e2e` as base name. If a file is specified explicitly, it must be present.") + +// handleFlags sets up all flags and parses the command line. +func handleFlags() { + config.CopyFlags(config.Flags, flag.CommandLine) + framework.RegisterCommonFlags(flag.CommandLine) + framework.RegisterClusterFlags(flag.CommandLine) + flag.Parse() +} + +func TestMain(m *testing.M) { + // Register test flags, then parse flags. + handleFlags() + + if framework.TestContext.ListImages { + for _, v := range image.GetImageConfigs() { + fmt.Println(v.GetE2EImage()) + } + os.Exit(0) + } + + framework.AfterReadingAllFlags(&framework.TestContext) + + // TODO: Deprecating repo-root over time... instead just use gobindata_util.go , see #23987. + // Right now it is still needed, for example by + // test/e2e/framework/ingress/ingress_utils.go + // for providing the optional secret.yaml file and by + // test/e2e/framework/util.go for cluster/log-dump. + if framework.TestContext.RepoRoot != "" { + testfiles.AddFileSource(testfiles.RootFileSource{Root: framework.TestContext.RepoRoot}) + } + + rand.Seed(time.Now().UnixNano()) + os.Exit(m.Run()) +} + +func TestE2E(t *testing.T) { + RunE2ETests(t) +} diff --git a/multidimensional-pod-autoscaler/e2e/v1alpha1/full_mpa.go b/multidimensional-pod-autoscaler/e2e/v1alpha1/full_mpa.go new file mode 100644 index 000000000000..b59185fc0471 --- /dev/null +++ b/multidimensional-pod-autoscaler/e2e/v1alpha1/full_mpa.go @@ -0,0 +1,367 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package autoscaling + +import ( + "context" + "fmt" + "time" + + autoscaling "k8s.io/api/autoscaling/v1" + apiv1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + mpa_types "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1alpha1" + "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/utils/test" + "k8s.io/kubernetes/test/e2e/framework" + podsecurity "k8s.io/pod-security-admission/api" + + ginkgo "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" +) + +const ( + minimalCPULowerBound = "0m" + minimalCPUUpperBound = "100m" + minimalMemoryLowerBound = "0Mi" + minimalMemoryUpperBound = "300Mi" + // the initial values should be outside minimal bounds + initialCPU = int64(10) // mCPU + initialMemory = int64(10) // MB + oomTestTimeout = 8 * time.Minute +) + +var _ = FullMpaE2eDescribe("Pods under MPA", func() { + var ( + rc *ResourceConsumer + ) + replicas := 3 + + ginkgo.AfterEach(func() { + rc.CleanUp() + }) + + // This schedules AfterEach block that needs to run after the AfterEach above and + // BeforeEach that needs to run before the BeforeEach below - thus the order of these matters. + f := framework.NewDefaultFramework("multidimensional-pod-autoscaling") + f.NamespacePodSecurityEnforceLevel = podsecurity.LevelBaseline + + ginkgo.BeforeEach(func() { + ns := f.Namespace.Name + ginkgo.By("Setting up a hamster deployment") + rc = NewDynamicResourceConsumer("hamster", ns, KindDeployment, + replicas, + 1, /*initCPUTotal*/ + 10, /*initMemoryTotal*/ + 1, /*initCustomMetric*/ + initialCPU, /*cpuRequest*/ + initialMemory, /*memRequest*/ + f.ClientSet, + f.ScalesGetter) + + ginkgo.By("Setting up a MPA CRD") + targetRef := &autoscaling.CrossVersionObjectReference{ + APIVersion: "apps/v1", + Kind: "Deployment", + Name: "hamster", + } + minReplicas := int32(1) + maxReplicas := int32(2) + hconstraints := mpa_types.HorizontalScalingConstraints{ + MinReplicas: &minReplicas, + MaxReplicas: &maxReplicas, + } + + containerName := GetHamsterContainerNameByIndex(0) + mpaCRD := test.MultidimPodAutoscaler(). + WithName("hamster-mpa"). + WithNamespace(f.Namespace.Name). + WithScaleTargetRef(targetRef). + WithContainer(containerName). + WithHorizontalScalingConstraints(hconstraints). + AppendRecommendation( + test.Recommendation(). + WithContainer(containerName). + WithTarget("250m", "200Mi"). + WithLowerBound("250m", "200Mi"). + WithUpperBound("250m", "200Mi"). + GetContainerResources()). + Get() + + InstallMPA(f, mpaCRD) + }) + + ginkgo.It("have cpu requests growing with usage", func() { + // initial CPU usage is low so a minimal recommendation is expected + err := waitForResourceRequestInRangeInPods( + f, pollTimeout, metav1.ListOptions{LabelSelector: "name=hamster"}, apiv1.ResourceCPU, + ParseQuantityOrDie(minimalCPULowerBound), ParseQuantityOrDie(minimalCPUUpperBound)) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + // consume more CPU to get a higher recommendation + rc.ConsumeCPU(600 * replicas) + err = waitForResourceRequestInRangeInPods( + f, pollTimeout, metav1.ListOptions{LabelSelector: "name=hamster"}, apiv1.ResourceCPU, + ParseQuantityOrDie("500m"), ParseQuantityOrDie("1300m")) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) + + ginkgo.It("have memory requests growing with usage", func() { + // initial memory usage is low so a minimal recommendation is expected + err := waitForResourceRequestInRangeInPods( + f, pollTimeout, metav1.ListOptions{LabelSelector: "name=hamster"}, apiv1.ResourceMemory, + ParseQuantityOrDie(minimalMemoryLowerBound), ParseQuantityOrDie(minimalMemoryUpperBound)) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + // consume more memory to get a higher recommendation + // NOTE: large range given due to unpredictability of actual memory usage + rc.ConsumeMem(1024 * replicas) + err = waitForResourceRequestInRangeInPods( + f, pollTimeout, metav1.ListOptions{LabelSelector: "name=hamster"}, apiv1.ResourceMemory, + ParseQuantityOrDie("900Mi"), ParseQuantityOrDie("4000Mi")) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) +}) + +var _ = FullMpaE2eDescribe("Pods under MPA with default recommender explicitly configured", func() { + var ( + rc *ResourceConsumer + ) + replicas := 3 + + ginkgo.AfterEach(func() { + rc.CleanUp() + }) + + // This schedules AfterEach block that needs to run after the AfterEach above and + // BeforeEach that needs to run before the BeforeEach below - thus the order of these matters. + f := framework.NewDefaultFramework("multidimensional-pod-autoscaling") + f.NamespacePodSecurityEnforceLevel = podsecurity.LevelBaseline + + ginkgo.BeforeEach(func() { + ns := f.Namespace.Name + ginkgo.By("Setting up a hamster deployment") + rc = NewDynamicResourceConsumer("hamster", ns, KindDeployment, + replicas, + 1, /*initCPUTotal*/ + 10, /*initMemoryTotal*/ + 1, /*initCustomMetric*/ + initialCPU, /*cpuRequest*/ + initialMemory, /*memRequest*/ + f.ClientSet, + f.ScalesGetter) + + ginkgo.By("Setting up a MPA CRD with Recommender explicitly configured") + targetRef := &autoscaling.CrossVersionObjectReference{ + APIVersion: "apps/v1", + Kind: "Deployment", + Name: "hamster", + } + + containerName := GetHamsterContainerNameByIndex(0) + mpaCRD := test.MultidimPodAutoscaler(). + WithName("hamster-mpa"). + WithNamespace(f.Namespace.Name). + WithScaleTargetRef(targetRef). + WithContainer(containerName). + AppendRecommendation( + test.Recommendation(). + WithContainer(containerName). + WithTarget("250m", "200Mi"). + WithLowerBound("250m", "200Mi"). + WithUpperBound("250m", "200Mi"). + GetContainerResources()). + Get() + + InstallMPA(f, mpaCRD) + + }) + + ginkgo.It("have cpu requests growing with usage", func() { + // initial CPU usage is low so a minimal recommendation is expected + err := waitForResourceRequestInRangeInPods( + f, pollTimeout, metav1.ListOptions{LabelSelector: "name=hamster"}, apiv1.ResourceCPU, + ParseQuantityOrDie(minimalCPULowerBound), ParseQuantityOrDie(minimalCPUUpperBound)) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + // consume more CPU to get a higher recommendation + rc.ConsumeCPU(600 * replicas) + err = waitForResourceRequestInRangeInPods( + f, pollTimeout, metav1.ListOptions{LabelSelector: "name=hamster"}, apiv1.ResourceCPU, + ParseQuantityOrDie("500m"), ParseQuantityOrDie("1300m")) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) +}) + +var _ = FullMpaE2eDescribe("Pods under MPA with non-recognized recommender explicitly configured", func() { + var ( + rc *ResourceConsumer + ) + replicas := 3 + + ginkgo.AfterEach(func() { + rc.CleanUp() + }) + + // This schedules AfterEach block that needs to run after the AfterEach above and + // BeforeEach that needs to run before the BeforeEach below - thus the order of these matters. + f := framework.NewDefaultFramework("multidimensional-pod-autoscaling") + f.NamespacePodSecurityEnforceLevel = podsecurity.LevelBaseline + + ginkgo.BeforeEach(func() { + ns := f.Namespace.Name + ginkgo.By("Setting up a hamster deployment") + rc = NewDynamicResourceConsumer("hamster", ns, KindDeployment, + replicas, + 1, /*initCPUTotal*/ + 10, /*initMemoryTotal*/ + 1, /*initCustomMetric*/ + initialCPU, /*cpuRequest*/ + initialMemory, /*memRequest*/ + f.ClientSet, + f.ScalesGetter) + + ginkgo.By("Setting up a MPA CRD with Recommender explicitly configured") + targetRef := &autoscaling.CrossVersionObjectReference{ + APIVersion: "apps/v1", + Kind: "Deployment", + Name: "hamster", + } + + containerName := GetHamsterContainerNameByIndex(0) + mpaCRD := test.MultidimPodAutoscaler(). + WithName("hamster-mpa"). + WithRecommender("non-recognized"). + WithNamespace(f.Namespace.Name). + WithScaleTargetRef(targetRef). + WithContainer(containerName). + AppendRecommendation( + test.Recommendation(). + WithContainer(containerName). + WithTarget("250m", "200Mi"). + WithLowerBound("250m", "200Mi"). + WithUpperBound("250m", "200Mi"). + GetContainerResources()). + Get() + + InstallMPA(f, mpaCRD) + + }) + + ginkgo.It("deployment not updated by non-recognized recommender", func() { + err := waitForResourceRequestInRangeInPods( + f, pollTimeout, metav1.ListOptions{LabelSelector: "name=hamster"}, apiv1.ResourceCPU, + ParseQuantityOrDie(minimalCPULowerBound), ParseQuantityOrDie(minimalCPUUpperBound)) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + // consume more CPU to get a higher recommendation + rc.ConsumeCPU(600 * replicas) + err = waitForResourceRequestInRangeInPods( + f, pollTimeout, metav1.ListOptions{LabelSelector: "name=hamster"}, apiv1.ResourceCPU, + ParseQuantityOrDie("500m"), ParseQuantityOrDie("1000m")) + gomega.Expect(err).To(gomega.HaveOccurred()) + }) +}) + +var _ = FullMpaE2eDescribe("OOMing pods under MPA", func() { + const replicas = 3 + + f := framework.NewDefaultFramework("multidimensional-pod-autoscaling") + f.NamespacePodSecurityEnforceLevel = podsecurity.LevelBaseline + + ginkgo.BeforeEach(func() { + ns := f.Namespace.Name + ginkgo.By("Setting up a hamster deployment") + + runOomingReplicationController( + f.ClientSet, + ns, + "hamster", + replicas) + ginkgo.By("Setting up a MPA CRD") + targetRef := &autoscaling.CrossVersionObjectReference{ + APIVersion: "v1", + Kind: "Deployment", + Name: "hamster", + } + + containerName := GetHamsterContainerNameByIndex(0) + mpaCRD := test.MultidimPodAutoscaler(). + WithName("hamster-mpa"). + WithNamespace(f.Namespace.Name). + WithScaleTargetRef(targetRef). + WithContainer(containerName). + Get() + + InstallMPA(f, mpaCRD) + }) + + ginkgo.It("have memory requests growing with OOMs", func() { + listOptions := metav1.ListOptions{ + LabelSelector: "name=hamster", + FieldSelector: getPodSelectorExcludingDonePodsOrDie(), + } + err := waitForResourceRequestInRangeInPods( + f, oomTestTimeout, listOptions, apiv1.ResourceMemory, + ParseQuantityOrDie("1400Mi"), ParseQuantityOrDie("10000Mi")) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) +}) + +func waitForPodsMatch(f *framework.Framework, timeout time.Duration, listOptions metav1.ListOptions, matcher func(pod apiv1.Pod) bool) error { + return wait.PollUntilContextTimeout(context.Background(), pollInterval, timeout, true, func(ctx context.Context) (done bool, err error) { + ns := f.Namespace.Name + c := f.ClientSet + + podList, err := c.CoreV1().Pods(ns).List(ctx, listOptions) + if err != nil { + return false, err + } + + if len(podList.Items) == 0 { + return false, nil + } + + // Run matcher on all pods, even if we find pod that doesn't match early. + // This allows the matcher to write logs for all pods. This in turns makes + // it easier to spot some problems (for example unexpected pods in the list + // results). + result := true + for _, pod := range podList.Items { + if !matcher(pod) { + result = false + } + } + return result, nil + + }) +} + +func waitForResourceRequestInRangeInPods(f *framework.Framework, timeout time.Duration, listOptions metav1.ListOptions, resourceName apiv1.ResourceName, lowerBound, upperBound resource.Quantity) error { + err := waitForPodsMatch(f, timeout, listOptions, + func(pod apiv1.Pod) bool { + resourceRequest, found := pod.Spec.Containers[0].Resources.Requests[resourceName] + framework.Logf("Comparing %v request %v against range of (%v, %v)", resourceName, resourceRequest, lowerBound, upperBound) + return found && resourceRequest.MilliValue() > lowerBound.MilliValue() && resourceRequest.MilliValue() < upperBound.MilliValue() + }) + + if err != nil { + return fmt.Errorf("error waiting for %s request in range of (%v,%v) for pods: %+v", resourceName, lowerBound, upperBound, listOptions) + } + return nil +} diff --git a/multidimensional-pod-autoscaler/e2e/v1alpha1/recommender.go b/multidimensional-pod-autoscaler/e2e/v1alpha1/recommender.go new file mode 100644 index 000000000000..c36edf868844 --- /dev/null +++ b/multidimensional-pod-autoscaler/e2e/v1alpha1/recommender.go @@ -0,0 +1,435 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package autoscaling + +import ( + "context" + "fmt" + "strings" + "time" + + autoscaling "k8s.io/api/autoscaling/v1" + apiv1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + mpa_types "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1alpha1" + mpa_clientset "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/client/clientset/versioned" + "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/utils/test" + vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" + klog "k8s.io/klog/v2" + "k8s.io/kubernetes/test/e2e/framework" + podsecurity "k8s.io/pod-security-admission/api" + + ginkgo "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" +) + +type resourceRecommendation struct { + target, lower, upper int64 +} + +func (r *resourceRecommendation) sub(other *resourceRecommendation) resourceRecommendation { + return resourceRecommendation{ + target: r.target - other.target, + lower: r.lower - other.lower, + upper: r.upper - other.upper, + } + +} + +func getResourceRecommendation(containerRecommendation *vpa_types.RecommendedContainerResources, r apiv1.ResourceName) resourceRecommendation { + getOrZero := func(resourceList apiv1.ResourceList) int64 { + value, found := resourceList[r] + if found { + return value.Value() + } + return 0 + } + return resourceRecommendation{ + target: getOrZero(containerRecommendation.Target), + lower: getOrZero(containerRecommendation.LowerBound), + upper: getOrZero(containerRecommendation.UpperBound), + } +} + +type recommendationChange struct { + oldMissing, newMissing bool + diff resourceRecommendation +} + +type observer struct { + channel chan recommendationChange +} + +func (*observer) OnAdd(obj interface{}, isInInitialList bool) {} +func (*observer) OnDelete(obj interface{}) {} + +func (o *observer) OnUpdate(oldObj, newObj interface{}) { + get := func(mpa *mpa_types.MultidimPodAutoscaler) (result resourceRecommendation, found bool) { + if mpa.Status.Recommendation == nil || len(mpa.Status.Recommendation.ContainerRecommendations) == 0 { + found = false + result = resourceRecommendation{} + } else { + found = true + result = getResourceRecommendation(&mpa.Status.Recommendation.ContainerRecommendations[0], apiv1.ResourceCPU) + } + return + } + oldMPA, _ := oldObj.(*mpa_types.MultidimPodAutoscaler) + NewMPA, _ := newObj.(*mpa_types.MultidimPodAutoscaler) + oldRecommendation, oldFound := get(oldMPA) + newRecommendation, newFound := get(NewMPA) + result := recommendationChange{ + oldMissing: !oldFound, + newMissing: !newFound, + diff: newRecommendation.sub(&oldRecommendation), + } + go func() { o.channel <- result }() +} + +func getMpaObserver(mpaClientSet mpa_clientset.Interface) *observer { + mpaListWatch := cache.NewListWatchFromClient(mpaClientSet.AutoscalingV1alpha1().RESTClient(), "multidimpodautoscalers", apiv1.NamespaceAll, fields.Everything()) + mpaObserver := observer{channel: make(chan recommendationChange)} + _, controller := cache.NewIndexerInformer(mpaListWatch, + &mpa_types.MultidimPodAutoscaler{}, + 1*time.Hour, + &mpaObserver, + cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + go controller.Run(make(chan struct{})) + if !cache.WaitForCacheSync(make(chan struct{}), controller.HasSynced) { + klog.Fatalf("Failed to sync MPA cache during initialization") + } else { + klog.InfoS("Initial MPA synced successfully") + } + return &mpaObserver +} + +var _ = RecommenderE2eDescribe("Checkpoints", func() { + f := framework.NewDefaultFramework("multidimensional-pod-autoscaling") + f.NamespacePodSecurityEnforceLevel = podsecurity.LevelBaseline + + ginkgo.It("with missing MPA objects are garbage collected", func() { + ns := f.Namespace.Name + mpaClientSet := getMpaClientSet(f) + + checkpoint := mpa_types.MultidimPodAutoscalerCheckpoint{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: ns, + }, + Spec: mpa_types.MultidimPodAutoscalerCheckpointSpec{ + MPAObjectName: "some-mpa", + }, + } + + _, err := mpaClientSet.AutoscalingV1alpha1().MultidimPodAutoscalerCheckpoints(ns).Create(context.TODO(), &checkpoint, metav1.CreateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + klog.InfoS("Sleeping for up to 15 minutes...") + + maxRetries := 90 + retryDelay := 10 * time.Second + for i := 0; i < maxRetries; i++ { + list, err := mpaClientSet.AutoscalingV1alpha1().MultidimPodAutoscalerCheckpoints(ns).List(context.TODO(), metav1.ListOptions{}) + if err == nil && len(list.Items) == 0 { + break + } + klog.InfoS("Still waiting...") + time.Sleep(retryDelay) + } + + list, err := mpaClientSet.AutoscalingV1alpha1().MultidimPodAutoscalerCheckpoints(ns).List(context.TODO(), metav1.ListOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(list.Items).To(gomega.BeEmpty()) + }) +}) + +var _ = RecommenderE2eDescribe("MPA CRD object", func() { + f := framework.NewDefaultFramework("multidimensional-pod-autoscaling") + f.NamespacePodSecurityEnforceLevel = podsecurity.LevelBaseline + + ginkgo.It("serves recommendation for CronJob", func() { + ginkgo.By("Setting up hamster CronJob") + SetupHamsterCronJob(f, "*/5 * * * *", "100m", "100Mi", defaultHamsterReplicas) + + mpaClientSet := getMpaClientSet(f) + + ginkgo.By("Setting up MPA") + targetRef := &autoscaling.CrossVersionObjectReference{ + APIVersion: "batch/v1", + Kind: "CronJob", + Name: "hamster-cronjob", + } + + containerName := GetHamsterContainerNameByIndex(0) + mpaCRD := test.MultidimPodAutoscaler(). + WithName("hamster-mpa"). + WithNamespace(f.Namespace.Name). + WithScaleTargetRef(targetRef). + WithContainer(containerName). + Get() + + InstallMPA(f, mpaCRD) + + ginkgo.By("Waiting for recommendation to be filled") + _, err := WaitForRecommendationPresent(mpaClientSet, mpaCRD) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) +}) + +var _ = RecommenderE2eDescribe("MPA CRD object", func() { + f := framework.NewDefaultFramework("multidimensional-pod-autoscaling") + f.NamespacePodSecurityEnforceLevel = podsecurity.LevelBaseline + + var ( + mpaCRD *mpa_types.MultidimPodAutoscaler + mpaClientSet mpa_clientset.Interface + ) + + ginkgo.BeforeEach(func() { + ginkgo.By("Setting up a hamster deployment") + _ = SetupHamsterDeployment( + f, /* framework */ + "100m", /* cpu */ + "100Mi", /* memory */ + 1, /* number of replicas */ + ) + + ginkgo.By("Setting up a MPA CRD") + containerName := GetHamsterContainerNameByIndex(0) + mpaCRD = test.MultidimPodAutoscaler(). + WithName("hamster-mpa"). + WithNamespace(f.Namespace.Name). + WithScaleTargetRef(hamsterTargetRef). + WithContainer(containerName). + Get() + + InstallMPA(f, mpaCRD) + + mpaClientSet = getMpaClientSet(f) + }) + + ginkgo.It("serves recommendation", func() { + ginkgo.By("Waiting for recommendation to be filled") + _, err := WaitForRecommendationPresent(mpaClientSet, mpaCRD) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) + + ginkgo.It("doesn't drop lower/upper after recommender's restart", func() { + + o := getMpaObserver(mpaClientSet) + + ginkgo.By("Waiting for recommendation to be filled") + _, err := WaitForRecommendationPresent(mpaClientSet, mpaCRD) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + ginkgo.By("Drain diffs") + out: + for { + select { + case recommendationDiff := <-o.channel: + fmt.Println("Dropping recommendation diff", recommendationDiff) + default: + break out + } + } + ginkgo.By("Deleting recommender") + gomega.Expect(deleteRecommender(f.ClientSet)).To(gomega.BeNil()) + ginkgo.By("Accumulating diffs after restart, sleeping for 5 minutes...") + time.Sleep(5 * time.Minute) + changeDetected := false + finish: + for { + select { + case recommendationDiff := <-o.channel: + fmt.Println("checking recommendation diff", recommendationDiff) + changeDetected = true + gomega.Expect(recommendationDiff.oldMissing).To(gomega.Equal(false)) + gomega.Expect(recommendationDiff.newMissing).To(gomega.Equal(false)) + gomega.Expect(recommendationDiff.diff.lower).Should(gomega.BeNumerically(">=", 0)) + gomega.Expect(recommendationDiff.diff.upper).Should(gomega.BeNumerically("<=", 0)) + default: + break finish + } + } + gomega.Expect(changeDetected).To(gomega.Equal(true)) + }) +}) + +var _ = RecommenderE2eDescribe("MPA CRD object", func() { + f := framework.NewDefaultFramework("multidimensional-pod-autoscaling") + f.NamespacePodSecurityEnforceLevel = podsecurity.LevelBaseline + + var ( + mpaClientSet mpa_clientset.Interface + ) + + ginkgo.BeforeEach(func() { + ginkgo.By("Setting up a hamster deployment") + _ = SetupHamsterDeployment( + f, /* framework */ + "100m", /* cpu */ + "100Mi", /* memory */ + 1, /* number of replicas */ + ) + + mpaClientSet = getMpaClientSet(f) + }) + + ginkgo.It("respects min allowed recommendation", func() { + const minMilliCpu = 10000 + ginkgo.By("Setting up a MPA CRD") + containerName := GetHamsterContainerNameByIndex(0) + mpaCRD2 := test.MultidimPodAutoscaler(). + WithName("hamster-mpa"). + WithNamespace(f.Namespace.Name). + WithScaleTargetRef(hamsterTargetRef). + WithContainer(containerName). + WithMinAllowed(containerName, "10000", ""). + Get() + + InstallMPA(f, mpaCRD2) + mpaCRD := mpaCRD2 + + ginkgo.By("Waiting for recommendation to be filled") + mpa, err := WaitForRecommendationPresent(mpaClientSet, mpaCRD) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(mpa.Status.Recommendation.ContainerRecommendations).Should(gomega.HaveLen(1)) + cpu := getMilliCpu(mpa.Status.Recommendation.ContainerRecommendations[0].Target) + gomega.Expect(cpu).Should(gomega.BeNumerically(">=", minMilliCpu), + fmt.Sprintf("target cpu recommendation should be greater than or equal to %dm", minMilliCpu)) + cpuUncapped := getMilliCpu(mpa.Status.Recommendation.ContainerRecommendations[0].UncappedTarget) + gomega.Expect(cpuUncapped).Should(gomega.BeNumerically("<", minMilliCpu), + fmt.Sprintf("uncapped target cpu recommendation should be less than %dm", minMilliCpu)) + }) + + ginkgo.It("respects max allowed recommendation", func() { + const maxMilliCpu = 1 + ginkgo.By("Setting up a MPA CRD") + containerName := GetHamsterContainerNameByIndex(0) + mpaCRD := test.MultidimPodAutoscaler(). + WithName("hamster-mpa"). + WithNamespace(f.Namespace.Name). + WithScaleTargetRef(hamsterTargetRef). + WithContainer(containerName). + WithMaxAllowed(containerName, "1m", ""). + Get() + + InstallMPA(f, mpaCRD) + + ginkgo.By("Waiting for recommendation to be filled") + mpa, err := WaitForUncappedCPURecommendationAbove(mpaClientSet, mpaCRD, maxMilliCpu) + gomega.Expect(err).NotTo(gomega.HaveOccurred(), fmt.Sprintf( + "Timed out waiting for uncapped cpu recommendation above %d mCPU", maxMilliCpu)) + gomega.Expect(mpa.Status.Recommendation.ContainerRecommendations).Should(gomega.HaveLen(1)) + cpu := getMilliCpu(mpa.Status.Recommendation.ContainerRecommendations[0].Target) + gomega.Expect(cpu).Should(gomega.BeNumerically("<=", maxMilliCpu), + fmt.Sprintf("target cpu recommendation should be less than or equal to %dm", maxMilliCpu)) + }) +}) + +func getMilliCpu(resources apiv1.ResourceList) int64 { + cpu := resources[apiv1.ResourceCPU] + return cpu.MilliValue() +} + +var _ = RecommenderE2eDescribe("MPA CRD object", func() { + f := framework.NewDefaultFramework("multidimensional-pod-autoscaling") + f.NamespacePodSecurityEnforceLevel = podsecurity.LevelBaseline + + var mpaClientSet mpa_clientset.Interface + + ginkgo.BeforeEach(func() { + mpaClientSet = getMpaClientSet(f) + }) + + ginkgo.It("with no containers opted out all containers get recommendations", func() { + ginkgo.By("Setting up a hamster deployment") + d := NewNHamstersDeployment(f, 2 /*number of containers*/) + _ = startDeploymentPods(f, d) + + ginkgo.By("Setting up MPA CRD") + container1Name := GetHamsterContainerNameByIndex(0) + container2Name := GetHamsterContainerNameByIndex(1) + mpaCRD := test.MultidimPodAutoscaler(). + WithName("hamster-mpa"). + WithNamespace(f.Namespace.Name). + WithScaleTargetRef(hamsterTargetRef). + WithContainer(container1Name). + WithContainer(container2Name). + Get() + + InstallMPA(f, mpaCRD) + + ginkgo.By("Waiting for recommendation to be filled for both containers") + mpa, err := WaitForRecommendationPresent(mpaClientSet, mpaCRD) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(mpa.Status.Recommendation.ContainerRecommendations).Should(gomega.HaveLen(2)) + }) + + ginkgo.It("only containers not-opted-out get recommendations", func() { + ginkgo.By("Setting up a hamster deployment") + d := NewNHamstersDeployment(f, 2 /*number of containers*/) + _ = startDeploymentPods(f, d) + + ginkgo.By("Setting up MPA CRD") + container1Name := GetHamsterContainerNameByIndex(0) + container2Name := GetHamsterContainerNameByIndex(1) + mpaCRD := test.MultidimPodAutoscaler(). + WithName("hamster-mpa"). + WithNamespace(f.Namespace.Name). + WithScaleTargetRef(hamsterTargetRef). + WithContainer(container1Name). + WithScalingMode(container1Name, vpa_types.ContainerScalingModeOff). + WithContainer(container2Name). + Get() + + InstallMPA(f, mpaCRD) + + ginkgo.By("Waiting for recommendation to be filled for just one container") + mpa, err := WaitForRecommendationPresent(mpaClientSet, mpaCRD) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + errMsg := fmt.Sprintf("%s container has recommendations turned off. We expect expect only recommendations for %s", + GetHamsterContainerNameByIndex(0), + GetHamsterContainerNameByIndex(1)) + gomega.Expect(mpa.Status.Recommendation.ContainerRecommendations).Should(gomega.HaveLen(1), errMsg) + gomega.Expect(mpa.Status.Recommendation.ContainerRecommendations[0].ContainerName).To(gomega.Equal(GetHamsterContainerNameByIndex(1)), errMsg) + }) +}) + +func deleteRecommender(c clientset.Interface) error { + namespace := "kube-system" + listOptions := metav1.ListOptions{} + podList, err := c.CoreV1().Pods(namespace).List(context.TODO(), listOptions) + if err != nil { + fmt.Println("Could not list pods.", err) + return err + } + fmt.Println("Pods list items:", len(podList.Items)) + for _, pod := range podList.Items { + if strings.HasPrefix(pod.Name, "mpa-recommender") { + fmt.Println("Deleting pod.", namespace, pod.Name) + err := c.CoreV1().Pods(namespace).Delete(context.TODO(), pod.Name, metav1.DeleteOptions{}) + if err != nil { + return err + } + return nil + } + } + return fmt.Errorf("mpa recommender not found") +} diff --git a/multidimensional-pod-autoscaler/e2e/v1alpha1/updater.go b/multidimensional-pod-autoscaler/e2e/v1alpha1/updater.go new file mode 100644 index 000000000000..fbbc7b871d98 --- /dev/null +++ b/multidimensional-pod-autoscaler/e2e/v1alpha1/updater.go @@ -0,0 +1,182 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package autoscaling + +import ( + "context" + "fmt" + "time" + + autoscaling "k8s.io/api/autoscaling/v1" + apiv1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/utils/test" + vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/status" + "k8s.io/kubernetes/test/e2e/framework" + podsecurity "k8s.io/pod-security-admission/api" + + ginkgo "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" +) + +var _ = UpdaterE2eDescribe("Updater", func() { + f := framework.NewDefaultFramework("multidimensional-pod-autoscaling") + f.NamespacePodSecurityEnforceLevel = podsecurity.LevelBaseline + + ginkgo.It("evicts pods when Admission Controller status available", func() { + const statusUpdateInterval = 10 * time.Second + + ginkgo.By("Setting up the Admission Controller status") + stopCh := make(chan struct{}) + statusUpdater := status.NewUpdater( + f.ClientSet, + status.AdmissionControllerStatusName, + status.AdmissionControllerStatusNamespace, + statusUpdateInterval, + "e2e test", + ) + defer func() { + // Schedule a cleanup of the Admission Controller status. + // Status is created outside the test namespace. + ginkgo.By("Deleting the Admission Controller status") + close(stopCh) + err := f.ClientSet.CoordinationV1().Leases(status.AdmissionControllerStatusNamespace). + Delete(context.TODO(), status.AdmissionControllerStatusName, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }() + statusUpdater.Run(stopCh) + + podList := setupPodsForUpscalingEviction(f) + + ginkgo.By("Waiting for pods to be evicted") + err := WaitForPodsEvicted(f, podList) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) + + ginkgo.It("evicts pods for downscaling", func() { + const statusUpdateInterval = 10 * time.Second + + ginkgo.By("Setting up the Admission Controller status") + stopCh := make(chan struct{}) + statusUpdater := status.NewUpdater( + f.ClientSet, + status.AdmissionControllerStatusName, + status.AdmissionControllerStatusNamespace, + statusUpdateInterval, + "e2e test", + ) + defer func() { + // Schedule a cleanup of the Admission Controller status. + // Status is created outside the test namespace. + ginkgo.By("Deleting the Admission Controller status") + close(stopCh) + err := f.ClientSet.CoordinationV1().Leases(status.AdmissionControllerStatusNamespace). + Delete(context.TODO(), status.AdmissionControllerStatusName, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }() + statusUpdater.Run(stopCh) + + podList := setupPodsForDownscalingEviction(f, nil) + + ginkgo.By("Waiting for pods to be evicted") + err := WaitForPodsEvicted(f, podList) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) + + ginkgo.It("does not evict pods for downscaling when EvictionRequirement prevents it", func() { + const statusUpdateInterval = 10 * time.Second + + ginkgo.By("Setting up the Admission Controller status") + stopCh := make(chan struct{}) + statusUpdater := status.NewUpdater( + f.ClientSet, + status.AdmissionControllerStatusName, + status.AdmissionControllerStatusNamespace, + statusUpdateInterval, + "e2e test", + ) + defer func() { + // Schedule a cleanup of the Admission Controller status. + // Status is created outside the test namespace. + ginkgo.By("Deleting the Admission Controller status") + close(stopCh) + err := f.ClientSet.CoordinationV1().Leases(status.AdmissionControllerStatusNamespace). + Delete(context.TODO(), status.AdmissionControllerStatusName, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }() + statusUpdater.Run(stopCh) + er := []*vpa_types.EvictionRequirement{ + { + Resources: []apiv1.ResourceName{apiv1.ResourceCPU}, + ChangeRequirement: vpa_types.TargetHigherThanRequests, + }, + } + podList := setupPodsForDownscalingEviction(f, er) + + ginkgo.By(fmt.Sprintf("Waiting for pods to be evicted, hoping it won't happen, sleep for %s", MpaEvictionTimeout.String())) + CheckNoPodsEvicted(f, MakePodSet(podList)) + }) + + ginkgo.It("doesn't evict pods when Admission Controller status unavailable", func() { + podList := setupPodsForUpscalingEviction(f) + + ginkgo.By(fmt.Sprintf("Waiting for pods to be evicted, hoping it won't happen, sleep for %s", MpaEvictionTimeout.String())) + CheckNoPodsEvicted(f, MakePodSet(podList)) + }) +}) + +func setupPodsForUpscalingEviction(f *framework.Framework) *apiv1.PodList { + return setupPodsForEviction(f, "100m", "100Mi", nil) +} + +func setupPodsForDownscalingEviction(f *framework.Framework, er []*vpa_types.EvictionRequirement) *apiv1.PodList { + return setupPodsForEviction(f, "500m", "500Mi", er) +} + +func setupPodsForEviction(f *framework.Framework, hamsterCPU, hamsterMemory string, er []*vpa_types.EvictionRequirement) *apiv1.PodList { + controller := &autoscaling.CrossVersionObjectReference{ + APIVersion: "apps/v1", + Kind: "Deployment", + Name: "hamster-deployment", + } + ginkgo.By(fmt.Sprintf("Setting up a hamster %v", controller.Kind)) + setupHamsterController(f, controller.Kind, hamsterCPU, hamsterMemory, defaultHamsterReplicas) + podList, err := GetHamsterPods(f) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + ginkgo.By("Setting up a MPA CRD") + containerName := GetHamsterContainerNameByIndex(0) + mpaCRD := test.MultidimPodAutoscaler(). + WithName("hamster-mpa"). + WithNamespace(f.Namespace.Name). + WithScaleTargetRef(controller). + // WithEvictionRequirements(er). // TODO: add this functionality + WithContainer(containerName). + AppendRecommendation( + test.Recommendation(). + WithContainer(containerName). + WithTarget(containerName, "200m"). + WithLowerBound(containerName, "200m"). + WithUpperBound(containerName, "200m"). + GetContainerResources()). + Get() + + InstallMPA(f, mpaCRD) + + return podList +} diff --git a/multidimensional-pod-autoscaler/examples/hamster-mpa.yaml b/multidimensional-pod-autoscaler/examples/hamster-mpa.yaml new file mode 100644 index 000000000000..ebab79b60923 --- /dev/null +++ b/multidimensional-pod-autoscaler/examples/hamster-mpa.yaml @@ -0,0 +1,40 @@ +# This config creates a corresponding Multidimensional Pod Autoscaler. +# Note that the update mode is left unset, so it defaults to "Auto" mode. +--- +apiVersion: "autoscaling.k8s.io/v1alpha1" +kind: MultidimPodAutoscaler +metadata: + name: hamster-mpa + namespace: default +spec: + # recommenders field can be unset when using the default recommender. + # When using an alternative recommender, the alternative recommender's name + # can be specified as the following in a list. + # recommenders: + # - name: 'hamster-recommender' + scaleTargetRef: + apiVersion: "apps/v1" + kind: Deployment + name: hamster + resourcePolicy: + containerPolicies: + - containerName: '*' + minAllowed: + cpu: 100m + memory: 50Mi + maxAllowed: + cpu: 1 + memory: 500Mi + controlledResources: ["cpu", "memory"] + constraints: + global: + minReplicas: 1 + maxReplicas: 6 + goals: + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 30 diff --git a/multidimensional-pod-autoscaler/examples/hamster.yaml b/multidimensional-pod-autoscaler/examples/hamster.yaml new file mode 100644 index 000000000000..5319d52dd10c --- /dev/null +++ b/multidimensional-pod-autoscaler/examples/hamster.yaml @@ -0,0 +1,36 @@ +# This config creates a deployment with two pods, each requesting 100 millicores +# and trying to utilize slightly above 500 millicores (repeatedly using CPU for +# 0.5s and sleeping 0.5s). +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: hamster + namespace: default +spec: + selector: + matchLabels: + app: hamster + replicas: 2 + template: + metadata: + labels: + app: hamster + spec: + securityContext: + runAsNonRoot: true + runAsUser: 65534 # nobody + containers: + - name: hamster + image: k8s.gcr.io/ubuntu-slim:0.1 + resources: + requests: + cpu: 100m + memory: 50Mi + limits: + cpu: 200m + memory: 100Mi + command: ["/bin/sh"] + args: + - "-c" + - "while true; do timeout 0.5s yes >/dev/null; sleep 0.5s; done" diff --git a/multidimensional-pod-autoscaler/go.mod b/multidimensional-pod-autoscaler/go.mod new file mode 100644 index 000000000000..e807f030caee --- /dev/null +++ b/multidimensional-pod-autoscaler/go.mod @@ -0,0 +1,151 @@ +module k8s.io/autoscaler/multidimensional-pod-autoscaler + +go 1.23.0 + +toolchain go1.23.3 + +require ( + github.com/fsnotify/fsnotify v1.8.0 + github.com/golang/mock v1.6.0 + github.com/prometheus/client_golang v1.20.5 + github.com/spf13/pflag v1.0.5 + github.com/stretchr/testify v1.10.0 + golang.org/x/time v0.8.0 + k8s.io/api v0.32.0 + k8s.io/apimachinery v0.32.0 + k8s.io/autoscaler/vertical-pod-autoscaler v1.2.1 + k8s.io/client-go v0.32.0 + k8s.io/code-generator v0.32.0 + k8s.io/component-base v0.32.0 + k8s.io/klog/v2 v2.130.1 + k8s.io/kubernetes v1.32.0 + k8s.io/metrics v0.32.0 +) + +require ( + cel.dev/expr v0.18.0 // indirect + github.com/NYTimes/gziphandler v1.1.1 // indirect + github.com/antlr4-go/antlr/v4 v4.13.0 // indirect + github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/blang/semver/v4 v4.0.0 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/coreos/go-semver v0.3.1 // indirect + github.com/coreos/go-systemd/v22 v22.5.0 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/distribution/reference v0.6.0 // indirect + github.com/emicklei/go-restful/v3 v3.11.0 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/fxamacker/cbor/v2 v2.7.0 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.23.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/google/btree v1.0.1 // indirect + github.com/google/cel-go v0.22.0 // indirect + github.com/google/gnostic-models v0.6.8 // indirect + github.com/google/go-cmp v0.6.0 // indirect + github.com/google/gofuzz v1.2.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/compress v1.17.9 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.61.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect + github.com/spf13/cobra v1.8.1 // indirect + github.com/stoewer/go-strcase v1.3.0 // indirect + github.com/stretchr/objx v0.5.2 // indirect + github.com/x448/float16 v0.8.4 // indirect + go.etcd.io/etcd/api/v3 v3.5.16 // indirect + go.etcd.io/etcd/client/pkg/v3 v3.5.16 // indirect + go.etcd.io/etcd/client/v3 v3.5.16 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.53.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 // indirect + go.opentelemetry.io/otel v1.28.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0 // indirect + go.opentelemetry.io/otel/metric v1.28.0 // indirect + go.opentelemetry.io/otel/sdk v1.28.0 // indirect + go.opentelemetry.io/otel/trace v1.28.0 // indirect + go.opentelemetry.io/proto/otlp v1.3.1 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.27.0 // indirect + golang.org/x/crypto v0.30.0 // indirect + golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect + golang.org/x/mod v0.21.0 // indirect + golang.org/x/net v0.32.0 // indirect + golang.org/x/oauth2 v0.24.0 // indirect + golang.org/x/sync v0.10.0 // indirect + golang.org/x/sys v0.28.0 // indirect + golang.org/x/term v0.27.0 // indirect + golang.org/x/text v0.21.0 // indirect + golang.org/x/tools v0.26.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240826202546-f6391c0de4c7 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240826202546-f6391c0de4c7 // indirect + google.golang.org/grpc v1.65.0 // indirect + google.golang.org/protobuf v1.35.2 // indirect + gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/apiextensions-apiserver v0.0.0 // indirect + k8s.io/apiserver v0.32.0 // indirect + k8s.io/cloud-provider v0.0.0 // indirect + k8s.io/component-helpers v0.32.0 // indirect + k8s.io/controller-manager v0.32.0 // indirect + k8s.io/gengo/v2 v2.0.0-20240911193312-2b36238f13e9 // indirect + k8s.io/kms v0.32.0 // indirect + k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f // indirect + k8s.io/kubelet v0.0.0 // indirect + k8s.io/utils v0.0.0-20241210054802-24370beab758 // indirect + sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.0 // indirect + sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.4.2 // indirect + sigs.k8s.io/yaml v1.4.0 // indirect +) + +replace ( + k8s.io/api => k8s.io/api v0.32.0 + k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.32.0 + k8s.io/apimachinery => k8s.io/apimachinery v0.32.0 + k8s.io/apiserver => k8s.io/apiserver v0.32.0 + k8s.io/autoscaler/vertical-pod-autoscaler => ../vertical-pod-autoscaler + k8s.io/cli-runtime => k8s.io/cli-runtime v0.32.0 + k8s.io/client-go => k8s.io/client-go v0.32.0 + k8s.io/cloud-provider => k8s.io/cloud-provider v0.32.0 + k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.32.0 + k8s.io/code-generator => k8s.io/code-generator v0.32.0 + k8s.io/component-base => k8s.io/component-base v0.32.0 + k8s.io/component-helpers => k8s.io/component-helpers v0.32.0 + k8s.io/controller-manager => k8s.io/controller-manager v0.32.0 + k8s.io/cri-api => k8s.io/cri-api v0.32.0 + k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.32.0 + k8s.io/dynamic-resource-allocation => k8s.io/dynamic-resource-allocation v0.32.0 + k8s.io/kms => k8s.io/kms v0.32.0 + k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.32.0 + k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.32.0 + k8s.io/kube-proxy => k8s.io/kube-proxy v0.32.0 + k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.32.0 + k8s.io/kubectl => k8s.io/kubectl v0.32.0 + k8s.io/kubelet => k8s.io/kubelet v0.32.0 + k8s.io/metrics => k8s.io/metrics v0.32.0 + k8s.io/mount-utils => k8s.io/mount-utils v0.32.0 + k8s.io/pod-security-admission => k8s.io/pod-security-admission v0.32.0 + k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.32.0 +) diff --git a/multidimensional-pod-autoscaler/go.sum b/multidimensional-pod-autoscaler/go.sum new file mode 100644 index 000000000000..40730ab6c9c7 --- /dev/null +++ b/multidimensional-pod-autoscaler/go.sum @@ -0,0 +1,339 @@ +cel.dev/expr v0.18.0 h1:CJ6drgk+Hf96lkLikr4rFf19WrU0BOWEihyZnI2TAzo= +cel.dev/expr v0.18.0/go.mod h1:MrpN08Q+lEBs+bGYdLxxHkZoUSsCp0nSKTs0nTymJgw= +github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I= +github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= +github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI= +github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= +github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA= +github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= +github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= +github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= +github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= +github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= +github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= +github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= +github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= +github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= +github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= +github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= +github.com/google/cel-go v0.22.0 h1:b3FJZxpiv1vTMo2/5RDUqAHPxkT8mmMfJIrq1llbf7g= +github.com/google/cel-go v0.22.0/go.mod h1:BuznPXXfQDpXKWQ9sPW3TzlAJN5zzFe+i9tIs0yC4s8= +github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= +github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= +github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= +github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 h1:bkypFPDjIYGfCYD5mRBvpqxfYX1YCS1PXdKYWi8FsN0= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737NnSbmxQAppXMRziHUxPOC8k= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jonboulle/clockwork v0.4.0 h1:p4Cf1aMWXnXAUh8lVfewRBx1zaTSYKrKMF2g3ST4RZ4= +github.com/jonboulle/clockwork v0.4.0/go.mod h1:xgRqUGwRcjKCO1vbZUEtSLrqKoPSsUpK7fnezOII0kc= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/onsi/ginkgo/v2 v2.21.0 h1:7rg/4f3rB88pb5obDgNZrNHrQ4e6WpjonchcpuBRnZM= +github.com/onsi/ginkgo/v2 v2.21.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= +github.com/onsi/gomega v1.35.1 h1:Cwbd75ZBPxFSuZ6T+rN/WCb/gOc6YgFBXLlZLhC7Ds4= +github.com/onsi/gomega v1.35.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= +github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.61.0 h1:3gv/GThfX0cV2lpO7gkTUwZru38mxevy90Bj8YFSRQQ= +github.com/prometheus/common v0.61.0/go.mod h1:zr29OCN/2BsJRaFwG8QOBr41D6kkchKbpeNH7pAjb/s= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= +github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= +github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= +github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stoewer/go-strcase v1.3.0 h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AVEzs= +github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7otjonDflCTK0BCfls4SPy3NcCVb5dqqmbRknE= +github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75/go.mod h1:KO6IkyS8Y3j8OdNO85qEYBsRPuteD+YciPomcXdrMnk= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510 h1:S2dVYn90KE98chqDkyE9Z4N61UnQd+KOfgp5Iu53llk= +github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +go.etcd.io/bbolt v1.3.11 h1:yGEzV1wPz2yVCLsD8ZAiGHhHVlczyC9d1rP43/VCRJ0= +go.etcd.io/bbolt v1.3.11/go.mod h1:dksAq7YMXoljX0xu6VF5DMZGbhYYoLUalEiSySYAS4I= +go.etcd.io/etcd/api/v3 v3.5.16 h1:WvmyJVbjWqK4R1E+B12RRHz3bRGy9XVfh++MgbN+6n0= +go.etcd.io/etcd/api/v3 v3.5.16/go.mod h1:1P4SlIP/VwkDmGo3OlOD7faPeP8KDIFhqvciH5EfN28= +go.etcd.io/etcd/client/pkg/v3 v3.5.16 h1:ZgY48uH6UvB+/7R9Yf4x574uCO3jIx0TRDyetSfId3Q= +go.etcd.io/etcd/client/pkg/v3 v3.5.16/go.mod h1:V8acl8pcEK0Y2g19YlOV9m9ssUe6MgiDSobSoaBAM0E= +go.etcd.io/etcd/client/v2 v2.305.16 h1:kQrn9o5czVNaukf2A2At43cE9ZtWauOtf9vRZuiKXow= +go.etcd.io/etcd/client/v2 v2.305.16/go.mod h1:h9YxWCzcdvZENbfzBTFCnoNumr2ax3F19sKMqHFmXHE= +go.etcd.io/etcd/client/v3 v3.5.16 h1:sSmVYOAHeC9doqi0gv7v86oY/BTld0SEFGaxsU9eRhE= +go.etcd.io/etcd/client/v3 v3.5.16/go.mod h1:X+rExSGkyqxvu276cr2OwPLBaeqFu1cIl4vmRjAD/50= +go.etcd.io/etcd/pkg/v3 v3.5.16 h1:cnavs5WSPWeK4TYwPYfmcr3Joz9BH+TZ6qoUtz6/+mc= +go.etcd.io/etcd/pkg/v3 v3.5.16/go.mod h1:+lutCZHG5MBBFI/U4eYT5yL7sJfnexsoM20Y0t2uNuY= +go.etcd.io/etcd/raft/v3 v3.5.16 h1:zBXA3ZUpYs1AwiLGPafYAKKl/CORn/uaxYDwlNwndAk= +go.etcd.io/etcd/raft/v3 v3.5.16/go.mod h1:P4UP14AxofMJ/54boWilabqqWoW9eLodl6I5GdGzazI= +go.etcd.io/etcd/server/v3 v3.5.16 h1:d0/SAdJ3vVsZvF8IFVb1k8zqMZ+heGcNfft71ul9GWE= +go.etcd.io/etcd/server/v3 v3.5.16/go.mod h1:ynhyZZpdDp1Gq49jkUg5mfkDWZwXnn3eIqCqtJnrD/s= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.53.0 h1:9G6E0TXzGFVfTnawRzrPl83iHOAV7L8NJiR8RSGYV1g= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.53.0/go.mod h1:azvtTADFQJA8mX80jIH/akaE7h+dbm/sVuaHqN13w74= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 h1:4K4tsIXefpVJtvA/8srF4V4y0akAoPHkIslgAkjixJA= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0/go.mod h1:jjdQuTGVsXV4vSs+CJ2qYDeDPf9yIJV23qlIzBm73Vg= +go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo= +go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 h1:3Q/xZUyC1BBkualc9ROb4G8qkH90LXEIICcs5zv1OYY= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0/go.mod h1:s75jGIWA9OfCMzF0xr+ZgfrB5FEbbV7UuYo32ahUiFI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0 h1:qFffATk0X+HD+f1Z8lswGiOQYKHRlzfmdJm0wEaVrFA= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0/go.mod h1:MOiCmryaYtc+V0Ei+Tx9o5S1ZjA7kzLucuVuyzBZloQ= +go.opentelemetry.io/otel/metric v1.28.0 h1:f0HGvSl1KRAU1DLgLGFjrwVyismPlnuU6JD6bOeuA5Q= +go.opentelemetry.io/otel/metric v1.28.0/go.mod h1:Fb1eVBFZmLVTMb6PPohq3TO9IIhUisDsbJoL/+uQW4s= +go.opentelemetry.io/otel/sdk v1.28.0 h1:b9d7hIry8yZsgtbmM0DKyPWMMUMlK9NEKuIG4aBqWyE= +go.opentelemetry.io/otel/sdk v1.28.0/go.mod h1:oYj7ClPUA7Iw3m+r7GeEjz0qckQRJK2B8zjcZEfu7Pg= +go.opentelemetry.io/otel/trace v1.28.0 h1:GhQ9cUuQGmNDd5BTCP2dAvv75RdMxEfTmYejp+lkx9g= +go.opentelemetry.io/otel/trace v1.28.0/go.mod h1:jPyXzNPg6da9+38HEwElrQiHlVMTnVfM3/yv2OlIHaI= +go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0= +go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.30.0 h1:RwoQn3GkWiMkzlX562cLB7OxWvjH1L8xutO2WoJcRoY= +golang.org/x/crypto v0.30.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0= +golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= +golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/oauth2 v0.24.0 h1:KTBBxWqUa0ykRPLtV69rRto9TLXcqYkeswu48x/gvNE= +golang.org/x/oauth2 v0.24.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= +golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/time v0.8.0 h1:9i3RxcPv3PZnitoVGMPDKZSq1xW1gK1Xy3ArNOGZfEg= +golang.org/x/time v0.8.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ= +golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80 h1:KAeGQVN3M9nD0/bQXnr/ClcEMJ968gUXJQ9pwfSynuQ= +google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80/go.mod h1:cc8bqMqtv9gMOr0zHg2Vzff5ULhhL2IXP4sbcn32Dro= +google.golang.org/genproto/googleapis/api v0.0.0-20240826202546-f6391c0de4c7 h1:YcyjlL1PRr2Q17/I0dPk2JmYS5CDXfcdb2Z3YRioEbw= +google.golang.org/genproto/googleapis/api v0.0.0-20240826202546-f6391c0de4c7/go.mod h1:OCdP9MfskevB/rbYvHTsXTtKC+3bHWajPdoKgjcYkfo= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240826202546-f6391c0de4c7 h1:2035KHhUv+EpyB+hWgJnaWKJOdX1E95w2S8Rr4uWKTs= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240826202546-f6391c0de4c7/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= +google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc= +google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ= +google.golang.org/protobuf v1.35.2 h1:8Ar7bF+apOIoThw1EdZl0p1oWvMqTHmpA2fRTyZO8io= +google.golang.org/protobuf v1.35.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= +gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= +gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/api v0.32.0 h1:OL9JpbvAU5ny9ga2fb24X8H6xQlVp+aJMFlgtQjR9CE= +k8s.io/api v0.32.0/go.mod h1:4LEwHZEf6Q/cG96F3dqR965sYOfmPM7rq81BLgsE0p0= +k8s.io/apiextensions-apiserver v0.32.0 h1:S0Xlqt51qzzqjKPxfgX1xh4HBZE+p8KKBq+k2SWNOE0= +k8s.io/apiextensions-apiserver v0.32.0/go.mod h1:86hblMvN5yxMvZrZFX2OhIHAuFIMJIZ19bTvzkP+Fmw= +k8s.io/apimachinery v0.32.0 h1:cFSE7N3rmEEtv4ei5X6DaJPHHX0C+upp+v5lVPiEwpg= +k8s.io/apimachinery v0.32.0/go.mod h1:GpHVgxoKlTxClKcteaeuF1Ul/lDVb74KpZcxcmLDElE= +k8s.io/apiserver v0.32.0 h1:VJ89ZvQZ8p1sLeiWdRJpRD6oLozNZD2+qVSLi+ft5Qs= +k8s.io/apiserver v0.32.0/go.mod h1:HFh+dM1/BE/Hm4bS4nTXHVfN6Z6tFIZPi649n83b4Ag= +k8s.io/client-go v0.32.0 h1:DimtMcnN/JIKZcrSrstiwvvZvLjG0aSxy8PxN8IChp8= +k8s.io/client-go v0.32.0/go.mod h1:boDWvdM1Drk4NJj/VddSLnx59X3OPgwrOo0vGbtq9+8= +k8s.io/cloud-provider v0.32.0 h1:QXYJGmwME2q2rprymbmw2GroMChQYc/MWN6l/I4Kgp8= +k8s.io/cloud-provider v0.32.0/go.mod h1:cz3gVodkhgwi2ugj/JUPglIruLSdDaThxawuDyCHfr8= +k8s.io/code-generator v0.32.0 h1:s0lNN8VSWny8LBz5t5iy7MCdgwdOhdg7vAGVxvS+VWU= +k8s.io/code-generator v0.32.0/go.mod h1:b7Q7KMZkvsYFy72A79QYjiv4aTz3GvW0f1T3UfhFq4s= +k8s.io/component-base v0.32.0 h1:d6cWHZkCiiep41ObYQS6IcgzOUQUNpywm39KVYaUqzU= +k8s.io/component-base v0.32.0/go.mod h1:JLG2W5TUxUu5uDyKiH2R/7NnxJo1HlPoRIIbVLkK5eM= +k8s.io/component-helpers v0.32.0 h1:pQEEBmRt3pDJJX98cQvZshDgJFeKRM4YtYkMmfOlczw= +k8s.io/component-helpers v0.32.0/go.mod h1:9RuClQatbClcokXOcDWSzFKQm1huIf0FzQlPRpizlMc= +k8s.io/controller-manager v0.32.0 h1:tpQl1rvH4huFB6Avl1nhowZHtZoCNWqn6OYdZPl7Ybc= +k8s.io/controller-manager v0.32.0/go.mod h1:JRuYnYCkKj3NgBTy+KNQKIUm/lJRoDAvGbfdEmk9LhY= +k8s.io/gengo/v2 v2.0.0-20240911193312-2b36238f13e9 h1:si3PfKm8dDYxgfbeA6orqrtLkvvIeH8UqffFJDl0bz4= +k8s.io/gengo/v2 v2.0.0-20240911193312-2b36238f13e9/go.mod h1:EJykeLsmFC60UQbYJezXkEsG2FLrt0GPNkU5iK5GWxU= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kms v0.32.0 h1:jwOfunHIrcdYl5FRcA+uUKKtg6qiqoPCwmS2T3XTYL4= +k8s.io/kms v0.32.0/go.mod h1:Bk2evz/Yvk0oVrvm4MvZbgq8BD34Ksxs2SRHn4/UiOM= +k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f h1:GA7//TjRY9yWGy1poLzYYJJ4JRdzg3+O6e8I+e+8T5Y= +k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f/go.mod h1:R/HEjbvWI0qdfb8viZUeVZm0X6IZnxAydC7YU42CMw4= +k8s.io/kubelet v0.32.0 h1:uLyiKlz195Wo4an/K2tyge8o3QHx0ZkhVN3pevvp59A= +k8s.io/kubelet v0.32.0/go.mod h1:lAwuVZT/Hm7EdLn0jW2D+WdrJoorjJL2rVSdhOFnegw= +k8s.io/kubernetes v1.32.0 h1:4BDBWSolqPrv8GC3YfZw0CJvh5kA1TPnoX0FxDVd+qc= +k8s.io/kubernetes v1.32.0/go.mod h1:tiIKO63GcdPRBHW2WiUFm3C0eoLczl3f7qi56Dm1W8I= +k8s.io/metrics v0.32.0 h1:70qJ3ZS/9DrtH0UA0NVBI6gW2ip2GAn9e7NtoKERpns= +k8s.io/metrics v0.32.0/go.mod h1:skdg9pDjVjCPIQqmc5rBzDL4noY64ORhKu9KCPv1+QI= +k8s.io/utils v0.0.0-20241210054802-24370beab758 h1:sdbE21q2nlQtFh65saZY+rRM6x6aJJI8IUa1AmH/qa0= +k8s.io/utils v0.0.0-20241210054802-24370beab758/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.0 h1:CPT0ExVicCzcpeN4baWEV2ko2Z/AsiZgEdwgcfwLgMo= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.0/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= +sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 h1:/Rv+M11QRah1itp8VhT6HoVx1Ray9eB4DBr+K+/sCJ8= +sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3/go.mod h1:18nIHnGi6636UCz6m8i4DhaJ65T6EruyzmoQqI2BVDo= +sigs.k8s.io/structured-merge-diff/v4 v4.4.2 h1:MdmvkGuXi/8io6ixD5wud3vOLwc1rj0aNqRlpuvjmwA= +sigs.k8s.io/structured-merge-diff/v4 v4.4.2/go.mod h1:N8f93tFZh9U6vpxwRArLiikrE5/2tiu1w1AGfACIGE4= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/multidimensional-pod-autoscaler/hack/boilerplate.go.txt b/multidimensional-pod-autoscaler/hack/boilerplate.go.txt new file mode 100644 index 000000000000..0926592d3895 --- /dev/null +++ b/multidimensional-pod-autoscaler/hack/boilerplate.go.txt @@ -0,0 +1,15 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ diff --git a/multidimensional-pod-autoscaler/hack/deploy-for-e2e-locally.sh b/multidimensional-pod-autoscaler/hack/deploy-for-e2e-locally.sh new file mode 100755 index 000000000000..93537fdee4b0 --- /dev/null +++ b/multidimensional-pod-autoscaler/hack/deploy-for-e2e-locally.sh @@ -0,0 +1,97 @@ +#!/bin/bash + +# Copyright 2023 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +SCRIPT_ROOT=$(dirname ${BASH_SOURCE})/.. +BASE_NAME=$(basename $0) +source "${SCRIPT_ROOT}/hack/lib/util.sh" + +ARCH=$(kube::util::host_arch) + +function print_help { + echo "ERROR! Usage: $BASE_NAME [suite]*" + echo " should be one of:" + echo " - recommender" + echo " - recommender-externalmetrics" + echo " - updater" + echo " - admission-controller" + echo " - full-mpa" +} + +if [ $# -eq 0 ]; then + print_help + exit 1 +fi + +if [ $# -gt 1 ]; then + print_help + exit 1 +fi + +SUITE=$1 + +case ${SUITE} in + recommender|recommender-externalmetrics|updater|admission-controller) + COMPONENTS="${SUITE}" + ;; + full-mpa) + COMPONENTS="recommender updater admission-controller" + ;; + *) + print_help + exit 1 + ;; +esac + +# Local KIND images +export REGISTRY=${REGISTRY:-localhost:5001} +export TAG=${TAG:-latest} + +rm -f ${SCRIPT_ROOT}/hack/e2e/mpa-rbac.yaml +patch -c ${SCRIPT_ROOT}/deploy/mpa-rbac.yaml -i ${SCRIPT_ROOT}/hack/e2e/mpa-rbac.diff -o ${SCRIPT_ROOT}/hack/e2e/mpa-rbac.yaml +kubectl apply -f ${SCRIPT_ROOT}/hack/e2e/mpa-rbac.yaml +# Other-versioned CRDs are irrelevant as we're running a modern-ish cluster. +kubectl apply -f ${SCRIPT_ROOT}/deploy/mpa-v1alpha1-crd-gen.yaml +kubectl apply -f ${SCRIPT_ROOT}/hack/e2e/k8s-metrics-server.yaml + +for i in ${COMPONENTS}; do + if [ $i == recommender-externalmetrics ] ; then + i=recommender + fi + if [ $i == admission-controller ] ; then + (cd ${SCRIPT_ROOT}/pkg/${i} && bash ./gencerts.sh e2e || true) + fi + ALL_ARCHITECTURES=${ARCH} make --directory ${SCRIPT_ROOT}/pkg/${i} docker-build REGISTRY=${REGISTRY} TAG=${TAG} + docker tag ${REGISTRY}/mpa-${i}-${ARCH}:${TAG} ${REGISTRY}/mpa-${i}:${TAG} + kind load docker-image ${REGISTRY}/mpa-${i}:${TAG} +done + + +for i in ${COMPONENTS}; do + if [ $i == recommender-externalmetrics ] ; then + kubectl delete namespace monitoring --ignore-not-found=true + kubectl create namespace monitoring + kubectl apply -f ${SCRIPT_ROOT}/hack/e2e/prometheus.yaml + kubectl apply -f ${SCRIPT_ROOT}/hack/e2e/prometheus-adapter.yaml + kubectl apply -f ${SCRIPT_ROOT}/hack/e2e/metrics-pump.yaml + kubectl apply -f ${SCRIPT_ROOT}/hack/e2e/${i}-deployment.yaml + else + REGISTRY=${REGISTRY} TAG=${TAG} ${SCRIPT_ROOT}/hack/mpa-process-yaml.sh ${SCRIPT_ROOT}/deploy/${i}-deployment.yaml | kubectl apply -f - + fi +done diff --git a/multidimensional-pod-autoscaler/hack/deploy-for-e2e.sh b/multidimensional-pod-autoscaler/hack/deploy-for-e2e.sh new file mode 100755 index 000000000000..695c032072fc --- /dev/null +++ b/multidimensional-pod-autoscaler/hack/deploy-for-e2e.sh @@ -0,0 +1,81 @@ +#!/bin/bash + +# Copyright 2018 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +SCRIPT_ROOT=$(dirname ${BASH_SOURCE})/.. + +function print_help { + echo "ERROR! Usage: deploy-for-e2e.sh [suite]*" + echo " should be one of:" + echo " - recommender" + echo " - updater" + echo " - admission-controller" + echo " - actuation" + echo " - full-vpa" + echo "If component is not specified all above will be started." +} + +if [ $# -eq 0 ]; then + print_help + exit 1 +fi + +if [ $# -gt 1 ]; then + print_help + exit 1 +fi + +SUITE=$1 + +case ${SUITE} in + recommender|updater|admission-controller) + COMPONENTS="${SUITE}" + ;; + full-vpa) + COMPONENTS="recommender updater admission-controller" + ;; + actuation) + COMPONENTS="updater admission-controller" + ;; + *) + print_help + exit 1 + ;; +esac + +export REGISTRY=gcr.io/`gcloud config get-value core/project` +export TAG=latest + +echo "Configuring registry authentication" +mkdir -p "${HOME}/.docker" +gcloud auth configure-docker -q + +for i in ${COMPONENTS}; do + if [ $i == admission-controller ] ; then + (cd ${SCRIPT_ROOT}/pkg/${i} && bash ./gencerts.sh e2e || true) + fi + ALL_ARCHITECTURES=amd64 make --directory ${SCRIPT_ROOT}/pkg/${i} release +done + +kubectl create -f ${SCRIPT_ROOT}/deploy/mpa-v1alpha1-crd-gen.yaml +kubectl create -f ${SCRIPT_ROOT}/deploy/mpa-rbac.yaml + +for i in ${COMPONENTS}; do + ${SCRIPT_ROOT}/hack/vpa-process-yaml.sh ${SCRIPT_ROOT}/deploy/${i}-deployment.yaml | kubectl create -f - +done diff --git a/multidimensional-pod-autoscaler/hack/e2e/Dockerfile.externalmetrics-writer b/multidimensional-pod-autoscaler/hack/e2e/Dockerfile.externalmetrics-writer new file mode 100644 index 000000000000..e4fe233b064f --- /dev/null +++ b/multidimensional-pod-autoscaler/hack/e2e/Dockerfile.externalmetrics-writer @@ -0,0 +1,18 @@ +# Copyright 2024 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM python:3.10-slim +RUN pip3 install kubernetes argparse requests + +COPY emit-metrics.py / diff --git a/multidimensional-pod-autoscaler/hack/e2e/k8s-metrics-server.yaml b/multidimensional-pod-autoscaler/hack/e2e/k8s-metrics-server.yaml new file mode 100644 index 000000000000..25984da14287 --- /dev/null +++ b/multidimensional-pod-autoscaler/hack/e2e/k8s-metrics-server.yaml @@ -0,0 +1,244 @@ +--- +# Source: metrics-server/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: local-metrics-server + namespace: default + labels: + helm.sh/chart: metrics-server-3.8.3 + app.kubernetes.io/name: metrics-server + app.kubernetes.io/instance: local-metrics-server + app.kubernetes.io/version: "0.6.2" + app.kubernetes.io/managed-by: Helm +--- +# Source: metrics-server/templates/clusterrole-aggregated-reader.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: system:metrics-server-aggregated-reader + labels: + helm.sh/chart: metrics-server-3.8.3 + app.kubernetes.io/name: metrics-server + app.kubernetes.io/instance: local-metrics-server + app.kubernetes.io/version: "0.6.2" + app.kubernetes.io/managed-by: Helm + rbac.authorization.k8s.io/aggregate-to-admin: "true" + rbac.authorization.k8s.io/aggregate-to-edit: "true" + rbac.authorization.k8s.io/aggregate-to-view: "true" +rules: + - apiGroups: + - metrics.k8s.io + resources: + - pods + - nodes + verbs: + - get + - list + - watch +--- +# Source: metrics-server/templates/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: system:local-metrics-server + labels: + helm.sh/chart: metrics-server-3.8.3 + app.kubernetes.io/name: metrics-server + app.kubernetes.io/instance: local-metrics-server + app.kubernetes.io/version: "0.6.2" + app.kubernetes.io/managed-by: Helm +rules: + - apiGroups: + - "" + resources: + - nodes/metrics + verbs: + - get + - apiGroups: + - "" + resources: + - pods + - nodes + - namespaces + - configmaps + verbs: + - get + - list + - watch +--- +# Source: metrics-server/templates/clusterrolebinding-auth-delegator.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: local-metrics-server:system:auth-delegator + labels: + helm.sh/chart: metrics-server-3.8.3 + app.kubernetes.io/name: metrics-server + app.kubernetes.io/instance: local-metrics-server + app.kubernetes.io/version: "0.6.2" + app.kubernetes.io/managed-by: Helm +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator +subjects: + - kind: ServiceAccount + name: local-metrics-server + namespace: default +--- +# Source: metrics-server/templates/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: system:local-metrics-server + labels: + helm.sh/chart: metrics-server-3.8.3 + app.kubernetes.io/name: metrics-server + app.kubernetes.io/instance: local-metrics-server + app.kubernetes.io/version: "0.6.2" + app.kubernetes.io/managed-by: Helm +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:local-metrics-server +subjects: + - kind: ServiceAccount + name: local-metrics-server + namespace: default +--- +# Source: metrics-server/templates/rolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: local-metrics-server-auth-reader + namespace: default + labels: + helm.sh/chart: metrics-server-3.8.3 + app.kubernetes.io/name: metrics-server + app.kubernetes.io/instance: local-metrics-server + app.kubernetes.io/version: "0.6.2" + app.kubernetes.io/managed-by: Helm +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: + - kind: ServiceAccount + name: local-metrics-server + namespace: default +--- +# Source: metrics-server/templates/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: local-metrics-server + namespace: default + labels: + helm.sh/chart: metrics-server-3.8.3 + app.kubernetes.io/name: metrics-server + app.kubernetes.io/instance: local-metrics-server + app.kubernetes.io/version: "0.6.2" + app.kubernetes.io/managed-by: Helm +spec: + type: ClusterIP + ports: + - name: https + port: 443 + protocol: TCP + targetPort: https + selector: + app.kubernetes.io/name: metrics-server + app.kubernetes.io/instance: local-metrics-server +--- +# Source: metrics-server/templates/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: local-metrics-server + namespace: default + labels: + helm.sh/chart: metrics-server-3.8.3 + app.kubernetes.io/name: metrics-server + app.kubernetes.io/instance: local-metrics-server + app.kubernetes.io/version: "0.6.2" + app.kubernetes.io/managed-by: Helm +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: metrics-server + app.kubernetes.io/instance: local-metrics-server + template: + metadata: + labels: + app.kubernetes.io/name: metrics-server + app.kubernetes.io/instance: local-metrics-server + spec: + schedulerName: + serviceAccountName: local-metrics-server + priorityClassName: "system-cluster-critical" + containers: + - name: metrics-server + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + image: registry.k8s.io/metrics-server/metrics-server:v0.6.2 + imagePullPolicy: IfNotPresent + args: + - --secure-port=10250 + - --cert-dir=/tmp + - --kubelet-insecure-tls + - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname + - --kubelet-use-node-status-port + - --metric-resolution=15s + ports: + - name: https + protocol: TCP + containerPort: 10250 + livenessProbe: + failureThreshold: 3 + httpGet: + path: /livez + port: https + scheme: HTTPS + initialDelaySeconds: 0 + periodSeconds: 10 + readinessProbe: + failureThreshold: 3 + httpGet: + path: /readyz + port: https + scheme: HTTPS + initialDelaySeconds: 20 + periodSeconds: 10 + volumeMounts: + - name: tmp + mountPath: /tmp + volumes: + - name: tmp + emptyDir: {} +--- +# Source: metrics-server/templates/apiservice.yaml +apiVersion: apiregistration.k8s.io/v1 +kind: APIService +metadata: + name: v1beta1.metrics.k8s.io + labels: + helm.sh/chart: metrics-server-3.8.3 + app.kubernetes.io/name: metrics-server + app.kubernetes.io/instance: local-metrics-server + app.kubernetes.io/version: "0.6.2" + app.kubernetes.io/managed-by: Helm +spec: + group: metrics.k8s.io + groupPriorityMinimum: 100 + insecureSkipTLSVerify: true + service: + name: local-metrics-server + namespace: default + port: 443 + version: v1beta1 + versionPriority: 100 diff --git a/multidimensional-pod-autoscaler/hack/e2e/kind-with-registry.sh b/multidimensional-pod-autoscaler/hack/e2e/kind-with-registry.sh new file mode 100755 index 000000000000..151cf40f7503 --- /dev/null +++ b/multidimensional-pod-autoscaler/hack/e2e/kind-with-registry.sh @@ -0,0 +1,57 @@ +#!/bin/sh + +# Copyright 2024 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Based on https://kind.sigs.k8s.io/examples/kind-with-registry.sh +set -o errexit + +# Create registry container unless it already exists +reg_name='kind-registry' +reg_port='5001' + +if [ "$(docker inspect -f '{{.State.Running}}' "${reg_name}" 2>/dev/null || true)" != 'true' ]; then + docker run \ + -d --restart=always -p "127.0.0.1:${reg_port}:5000" --name "${reg_name}" \ + registry:2 +fi + +# Create a cluster with the local registry enabled in containerd +cat <>{<<.LabelMatchers>>}) by (name) + resources: + overrides: + kubernetes_namespace: { resource: namespace } + kubernetes_pod_name: { resource: pod } + - seriesQuery: '{__name__="mem"}' + metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (name) + resources: + overrides: + kubernetes_namespace: { resource: namespace } + kubernetes_pod_name: { resource: pod } + rules: + - seriesQuery: '{__name__=~"^container_.*",container!="POD",namespace!="",pod!=""}' + seriesFilters: [] + resources: + overrides: + namespace: + resource: namespace + pod: + resource: pod + name: + matches: ^container_(.*)_seconds_total$ + as: "" + metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>,container!="POD"}[5m])) + by (<<.GroupBy>>) + - seriesQuery: '{__name__=~"^container_.*",container!="POD",namespace!="",pod!=""}' + seriesFilters: + - isNot: ^container_.*_seconds_total$ + resources: + overrides: + namespace: + resource: namespace + pod: + resource: pod + name: + matches: ^container_(.*)_total$ + as: "" + metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>,container!="POD"}[5m])) + by (<<.GroupBy>>) + - seriesQuery: '{__name__=~"^container_.*",container!="POD",namespace!="",pod!=""}' + seriesFilters: + - isNot: ^container_.*_total$ + resources: + overrides: + namespace: + resource: namespace + pod: + resource: pod + name: + matches: ^container_(.*)$ + as: "" + metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>,container!="POD"}) by (<<.GroupBy>>) + - seriesQuery: '{namespace!="",__name__!~"^container_.*"}' + seriesFilters: + - isNot: .*_total$ + resources: + template: <<.Resource>> + name: + matches: "" + as: "" + metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>) + - seriesQuery: '{namespace!="",__name__!~"^container_.*"}' + seriesFilters: + - isNot: .*_seconds_total + resources: + template: <<.Resource>> + name: + matches: ^(.*)_total$ + as: "" + metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>}[5m])) by (<<.GroupBy>>) + - seriesQuery: '{namespace!="",__name__!~"^container_.*"}' + seriesFilters: [] + resources: + template: <<.Resource>> + name: + matches: ^(.*)_seconds_total$ + as: "" + metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>}[5m])) by (<<.GroupBy>>) +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/component: metrics + app.kubernetes.io/instance: prometheus-adapter + app.kubernetes.io/name: prometheus-adapter + name: prometheus-adapter + namespace: monitoring +spec: + progressDeadlineSeconds: 600 + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/instance: prometheus-adapter + app.kubernetes.io/name: prometheus-adapter + template: + metadata: + labels: + app.kubernetes.io/component: metrics + app.kubernetes.io/instance: prometheus-adapter + app.kubernetes.io/name: prometheus-adapter + name: prometheus-adapter + spec: + containers: + - name: prometheus-adapter + args: + - /adapter + - --secure-port=6443 + - --cert-dir=/tmp/cert + - --logtostderr=true + - --prometheus-url=http://prometheus.monitoring.svc:9090 + - --metrics-relist-interval=1m + - --v=4 + - --config=/etc/adapter/config.yaml + image: registry.k8s.io/prometheus-adapter/prometheus-adapter:v0.10.0 + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: https + scheme: HTTPS + initialDelaySeconds: 30 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + ports: + - containerPort: 6443 + name: https + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: https + scheme: HTTPS + initialDelaySeconds: 30 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - all + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 10001 + seccompProfile: + type: RuntimeDefault + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /etc/adapter/ + name: config + readOnly: true + - mountPath: /tmp + name: tmp + dnsPolicy: ClusterFirst + restartPolicy: Always + schedulerName: default-scheduler + serviceAccount: prometheus-adapter + serviceAccountName: prometheus-adapter + terminationGracePeriodSeconds: 30 + volumes: + - configMap: + defaultMode: 420 + name: prometheus-adapter + name: config + - emptyDir: {} + name: tmp diff --git a/multidimensional-pod-autoscaler/hack/e2e/prometheus.yaml b/multidimensional-pod-autoscaler/hack/e2e/prometheus.yaml new file mode 100644 index 000000000000..651cf81ab256 --- /dev/null +++ b/multidimensional-pod-autoscaler/hack/e2e/prometheus.yaml @@ -0,0 +1,264 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: prometheus-server-conf + labels: + name: prometheus-server-conf + namespace: monitoring +data: + prometheus.rules: |- + groups: + - name: devopscube demo alert + rules: + - alert: High Pod Memory + expr: sum(container_memory_usage_bytes) > 1 + for: 1m + labels: + severity: slack + annotations: + summary: High Memory Usage + prometheus.yml: |- + global: + scrape_interval: 5s + evaluation_interval: 5s + rule_files: + - /etc/prometheus/prometheus.rules + + scrape_configs: + - job_name: 'kubernetes-apiservers' + + kubernetes_sd_configs: + - role: endpoints + scheme: https + + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + + relabel_configs: + - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name] + action: keep + regex: default;kubernetes;https + + - job_name: 'kubernetes-pods' + + kubernetes_sd_configs: + - role: pod + + relabel_configs: + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape] + action: keep + regex: true + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path] + action: replace + target_label: __metrics_path__ + regex: (.+) + - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port] + action: replace + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:$2 + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - source_labels: [__meta_kubernetes_namespace] + action: replace + target_label: kubernetes_namespace + - source_labels: [__meta_kubernetes_pod_name] + action: replace + target_label: kubernetes_pod_name + + - job_name: 'kubernetes-service-endpoints' + + kubernetes_sd_configs: + - role: endpoints + + relabel_configs: + - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape] + action: keep + regex: true + - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme] + action: replace + target_label: __scheme__ + regex: (https?) + - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path] + action: replace + target_label: __metrics_path__ + regex: (.+) + - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port] + action: replace + target_label: __address__ + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:$2 + - action: labelmap + regex: __meta_kubernetes_service_label_(.+) + - source_labels: [__meta_kubernetes_namespace] + action: replace + target_label: kubernetes_namespace + - source_labels: [__meta_kubernetes_service_name] + action: replace + target_label: kubernetes_name + + - job_name: "pushgateway" + honor_labels: true + static_configs: + - targets: ['localhost:9091'] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: prometheus +rules: +- apiGroups: [""] + resources: + - nodes + - nodes/proxy + - services + - endpoints + - pods + - configmaps + verbs: ["get", "list", "watch"] +- apiGroups: + - extensions + resources: + - ingresses + verbs: ["get", "list", "watch"] +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: ["create"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: prometheus +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: prometheus +subjects: +- kind: ServiceAccount + name: prometheus + namespace: monitoring +- kind: ServiceAccount + name: metrics-pump + namespace: monitoring +- kind: ServiceAccount + name: prometheus-adapter + namespace: monitoring + +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: prometheus + namespace: monitoring +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: metrics-pump + namespace: monitoring +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: prometheus-adapter + namespace: monitoring + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: prometheus + namespace: monitoring +spec: + replicas: 1 + selector: + matchLabels: + app: prometheus + template: + metadata: + labels: + app: prometheus + spec: + serviceAccountName: prometheus + securityContext: + runAsNonRoot: true + runAsUser: 65534 # nobody + containers: + - name: prometheus + image: prom/prometheus + args: + - "--storage.tsdb.retention.time=12h" + - "--config.file=/etc/prometheus/prometheus.yml" + - "--storage.tsdb.path=/prometheus/" + ports: + - containerPort: 9090 + name: prometheus + resources: + requests: + cpu: 500m + memory: 500M + limits: + cpu: 1 + memory: 1Gi + volumeMounts: + - name: prometheus-config-volume + mountPath: /etc/prometheus/ + - name: prometheus-storage-volume + mountPath: /prometheus/ + - name: pushgateway + image: prom/pushgateway + ports: + - containerPort: 9091 + name: pushgateway + resources: + requests: + cpu: 500m + memory: 500M + volumes: + - name: prometheus-config-volume + configMap: + defaultMode: 420 + name: prometheus-server-conf + - name: prometheus-storage-volume + emptyDir: {} + +--- +apiVersion: v1 +kind: Service +metadata: + name: prometheus + namespace: monitoring +spec: + selector: + app: prometheus + ports: + - name: prometheus + protocol: TCP + port: 9090 + targetPort: prometheus + - name: pushgateway + protocol: TCP + port: 9091 + targetPort: pushgateway +--- +apiVersion: v1 +kind: Service +metadata: + name: prometheus-adapter + namespace: monitoring +spec: + selector: + app.kubernetes.io/name: prometheus-adapter + ports: + - name: api + protocol: TCP + port: 443 + targetPort: https + diff --git a/multidimensional-pod-autoscaler/hack/e2e/recommender-externalmetrics-deployment.yaml b/multidimensional-pod-autoscaler/hack/e2e/recommender-externalmetrics-deployment.yaml new file mode 100644 index 000000000000..a9360e225764 --- /dev/null +++ b/multidimensional-pod-autoscaler/hack/e2e/recommender-externalmetrics-deployment.yaml @@ -0,0 +1,42 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mpa-recommender + namespace: kube-system +spec: + replicas: 1 + selector: + matchLabels: + app: mpa-recommender + template: + metadata: + labels: + app: mpa-recommender + spec: + serviceAccountName: mpa-recommender + securityContext: + runAsNonRoot: true + runAsUser: 65534 # nobody + containers: + - name: recommender + image: localhost:5001/mpa-recommender-amd64:latest + imagePullPolicy: Never + args: + - /recommender-amd64 + - --use-external-metrics=true + - --external-metrics-cpu-metric=cpu + - --external-metrics-memory-metric=mem + - --v=4 + resources: + limits: + cpu: 200m + memory: 1000Mi + requests: + cpu: 50m + memory: 500Mi + ports: + - name: prometheus + containerPort: 8942 + - name: delve + containerPort: 40000 diff --git a/multidimensional-pod-autoscaler/hack/emit-metrics.py b/multidimensional-pod-autoscaler/hack/emit-metrics.py new file mode 100755 index 000000000000..c568536f566f --- /dev/null +++ b/multidimensional-pod-autoscaler/hack/emit-metrics.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python3 + +# Copyright 2023 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script runs as a pod, scanning the cluster for other pods. +# It then publishes fake metrics (Gaussian by --mean_* and --stddev_*) +# for each pod into a Prometheus Pushgateway. The Prometheus instance +# connected to that Pushgateway can have a Prometheus Adapter connected +# to it that serves as an External Metrics Provider to Kubernetes. + +import argparse +import base64 +from collections import defaultdict +from kubernetes import client, config +import math +import random +import re +import requests +import sys +import time +import urllib.parse +import pprint + +def parse_arguments(): + parser = argparse.ArgumentParser(description='') + parser.add_argument('--dest', type=str, default='pushservice') + parser.add_argument('--mean_cpu', type=int, default='1000', help='Mean millicores for cpu.') + parser.add_argument('--mean_mem', type=int, default='128', help='Mean megabytes for memory.') + parser.add_argument('--stddev_cpu', type=int, default=150, help='Standard deviation for cpu.') + parser.add_argument('--stddev_mem', type=int, default=15, help='Standard deviation for mem.') + parser.add_argument('--sleep_sec', type=int, default=30, help='Delay between metric-sends, in seconds.') + parser.add_argument('-t','--tags', action='append', nargs=2, metavar=('key','value'), default=[['data', 'emit-metrics']], + help='Additional tags to attach to metrics.') + parser.add_argument('--namespace_pattern', default='monitoring', help='Regex to match namespace names.') + parser.add_argument('--pod_pattern', default='prometheus-[0-9a-f]{9}-[0-9a-z]{5}', help='Regex to match pod names.') + parser.add_argument('--all', default=False, action='store_true', help='Write metrics for all pods.') + parser.add_argument('--job', default='emit-metrics', help='Job name to submit under.') + return parser.parse_args() + +def safestr(s): + '''Is s a URL-safe string?''' + return s.strip('_').isalnum() + +def urlify(key, value): + replacements = { ".": "%2E", "-": "%2D" } + def encode(s): + s = urllib.parse.quote(s, safe='') + for c,repl in replacements.items(): + s = s.replace(c, repl) + return s + if safestr(key) and safestr(value): + return f"{key}/{value}" + elif len(value) == 0: + # Possibly encode the key using URI encoding, but + # definitely use base64 for the value. + return encode(key)+"@base64/=" + else: + return f"{encode(key)}/{encode(value)}" + +def valid_key(key): + invalid_char_re = re.compile(r'.*[./-].*') + invalid_keys = set(["pod-template-hash", "k8s-app", "controller-uid", + "controller-revision-hash", "pod-template-generation"]) + return (key not in invalid_keys) and invalid_char_re.match(key) == None + +def send_metrics(args, job, path, cpuval, memval): + cpuval = cpuval / 1000.0 # Scale from millicores to cores + payload = f"cpu {cpuval:.3f}\nmem {memval:d}.0\n" + path_str = '/'.join([urlify(key,value) for key, value in path.items()]) + url = f'http://{args.dest}/metrics/job/{job}/namespace/{path["namespace"]}/{path_str}' + response = requests.put(url=url, data=bytes(payload, 'utf-8')) + if response.status_code != 200: + print (f"Writing to {url}.\n>> Got {response.status_code}: {response.reason}, {response.text}\n>> Dict was:") + pprint.pprint(path) + else: + print (f"Wrote to {url}: {payload}") + sys.stdout.flush() + +def main(args): + print (f"Starting up.") + sys.stdout.flush() + pod_name_pattern = re.compile(args.pod_pattern) + namespace_name_pattern = re.compile(args.namespace_pattern) + try: + config.load_kube_config() + except: + config.load_incluster_config() + v1 = client.CoreV1Api() + print (f"Initialized. Sleep interval is for {args.sleep_sec} seconds.") + sys.stdout.flush() + pod_cache = dict() + while True: + skipped_keys= set() + time.sleep(args.sleep_sec) + pods = v1.list_pod_for_all_namespaces(watch=False) + all = 0 + found = 0 + for pod in pods.items: + all += 1 + job = args.job + if args.all or (namespace_name_pattern.match(pod.metadata.namespace) and pod_name_pattern.match(pod.metadata.name)): + # Get container names and send metrics for each. + key = f"{pod.metadata.namespace}/{pod.metadata.name}" + if key not in pod_cache: + v1pod = v1.read_namespaced_pod(pod.metadata.name, pod.metadata.namespace, pretty=False) + pod_cache[key] = v1pod + else: + v1pod = pod_cache[key] + containers = [ c.name for c in v1pod.spec.containers ] + found += 1 + path = { "kubernetes_namespace": pod.metadata.namespace, + "kubernetes_pod_name": pod.metadata.name, + "pod": pod.metadata.name, + "namespace": pod.metadata.namespace} + # Append metadata to the data point, add the labels second to overwrite annotations on + # conflict + try: + if v1pod.metadata.annotations: + for k,v in v1pod.metadata.annotations.items(): + if valid_key(k): + path[k] = v + else: + skipped_keys |= set([k]) + if v1pod.metadata.labels: + for k,v in v1pod.metadata.labels.items(): + if valid_key(k): + path[k] = v + else: + skipped_keys |= set([k]) + except ValueError as err: + print (f"{err} on {v1pod.metadata} when getting annotations/labels") + if "job" in path: + job = path["job"] + for container in containers: + cpuval = random.normalvariate(args.mean_cpu, args.stddev_cpu) + memval = random.normalvariate(args.mean_mem, args.stddev_mem) + path['name'] = container + path['container'] = container + send_metrics(args, job, path, math.floor(cpuval), math.floor(memval * 1048576.0)) + print(f"Found {found} out of {all} pods. Skipped keys:") + pprint.pprint(skipped_keys) + +if __name__ == '__main__': + main(parse_arguments()) diff --git a/multidimensional-pod-autoscaler/hack/generate-crd-yaml.sh b/multidimensional-pod-autoscaler/hack/generate-crd-yaml.sh new file mode 100755 index 000000000000..85d213c9f8e3 --- /dev/null +++ b/multidimensional-pod-autoscaler/hack/generate-crd-yaml.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +# Copyright 2020 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +REPOSITORY_ROOT=$(realpath $(dirname ${BASH_SOURCE})/..) +CRD_OPTS=crd:allowDangerousTypes=true +APIS_PATH=${REPOSITORY_ROOT}/pkg/apis +OUTPUT=${REPOSITORY_ROOT}/deploy/mpa-v1alpha1-crd-gen.yaml +WORKSPACE=$(mktemp -d) + +function cleanup() { + rm -r ${WORKSPACE} +} +trap cleanup EXIT + +if [[ -z $(which controller-gen) ]]; then + ( + cd $WORKSPACE + go install sigs.k8s.io/controller-tools/cmd/controller-gen@v0.16.5 + ) + CONTROLLER_GEN=${GOBIN:-$(go env GOPATH)/bin}/controller-gen +else + CONTROLLER_GEN=$(which controller-gen) +fi + +# The following commands always returns an error because controller-gen does not accept keys other than strings. +${CONTROLLER_GEN} ${CRD_OPTS} paths="${APIS_PATH}/..." output:crd:dir="\"${WORKSPACE}\"" >& ${WORKSPACE}/errors.log ||: +grep -v -e 'map keys must be strings, not int' -e 'not all generators ran successfully' -e 'usage' ${WORKSPACE}/errors.log \ + && { echo "Failed to generate CRD YAMLs."; exit 1; } + +cd ${WORKSPACE} +cat "${WORKSPACE}/autoscaling.k8s.io_multidimpodautoscalercheckpoints.yaml" > ${OUTPUT} +cat "${WORKSPACE}/autoscaling.k8s.io_multidimpodautoscalers.yaml" >> ${OUTPUT} diff --git a/multidimensional-pod-autoscaler/hack/lib/util.sh b/multidimensional-pod-autoscaler/hack/lib/util.sh new file mode 100644 index 000000000000..04e59f3d3611 --- /dev/null +++ b/multidimensional-pod-autoscaler/hack/lib/util.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash + +# Copyright 2024 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +kube::util::host_arch() { + local host_arch + case "$(uname -m)" in + x86_64*) + host_arch=amd64 + ;; + i?86_64*) + host_arch=amd64 + ;; + amd64*) + host_arch=amd64 + ;; + aarch64*) + host_arch=arm64 + ;; + arm64*) + host_arch=arm64 + ;; + arm*) + host_arch=arm + ;; + i?86*) + host_arch=x86 + ;; + s390x*) + host_arch=s390x + ;; + ppc64le*) + host_arch=ppc64le + ;; + *) + kube::log::error "Unsupported host arch. Must be x86_64, 386, arm, arm64, s390x or ppc64le." + exit 1 + ;; + esac + echo "${host_arch}" +} diff --git a/multidimensional-pod-autoscaler/hack/local-cluster.md b/multidimensional-pod-autoscaler/hack/local-cluster.md new file mode 100644 index 000000000000..39be865fed24 --- /dev/null +++ b/multidimensional-pod-autoscaler/hack/local-cluster.md @@ -0,0 +1,31 @@ +# Running Integration Tests locally +Included in parallel with `run-e2e.sh` and `deploy-for-e2e.sh` are two alternate versions +with `-locally` as part of their names. They use Kubernetes in Docker (`kind`) to run a local +cluster in Docker. Using them will require `docker` and `kind` in your `PATH`. + +## External Metrics Tests +The external metrics tests (`recommender-externalmetrics`, available on the `-locally` variants) +use a stack of 4 additional programs to support testing: + +1. `hack/emit-metrics.py` to generate random CPU and RAM metrics for every pod in the local cluster. +2. Prometheus Pushgateway to accept metrics from `hack/emit-metrics`. +3. Prometheus to store the metrics accepted by the Pushgateway. +4. Prometheus Adapter to provide an External Metrics interface to Prometheus. + +The External Metrics tests run by configuring a `recommender` to use the External Metrics interface +from the Prometheus Adapter. With that configuration, it runs the standard `recommender` test suite. + +## Non-recommender tests +The `recommender` and `recommender-externalmetrics` test work locally, but none of the others do; +they require more Makefile work. + +# Configuration Notes +To support the regular `recommender` tests locally, we've added the stock Kubernetes Metrics Server. +Unfortunately, it doesn't work with TLS turned on. The metrics server is being run in insecure mode +to work around this. This only runs in the local `kind` case, not in a real cluster. + +# RBAC Changes +The local test cases support running the `recommender` with external metrics. This requires +additional permissions we don't want to automatically enable for all customers via the +configuration given in `deploy/mpa-rbac.yaml`. The scripts use a context diff `hack/e2e/mpa-rbac.diff` +to enable those permission when running locally. diff --git a/multidimensional-pod-autoscaler/hack/mpa-down.sh b/multidimensional-pod-autoscaler/hack/mpa-down.sh new file mode 100755 index 000000000000..d83659930483 --- /dev/null +++ b/multidimensional-pod-autoscaler/hack/mpa-down.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# Copyright 2024 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +SCRIPT_ROOT=$(dirname ${BASH_SOURCE})/.. + +$SCRIPT_ROOT/hack/mpa-process-yamls.sh delete $* diff --git a/multidimensional-pod-autoscaler/hack/mpa-process-yaml.sh b/multidimensional-pod-autoscaler/hack/mpa-process-yaml.sh new file mode 100755 index 000000000000..890edb58a52f --- /dev/null +++ b/multidimensional-pod-autoscaler/hack/mpa-process-yaml.sh @@ -0,0 +1,52 @@ +#!/bin/bash + +# Copyright 2024 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +SCRIPT_ROOT=$(dirname ${BASH_SOURCE})/.. + +function print_help { + echo "ERROR! Usage: mpa-process-yaml.sh +" + echo "Script will output content of YAML files separated with YAML document" + echo "separator and substituting REGISTRY and TAG for pod images" +} + +if [ $# -eq 0 ]; then + print_help + exit 1 +fi + +DEFAULT_REGISTRY="registry.k8s.io/autoscaling" +DEFAULT_TAG="0.1.0" + +REGISTRY_TO_APPLY=${REGISTRY-$DEFAULT_REGISTRY} +TAG_TO_APPLY=${TAG-$DEFAULT_TAG} + +if [ "${REGISTRY_TO_APPLY}" != "${DEFAULT_REGISTRY}" ]; then + (>&2 echo "WARNING! Using image repository from REGISTRY env variable (${REGISTRY_TO_APPLY}) instead of ${DEFAULT_REGISTRY}.") +fi + +if [ "${TAG_TO_APPLY}" != "${DEFAULT_TAG}" ]; then + (>&2 echo "WARNING! Using tag from TAG env variable (${TAG_TO_APPLY}) instead of the default (${DEFAULT_TAG}).") +fi + +for i in $*; do + sed -e "s,${DEFAULT_REGISTRY}/\([a-z-]*\):.*,${REGISTRY_TO_APPLY}/\1:${TAG_TO_APPLY}," $i + echo "" + echo "---" +done diff --git a/multidimensional-pod-autoscaler/hack/mpa-process-yamls.sh b/multidimensional-pod-autoscaler/hack/mpa-process-yamls.sh new file mode 100755 index 000000000000..0a3408ac5230 --- /dev/null +++ b/multidimensional-pod-autoscaler/hack/mpa-process-yamls.sh @@ -0,0 +1,58 @@ +#!/bin/bash + +# Copyright 2024 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +SCRIPT_ROOT=$(dirname ${BASH_SOURCE})/.. + +function print_help { + echo "ERROR! Usage: mpa-process-yamls.sh []" + echo " should be either 'create' or 'delete'." + echo " might be one of 'admission-controller', 'updater', 'recommender'." + echo "If is set, only the deployment of that component will be processed," + echo "otherwise all components and configs will be processed." +} + +if [ $# -eq 0 ]; then + print_help + exit 1 +fi + +if [ $# -gt 2 ]; then + print_help + exit 1 +fi + +ACTION=$1 +COMPONENTS="mpa-v1alpha1-crd-gen mpa-rbac updater-deployment recommender-deployment admission-controller-deployment" + +if [ $# -gt 1 ]; then + COMPONENTS="$2-deployment" +fi + +for i in $COMPONENTS; do + if [ $i == admission-controller-deployment ] ; then + if [ ${ACTION} == create ] ; then + (bash ${SCRIPT_ROOT}/pkg/admission-controller/gencerts.sh || true) + elif [ ${ACTION} == delete ] ; then + (bash ${SCRIPT_ROOT}/pkg/admission-controller/rmcerts.sh || true) + (bash ${SCRIPT_ROOT}/pkg/admission-controller/delete-webhook.sh || true) + fi + fi + kubectl ${ACTION} -f ${SCRIPT_ROOT}/deploy/$i.yaml || true +done diff --git a/multidimensional-pod-autoscaler/hack/mpa-up.sh b/multidimensional-pod-autoscaler/hack/mpa-up.sh new file mode 100755 index 000000000000..ec9d2dd96614 --- /dev/null +++ b/multidimensional-pod-autoscaler/hack/mpa-up.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# Copyright 2024 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +SCRIPT_ROOT=$(dirname ${BASH_SOURCE})/.. + +$SCRIPT_ROOT/hack/mpa-process-yamls.sh create $* diff --git a/multidimensional-pod-autoscaler/hack/run-e2e-locally.sh b/multidimensional-pod-autoscaler/hack/run-e2e-locally.sh new file mode 100755 index 000000000000..4e0e60881271 --- /dev/null +++ b/multidimensional-pod-autoscaler/hack/run-e2e-locally.sh @@ -0,0 +1,103 @@ +#!/bin/bash + +# Copyright 2024 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o nounset +set -o pipefail + +BASE_NAME=$(basename $0) +SCRIPT_ROOT=$(dirname ${BASH_SOURCE})/.. + +function print_help { + echo "ERROR! Usage: $BASE_NAME " + echo " should be one of:" + echo " - recommender" + echo " - recommender-externalmetrics" + echo " - updater" + echo " - admission-controller" + echo " - full-mpa" + +} + +if [ $# -eq 0 ]; then + print_help + exit 1 +fi + +if [ $# -gt 1 ]; then + print_help + exit 1 +fi + +SUITE=$1 +REQUIRED_COMMANDS=" +docker +go +kind +kubectl +make +" + +for i in $REQUIRED_COMMANDS; do + if ! command -v $i > /dev/null 2>&1 + then + echo "$i could not be found, please ensure it is installed" + echo + echo "The following commands are required to run these tests:" + echo $REQUIRED_COMMANDS + exit 1; + fi +done + +if ! docker ps >/dev/null 2>&1 +then + echo "docker isn't running" + echo + echo "Please ensure that docker is running" + exit 1 +fi + + +echo "Deleting KIND cluster 'kind'." +kind delete cluster -n kind -q + +echo "Creating KIND cluster 'kind'" +KIND_VERSION="kindest/node:v1.31.2" +kind create cluster --image=${KIND_VERSION} + +echo "Building metrics-pump image" +docker build -t localhost:5001/write-metrics:dev -f ${SCRIPT_ROOT}/hack/e2e/Dockerfile.externalmetrics-writer ${SCRIPT_ROOT}/hack +echo " loading image into kind" +kind load docker-image localhost:5001/write-metrics:dev + + +case ${SUITE} in + recommender|recommender-externalmetrics|updater|admission-controller|full-mpa) + ${SCRIPT_ROOT}/hack/mpa-down.sh + echo " ** Deploying for suite ${SUITE}" + ${SCRIPT_ROOT}/hack/deploy-for-e2e-locally.sh ${SUITE} + + echo " ** Running suite ${SUITE}" + if [ ${SUITE} == recommender-externalmetrics ]; then + WORKSPACE=./workspace/_artifacts ${SCRIPT_ROOT}/hack/run-e2e-tests.sh recommender + else + WORKSPACE=./workspace/_artifacts ${SCRIPT_ROOT}/hack/run-e2e-tests.sh ${SUITE} + fi + ;; + *) + print_help + exit 1 + ;; +esac diff --git a/multidimensional-pod-autoscaler/hack/run-e2e-tests.sh b/multidimensional-pod-autoscaler/hack/run-e2e-tests.sh new file mode 100755 index 000000000000..27890ae95667 --- /dev/null +++ b/multidimensional-pod-autoscaler/hack/run-e2e-tests.sh @@ -0,0 +1,69 @@ +#!/bin/bash + +# Copyright 2024 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o nounset +set -o pipefail + +SCRIPT_ROOT=$(dirname ${BASH_SOURCE})/.. + +function print_help { + echo "ERROR! Usage: run-e2e-tests.sh " + echo " should be one of:" + echo " - recommender" + echo " - updater" + echo " - admission-controller" + echo " - actuation" + echo " - full-mpa" +} + + +if [ $# -eq 0 ]; then + print_help + exit 1 +fi + +if [ $# -gt 1 ]; then + print_help + exit 1 +fi + +SUITE=$1 + +export GO111MODULE=on + +export WORKSPACE=${WORKSPACE:-/workspace/_artifacts} + +case ${SUITE} in + recommender|updater|admission-controller|actuation|full-mpa) + export KUBECONFIG=$HOME/.kube/config + pushd ${SCRIPT_ROOT}/e2e + go test ./v1alpha1/*go -v --test.timeout=90m --args --ginkgo.v=true --ginkgo.focus="\[MPA\] \[${SUITE}\]" --report-dir=${WORKSPACE} --disable-log-dump --ginkgo.timeout=90m + V1ALPHA1_RESULT=$? + popd + echo v1alpha1 test result: ${V1ALPHA1_RESULT} + if [ $V1ALPHA1_RESULT -gt 0 ]; then + echo "Please check v1alpha1 \"go test\" logs!" + fi + if [ $V1ALPHA1_RESULT -gt 0 ]; then + echo "Tests failed" + exit 1 + fi + ;; + *) + print_help + exit 1 + ;; +esac diff --git a/multidimensional-pod-autoscaler/hack/run-e2e.sh b/multidimensional-pod-autoscaler/hack/run-e2e.sh new file mode 100755 index 000000000000..eede52070615 --- /dev/null +++ b/multidimensional-pod-autoscaler/hack/run-e2e.sh @@ -0,0 +1,54 @@ +#!/bin/bash + +# Copyright 2024 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o nounset +set -o pipefail + +SCRIPT_ROOT=$(dirname ${BASH_SOURCE})/.. + +function print_help { + echo "ERROR! Usage: run-e2e.sh " + echo " should be one of:" + echo " - recommender" + echo " - updater" + echo " - admission-controller" + echo " - actuation" + echo " - full-mpa" +} + +if [ $# -eq 0 ]; then + print_help + exit 1 +fi + +if [ $# -gt 1 ]; then + print_help + exit 1 +fi + +SUITE=$1 + +case ${SUITE} in + recommender|updater|admission-controller|actuation|full-mpa) + ${SCRIPT_ROOT}/hack/mpa-down.sh + ${SCRIPT_ROOT}/hack/deploy-for-e2e.sh ${SUITE} + ${SCRIPT_ROOT}/hack/run-e2e-tests.sh ${SUITE} + ;; + *) + print_help + exit 1 + ;; +esac diff --git a/multidimensional-pod-autoscaler/hack/tools.go b/multidimensional-pod-autoscaler/hack/tools.go new file mode 100644 index 000000000000..5b62bce81c49 --- /dev/null +++ b/multidimensional-pod-autoscaler/hack/tools.go @@ -0,0 +1,25 @@ +//go:build tools +// +build tools + +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package tools is here, such that we can version dependencies that our helpers in the Makefile and hack/ directory +// use with the regular go.mod mechanism. They get downloaded with `go mod vendor` and scripts can directly reference +// the executables used e.g. for generating CRDs, clients and informers. +package tools + +import _ "k8s.io/code-generator" diff --git a/multidimensional-pod-autoscaler/hack/update-codegen.sh b/multidimensional-pod-autoscaler/hack/update-codegen.sh new file mode 100755 index 000000000000..6a2a0b12b0d1 --- /dev/null +++ b/multidimensional-pod-autoscaler/hack/update-codegen.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +# Copyright 2024 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +GO_CMD=${1:-go} +CURRENT_DIR=$(dirname "${BASH_SOURCE[0]}") +REPO_ROOT="$(git rev-parse --show-toplevel)" +CODEGEN_PKG=$($GO_CMD list -m -mod=readonly -f "{{.Dir}}" k8s.io/code-generator) +cd "${CURRENT_DIR}/.." + +# shellcheck source=/dev/null +source "${CODEGEN_PKG}/kube_codegen.sh" + +kube::codegen::gen_helpers \ + "$(dirname ${BASH_SOURCE})/../pkg/apis" \ + --boilerplate "${REPO_ROOT}/hack/boilerplate/boilerplate.generatego.txt" + +echo "Ran gen helpers, moving on to generating client code..." + +kube::codegen::gen_client \ + "$(dirname ${BASH_SOURCE})/../pkg/apis" \ + --output-pkg k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/client \ + --output-dir "$(dirname ${BASH_SOURCE})/../pkg/client" \ + --boilerplate "${REPO_ROOT}/hack/boilerplate/boilerplate.generatego.txt" \ + --with-watch + +echo "Generated client code, running `go mod tidy`..." + +# We need to clean up the go.mod file since code-generator adds temporary library to the go.mod file. +"${GO_CMD}" mod tidy diff --git a/multidimensional-pod-autoscaler/hack/update-kubernetes-deps-in-e2e.sh b/multidimensional-pod-autoscaler/hack/update-kubernetes-deps-in-e2e.sh new file mode 100755 index 000000000000..cf4fb4157ea9 --- /dev/null +++ b/multidimensional-pod-autoscaler/hack/update-kubernetes-deps-in-e2e.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash + +# Copyright 2024 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Usage: +# $ K8S_TAG= ./hack/update-kubernetes-deps-in-e2e.sh +# K8S_TAG - k8s version to use for the dependencies update. +# Suggested format is K8S_TAG=v1.10.3 + +set -euo pipefail + +K8S_TAG=${K8S_TAG:-v1.25.0} +K8S_TAG=${K8S_TAG#v} +K8S_FORK="git@github.com:kubernetes/kubernetes.git" + +export GO111MODULE=on + +function update_deps() { + # list staged k8s.io repos + MODS=($( + curl -sS https://raw.githubusercontent.com/kubernetes/kubernetes/v${K8S_TAG}/go.mod | + sed -n 's|.*k8s.io/\(.*\) => ./staging/src/k8s.io/.*|k8s.io/\1|p' + )) + + # get matching tag for each staged k8s.io repo + for MOD in "${MODS[@]}"; do + V=$( + go mod download -json "${MOD}@kubernetes-${K8S_TAG}" | + sed -n 's|.*"Version": "\(.*\)".*|\1|p' + ) + echo "Replacing ${MOD} with version ${V}" + go mod edit "-replace=${MOD}=${MOD}@${V}" + done +} + +# execute in subshell to keep CWD even in case of failures +( + # find script directory invariantly of CWD + DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + + # cd to e2e tests + cd ${DIR}/../e2e + + echo "Updating MPA e2e dependencies to k8s ${K8S_TAG}" + update_deps + + echo "Updating k8s to ${K8S_TAG}" + go get "k8s.io/kubernetes@v${K8S_TAG}" + + echo "Running go mod tidy and vendoring deps" + # tidy and vendor modules + go mod tidy + go mod vendor +) diff --git a/multidimensional-pod-autoscaler/hack/update-kubernetes-deps.sh b/multidimensional-pod-autoscaler/hack/update-kubernetes-deps.sh new file mode 100755 index 000000000000..9903740611a1 --- /dev/null +++ b/multidimensional-pod-autoscaler/hack/update-kubernetes-deps.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash + +# Copyright 2024 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Usage: +# $ K8S_TAG= ./hack/update-kubernetes-deps-in-e2e.sh +# K8S_TAG - k8s version to use for the dependencies update. +# Suggested format is K8S_TAG=v1.10.3 + +set -euo pipefail + +K8S_TAG=${K8S_TAG:-v1.26.1} +K8S_TAG=${K8S_TAG#v} +K8S_FORK="git@github.com:kubernetes/kubernetes.git" + +export GO111MODULE=on + +function update_deps() { + # list staged k8s.io repos + MODS=($( + curl -sS https://raw.githubusercontent.com/kubernetes/kubernetes/v${K8S_TAG}/go.mod | + sed -n 's|.*k8s.io/\(.*\) => ./staging/src/k8s.io/.*|k8s.io/\1|p' + )) + + # get matching tag for each staged k8s.io repo + for MOD in "${MODS[@]}"; do + V=$( + go mod download -json "${MOD}@kubernetes-${K8S_TAG}" | + sed -n 's|.*"Version": "\(.*\)".*|\1|p' + ) + echo "Replacing ${MOD} with version ${V}" + go mod edit "-replace=${MOD}=${MOD}@${V}" + done +} + +# execute in subshell to keep CWD even in case of failures +( + # find script directory invariantly of CWD + DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + + # cd to e2e tests + cd ${DIR}/.. + + echo "Updating MPA dependencies to k8s ${K8S_TAG}" + update_deps + + echo "Updating k8s to ${K8S_TAG}" + go get "k8s.io/kubernetes@v${K8S_TAG}" + + echo "Running go mod tidy and vendoring deps" + # tidy and vendor modules + go mod tidy + go mod vendor +) diff --git a/multidimensional-pod-autoscaler/hack/verify-codegen.sh b/multidimensional-pod-autoscaler/hack/verify-codegen.sh new file mode 100755 index 000000000000..1b2f9789873e --- /dev/null +++ b/multidimensional-pod-autoscaler/hack/verify-codegen.sh @@ -0,0 +1,48 @@ +#!/bin/bash + +# Copyright 2024 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +SCRIPT_ROOT=$(dirname "${BASH_SOURCE}")/.. + +DIFFROOT="${SCRIPT_ROOT}/pkg" +TMP_DIFFROOT="${SCRIPT_ROOT}/_tmp/pkg" +_tmp="${SCRIPT_ROOT}/_tmp" + +cleanup() { + rm -rf "${_tmp}" +} +trap "cleanup" EXIT SIGINT + +cleanup + +mkdir -p "${TMP_DIFFROOT}" +cp -a "${DIFFROOT}"/* "${TMP_DIFFROOT}" + +"${SCRIPT_ROOT}/hack/update-codegen.sh" +echo "diffing ${DIFFROOT} against freshly generated codegen" +ret=0 +diff -Naupr "${DIFFROOT}" "${TMP_DIFFROOT}" || ret=$? +cp -a "${TMP_DIFFROOT}"/* "${DIFFROOT}" +if [[ $ret -eq 0 ]] +then + echo "${DIFFROOT} up to date." +else + echo "${DIFFROOT} is out of date. Please run hack/update-codegen.sh" + exit 1 +fi diff --git a/multidimensional-pod-autoscaler/pkg/admission-controller/.gitignore b/multidimensional-pod-autoscaler/pkg/admission-controller/.gitignore new file mode 100644 index 000000000000..03f13774a440 --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/admission-controller/.gitignore @@ -0,0 +1,7 @@ +# Admission Controller binary +admission-controller +admission-controller-amd64 +admission-controller-arm64 +admission-controller-arm +admission-controller-ppc64le +admission-controller-s390x diff --git a/multidimensional-pod-autoscaler/pkg/admission-controller/Dockerfile b/multidimensional-pod-autoscaler/pkg/admission-controller/Dockerfile new file mode 100644 index 000000000000..62c205ea63b3 --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/admission-controller/Dockerfile @@ -0,0 +1,40 @@ +# Copyright 2018 The Kubernetes Authors. All rights reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM --platform=$BUILDPLATFORM golang:1.23.4 AS builder + +WORKDIR /workspace + +# Copy the Go Modules manifests +COPY multidimensional-pod-autoscaler/go.mod go.mod +COPY multidimensional-pod-autoscaler/go.sum go.sum +# TODO: This is temporary until the VPA has cut a new release +COPY vertical-pod-autoscaler /vertical-pod-autoscaler + +RUN go mod download + +COPY multidimensional-pod-autoscaler/common common +COPY multidimensional-pod-autoscaler/pkg pkg + +ARG TARGETOS TARGETARCH + +RUN CGO_ENABLED=0 LD_FLAGS=-s GOARCH=$TARGETARCH GOOS=$TARGETOS go build -C pkg/admission-controller -o admission-controller-$TARGETARCH + +FROM gcr.io/distroless/static:nonroot + +ARG TARGETARCH + +COPY --from=builder /workspace/pkg/admission-controller/admission-controller-$TARGETARCH /admission-controller + +ENTRYPOINT ["/admission-controller"] diff --git a/multidimensional-pod-autoscaler/pkg/admission-controller/Makefile b/multidimensional-pod-autoscaler/pkg/admission-controller/Makefile new file mode 100644 index 000000000000..c0c23b1b8015 --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/admission-controller/Makefile @@ -0,0 +1,90 @@ +all: build + +TAG?=dev +REGISTRY?=staging-k8s.gcr.io +FLAGS= +TEST_ENVVAR=LD_FLAGS=-s GO111MODULE=on +ENVVAR=CGO_ENABLED=0 $(TEST_ENVVAR) +GOOS?=linux +COMPONENT=admission-controller +FULL_COMPONENT=mpa-${COMPONENT} + +ALL_ARCHITECTURES?=amd64 arm arm64 ppc64le s390x +export DOCKER_CLI_EXPERIMENTAL=enabled + +build: clean + $(ENVVAR) GOOS=$(GOOS) go build ./... + $(ENVVAR) GOOS=$(GOOS) go build -o ${COMPONENT} + +build-binary: clean + $(ENVVAR) GOOS=$(GOOS) go build -o ${COMPONENT} + +test-unit: clean build + $(TEST_ENVVAR) go test --test.short -race ./... $(FLAGS) + +.PHONY: docker-build +docker-build: $(addprefix docker-build-,$(ALL_ARCHITECTURES)) + +.PHONY: docker-build-* +docker-build-%: +ifndef REGISTRY + ERR = $(error REGISTRY is undefined) + $(ERR) +endif +ifndef TAG + ERR = $(error TAG is undefined) + $(ERR) +endif + docker buildx build --pull --load --platform linux/$* -t ${REGISTRY}/${FULL_COMPONENT}-$*:${TAG} -f ./Dockerfile ../../../ + +.PHONY: docker-push +docker-push: $(addprefix do-push-,$(ALL_ARCHITECTURES)) push-multi-arch; + +.PHONY: do-push-* +do-push-%: +ifndef REGISTRY + ERR = $(error REGISTRY is undefined) + $(ERR) +endif +ifndef TAG + ERR = $(error TAG is undefined) + $(ERR) +endif + docker push ${REGISTRY}/${FULL_COMPONENT}-$*:${TAG} + +.PHONY: push-multi-arch +push-multi-arch: + docker manifest create --amend $(REGISTRY)/${FULL_COMPONENT}:$(TAG) $(shell echo $(ALL_ARCHITECTURES) | sed -e "s~[^ ]*~$(REGISTRY)/${FULL_COMPONENT}\-&:$(TAG)~g") + @for arch in $(ALL_ARCHITECTURES); do docker manifest annotate --arch $${arch} $(REGISTRY)/${FULL_COMPONENT}:$(TAG) $(REGISTRY)/${FULL_COMPONENT}-$${arch}:${TAG}; done + docker manifest push --purge $(REGISTRY)/${FULL_COMPONENT}:$(TAG) + +.PHONY: show-git-info +show-git-info: + echo '=============== local git status ===============' + git status + echo '=============== last commit ===============' + git log -1 + echo '=============== bulding from the above ===============' + +.PHONY: create-buildx-builder +create-buildx-builder: + BUILDER=$(shell docker buildx create --driver=docker-container --use) + +.PHONY: remove-buildx-builder +remove-buildx-builder: + docker buildx rm ${BUILDER} + +.PHONY: release +release: show-git-info create-buildx-builder docker-build remove-buildx-builder docker-push + @echo "Full in-docker release ${FULL_COMPONENT}:${TAG} completed" + +clean: $(addprefix clean-,$(ALL_ARCHITECTURES)) + +clean-%: + rm -f ${COMPONENT}-$* + +format: + test -z "$$(find . -path ./vendor -prune -type f -o -name '*.go' -exec gofmt -s -d {} + | tee /dev/stderr)" || \ + test -z "$$(find . -path ./vendor -prune -type f -o -name '*.go' -exec gofmt -s -w {} + | tee /dev/stderr)" + +.PHONY: all build test-unit clean format release diff --git a/multidimensional-pod-autoscaler/pkg/admission-controller/README.md b/multidimensional-pod-autoscaler/pkg/admission-controller/README.md new file mode 100644 index 000000000000..cdf27f746197 --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/admission-controller/README.md @@ -0,0 +1,50 @@ +# MPA Admission Controller + +- [Intro](#intro) +- [Running](#running) +- [Implementation](#implmentation) + +## Intro + +This is a binary that registers itself as a Mutating Admission Webhook +and because of that is on the path of creating all pods. +For each pod creation, it will get a request from the apiserver and it will +either decide there's no matching MPA configuration or find the corresponding +one and use current recommendation to set resource requests in the pod. + +## Running + +1. You should make sure your API server supports Mutating Webhooks. +Its `--admission-control` flag should have `MutatingAdmissionWebhook` as one of +the values on the list and its `--runtime-config` flag should include +`admissionregistration.k8s.io/v1beta1=true`. +To change those flags, ssh to your API Server instance, edit +`/etc/kubernetes/manifests/kube-apiserver.manifest` and restart kubelet to pick +up the changes: ```sudo systemctl restart kubelet.service``` +1. Generate certs by running `bash gencerts.sh`. This will use kubectl to create + a secret in your cluster with the certs. +1. Create RBAC configuration for the admission controller pod by running + `kubectl create -f ../../deploy/mpa-rbac.yaml` +1. Create the pod: + `kubectl create -f ../../deploy/admission-controller-deployment.yaml`. + The first thing this will do is it will register itself with the apiserver as + Webhook Admission Controller and start changing resource requirements + for pods on their creation & updates. +1. You can specify a path for it to register as a part of the installation process + by setting `--register-by-url=true` and passing `--webhook-address` and `--webhook-port`. + +## Implementation + +All MPA configurations in the cluster are watched with a lister. +In the context of pod creation, there is an incoming https request from +apiserver. +The logic to serve that request involves finding the appropriate MPA, retrieving +current recommendation from it and encoding the recommendation as a json patch to +the Pod resource. + +## Building the Docker Image + +``` +make build-binary-with-vendor-amd64 +make docker-build-amd64 +``` diff --git a/multidimensional-pod-autoscaler/pkg/admission-controller/certs.go b/multidimensional-pod-autoscaler/pkg/admission-controller/certs.go new file mode 100644 index 000000000000..69b67862cdb0 --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/admission-controller/certs.go @@ -0,0 +1,99 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "crypto/tls" + "os" + "path" + "sync" + + "github.com/fsnotify/fsnotify" + "k8s.io/klog/v2" +) + +type certsConfig struct { + clientCaFile, tlsCertFile, tlsPrivateKey *string + reload *bool +} + +func readFile(filePath string) []byte { + res, err := os.ReadFile(filePath) + if err != nil { + klog.ErrorS(err, "Error reading certificate file", "file", filePath) + return nil + } + klog.V(3).InfoS("Successfully read bytes from file", "bytes", len(res), "file", filePath) + return res +} + +type certReloader struct { + tlsCertPath string + tlsKeyPath string + cert *tls.Certificate + mu sync.RWMutex +} + +func (cr *certReloader) start(stop <-chan struct{}) error { + watcher, err := fsnotify.NewWatcher() + if err != nil { + return err + } + + if err = watcher.Add(path.Dir(cr.tlsCertPath)); err != nil { + return err + } + if err = watcher.Add(path.Dir(cr.tlsKeyPath)); err != nil { + return err + } + go func() { + defer watcher.Close() + for { + select { + case event := <-watcher.Events: + if event.Has(fsnotify.Create) || event.Has(fsnotify.Write) { + klog.V(2).InfoS("New certificate found, reloading") + if err := cr.load(); err != nil { + klog.ErrorS(err, "Failed to reload certificate") + } + } + case err := <-watcher.Errors: + klog.Warningf("Error watching certificate files: %s", err) + case <-stop: + return + } + } + }() + return nil +} + +func (cr *certReloader) load() error { + cert, err := tls.LoadX509KeyPair(cr.tlsCertPath, cr.tlsKeyPath) + if err != nil { + return err + } + cr.mu.Lock() + defer cr.mu.Unlock() + cr.cert = &cert + return nil +} + +func (cr *certReloader) getCertificate(_ *tls.ClientHelloInfo) (*tls.Certificate, error) { + cr.mu.RLock() + defer cr.mu.RUnlock() + return cr.cert, nil +} diff --git a/multidimensional-pod-autoscaler/pkg/admission-controller/certs_test.go b/multidimensional-pod-autoscaler/pkg/admission-controller/certs_test.go new file mode 100644 index 000000000000..078de49fa23c --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/admission-controller/certs_test.go @@ -0,0 +1,150 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "bytes" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math/big" + "net" + "os" + "path" + "testing" + "time" +) + +func generateCerts(t *testing.T, org string, caCert *x509.Certificate, caKey *rsa.PrivateKey) ([]byte, []byte) { + cert := &x509.Certificate{ + SerialNumber: big.NewInt(0), + Subject: pkix.Name{ + Organization: []string{org}, + }, + IPAddresses: []net.IP{net.IPv4(127, 0, 0, 1), net.IPv6loopback}, + NotBefore: time.Now(), + NotAfter: time.Now().AddDate(1, 0, 0), + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, + KeyUsage: x509.KeyUsageDigitalSignature, + } + certKey, err := rsa.GenerateKey(rand.Reader, 4096) + if err != nil { + t.Error(err) + } + certBytes, err := x509.CreateCertificate(rand.Reader, cert, caCert, &certKey.PublicKey, caKey) + if err != nil { + t.Error(err) + } + + var certPem bytes.Buffer + err = pem.Encode(&certPem, &pem.Block{ + Type: "CERTIFICATE", + Bytes: certBytes, + }) + if err != nil { + t.Error(err) + } + + var certKeyPem bytes.Buffer + err = pem.Encode(&certKeyPem, &pem.Block{ + Type: "RSA PRIVATE KEY", + Bytes: x509.MarshalPKCS1PrivateKey(certKey), + }) + if err != nil { + t.Error(err) + } + return certPem.Bytes(), certKeyPem.Bytes() +} + +func TestKeypairReloader(t *testing.T) { + tempDir := t.TempDir() + caCert := &x509.Certificate{ + SerialNumber: big.NewInt(0), + Subject: pkix.Name{ + Organization: []string{"ca"}, + }, + NotBefore: time.Now(), + NotAfter: time.Now().AddDate(2, 0, 0), + IsCA: true, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, + BasicConstraintsValid: true, + } + caKey, err := rsa.GenerateKey(rand.Reader, 4096) + if err != nil { + t.Error(err) + } + caBytes, err := x509.CreateCertificate(rand.Reader, caCert, caCert, &caKey.PublicKey, caKey) + if err != nil { + t.Error(err) + } + caPath := path.Join(tempDir, "ca.crt") + caFile, err := os.Create(caPath) + if err != nil { + t.Error(err) + } + err = pem.Encode(caFile, &pem.Block{ + Type: "CERTIFICATE", + Bytes: caBytes, + }) + if err != nil { + t.Error(err) + } + + pub, privateKey := generateCerts(t, "first", caCert, caKey) + certPath := path.Join(tempDir, "cert.crt") + if err = os.WriteFile(certPath, pub, 0666); err != nil { + t.Error(err) + } + keyPath := path.Join(tempDir, "cert.key") + if err = os.WriteFile(keyPath, privateKey, 0666); err != nil { + t.Error(err) + } + + reloader := certReloader{ + tlsCertPath: certPath, + tlsKeyPath: keyPath, + } + stop := make(chan struct{}) + defer close(stop) + if err = reloader.start(stop); err != nil { + t.Error(err) + } + + pub, privateKey = generateCerts(t, "second", caCert, caKey) + if err = os.WriteFile(certPath, pub, 0666); err != nil { + t.Error(err) + } + if err = os.WriteFile(keyPath, privateKey, 0666); err != nil { + t.Error(err) + } + for { + tlsCert, err := reloader.getCertificate(nil) + if err != nil { + t.Error(err) + } + if tlsCert == nil { + continue + } + pubDER, _ := pem.Decode(pub) + if string(tlsCert.Certificate[0]) == string(pubDER.Bytes) { + return + } + } +} diff --git a/multidimensional-pod-autoscaler/pkg/admission-controller/config.go b/multidimensional-pod-autoscaler/pkg/admission-controller/config.go new file mode 100644 index 000000000000..d1426567c76d --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/admission-controller/config.go @@ -0,0 +1,215 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + "crypto/tls" + "fmt" + "strings" + "time" + + admissionregistration "k8s.io/api/admissionregistration/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/klog/v2" +) + +const ( + webhookConfigName = "mpa-webhook-config" +) + +func configTLS(cfg certsConfig, minTlsVersion, ciphers string, stop <-chan struct{}) *tls.Config { + var tlsVersion uint16 + var ciphersuites []uint16 + reverseCipherMap := make(map[string]uint16) + + for _, c := range tls.CipherSuites() { + reverseCipherMap[c.Name] = c.ID + } + for _, c := range strings.Split(strings.ReplaceAll(ciphers, ",", ":"), ":") { + cipher, ok := reverseCipherMap[c] + if ok { + ciphersuites = append(ciphersuites, cipher) + } + } + if len(ciphersuites) == 0 { + ciphersuites = nil + } + + switch minTlsVersion { + case "": + fallthrough + case "tls1_2": + tlsVersion = tls.VersionTLS12 + case "tls1_3": + tlsVersion = tls.VersionTLS13 + default: + klog.Fatal(fmt.Errorf("Unable to determine value for --min-tls-version (%s), must be either tls1_2 or tls1_3", minTlsVersion)) + } + + config := &tls.Config{ + MinVersion: tlsVersion, + CipherSuites: ciphersuites, + } + if *cfg.reload { + cr := certReloader{ + tlsCertPath: *cfg.tlsCertFile, + tlsKeyPath: *cfg.tlsPrivateKey, + } + if err := cr.load(); err != nil { + klog.Fatal(err) + } + if err := cr.start(stop); err != nil { + klog.Fatal(err) + } + config.GetCertificate = cr.getCertificate + } else { + cert, err := tls.LoadX509KeyPair(*cfg.tlsCertFile, *cfg.tlsPrivateKey) + if err != nil { + klog.Fatal(err) + } + config.Certificates = []tls.Certificate{cert} + } + return config +} + +// register this webhook admission controller with the kube-apiserver +// by creating MutatingWebhookConfiguration. +func selfRegistration(clientset kubernetes.Interface, caCert []byte, webHookDelay time.Duration, namespace, serviceName, url string, registerByURL bool, timeoutSeconds int32, selectedNamespace string, ignoredNamespaces []string, webHookFailurePolicy bool, webHookLabels string) { + time.Sleep(webHookDelay) + client := clientset.AdmissionregistrationV1().MutatingWebhookConfigurations() + _, err := client.Get(context.TODO(), webhookConfigName, metav1.GetOptions{}) + if err == nil { + if err2 := client.Delete(context.TODO(), webhookConfigName, metav1.DeleteOptions{}); err2 != nil { + klog.Fatal(err2) + } + } + RegisterClientConfig := admissionregistration.WebhookClientConfig{} + if !registerByURL { + RegisterClientConfig.Service = &admissionregistration.ServiceReference{ + Namespace: namespace, + Name: serviceName, + } + } else { + RegisterClientConfig.URL = &url + } + sideEffects := admissionregistration.SideEffectClassNone + + var failurePolicy admissionregistration.FailurePolicyType + if webHookFailurePolicy { + failurePolicy = admissionregistration.Fail + } else { + failurePolicy = admissionregistration.Ignore + } + + RegisterClientConfig.CABundle = caCert + + var namespaceSelector metav1.LabelSelector + if len(ignoredNamespaces) > 0 { + namespaceSelector = metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "kubernetes.io/metadata.name", + Operator: metav1.LabelSelectorOpNotIn, + Values: ignoredNamespaces, + }, + }, + } + } else if len(selectedNamespace) > 0 { + namespaceSelector = metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "kubernetes.io/metadata.name", + Operator: metav1.LabelSelectorOpIn, + Values: []string{selectedNamespace}, + }, + }, + } + } + webhookLabelsMap, err := convertLabelsToMap(webHookLabels) + if err != nil { + klog.ErrorS(err, "Unable to parse webhook labels") + webhookLabelsMap = map[string]string{} + } + webhookConfig := &admissionregistration.MutatingWebhookConfiguration{ + ObjectMeta: metav1.ObjectMeta{ + Name: webhookConfigName, + Labels: webhookLabelsMap, + }, + Webhooks: []admissionregistration.MutatingWebhook{ + { + Name: "mpa.k8s.io", + AdmissionReviewVersions: []string{"v1"}, + Rules: []admissionregistration.RuleWithOperations{ + { + Operations: []admissionregistration.OperationType{admissionregistration.Create}, + Rule: admissionregistration.Rule{ + APIGroups: []string{""}, + APIVersions: []string{"v1"}, + Resources: []string{"pods"}, + }, + }, + { + Operations: []admissionregistration.OperationType{admissionregistration.Create, admissionregistration.Update}, + Rule: admissionregistration.Rule{ + APIGroups: []string{"autoscaling.k8s.io"}, + APIVersions: []string{"*"}, + Resources: []string{"multidimpodautoscalers"}, + }, + }, + }, + FailurePolicy: &failurePolicy, + ClientConfig: RegisterClientConfig, + SideEffects: &sideEffects, + TimeoutSeconds: &timeoutSeconds, + NamespaceSelector: &namespaceSelector, + }, + }, + } + if _, err := client.Create(context.TODO(), webhookConfig, metav1.CreateOptions{}); err != nil { + klog.Fatal(err) + } else { + klog.V(3).Info("Self registration as MutatingWebhook succeeded.") + } +} + +// convertLabelsToMap convert the labels from string to map +// the valid labels format is "key1:value1,key2:value2", which could be converted to +// {"key1": "value1", "key2": "value2"} +func convertLabelsToMap(labels string) (map[string]string, error) { + m := make(map[string]string) + if labels == "" { + return m, nil + } + labels = strings.Trim(labels, "\"") + s := strings.Split(labels, ",") + for _, tag := range s { + kv := strings.SplitN(tag, ":", 2) + if len(kv) != 2 { + return map[string]string{}, fmt.Errorf("labels '%s' are invalid, the format should be: 'key1:value1,key2:value2'", labels) + } + key := strings.TrimSpace(kv[0]) + if key == "" { + return map[string]string{}, fmt.Errorf("labels '%s' are invalid, the format should be: 'key1:value1,key2:value2'", labels) + } + value := strings.TrimSpace(kv[1]) + m[key] = value + } + + return m, nil +} diff --git a/multidimensional-pod-autoscaler/pkg/admission-controller/config_test.go b/multidimensional-pod-autoscaler/pkg/admission-controller/config_test.go new file mode 100644 index 000000000000..69ed696519a8 --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/admission-controller/config_test.go @@ -0,0 +1,373 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + admissionregistration "k8s.io/api/admissionregistration/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" +) + +func TestSelfRegistrationBase(t *testing.T) { + + testClientSet := fake.NewSimpleClientset() + caCert := []byte("fake") + webHookDelay := 0 * time.Second + namespace := "default" + serviceName := "mpa-service" + url := "http://example.com/" + registerByURL := true + timeoutSeconds := int32(32) + selectedNamespace := "" + ignoredNamespaces := []string{} + + selfRegistration(testClientSet, caCert, webHookDelay, namespace, serviceName, url, registerByURL, timeoutSeconds, selectedNamespace, ignoredNamespaces, false, "key1:value1,key2:value2") + + webhookConfigInterface := testClientSet.AdmissionregistrationV1().MutatingWebhookConfigurations() + webhookConfig, err := webhookConfigInterface.Get(context.TODO(), webhookConfigName, metav1.GetOptions{}) + + assert.NoError(t, err, "expected no error fetching webhook configuration") + assert.Equal(t, webhookConfigName, webhookConfig.Name, "expected webhook configuration name to match") + assert.Equal(t, webhookConfig.Labels, map[string]string{"key1": "value1", "key2": "value2"}, "expected webhook configuration labels to match") + + assert.Len(t, webhookConfig.Webhooks, 1, "expected one webhook configuration") + webhook := webhookConfig.Webhooks[0] + assert.Equal(t, "mpa.k8s.io", webhook.Name, "expected webhook name to match") + + PodRule := webhook.Rules[0] + assert.Equal(t, []admissionregistration.OperationType{admissionregistration.Create}, PodRule.Operations, "expected operations to match") + assert.Equal(t, []string{""}, PodRule.APIGroups, "expected API groups to match") + assert.Equal(t, []string{"v1"}, PodRule.APIVersions, "expected API versions to match") + assert.Equal(t, []string{"pods"}, PodRule.Resources, "expected resources to match") + + MPARule := webhook.Rules[1] + assert.Equal(t, []admissionregistration.OperationType{admissionregistration.Create, admissionregistration.Update}, MPARule.Operations, "expected operations to match") + assert.Equal(t, []string{"autoscaling.k8s.io"}, MPARule.APIGroups, "expected API groups to match") + assert.Equal(t, []string{"*"}, MPARule.APIVersions, "ehook.Rulxpected API versions to match") + assert.Equal(t, []string{"verticalpodautoscalers"}, MPARule.Resources, "expected resources to match") + + assert.Equal(t, admissionregistration.SideEffectClassNone, *webhook.SideEffects, "expected side effects to match") + assert.Equal(t, admissionregistration.Ignore, *webhook.FailurePolicy, "expected failure policy to match") + assert.Equal(t, caCert, webhook.ClientConfig.CABundle, "expected CA bundle to match") + assert.Equal(t, timeoutSeconds, *webhook.TimeoutSeconds, "expected timeout seconds to match") +} + +func TestSelfRegistrationWithURL(t *testing.T) { + + testClientSet := fake.NewSimpleClientset() + caCert := []byte("fake") + webHookDelay := 0 * time.Second + namespace := "default" + serviceName := "mpa-service" + url := "http://example.com/" + registerByURL := true + timeoutSeconds := int32(32) + selectedNamespace := "" + ignoredNamespaces := []string{} + + selfRegistration(testClientSet, caCert, webHookDelay, namespace, serviceName, url, registerByURL, timeoutSeconds, selectedNamespace, ignoredNamespaces, false, "") + + webhookConfigInterface := testClientSet.AdmissionregistrationV1().MutatingWebhookConfigurations() + webhookConfig, err := webhookConfigInterface.Get(context.TODO(), webhookConfigName, metav1.GetOptions{}) + + assert.NoError(t, err, "expected no error fetching webhook configuration") + + assert.Len(t, webhookConfig.Webhooks, 1, "expected one webhook configuration") + webhook := webhookConfig.Webhooks[0] + + assert.Nil(t, webhook.ClientConfig.Service, "expected service reference to be nil") + assert.NotNil(t, webhook.ClientConfig.URL, "expected URL to be set") + assert.Equal(t, url, *webhook.ClientConfig.URL, "expected URL to match") +} + +func TestSelfRegistrationWithOutURL(t *testing.T) { + + testClientSet := fake.NewSimpleClientset() + caCert := []byte("fake") + webHookDelay := 0 * time.Second + namespace := "default" + serviceName := "mpa-service" + url := "http://example.com/" + registerByURL := false + timeoutSeconds := int32(32) + selectedNamespace := "" + ignoredNamespaces := []string{} + + selfRegistration(testClientSet, caCert, webHookDelay, namespace, serviceName, url, registerByURL, timeoutSeconds, selectedNamespace, ignoredNamespaces, false, "") + + webhookConfigInterface := testClientSet.AdmissionregistrationV1().MutatingWebhookConfigurations() + webhookConfig, err := webhookConfigInterface.Get(context.TODO(), webhookConfigName, metav1.GetOptions{}) + + assert.NoError(t, err, "expected no error fetching webhook configuration") + + assert.Len(t, webhookConfig.Webhooks, 1, "expected one webhook configuration") + webhook := webhookConfig.Webhooks[0] + + assert.NotNil(t, webhook.ClientConfig.Service, "expected service reference to be nil") + assert.Equal(t, webhook.ClientConfig.Service.Name, serviceName, "expected service name to be equal") + assert.Equal(t, webhook.ClientConfig.Service.Namespace, namespace, "expected service namespace to be equal") + + assert.Nil(t, webhook.ClientConfig.URL, "expected URL to be set") +} + +func TestSelfRegistrationWithIgnoredNamespaces(t *testing.T) { + + testClientSet := fake.NewSimpleClientset() + caCert := []byte("fake") + webHookDelay := 0 * time.Second + namespace := "default" + serviceName := "mpa-service" + url := "http://example.com/" + registerByURL := false + timeoutSeconds := int32(32) + selectedNamespace := "" + ignoredNamespaces := []string{"test"} + + selfRegistration(testClientSet, caCert, webHookDelay, namespace, serviceName, url, registerByURL, timeoutSeconds, selectedNamespace, ignoredNamespaces, false, "") + + webhookConfigInterface := testClientSet.AdmissionregistrationV1().MutatingWebhookConfigurations() + webhookConfig, err := webhookConfigInterface.Get(context.TODO(), webhookConfigName, metav1.GetOptions{}) + + assert.NoError(t, err, "expected no error fetching webhook configuration") + + assert.Len(t, webhookConfig.Webhooks, 1, "expected one webhook configuration") + webhook := webhookConfig.Webhooks[0] + + assert.NotNil(t, webhook.NamespaceSelector.MatchExpressions, "expected namespace selector not to be nil") + assert.Len(t, webhook.NamespaceSelector.MatchExpressions, 1, "expected one match expression") + + matchExpression := webhook.NamespaceSelector.MatchExpressions[0] + assert.Equal(t, matchExpression.Operator, metav1.LabelSelectorOpNotIn, "expected namespace operator to be OpNotIn") + assert.Equal(t, matchExpression.Values, ignoredNamespaces, "expected namespace selector match expression to be equal") +} + +func TestSelfRegistrationWithSelectedNamespaces(t *testing.T) { + + testClientSet := fake.NewSimpleClientset() + caCert := []byte("fake") + webHookDelay := 0 * time.Second + namespace := "default" + serviceName := "mpa-service" + url := "http://example.com/" + registerByURL := false + timeoutSeconds := int32(32) + selectedNamespace := "test" + ignoredNamespaces := []string{} + + selfRegistration(testClientSet, caCert, webHookDelay, namespace, serviceName, url, registerByURL, timeoutSeconds, selectedNamespace, ignoredNamespaces, false, "") + + webhookConfigInterface := testClientSet.AdmissionregistrationV1().MutatingWebhookConfigurations() + webhookConfig, err := webhookConfigInterface.Get(context.TODO(), webhookConfigName, metav1.GetOptions{}) + + assert.NoError(t, err, "expected no error fetching webhook configuration") + + assert.Len(t, webhookConfig.Webhooks, 1, "expected one webhook configuration") + webhook := webhookConfig.Webhooks[0] + + assert.NotNil(t, webhook.NamespaceSelector.MatchExpressions, "expected namespace selector not to be nil") + assert.Len(t, webhook.NamespaceSelector.MatchExpressions, 1, "expected one match expression") + + matchExpression := webhook.NamespaceSelector.MatchExpressions[0] + assert.Equal(t, metav1.LabelSelectorOpIn, matchExpression.Operator, "expected namespace operator to be OpIn") + assert.Equal(t, matchExpression.Operator, metav1.LabelSelectorOpIn, "expected namespace operator to be OpIn") + assert.Equal(t, matchExpression.Values, []string{selectedNamespace}, "expected namespace selector match expression to be equal") +} + +func TestSelfRegistrationWithFailurePolicy(t *testing.T) { + + testClientSet := fake.NewSimpleClientset() + caCert := []byte("fake") + webHookDelay := 0 * time.Second + namespace := "default" + serviceName := "mpa-service" + url := "http://example.com/" + registerByURL := false + timeoutSeconds := int32(32) + selectedNamespace := "test" + ignoredNamespaces := []string{} + + selfRegistration(testClientSet, caCert, webHookDelay, namespace, serviceName, url, registerByURL, timeoutSeconds, selectedNamespace, ignoredNamespaces, true, "") + + webhookConfigInterface := testClientSet.AdmissionregistrationV1().MutatingWebhookConfigurations() + webhookConfig, err := webhookConfigInterface.Get(context.TODO(), webhookConfigName, metav1.GetOptions{}) + + assert.NoError(t, err, "expected no error fetching webhook configuration") + + assert.Len(t, webhookConfig.Webhooks, 1, "expected one webhook configuration") + webhook := webhookConfig.Webhooks[0] + + assert.NotNil(t, *webhook.FailurePolicy, "expected failurePolicy not to be nil") + assert.Equal(t, *webhook.FailurePolicy, admissionregistration.Fail, "expected failurePolicy to be Fail") +} + +func TestSelfRegistrationWithOutFailurePolicy(t *testing.T) { + + testClientSet := fake.NewSimpleClientset() + caCert := []byte("fake") + webHookDelay := 0 * time.Second + namespace := "default" + serviceName := "mpa-service" + url := "http://example.com/" + registerByURL := false + timeoutSeconds := int32(32) + selectedNamespace := "test" + ignoredNamespaces := []string{} + + selfRegistration(testClientSet, caCert, webHookDelay, namespace, serviceName, url, registerByURL, timeoutSeconds, selectedNamespace, ignoredNamespaces, false, "") + + webhookConfigInterface := testClientSet.AdmissionregistrationV1().MutatingWebhookConfigurations() + webhookConfig, err := webhookConfigInterface.Get(context.TODO(), webhookConfigName, metav1.GetOptions{}) + + assert.NoError(t, err, "expected no error fetching webhook configuration") + + assert.Len(t, webhookConfig.Webhooks, 1, "expected one webhook configuration") + webhook := webhookConfig.Webhooks[0] + + assert.NotNil(t, *webhook.FailurePolicy, "expected namespace selector not to be nil") + assert.Equal(t, *webhook.FailurePolicy, admissionregistration.Ignore, "expected failurePolicy to be Ignore") +} + +func TestSelfRegistrationWithInvalidLabels(t *testing.T) { + + testClientSet := fake.NewSimpleClientset() + caCert := []byte("fake") + webHookDelay := 0 * time.Second + namespace := "default" + serviceName := "mpa-service" + url := "http://example.com/" + registerByURL := true + timeoutSeconds := int32(32) + selectedNamespace := "" + ignoredNamespaces := []string{} + + selfRegistration(testClientSet, caCert, webHookDelay, namespace, serviceName, url, registerByURL, timeoutSeconds, selectedNamespace, ignoredNamespaces, false, "foo,bar") + + webhookConfigInterface := testClientSet.AdmissionregistrationV1().MutatingWebhookConfigurations() + webhookConfig, err := webhookConfigInterface.Get(context.TODO(), webhookConfigName, metav1.GetOptions{}) + + assert.NoError(t, err, "expected no error fetching webhook configuration") + assert.Equal(t, webhookConfigName, webhookConfig.Name, "expected webhook configuration name to match") + assert.Equal(t, webhookConfig.Labels, map[string]string{}, "expected invalid webhook configuration labels to match") + + assert.Len(t, webhookConfig.Webhooks, 1, "expected one webhook configuration") + webhook := webhookConfig.Webhooks[0] + assert.Equal(t, "mpa.k8s.io", webhook.Name, "expected webhook name to match") + + PodRule := webhook.Rules[0] + assert.Equal(t, []admissionregistration.OperationType{admissionregistration.Create}, PodRule.Operations, "expected operations to match") + assert.Equal(t, []string{""}, PodRule.APIGroups, "expected API groups to match") + assert.Equal(t, []string{"v1"}, PodRule.APIVersions, "expected API versions to match") + assert.Equal(t, []string{"pods"}, PodRule.Resources, "expected resources to match") + + MPARule := webhook.Rules[1] + assert.Equal(t, []admissionregistration.OperationType{admissionregistration.Create, admissionregistration.Update}, MPARule.Operations, "expected operations to match") + assert.Equal(t, []string{"autoscaling.k8s.io"}, MPARule.APIGroups, "expected API groups to match") + assert.Equal(t, []string{"*"}, MPARule.APIVersions, "ehook.Rulxpected API versions to match") + assert.Equal(t, []string{"verticalpodautoscalers"}, MPARule.Resources, "expected resources to match") + + assert.Equal(t, admissionregistration.SideEffectClassNone, *webhook.SideEffects, "expected side effects to match") + assert.Equal(t, admissionregistration.Ignore, *webhook.FailurePolicy, "expected failure policy to match") + assert.Equal(t, caCert, webhook.ClientConfig.CABundle, "expected CA bundle to match") + assert.Equal(t, timeoutSeconds, *webhook.TimeoutSeconds, "expected timeout seconds to match") +} + +func TestConvertLabelsToMap(t *testing.T) { + testCases := []struct { + desc string + labels string + expectedOutput map[string]string + expectedError bool + }{ + { + desc: "should return empty map when tag is empty", + labels: "", + expectedOutput: map[string]string{}, + expectedError: false, + }, + { + desc: "single valid tag should be converted", + labels: "key:value", + expectedOutput: map[string]string{ + "key": "value", + }, + expectedError: false, + }, + { + desc: "multiple valid labels should be converted", + labels: "key1:value1,key2:value2", + expectedOutput: map[string]string{ + "key1": "value1", + "key2": "value2", + }, + expectedError: false, + }, + { + desc: "whitespaces should be trimmed", + labels: "key1:value1, key2:value2", + expectedOutput: map[string]string{ + "key1": "value1", + "key2": "value2", + }, + expectedError: false, + }, + { + desc: "whitespaces between keys and values should be trimmed", + labels: "key1 : value1,key2 : value2", + expectedOutput: map[string]string{ + "key1": "value1", + "key2": "value2", + }, + expectedError: false, + }, + { + desc: "should return error for invalid format", + labels: "foo,bar", + expectedOutput: nil, + expectedError: true, + }, + { + desc: "should return error for when key is missed", + labels: "key1:value1,:bar", + expectedOutput: nil, + expectedError: true, + }, + { + desc: "should strip additional quotes", + labels: "\"key1:value1,key2:value2\"", + expectedOutput: map[string]string{ + "key1": "value1", + "key2": "value2", + }, + expectedError: false, + }, + } + + for i, c := range testCases { + m, err := convertLabelsToMap(c.labels) + if c.expectedError { + assert.NotNil(t, err, "TestCase[%d]: %s", i, c.desc) + } else { + assert.Nil(t, err, "TestCase[%d]: %s", i, c.desc) + assert.Equal(t, m, c.expectedOutput, "expected labels map") + } + } +} diff --git a/multidimensional-pod-autoscaler/pkg/admission-controller/delete-webhook.sh b/multidimensional-pod-autoscaler/pkg/admission-controller/delete-webhook.sh new file mode 100644 index 000000000000..ab9ab3209dfb --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/admission-controller/delete-webhook.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# Copyright 2024 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Unregisters the admission controller webhook. +set -e + +echo "Unregistering MPA admission controller webhook" + +kubectl delete -n kube-system mutatingwebhookconfiguration.v1.admissionregistration.k8s.io mpa-webhook-config diff --git a/multidimensional-pod-autoscaler/pkg/admission-controller/gencerts.sh b/multidimensional-pod-autoscaler/pkg/admission-controller/gencerts.sh new file mode 100755 index 000000000000..1a5b2d899b4b --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/admission-controller/gencerts.sh @@ -0,0 +1,70 @@ +#!/bin/bash + +# Copyright 2024 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Generates the a CA cert, a server key, and a server cert signed by the CA. +# reference: +# https://github.com/kubernetes/kubernetes/blob/master/plugin/pkg/admission/webhook/gencerts.sh +set -o errexit +set -o nounset +set -o pipefail + +CN_BASE="mpa_webhook" +TMP_DIR="/tmp/mpa-certs" + +echo "Generating certs for the MPA Admission Controller in ${TMP_DIR}." +mkdir -p ${TMP_DIR} +cat > ${TMP_DIR}/server.conf << EOF +[req] +req_extensions = v3_req +distinguished_name = req_distinguished_name +[req_distinguished_name] +[ v3_req ] +basicConstraints = CA:FALSE +keyUsage = nonRepudiation, digitalSignature, keyEncipherment +extendedKeyUsage = clientAuth, serverAuth +subjectAltName = DNS:mpa-webhook.kube-system.svc +EOF + +# Create a certificate authority +openssl genrsa -out ${TMP_DIR}/caKey.pem 2048 +set +o errexit +openssl req -x509 -new -nodes -key ${TMP_DIR}/caKey.pem -days 100000 -out ${TMP_DIR}/caCert.pem -subj "/CN=${CN_BASE}_ca" -addext "subjectAltName = DNS:${CN_BASE}_ca" +if [[ $? -ne 0 ]]; then + echo "ERROR: Failed to create CA certificate for self-signing." + exit 1 +fi +set -o errexit + +# Create a server certificate +openssl genrsa -out ${TMP_DIR}/serverKey.pem 2048 +# Note the CN is the DNS name of the service of the webhook. +openssl req -new -key ${TMP_DIR}/serverKey.pem -out ${TMP_DIR}/server.csr -subj "/CN=mpa-webhook.kube-system.svc" -config ${TMP_DIR}/server.conf +openssl x509 -req -in ${TMP_DIR}/server.csr -CA ${TMP_DIR}/caCert.pem -CAkey ${TMP_DIR}/caKey.pem -CAcreateserial -out ${TMP_DIR}/serverCert.pem -days 100000 -extensions SAN -extensions v3_req -extfile ${TMP_DIR}/server.conf + +echo "Uploading certs to the cluster." +kubectl create secret --namespace=kube-system generic mpa-tls-certs --from-file=${TMP_DIR}/caKey.pem --from-file=${TMP_DIR}/caCert.pem --from-file=${TMP_DIR}/serverKey.pem --from-file=${TMP_DIR}/serverCert.pem + +if [ "${1:-unset}" = "e2e" ]; then + openssl genrsa -out ${TMP_DIR}/e2eKey.pem 2048 + openssl req -new -key ${TMP_DIR}/e2eKey.pem -out ${TMP_DIR}/e2e.csr -subj "/CN=mpa-webhook.kube-system.svc" -config ${TMP_DIR}/server.conf + openssl x509 -req -in ${TMP_DIR}/e2e.csr -CA ${TMP_DIR}/caCert.pem -CAkey ${TMP_DIR}/caKey.pem -CAcreateserial -out ${TMP_DIR}/e2eCert.pem -days 100000 -extensions SAN -extensions v3_req -extfile ${TMP_DIR}/server.conf + echo "Uploading rotation e2e test certs to the cluster." + kubectl create secret --namespace=kube-system generic mpa-e2e-certs --from-file=${TMP_DIR}/e2eKey.pem --from-file=${TMP_DIR}/e2eCert.pem +fi + +# Clean up after we're done. +echo "Deleting ${TMP_DIR}." +rm -rf ${TMP_DIR} diff --git a/multidimensional-pod-autoscaler/pkg/admission-controller/logic/server.go b/multidimensional-pod-autoscaler/pkg/admission-controller/logic/server.go new file mode 100644 index 000000000000..17d767f4190c --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/admission-controller/logic/server.go @@ -0,0 +1,179 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package logic + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + + admissionv1 "k8s.io/api/admission/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/admission-controller/resource/mpa" + "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/admission-controller/resource/pod" + "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/admission-controller/resource/pod/patch" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/admission-controller/resource" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/limitrange" + metrics_admission "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/metrics/admission" + "k8s.io/klog/v2" +) + +// AdmissionServer is an admission webhook server that modifies pod resources request based on MPA recommendation +type AdmissionServer struct { + limitsChecker limitrange.LimitRangeCalculator + resourceHandlers map[metav1.GroupResource]resource.Handler +} + +// NewAdmissionServer constructs new AdmissionServer +func NewAdmissionServer(podPreProcessor pod.PreProcessor, + mpaPreProcessor mpa.PreProcessor, + limitsChecker limitrange.LimitRangeCalculator, + mpaMatcher mpa.Matcher, + patchCalculators []patch.Calculator) *AdmissionServer { + as := &AdmissionServer{limitsChecker, map[metav1.GroupResource]resource.Handler{}} + as.RegisterResourceHandler(pod.NewResourceHandler(podPreProcessor, mpaMatcher, patchCalculators)) + as.RegisterResourceHandler(mpa.NewResourceHandler(mpaPreProcessor)) + return as +} + +// RegisterResourceHandler allows to register a custom logic for handling given types of resources. +func (s *AdmissionServer) RegisterResourceHandler(resourceHandler resource.Handler) { + s.resourceHandlers[resourceHandler.GroupResource()] = resourceHandler +} + +func (s *AdmissionServer) admit(ctx context.Context, data []byte) (*admissionv1.AdmissionResponse, metrics_admission.AdmissionStatus, metrics_admission.AdmissionResource) { + // we don't block the admission by default, even on unparsable JSON + response := admissionv1.AdmissionResponse{} + response.Allowed = true + + ar := admissionv1.AdmissionReview{} + if err := json.Unmarshal(data, &ar); err != nil { + klog.Error(err) + return &response, metrics_admission.Error, metrics_admission.Unknown + } + + response.UID = ar.Request.UID + + var patches []resource.PatchRecord + var err error + resource := metrics_admission.Unknown + + admittedGroupResource := metav1.GroupResource{ + Group: ar.Request.Resource.Group, + Resource: ar.Request.Resource.Resource, + } + + handler, ok := s.resourceHandlers[admittedGroupResource] + if ok { + patches, err = handler.GetPatches(ctx, ar.Request) + resource = handler.AdmissionResource() + + if handler.DisallowIncorrectObjects() && err != nil { + // we don't let in problematic objects - late validation + status := metav1.Status{} + status.Status = "Failure" + status.Message = err.Error() + response.Result = &status + response.Allowed = false + } + } else { + patches, err = nil, fmt.Errorf("not supported resource type: %v", admittedGroupResource) + } + + if err != nil { + klog.Error(err) + return &response, metrics_admission.Error, resource + } + + if len(patches) > 0 { + patch, err := json.Marshal(patches) + if err != nil { + klog.Errorf("Cannot marshal the patch %v: %v", patches, err) + return &response, metrics_admission.Error, resource + } + patchType := admissionv1.PatchTypeJSONPatch + response.PatchType = &patchType + response.Patch = patch + klog.V(4).InfoS("Sending patches", "patches", patches) + } + + var status metrics_admission.AdmissionStatus + if len(patches) > 0 { + status = metrics_admission.Applied + } else { + status = metrics_admission.Skipped + } + if resource == metrics_admission.Pod { + metrics_admission.OnAdmittedPod(status == metrics_admission.Applied) + } + + return &response, status, resource +} + +// Serve is a handler function of AdmissionServer +func (s *AdmissionServer) Serve(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + executionTimer := metrics_admission.NewExecutionTimer() + defer executionTimer.ObserveTotal() + admissionLatency := metrics_admission.NewAdmissionLatency() + + var body []byte + if r.Body != nil { + if data, err := io.ReadAll(r.Body); err == nil { + body = data + } + } + // verify the content type is accurate + contentType := r.Header.Get("Content-Type") + if contentType != "application/json" { + klog.Errorf("contentType=%s, expect application/json", contentType) + admissionLatency.Observe(metrics_admission.Error, metrics_admission.Unknown) + return + } + executionTimer.ObserveStep("read_request") + + reviewResponse, status, resource := s.admit(ctx, body) + ar := admissionv1.AdmissionReview{ + Response: reviewResponse, + TypeMeta: metav1.TypeMeta{ + Kind: "AdmissionReview", + APIVersion: "admission.k8s.io/v1", + }, + } + executionTimer.ObserveStep("admit") + + resp, err := json.Marshal(ar) + if err != nil { + klog.Error(err) + admissionLatency.Observe(metrics_admission.Error, resource) + return + } + executionTimer.ObserveStep("build_response") + + _, err = w.Write(resp) + if err != nil { + klog.Error(err) + admissionLatency.Observe(metrics_admission.Error, resource) + return + } + executionTimer.ObserveStep("write_response") + + admissionLatency.Observe(status, resource) +} diff --git a/multidimensional-pod-autoscaler/pkg/admission-controller/main.go b/multidimensional-pod-autoscaler/pkg/admission-controller/main.go new file mode 100644 index 000000000000..9de357d2a41b --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/admission-controller/main.go @@ -0,0 +1,163 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "flag" + "fmt" + "net/http" + "os" + "strings" + "time" + + apiv1 "k8s.io/api/core/v1" + "k8s.io/autoscaler/multidimensional-pod-autoscaler/common" + "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/admission-controller/logic" + mpa "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/admission-controller/resource/mpa" + "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/admission-controller/resource/pod" + "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/admission-controller/resource/pod/patch" + "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/admission-controller/resource/pod/recommendation" + mpa_clientset "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/client/clientset/versioned" + "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/target" + mpa_api_util "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/utils/mpa" + vpa_common "k8s.io/autoscaler/vertical-pod-autoscaler/common" + controllerfetcher "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/target/controller_fetcher" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/limitrange" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/metrics" + metrics_admission "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/metrics/admission" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/server" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/status" + "k8s.io/client-go/informers" + kube_client "k8s.io/client-go/kubernetes" + kube_flag "k8s.io/component-base/cli/flag" + "k8s.io/klog/v2" +) + +const ( + defaultResyncPeriod = 10 * time.Minute + statusUpdateInterval = 10 * time.Second + scaleCacheEntryLifetime time.Duration = time.Hour + scaleCacheEntryFreshnessTime time.Duration = 10 * time.Minute + scaleCacheEntryJitterFactor float64 = 1. + webHookDelay = 10 * time.Second +) + +var ( + certsConfiguration = &certsConfig{ + clientCaFile: flag.String("client-ca-file", "/etc/tls-certs/caCert.pem", "Path to CA PEM file."), + tlsCertFile: flag.String("tls-cert-file", "/etc/tls-certs/serverCert.pem", "Path to server certificate PEM file."), + tlsPrivateKey: flag.String("tls-private-key", "/etc/tls-certs/serverKey.pem", "Path to server certificate key PEM file."), + reload: flag.Bool("reload-cert", false, "If set to true, reload leaf certificate."), + } + + ciphers = flag.String("tls-ciphers", "", "A comma-separated or colon-separated list of ciphers to accept. Only works when min-tls-version is set to tls1_2.") + minTlsVersion = flag.String("min-tls-version", "tls1_2", "The minimum TLS version to accept. Must be set to either tls1_2 (default) or tls1_3.") + port = flag.Int("port", 8000, "The port to listen on.") + address = flag.String("address", ":8944", "The address to expose Prometheus metrics.") + // kubeconfig = flag.String("kubeconfig", "", "Path to a kubeconfig. Only required if out-of-cluster.") + // kubeApiQps = flag.Float64("kube-api-qps", 5.0, `QPS limit when making requests to Kubernetes apiserver`) + // kubeApiBurst = flag.Float64("kube-api-burst", 10.0, `QPS burst limit when making requests to Kubernetes apiserver`) + namespace = os.Getenv("NAMESPACE") + serviceName = flag.String("webhook-service", "mpa-webhook", "Kubernetes service under which webhook is registered. Used when registerByURL is set to false.") + webhookAddress = flag.String("webhook-address", "", "Address under which webhook is registered. Used when registerByURL is set to true.") + webhookPort = flag.String("webhook-port", "", "Server Port for Webhook") + webhookTimeout = flag.Int("webhook-timeout-seconds", 30, "Timeout in seconds that the API server should wait for this webhook to respond before failing.") + webHookFailurePolicy = flag.Bool("webhook-failure-policy-fail", false, "If set to true, will configure the admission webhook failurePolicy to \"Fail\". Use with caution.") + registerWebhook = flag.Bool("register-webhook", true, "If set to true, admission webhook object will be created on start up to register with the API server.") + webhookLabels = flag.String("webhook-labels", "", "Comma separated list of labels to add to the webhook object. Format: key1:value1,key2:value2") + registerByURL = flag.Bool("register-by-url", false, "If set to true, admission webhook will be registered by URL (webhookAddress:webhookPort) instead of by service name") + mpaObjectNamespace = flag.String("mpa-object-namespace", apiv1.NamespaceAll, "Namespace to search for MPA objects. Empty means all namespaces will be used.") + ignoredMpaObjectNamespaces = flag.String("ignored-mpa-object-namespaces", "", "Comma separated list of namespaces to ignore when searching for MPA objects. Empty means no namespaces will be ignored.") +) + +func main() { + commonFlags := vpa_common.InitCommonFlags() + klog.InitFlags(nil) + vpa_common.InitLoggingFlags() + kube_flag.InitFlags() + klog.V(1).Infof("Multi-dimensional Pod Autoscaler %s Admission Controller", common.MultidimPodAutoscalerVersion) + + if len(*mpaObjectNamespace) > 0 && len(*ignoredMpaObjectNamespaces) > 0 { + klog.Fatalf("--mpa-object-namespace and --ignored-mpa-object-namespaces are mutually exclusive and can't be set together.") + } + + healthCheck := metrics.NewHealthCheck(time.Minute) + metrics_admission.Register() + server.Initialize(&commonFlags.EnableProfiling, healthCheck, address) + + config := common.CreateKubeConfigOrDie(commonFlags.KubeConfig, float32(commonFlags.KubeApiQps), int(commonFlags.KubeApiBurst)) + + mpaClient := mpa_clientset.NewForConfigOrDie(config) + mpaLister := mpa_api_util.NewMpasLister(mpaClient, make(chan struct{}), *mpaObjectNamespace) + kubeClient := kube_client.NewForConfigOrDie(config) + factory := informers.NewSharedInformerFactory(kubeClient, defaultResyncPeriod) + targetSelectorFetcher := target.NewMpaTargetSelectorFetcher(config, kubeClient, factory) + controllerFetcher := controllerfetcher.NewControllerFetcher(config, kubeClient, factory, scaleCacheEntryFreshnessTime, scaleCacheEntryLifetime, scaleCacheEntryJitterFactor) + podPreprocessor := pod.NewDefaultPreProcessor() + mpaPreprocessor := mpa.NewDefaultPreProcessor() + var limitRangeCalculator limitrange.LimitRangeCalculator + limitRangeCalculator, err := limitrange.NewLimitsRangeCalculator(factory) + if err != nil { + klog.Errorf("Failed to create limitRangeCalculator, falling back to not checking limits. Error message: %s", err) + limitRangeCalculator = limitrange.NewNoopLimitsCalculator() + } + recommendationProvider := recommendation.NewProvider(limitRangeCalculator, mpa_api_util.NewCappingRecommendationProcessor(limitRangeCalculator)) + mpaMatcher := mpa.NewMatcher(mpaLister, targetSelectorFetcher, controllerFetcher) + + hostname, err := os.Hostname() + if err != nil { + klog.Fatalf("Unable to get hostname: %v", err) + } + + statusNamespace := status.AdmissionControllerStatusNamespace + if namespace != "" { + statusNamespace = namespace + } + stopCh := make(chan struct{}) + statusUpdater := status.NewUpdater( + kubeClient, + status.AdmissionControllerStatusName, + statusNamespace, + statusUpdateInterval, + hostname, + ) + defer close(stopCh) + + calculators := []patch.Calculator{patch.NewResourceUpdatesCalculator(recommendationProvider), patch.NewObservedContainersCalculator()} + as := logic.NewAdmissionServer(podPreprocessor, mpaPreprocessor, limitRangeCalculator, mpaMatcher, calculators) + http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + as.Serve(w, r) + healthCheck.UpdateLastActivity() + }) + server := &http.Server{ + Addr: fmt.Sprintf(":%d", *port), + TLSConfig: configTLS(*certsConfiguration, *minTlsVersion, *ciphers, stopCh), + } + url := fmt.Sprintf("%v:%v", *webhookAddress, *webhookPort) + ignoredNamespaces := strings.Split(*ignoredMpaObjectNamespaces, ",") + go func() { + if *registerWebhook { + selfRegistration(kubeClient, readFile(*certsConfiguration.clientCaFile), webHookDelay, namespace, *serviceName, url, *registerByURL, int32(*webhookTimeout), commonFlags.VpaObjectNamespace, ignoredNamespaces, *webHookFailurePolicy, *webhookLabels) + } + // Start status updates after the webhook is initialized. + statusUpdater.Run(stopCh) + }() + + if err = server.ListenAndServeTLS("", ""); err != nil { + klog.Fatalf("HTTPS Error: %s", err) + } +} diff --git a/multidimensional-pod-autoscaler/pkg/admission-controller/resource/mpa/handler.go b/multidimensional-pod-autoscaler/pkg/admission-controller/resource/mpa/handler.go new file mode 100644 index 000000000000..675178dbe7c2 --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/admission-controller/resource/mpa/handler.go @@ -0,0 +1,202 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mpa + +import ( + "context" + "encoding/json" + "fmt" + + v1 "k8s.io/api/admission/v1" + corev1 "k8s.io/api/core/v1" + apires "k8s.io/apimachinery/pkg/api/resource" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + mpa_types "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1alpha1" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/admission-controller/resource" + vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/metrics/admission" + "k8s.io/klog/v2" +) + +var ( + possibleUpdateModes = map[vpa_types.UpdateMode]interface{}{ + vpa_types.UpdateModeOff: struct{}{}, + vpa_types.UpdateModeInitial: struct{}{}, + vpa_types.UpdateModeRecreate: struct{}{}, + vpa_types.UpdateModeAuto: struct{}{}, + } + + possibleScalingModes = map[vpa_types.ContainerScalingMode]interface{}{ + vpa_types.ContainerScalingModeAuto: struct{}{}, + vpa_types.ContainerScalingModeOff: struct{}{}, + } +) + +// resourceHandler builds patches for VPAs. +type resourceHandler struct { + preProcessor PreProcessor +} + +// NewResourceHandler creates new instance of resourceHandler. +func NewResourceHandler(preProcessor PreProcessor) resource.Handler { + return &resourceHandler{preProcessor: preProcessor} +} + +// AdmissionResource returns resource type this handler accepts. +func (h *resourceHandler) AdmissionResource() admission.AdmissionResource { + return admission.Vpa +} + +// GroupResource returns Group and Resource type this handler accepts. +func (h *resourceHandler) GroupResource() metav1.GroupResource { + return metav1.GroupResource{Group: "autoscaling.k8s.io", Resource: "multidimpodautoscalers"} +} + +// DisallowIncorrectObjects decides whether incorrect objects (eg. unparsable, not passing validations) should be disallowed by Admission Server. +func (h *resourceHandler) DisallowIncorrectObjects() bool { + return true +} + +// GetPatches builds patches for VPA in given admission request. +func (h *resourceHandler) GetPatches(_ context.Context, ar *v1.AdmissionRequest) ([]resource.PatchRecord, error) { + raw, isCreate := ar.Object.Raw, ar.Operation == v1.Create + mpa, err := parseMPA(raw) + if err != nil { + return nil, err + } + + mpa, err = h.preProcessor.Process(mpa, isCreate) + if err != nil { + return nil, err + } + + err = ValidateMPA(mpa, isCreate) + if err != nil { + return nil, err + } + + klog.V(4).InfoS("Processing mpa", "mpa", mpa) + patches := []resource.PatchRecord{} + if mpa.Spec.Policy == nil { + // Sets the default updatePolicy. + defaultUpdateMode := vpa_types.UpdateModeAuto + patches = append(patches, resource.PatchRecord{ + Op: "add", + Path: "/spec/updatePolicy", + Value: vpa_types.PodUpdatePolicy{UpdateMode: &defaultUpdateMode}}) + } + return patches, nil +} + +func parseMPA(raw []byte) (*mpa_types.MultidimPodAutoscaler, error) { + mpa := mpa_types.MultidimPodAutoscaler{} + if err := json.Unmarshal(raw, &mpa); err != nil { + return nil, err + } + return &mpa, nil +} + +// ValidateMPA checks the correctness of MPA Spec and returns an error if there is a problem. +func ValidateMPA(mpa *mpa_types.MultidimPodAutoscaler, isCreate bool) error { + if mpa.Spec.Policy != nil { + mode := mpa.Spec.Policy.UpdateMode + if mode == nil { + return fmt.Errorf("UpdateMode is required if UpdatePolicy is used") + } + if _, found := possibleUpdateModes[*mode]; !found { + return fmt.Errorf("unexpected UpdateMode value %s", *mode) + } + } + + if mpa.Spec.Constraints != nil { + if mpa.Spec.Constraints.Global != nil { + if minReplicas := mpa.Spec.Constraints.Global.MinReplicas; minReplicas != nil && *minReplicas <= 0 { + return fmt.Errorf("MinReplicas has to be positive, got %v", *minReplicas) + } + } + } + + if mpa.Spec.ResourcePolicy != nil { + for _, policy := range mpa.Spec.ResourcePolicy.ContainerPolicies { + if policy.ContainerName == "" { + return fmt.Errorf("ContainerPolicies.ContainerName is required") + } + mode := policy.Mode + if mode != nil { + if _, found := possibleScalingModes[*mode]; !found { + return fmt.Errorf("unexpected Mode value %s", *mode) + } + } + for resource, min := range policy.MinAllowed { + if err := validateResourceResolution(resource, min); err != nil { + return fmt.Errorf("MinAllowed: %v", err) + } + max, found := policy.MaxAllowed[resource] + if found && max.Cmp(min) < 0 { + return fmt.Errorf("max resource for %v is lower than min", resource) + } + } + + for resource, max := range policy.MaxAllowed { + if err := validateResourceResolution(resource, max); err != nil { + return fmt.Errorf("MaxAllowed: %v", err) + } + } + ControlledValues := policy.ControlledValues + if mode != nil && ControlledValues != nil { + if *mode == vpa_types.ContainerScalingModeOff && *ControlledValues == vpa_types.ContainerControlledValuesRequestsAndLimits { + return fmt.Errorf("ControlledValues shouldn't be specified if container scaling mode is off.") + } + } + } + } + + if isCreate && mpa.Spec.ScaleTargetRef == nil { + return fmt.Errorf("ScaleTargetRef is required.") + } + + if len(mpa.Spec.Recommenders) > 1 { + return fmt.Errorf("The current version of MPA object shouldn't specify more than one recommenders.") + } + + return nil +} + +func validateResourceResolution(name corev1.ResourceName, val apires.Quantity) error { + switch name { + case corev1.ResourceCPU: + return validateCPUResolution(val) + case corev1.ResourceMemory: + return validateMemoryResolution(val) + } + return nil +} + +func validateCPUResolution(val apires.Quantity) error { + if _, precissionPreserved := val.AsScale(apires.Milli); !precissionPreserved { + return fmt.Errorf("CPU [%s] must be a whole number of milli CPUs", val.String()) + } + return nil +} + +func validateMemoryResolution(val apires.Quantity) error { + if _, precissionPreserved := val.AsScale(0); !precissionPreserved { + return fmt.Errorf("Memory [%v] must be a whole number of bytes", val) + } + return nil +} diff --git a/multidimensional-pod-autoscaler/pkg/admission-controller/resource/mpa/handler_test.go b/multidimensional-pod-autoscaler/pkg/admission-controller/resource/mpa/handler_test.go new file mode 100644 index 000000000000..1ee9b5f5461b --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/admission-controller/resource/mpa/handler_test.go @@ -0,0 +1,314 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mpa + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + apiv1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + mpa_types "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1alpha1" + vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" +) + +const ( + cpu = apiv1.ResourceCPU + memory = apiv1.ResourceMemory +) + +func TestValidateVPA(t *testing.T) { + badUpdateMode := vpa_types.UpdateMode("bad") + validUpdateMode := vpa_types.UpdateModeOff + badMinReplicas := int32(0) + validMinReplicas := int32(1) + badScalingMode := vpa_types.ContainerScalingMode("bad") + badCPUResource := resource.MustParse("187500u") + validScalingMode := vpa_types.ContainerScalingModeAuto + scalingModeOff := vpa_types.ContainerScalingModeOff + controlledValuesRequestsAndLimits := vpa_types.ContainerControlledValuesRequestsAndLimits + tests := []struct { + name string + mpa mpa_types.MultidimPodAutoscaler + isCreate bool + expectError error + }{ + { + name: "empty update", + mpa: mpa_types.MultidimPodAutoscaler{}, + }, + { + name: "empty create", + mpa: mpa_types.MultidimPodAutoscaler{}, + isCreate: true, + expectError: fmt.Errorf("ScaleTargetRef is required."), + }, + { + name: "no update mode", + mpa: mpa_types.MultidimPodAutoscaler{ + Spec: mpa_types.MultidimPodAutoscalerSpec{ + Policy: &mpa_types.PodUpdatePolicy{}, + Constraints: &mpa_types.ScalingConstraints{}, + }, + }, + expectError: fmt.Errorf("UpdateMode is required if UpdatePolicy is used"), + }, + { + name: "bad update mode", + mpa: mpa_types.MultidimPodAutoscaler{ + Spec: mpa_types.MultidimPodAutoscalerSpec{ + Policy: &mpa_types.PodUpdatePolicy{ + UpdateMode: &badUpdateMode, + }, + Constraints: &mpa_types.ScalingConstraints{}, + }, + }, + expectError: fmt.Errorf("unexpected UpdateMode value bad"), + }, + { + name: "zero minReplicas", + mpa: mpa_types.MultidimPodAutoscaler{ + Spec: mpa_types.MultidimPodAutoscalerSpec{ + Policy: &mpa_types.PodUpdatePolicy{ + UpdateMode: &validUpdateMode, + }, + Constraints: &mpa_types.ScalingConstraints{ + Global: &mpa_types.HorizontalScalingConstraints{ + MinReplicas: &badMinReplicas, + }, + }, + }, + }, + expectError: fmt.Errorf("MinReplicas has to be positive, got 0"), + }, + { + name: "no policy name", + mpa: mpa_types.MultidimPodAutoscaler{ + Spec: mpa_types.MultidimPodAutoscalerSpec{ + ResourcePolicy: &vpa_types.PodResourcePolicy{ + ContainerPolicies: []vpa_types.ContainerResourcePolicy{{}}, + }, + Constraints: &mpa_types.ScalingConstraints{}, + }, + }, + expectError: fmt.Errorf("ContainerPolicies.ContainerName is required"), + }, + { + name: "invalid scaling mode", + mpa: mpa_types.MultidimPodAutoscaler{ + Spec: mpa_types.MultidimPodAutoscalerSpec{ + ResourcePolicy: &vpa_types.PodResourcePolicy{ + ContainerPolicies: []vpa_types.ContainerResourcePolicy{ + { + ContainerName: "loot box", + Mode: &badScalingMode, + }, + }, + }, + Constraints: &mpa_types.ScalingConstraints{}, + }, + }, + expectError: fmt.Errorf("unexpected Mode value bad"), + }, + { + name: "more than one recommender", + mpa: mpa_types.MultidimPodAutoscaler{ + Spec: mpa_types.MultidimPodAutoscalerSpec{ + Policy: &mpa_types.PodUpdatePolicy{ + UpdateMode: &validUpdateMode, + }, + Recommenders: []*mpa_types.MultidimPodAutoscalerRecommenderSelector{ + {Name: "test1"}, + {Name: "test2"}, + }, + Constraints: &mpa_types.ScalingConstraints{}, + }, + }, + expectError: fmt.Errorf("The current version of MPA object shouldn't specify more than one recommenders."), + }, + { + name: "bad limits", + mpa: mpa_types.MultidimPodAutoscaler{ + Spec: mpa_types.MultidimPodAutoscalerSpec{ + ResourcePolicy: &vpa_types.PodResourcePolicy{ + ContainerPolicies: []vpa_types.ContainerResourcePolicy{ + { + ContainerName: "loot box", + MinAllowed: apiv1.ResourceList{ + cpu: resource.MustParse("100"), + }, + MaxAllowed: apiv1.ResourceList{ + cpu: resource.MustParse("10"), + }, + }, + }, + }, + Constraints: &mpa_types.ScalingConstraints{}, + }, + }, + expectError: fmt.Errorf("max resource for cpu is lower than min"), + }, + { + name: "bad minAllowed cpu value", + mpa: mpa_types.MultidimPodAutoscaler{ + Spec: mpa_types.MultidimPodAutoscalerSpec{ + ResourcePolicy: &vpa_types.PodResourcePolicy{ + ContainerPolicies: []vpa_types.ContainerResourcePolicy{ + { + ContainerName: "loot box", + MinAllowed: apiv1.ResourceList{ + cpu: resource.MustParse("187500u"), + }, + MaxAllowed: apiv1.ResourceList{ + cpu: resource.MustParse("275m"), + }, + }, + }, + }, + Constraints: &mpa_types.ScalingConstraints{}, + }, + }, + expectError: fmt.Errorf("MinAllowed: CPU [%v] must be a whole number of milli CPUs", badCPUResource.String()), + }, + { + name: "bad minAllowed memory value", + mpa: mpa_types.MultidimPodAutoscaler{ + Spec: mpa_types.MultidimPodAutoscalerSpec{ + ResourcePolicy: &vpa_types.PodResourcePolicy{ + ContainerPolicies: []vpa_types.ContainerResourcePolicy{ + { + ContainerName: "loot box", + MinAllowed: apiv1.ResourceList{ + cpu: resource.MustParse("1m"), + memory: resource.MustParse("100m"), + }, + MaxAllowed: apiv1.ResourceList{ + cpu: resource.MustParse("275m"), + memory: resource.MustParse("500M"), + }, + }, + }, + }, + Constraints: &mpa_types.ScalingConstraints{}, + }, + }, + expectError: fmt.Errorf("MinAllowed: Memory [%v] must be a whole number of bytes", resource.MustParse("100m")), + }, + { + name: "bad maxAllowed cpu value", + mpa: mpa_types.MultidimPodAutoscaler{ + Spec: mpa_types.MultidimPodAutoscalerSpec{ + ResourcePolicy: &vpa_types.PodResourcePolicy{ + ContainerPolicies: []vpa_types.ContainerResourcePolicy{ + { + ContainerName: "loot box", + MinAllowed: apiv1.ResourceList{}, + MaxAllowed: apiv1.ResourceList{ + cpu: resource.MustParse("187500u"), + }, + }, + }, + }, + Constraints: &mpa_types.ScalingConstraints{}, + }, + }, + expectError: fmt.Errorf("MaxAllowed: CPU [%s] must be a whole number of milli CPUs", badCPUResource.String()), + }, + { + name: "bad maxAllowed memory value", + mpa: mpa_types.MultidimPodAutoscaler{ + Spec: mpa_types.MultidimPodAutoscalerSpec{ + ResourcePolicy: &vpa_types.PodResourcePolicy{ + ContainerPolicies: []vpa_types.ContainerResourcePolicy{ + { + ContainerName: "loot box", + MinAllowed: apiv1.ResourceList{ + cpu: resource.MustParse("1m")}, + MaxAllowed: apiv1.ResourceList{ + cpu: resource.MustParse("275m"), + memory: resource.MustParse("500m"), + }, + }, + }, + }, + Constraints: &mpa_types.ScalingConstraints{}, + }, + }, + expectError: fmt.Errorf("MaxAllowed: Memory [%v] must be a whole number of bytes", resource.MustParse("500m")), + }, + { + name: "scaling off with controlled values requests and limits", + mpa: mpa_types.MultidimPodAutoscaler{ + Spec: mpa_types.MultidimPodAutoscalerSpec{ + ResourcePolicy: &vpa_types.PodResourcePolicy{ + ContainerPolicies: []vpa_types.ContainerResourcePolicy{ + { + ContainerName: "loot box", + Mode: &scalingModeOff, + ControlledValues: &controlledValuesRequestsAndLimits, + }, + }, + }, + Constraints: &mpa_types.ScalingConstraints{}, + }, + }, + expectError: fmt.Errorf("ControlledValues shouldn't be specified if container scaling mode is off."), + }, + { + name: "all valid", + mpa: mpa_types.MultidimPodAutoscaler{ + Spec: mpa_types.MultidimPodAutoscalerSpec{ + ResourcePolicy: &vpa_types.PodResourcePolicy{ + ContainerPolicies: []vpa_types.ContainerResourcePolicy{ + { + ContainerName: "loot box", + Mode: &validScalingMode, + MinAllowed: apiv1.ResourceList{ + cpu: resource.MustParse("10"), + }, + MaxAllowed: apiv1.ResourceList{ + cpu: resource.MustParse("100"), + }, + }, + }, + }, + Policy: &mpa_types.PodUpdatePolicy{ + UpdateMode: &validUpdateMode, + }, + Constraints: &mpa_types.ScalingConstraints{ + Global: &mpa_types.HorizontalScalingConstraints{ + MinReplicas: &validMinReplicas, + }, + }, + }, + }, + }, + } + for _, tc := range tests { + t.Run(fmt.Sprintf("test case: %s", tc.name), func(t *testing.T) { + err := ValidateMPA(&tc.mpa, tc.isCreate) + if tc.expectError == nil { + assert.NoError(t, err) + } else { + if assert.Error(t, err) { + assert.Equal(t, tc.expectError.Error(), err.Error()) + } + } + }) + } +} diff --git a/multidimensional-pod-autoscaler/pkg/admission-controller/resource/mpa/matcher.go b/multidimensional-pod-autoscaler/pkg/admission-controller/resource/mpa/matcher.go new file mode 100644 index 000000000000..46d05b7e4efb --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/admission-controller/resource/mpa/matcher.go @@ -0,0 +1,98 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mpa + +import ( + "context" + + core "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/labels" + mpa_types "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1alpha1" + mpa_lister "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/client/listers/autoscaling.k8s.io/v1alpha1" + "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/target" + mpa_api_util "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/utils/mpa" + vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" + controllerfetcher "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/target/controller_fetcher" + "k8s.io/klog/v2" +) + +// Matcher is capable of returning a single matching MPA object +// for a pod. Will return nil if no matching object is found. +type Matcher interface { + GetMatchingMPA(ctx context.Context, pod *core.Pod) *mpa_types.MultidimPodAutoscaler +} + +type matcher struct { + mpaLister mpa_lister.MultidimPodAutoscalerLister + selectorFetcher target.MpaTargetSelectorFetcher + controllerFetcher controllerfetcher.ControllerFetcher +} + +// NewMatcher returns a new MPA matcher. +func NewMatcher(mpaLister mpa_lister.MultidimPodAutoscalerLister, + selectorFetcher target.MpaTargetSelectorFetcher, + controllerFetcher controllerfetcher.ControllerFetcher) Matcher { + return &matcher{mpaLister: mpaLister, + selectorFetcher: selectorFetcher, + controllerFetcher: controllerFetcher} +} + +func (m *matcher) GetMatchingMPA(ctx context.Context, pod *core.Pod) *mpa_types.MultidimPodAutoscaler { + parentController, err := mpa_api_util.FindParentControllerForPod(ctx, pod, m.controllerFetcher) + if err != nil { + klog.ErrorS(err, "Failed to get parent controller for pod", "pod", klog.KObj(pod)) + return nil + } + if parentController == nil { + return nil + } + + configs, err := m.mpaLister.MultidimPodAutoscalers(pod.Namespace).List(labels.Everything()) + if err != nil { + klog.Errorf("failed to get mpa configs: %v", err) + return nil + } + + var controllingMpa *mpa_types.MultidimPodAutoscaler + for _, mpaConfig := range configs { + if mpa_api_util.GetUpdateMode(mpaConfig) == vpa_types.UpdateModeOff { + continue + } + if mpaConfig.Spec.ScaleTargetRef == nil { + klog.V(5).InfoS("Skipping MPA object because scaleTargetRef is not defined.", "mpa", klog.KObj(mpaConfig)) + continue + } + if mpaConfig.Spec.ScaleTargetRef.Kind != parentController.Kind || + mpaConfig.Namespace != parentController.Namespace || + mpaConfig.Spec.ScaleTargetRef.Name != parentController.Name { + continue // This pod is not associated to the right controller + } + + selector, err := m.selectorFetcher.Fetch(ctx, mpaConfig) + if err != nil { + klog.V(3).InfoS("Skipping MPA object because we cannot fetch selector", "mpa", klog.KObj(mpaConfig), "error", err) + continue + } + + mpaWithSelector := &mpa_api_util.MpaWithSelector{Mpa: mpaConfig, Selector: selector} + if mpa_api_util.PodMatchesMPA(pod, mpaWithSelector) && mpa_api_util.Stronger(mpaConfig, controllingMpa) { + controllingMpa = mpaConfig + } + } + + return controllingMpa +} diff --git a/multidimensional-pod-autoscaler/pkg/admission-controller/resource/mpa/matcher_test.go b/multidimensional-pod-autoscaler/pkg/admission-controller/resource/mpa/matcher_test.go new file mode 100644 index 000000000000..cba79b2d730a --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/admission-controller/resource/mpa/matcher_test.go @@ -0,0 +1,137 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mpa + +import ( + "context" + "testing" + + core "k8s.io/api/core/v1" + meta "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + mpa_types "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1alpha1" + target_mock "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/target/mock" + "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/utils/test" + vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" + controllerfetcher "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/target/controller_fetcher" + test_vpa "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/test" + + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/assert" +) + +func parseLabelSelector(selector string) labels.Selector { + labelSelector, _ := meta.ParseToLabelSelector(selector) + parsedSelector, _ := meta.LabelSelectorAsSelector(labelSelector) + return parsedSelector +} + +func TestGetMatchingVpa(t *testing.T) { + podBuilder := test_vpa.Pod().WithName("test-pod").WithLabels(map[string]string{"app": "test"}). + AddContainer(test.Container().WithName("i-am-container").Get()) + mpaBuilder := test.MultidimPodAutoscaler().WithContainer("i-am-container") + testCases := []struct { + name string + pod *core.Pod + mpas []*mpa_types.MultidimPodAutoscaler + labelSelector string + expectedFound bool + expectedVpaName string + }{ + { + name: "matching selector", + pod: podBuilder.Get(), + mpas: []*mpa_types.MultidimPodAutoscaler{ + mpaBuilder.WithUpdateMode(vpa_types.UpdateModeAuto).WithName("auto-mpa").Get(), + }, + labelSelector: "app = test", + expectedFound: true, + expectedVpaName: "auto-mpa", + }, { + name: "not matching selector", + pod: podBuilder.Get(), + mpas: []*mpa_types.MultidimPodAutoscaler{ + mpaBuilder.WithUpdateMode(vpa_types.UpdateModeAuto).WithName("auto-mpa").Get(), + }, + labelSelector: "app = differentApp", + expectedFound: false, + }, { + name: "off mode", + pod: podBuilder.Get(), + mpas: []*mpa_types.MultidimPodAutoscaler{ + mpaBuilder.WithUpdateMode(vpa_types.UpdateModeOff).WithName("off-mpa").Get(), + }, + labelSelector: "app = test", + expectedFound: false, + }, { + name: "two vpas one in off mode", + pod: podBuilder.Get(), + mpas: []*mpa_types.MultidimPodAutoscaler{ + mpaBuilder.WithUpdateMode(vpa_types.UpdateModeOff).WithName("off-mpa").Get(), + mpaBuilder.WithUpdateMode(vpa_types.UpdateModeAuto).WithName("auto-mpa").Get(), + }, + labelSelector: "app = test", + expectedFound: true, + expectedVpaName: "auto-mpa", + }, { + name: "initial mode", + pod: podBuilder.Get(), + mpas: []*mpa_types.MultidimPodAutoscaler{ + mpaBuilder.WithUpdateMode(vpa_types.UpdateModeInitial).WithName("initial-mpa").Get(), + }, + labelSelector: "app = test", + expectedFound: true, + expectedVpaName: "initial-mpa", + }, { + name: "no vpa objects", + pod: podBuilder.Get(), + mpas: []*mpa_types.MultidimPodAutoscaler{}, + labelSelector: "app = test", + expectedFound: false, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockSelectorFetcher := target_mock.NewMockMpaTargetSelectorFetcher(ctrl) + + mpaNamespaceLister := &test.MultidimPodAutoscalerListerMock{} + mpaNamespaceLister.On("List").Return(tc.mpas, nil) + + mpaLister := &test.MultidimPodAutoscalerListerMock{} + mpaLister.On("MultidimPodAutoscalers", "default").Return(mpaNamespaceLister) + + if tc.labelSelector != "" { + mockSelectorFetcher.EXPECT().Fetch(gomock.Any()).AnyTimes().Return(parseLabelSelector(tc.labelSelector), nil) + } + // This test is using a FakeControllerFetcher which returns the same ownerRef that is passed to it. + // In other words, it cannot go through the hierarchy of controllers like "ReplicaSet => Deployment" + // For this reason we are using "StatefulSet" as the ownerRef kind in the test, since it is a direct link. + // The hierarchy part is being test in the "TestControllerFetcher" test. + matcher := NewMatcher(mpaLister, mockSelectorFetcher, controllerfetcher.FakeControllerFetcher{}) + + mpa := matcher.GetMatchingMPA(context.Background(), tc.pod) + if tc.expectedFound && assert.NotNil(t, mpa) { + assert.Equal(t, tc.expectedVpaName, mpa.Name) + } else { + assert.Nil(t, mpa) + } + }) + } +} diff --git a/multidimensional-pod-autoscaler/pkg/admission-controller/resource/mpa/pre_processor.go b/multidimensional-pod-autoscaler/pkg/admission-controller/resource/mpa/pre_processor.go new file mode 100644 index 000000000000..c368f726b5ba --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/admission-controller/resource/mpa/pre_processor.go @@ -0,0 +1,39 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mpa + +import ( + mpa_types "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1alpha1" +) + +// PreProcessor processes the MPAs before applying default . +type PreProcessor interface { + Process(mpa *mpa_types.MultidimPodAutoscaler, isCreate bool) (*mpa_types.MultidimPodAutoscaler, error) +} + +// noopPreProcessor leaves pods unchanged when processing +type noopPreProcessor struct{} + +// Process leaves the pod unchanged +func (p *noopPreProcessor) Process(mpa *mpa_types.MultidimPodAutoscaler, isCreate bool) (*mpa_types.MultidimPodAutoscaler, error) { + return mpa, nil +} + +// NewDefaultPreProcessor creates a PreProcessor that leaves MPAs unchanged and returns no error +func NewDefaultPreProcessor() PreProcessor { + return &noopPreProcessor{} +} diff --git a/multidimensional-pod-autoscaler/pkg/admission-controller/resource/pod/handler.go b/multidimensional-pod-autoscaler/pkg/admission-controller/resource/pod/handler.go new file mode 100644 index 000000000000..fa076feae65f --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/admission-controller/resource/pod/handler.go @@ -0,0 +1,104 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package pod + +import ( + "context" + "encoding/json" + "fmt" + + admissionv1 "k8s.io/api/admission/v1" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/admission-controller/resource/mpa" + "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/admission-controller/resource/pod/patch" + resource_admission "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/admission-controller/resource" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/metrics/admission" + "k8s.io/klog/v2" +) + +// resourceHandler builds patches for Pods. +type resourceHandler struct { + preProcessor PreProcessor + mpaMatcher mpa.Matcher + patchCalculators []patch.Calculator +} + +// NewResourceHandler creates new instance of resourceHandler. +func NewResourceHandler(preProcessor PreProcessor, mpaMatcher mpa.Matcher, patchCalculators []patch.Calculator) resource_admission.Handler { + return &resourceHandler{ + preProcessor: preProcessor, + mpaMatcher: mpaMatcher, + patchCalculators: patchCalculators, + } +} + +// AdmissionResource returns resource type this handler accepts. +func (h *resourceHandler) AdmissionResource() admission.AdmissionResource { + return admission.Pod +} + +// GroupResource returns Group and Resource type this handler accepts. +func (h *resourceHandler) GroupResource() metav1.GroupResource { + return metav1.GroupResource{Group: "", Resource: "pods"} +} + +// DisallowIncorrectObjects decides whether incorrect objects (eg. unparsable, not passing validations) should be disallowed by Admission Server. +func (h *resourceHandler) DisallowIncorrectObjects() bool { + // Incorrect Pods are validated by API Server. + return false +} + +// GetPatches builds patches for Pod in given admission request. +func (h *resourceHandler) GetPatches(ctx context.Context, ar *admissionv1.AdmissionRequest) ([]resource_admission.PatchRecord, error) { + if ar.Resource.Version != "v1" { + return nil, fmt.Errorf("only v1 Pods are supported") + } + raw, namespace := ar.Object.Raw, ar.Namespace + pod := v1.Pod{} + if err := json.Unmarshal(raw, &pod); err != nil { + return nil, err + } + if len(pod.Name) == 0 { + pod.Name = pod.GenerateName + "%" + pod.Namespace = namespace + } + klog.V(4).Infof("Admitting pod %v", pod.ObjectMeta) + controllingMpa := h.mpaMatcher.GetMatchingMPA(ctx, &pod) + if controllingMpa == nil { + klog.V(4).Infof("No matching MPA found for pod %s/%s", pod.Namespace, pod.Name) + return []resource_admission.PatchRecord{}, nil + } + pod, err := h.preProcessor.Process(pod) + if err != nil { + return nil, err + } + + patches := []resource_admission.PatchRecord{} + if pod.Annotations == nil { + patches = append(patches, patch.GetAddEmptyAnnotationsPatch()) + } + for _, c := range h.patchCalculators { + partialPatches, err := c.CalculatePatches(&pod, controllingMpa) + if err != nil { + return []resource_admission.PatchRecord{}, err + } + patches = append(patches, partialPatches...) + } + + return patches, nil +} diff --git a/multidimensional-pod-autoscaler/pkg/admission-controller/resource/pod/handler_test.go b/multidimensional-pod-autoscaler/pkg/admission-controller/resource/pod/handler_test.go new file mode 100644 index 000000000000..868e542e91ad --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/admission-controller/resource/pod/handler_test.go @@ -0,0 +1,205 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package pod + +import ( + "context" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + admissionv1 "k8s.io/api/admission/v1" + apiv1 "k8s.io/api/core/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/admission-controller/resource/pod/patch" + mpa_types "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1alpha1" + "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/utils/test" + resource_admission "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/admission-controller/resource" +) + +type fakePodPreProcessor struct { + err error +} + +func (fpp *fakePodPreProcessor) Process(pod apiv1.Pod) (apiv1.Pod, error) { + return pod, fpp.err +} + +type fakeMpaMatcher struct { + mpa *mpa_types.MultidimPodAutoscaler +} + +func (m *fakeMpaMatcher) GetMatchingMPA(_ context.Context, _ *apiv1.Pod) *mpa_types.MultidimPodAutoscaler { + return m.mpa +} + +type fakePatchCalculator struct { + patches []resource_admission.PatchRecord + err error +} + +func (c *fakePatchCalculator) CalculatePatches(_ *apiv1.Pod, _ *mpa_types.MultidimPodAutoscaler) ( + []resource_admission.PatchRecord, error) { + return c.patches, c.err +} + +func TestGetPatches(t *testing.T) { + testMpa := test.MultidimPodAutoscaler().WithName("name").WithContainer("testy-container").Get() + testPatchRecord := resource_admission.PatchRecord{ + Op: "add", + Path: "some/path", + Value: "much", + } + testPatchRecord2 := resource_admission.PatchRecord{ + Op: "add", + Path: "other/path", + Value: "not so much", + } + tests := []struct { + name string + podJson []byte + namespace string + mpa *mpa_types.MultidimPodAutoscaler + podPreProcessorError error + calculators []patch.Calculator + expectPatches []resource_admission.PatchRecord + expectError error + }{ + { + name: "invalid JSON", + podJson: []byte("{"), + namespace: "default", + mpa: testMpa, + podPreProcessorError: nil, + expectError: fmt.Errorf("unexpected end of JSON input"), + }, + { + name: "invalid pod", + podJson: []byte("{}"), + namespace: "default", + mpa: testMpa, + podPreProcessorError: fmt.Errorf("bad pod"), + expectError: fmt.Errorf("bad pod"), + }, + { + name: "no vpa found", + podJson: []byte("{}"), + namespace: "test", + mpa: nil, + podPreProcessorError: nil, + expectError: nil, + expectPatches: []resource_admission.PatchRecord{}, + }, + { + name: "calculator returns error", + podJson: []byte("{}"), + namespace: "test", + mpa: testMpa, + calculators: []patch.Calculator{&fakePatchCalculator{ + []resource_admission.PatchRecord{}, fmt.Errorf("Can't calculate this"), + }}, + podPreProcessorError: nil, + expectError: fmt.Errorf("Can't calculate this"), + expectPatches: []resource_admission.PatchRecord{}, + }, + { + name: "second calculator returns error", + podJson: []byte("{}"), + namespace: "test", + mpa: testMpa, + calculators: []patch.Calculator{ + &fakePatchCalculator{[]resource_admission.PatchRecord{ + testPatchRecord, + }, nil}, + &fakePatchCalculator{ + []resource_admission.PatchRecord{}, fmt.Errorf("Can't calculate this"), + }}, + podPreProcessorError: nil, + expectError: fmt.Errorf("Can't calculate this"), + expectPatches: []resource_admission.PatchRecord{}, + }, + { + name: "patches returned correctly", + podJson: []byte("{}"), + namespace: "test", + mpa: testMpa, + calculators: []patch.Calculator{ + &fakePatchCalculator{[]resource_admission.PatchRecord{ + testPatchRecord, + testPatchRecord2, + }, nil}}, + podPreProcessorError: nil, + expectError: nil, + expectPatches: []resource_admission.PatchRecord{ + patch.GetAddEmptyAnnotationsPatch(), + testPatchRecord, + testPatchRecord2, + }, + }, + { + name: "patches returned correctly for multiple calculators", + podJson: []byte("{}"), + namespace: "test", + mpa: testMpa, + calculators: []patch.Calculator{ + &fakePatchCalculator{[]resource_admission.PatchRecord{ + testPatchRecord, + }, nil}, + &fakePatchCalculator{[]resource_admission.PatchRecord{ + testPatchRecord2, + }, nil}}, + podPreProcessorError: nil, + expectError: nil, + expectPatches: []resource_admission.PatchRecord{ + patch.GetAddEmptyAnnotationsPatch(), + testPatchRecord, + testPatchRecord2, + }, + }, + } + for _, tc := range tests { + t.Run(fmt.Sprintf("test case: %s", tc.name), func(t *testing.T) { + fppp := &fakePodPreProcessor{tc.podPreProcessorError} + fvm := &fakeMpaMatcher{mpa: tc.mpa} + h := NewResourceHandler(fppp, fvm, tc.calculators) + patches, err := h.GetPatches(context.Background(), &admissionv1.AdmissionRequest{ + Resource: v1.GroupVersionResource{ + Version: "v1", + }, + Namespace: tc.namespace, + Object: runtime.RawExtension{ + Raw: tc.podJson, + }, + }) + if tc.expectError == nil { + assert.NoError(t, err) + } else { + if assert.Error(t, err) { + assert.Equal(t, tc.expectError.Error(), err.Error()) + } + } + if assert.Equal(t, len(tc.expectPatches), len(patches), fmt.Sprintf("got %+v, want %+v", patches, tc.expectPatches)) { + for i, gotPatch := range patches { + if !patch.EqPatch(gotPatch, tc.expectPatches[i]) { + t.Errorf("Expected patch at position %d to be %+v, got %+v", i, tc.expectPatches[i], gotPatch) + } + } + } + }) + } +} diff --git a/multidimensional-pod-autoscaler/pkg/admission-controller/resource/pod/patch/calculator.go b/multidimensional-pod-autoscaler/pkg/admission-controller/resource/pod/patch/calculator.go new file mode 100644 index 000000000000..b0c8fa84c2e9 --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/admission-controller/resource/pod/patch/calculator.go @@ -0,0 +1,28 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package patch + +import ( + core "k8s.io/api/core/v1" + mpa_types "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1alpha1" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/admission-controller/resource" +) + +// Calculator is capable of calculating required patches for pod. +type Calculator interface { + CalculatePatches(pod *core.Pod, mpa *mpa_types.MultidimPodAutoscaler) ([]resource.PatchRecord, error) +} diff --git a/multidimensional-pod-autoscaler/pkg/admission-controller/resource/pod/patch/observed_containers.go b/multidimensional-pod-autoscaler/pkg/admission-controller/resource/pod/patch/observed_containers.go new file mode 100644 index 000000000000..95227d26e0fc --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/admission-controller/resource/pod/patch/observed_containers.go @@ -0,0 +1,37 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package patch + +import ( + core "k8s.io/api/core/v1" + mpa_types "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1alpha1" + resource_admission "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/admission-controller/resource" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/annotations" +) + +type observedContainers struct{} + +func (*observedContainers) CalculatePatches(pod *core.Pod, _ *mpa_types.MultidimPodAutoscaler) ([]resource_admission.PatchRecord, error) { + vpaObservedContainersValue := annotations.GetVpaObservedContainersValue(pod) + return []resource_admission.PatchRecord{GetAddAnnotationPatch(annotations.VpaObservedContainersLabel, vpaObservedContainersValue)}, nil +} + +// NewObservedContainersCalculator returns calculator for +// observed containers patches. +func NewObservedContainersCalculator() Calculator { + return &observedContainers{} +} diff --git a/multidimensional-pod-autoscaler/pkg/admission-controller/resource/pod/patch/observed_containers_test.go b/multidimensional-pod-autoscaler/pkg/admission-controller/resource/pod/patch/observed_containers_test.go new file mode 100644 index 000000000000..7778ffa0c756 --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/admission-controller/resource/pod/patch/observed_containers_test.go @@ -0,0 +1,66 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package patch + +import ( + "strings" + "testing" + + core "k8s.io/api/core/v1" + resource_admission "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/admission-controller/resource" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/annotations" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/test" + + "github.com/stretchr/testify/assert" +) + +func addVpaObservedContainersPatch(containerNames []string) resource_admission.PatchRecord { + return GetAddAnnotationPatch( + annotations.VpaObservedContainersLabel, + strings.Join(containerNames, ", "), + ) +} + +func TestCalculatePatches_ObservedContainers(t *testing.T) { + tests := []struct { + name string + pod *core.Pod + expectedPatch resource_admission.PatchRecord + }{ + { + name: "create vpa observed containers annotation", + pod: test.Pod().AddContainer(test.Container().WithName("test1").Get()). + AddContainer(test.Container().WithName("test2").Get()).Get(), + expectedPatch: addVpaObservedContainersPatch([]string{"test1", "test2"}), + }, + { + name: "create vpa observed containers annotation with no containers", + pod: test.Pod().Get(), + expectedPatch: addVpaObservedContainersPatch([]string{}), + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + c := NewObservedContainersCalculator() + patches, err := c.CalculatePatches(tc.pod, nil) + assert.NoError(t, err) + if assert.Len(t, patches, 1, "Unexpected number of patches.") { + AssertEqPatch(t, tc.expectedPatch, patches[0]) + } + }) + } +} diff --git a/multidimensional-pod-autoscaler/pkg/admission-controller/resource/pod/patch/resource_updates.go b/multidimensional-pod-autoscaler/pkg/admission-controller/resource/pod/patch/resource_updates.go new file mode 100644 index 000000000000..07c536394d86 --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/admission-controller/resource/pod/patch/resource_updates.go @@ -0,0 +1,128 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package patch + +import ( + "fmt" + "strings" + + core "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/admission-controller/resource/pod/recommendation" + mpa_types "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1alpha1" + resource_admission "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/admission-controller/resource" + vpa_api_util "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/vpa" +) + +const ( + // ResourceUpdatesAnnotation is the name of annotation + // containing resource updates performed by VPA. + ResourceUpdatesAnnotation = "vpaUpdates" +) + +type resourcesUpdatesPatchCalculator struct { + recommendationProvider recommendation.Provider +} + +// NewResourceUpdatesCalculator returns a calculator for +// resource update patches. +func NewResourceUpdatesCalculator(recommendationProvider recommendation.Provider) Calculator { + return &resourcesUpdatesPatchCalculator{ + recommendationProvider: recommendationProvider, + } +} + +func (c *resourcesUpdatesPatchCalculator) CalculatePatches(pod *core.Pod, mpa *mpa_types.MultidimPodAutoscaler) ([]resource_admission.PatchRecord, error) { + result := []resource_admission.PatchRecord{} + + containersResources, annotationsPerContainer, err := c.recommendationProvider.GetContainersResourcesForPod(pod, mpa) + if err != nil { + return []resource_admission.PatchRecord{}, fmt.Errorf("Failed to calculate resource patch for pod %v/%v: %v", pod.Namespace, pod.Name, err) + } + + if annotationsPerContainer == nil { + annotationsPerContainer = vpa_api_util.ContainerToAnnotationsMap{} + } + + updatesAnnotation := []string{} + for i, containerResources := range containersResources { + newPatches, newUpdatesAnnotation := getContainerPatch(pod, i, annotationsPerContainer, containerResources) + result = append(result, newPatches...) + updatesAnnotation = append(updatesAnnotation, newUpdatesAnnotation) + } + + if len(updatesAnnotation) > 0 { + vpaAnnotationValue := fmt.Sprintf("Pod resources updated by %s: %s", mpa.Name, strings.Join(updatesAnnotation, "; ")) + result = append(result, GetAddAnnotationPatch(ResourceUpdatesAnnotation, vpaAnnotationValue)) + } + return result, nil +} + +func getContainerPatch(pod *core.Pod, i int, annotationsPerContainer vpa_api_util.ContainerToAnnotationsMap, containerResources vpa_api_util.ContainerResources) ([]resource_admission.PatchRecord, string) { + var patches []resource_admission.PatchRecord + // Add empty resources object if missing. + if pod.Spec.Containers[i].Resources.Limits == nil && + pod.Spec.Containers[i].Resources.Requests == nil { + patches = append(patches, getPatchInitializingEmptyResources(i)) + } + + annotations, found := annotationsPerContainer[pod.Spec.Containers[i].Name] + if !found { + annotations = make([]string, 0) + } + + patches, annotations = appendPatchesAndAnnotations(patches, annotations, pod.Spec.Containers[i].Resources.Requests, i, containerResources.Requests, "requests", "request") + patches, annotations = appendPatchesAndAnnotations(patches, annotations, pod.Spec.Containers[i].Resources.Limits, i, containerResources.Limits, "limits", "limit") + + updatesAnnotation := fmt.Sprintf("container %d: ", i) + strings.Join(annotations, ", ") + return patches, updatesAnnotation +} + +func appendPatchesAndAnnotations(patches []resource_admission.PatchRecord, annotations []string, current core.ResourceList, containerIndex int, resources core.ResourceList, fieldName, resourceName string) ([]resource_admission.PatchRecord, []string) { + // Add empty object if it's missing and we're about to fill it. + if current == nil && len(resources) > 0 { + patches = append(patches, getPatchInitializingEmptyResourcesSubfield(containerIndex, fieldName)) + } + for resource, request := range resources { + patches = append(patches, getAddResourceRequirementValuePatch(containerIndex, fieldName, resource, request)) + annotations = append(annotations, fmt.Sprintf("%s %s", resource, resourceName)) + } + return patches, annotations +} + +func getAddResourceRequirementValuePatch(i int, kind string, resource core.ResourceName, quantity resource.Quantity) resource_admission.PatchRecord { + return resource_admission.PatchRecord{ + Op: "add", + Path: fmt.Sprintf("/spec/containers/%d/resources/%s/%s", i, kind, resource), + Value: quantity.String()} +} + +func getPatchInitializingEmptyResources(i int) resource_admission.PatchRecord { + return resource_admission.PatchRecord{ + Op: "add", + Path: fmt.Sprintf("/spec/containers/%d/resources", i), + Value: core.ResourceRequirements{}, + } +} + +func getPatchInitializingEmptyResourcesSubfield(i int, kind string) resource_admission.PatchRecord { + return resource_admission.PatchRecord{ + Op: "add", + Path: fmt.Sprintf("/spec/containers/%d/resources/%s", i, kind), + Value: core.ResourceList{}, + } +} diff --git a/multidimensional-pod-autoscaler/pkg/admission-controller/resource/pod/patch/resource_updates_test.go b/multidimensional-pod-autoscaler/pkg/admission-controller/resource/pod/patch/resource_updates_test.go new file mode 100644 index 000000000000..25a59e1958f5 --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/admission-controller/resource/pod/patch/resource_updates_test.go @@ -0,0 +1,311 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package patch + +import ( + "fmt" + "strings" + "testing" + + core "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + mpa_types "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1alpha1" + "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/utils/test" + resource_admission "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/admission-controller/resource" + vpa_api_util "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/vpa" + + "github.com/stretchr/testify/assert" +) + +const ( + cpu = "cpu" + unobtanium = "unobtanium" + limit = "limit" + request = "request" +) + +type fakeRecommendationProvider struct { + resources []vpa_api_util.ContainerResources + containerToAnnotations vpa_api_util.ContainerToAnnotationsMap + e error +} + +func (frp *fakeRecommendationProvider) GetContainersResourcesForPod(pod *core.Pod, mpa *mpa_types.MultidimPodAutoscaler) ([]vpa_api_util.ContainerResources, vpa_api_util.ContainerToAnnotationsMap, error) { + return frp.resources, frp.containerToAnnotations, frp.e +} + +func addResourcesPatch(idx int) resource_admission.PatchRecord { + return resource_admission.PatchRecord{ + Op: "add", + Path: fmt.Sprintf("/spec/containers/%d/resources", idx), + Value: core.ResourceRequirements{}, + } +} + +func addRequestsPatch(idx int) resource_admission.PatchRecord { + return resource_admission.PatchRecord{ + Op: "add", + Path: fmt.Sprintf("/spec/containers/%d/resources/requests", idx), + Value: core.ResourceList{}, + } +} + +func addLimitsPatch(idx int) resource_admission.PatchRecord { + return resource_admission.PatchRecord{ + Op: "add", + Path: fmt.Sprintf("/spec/containers/%d/resources/limits", idx), + Value: core.ResourceList{}, + } +} + +func addResourceRequestPatch(index int, res, amount string) resource_admission.PatchRecord { + return resource_admission.PatchRecord{ + Op: "add", + Path: fmt.Sprintf("/spec/containers/%d/resources/requests/%s", index, res), + Value: resource.MustParse(amount), + } +} + +func addResourceLimitPatch(index int, res, amount string) resource_admission.PatchRecord { + return resource_admission.PatchRecord{ + Op: "add", + Path: fmt.Sprintf("/spec/containers/%d/resources/limits/%s", index, res), + Value: resource.MustParse(amount), + } +} + +func addAnnotationRequest(updateResources [][]string, kind string) resource_admission.PatchRecord { + requests := make([]string, 0) + for idx, podResources := range updateResources { + podRequests := make([]string, 0) + for _, resource := range podResources { + podRequests = append(podRequests, resource+" "+kind) + } + requests = append(requests, fmt.Sprintf("container %d: %s", idx, strings.Join(podRequests, ", "))) + } + + vpaUpdates := fmt.Sprintf("Pod resources updated by name: %s", strings.Join(requests, "; ")) + return GetAddAnnotationPatch(ResourceUpdatesAnnotation, vpaUpdates) +} + +func TestClalculatePatches_ResourceUpdates(t *testing.T) { + tests := []struct { + name string + pod *core.Pod + namespace string + recommendResources []vpa_api_util.ContainerResources + recommendAnnotations vpa_api_util.ContainerToAnnotationsMap + recommendError error + expectPatches []resource_admission.PatchRecord + expectError error + }{ + { + name: "new cpu recommendation", + pod: &core.Pod{ + Spec: core.PodSpec{ + Containers: []core.Container{{}}, + }, + }, + namespace: "default", + recommendResources: []vpa_api_util.ContainerResources{ + { + Requests: core.ResourceList{ + cpu: resource.MustParse("1"), + }, + }, + }, + recommendAnnotations: vpa_api_util.ContainerToAnnotationsMap{}, + expectPatches: []resource_admission.PatchRecord{ + addResourcesPatch(0), + addRequestsPatch(0), + addResourceRequestPatch(0, cpu, "1"), + addAnnotationRequest([][]string{{cpu}}, request), + }, + }, + { + name: "replacement cpu recommendation", + pod: &core.Pod{ + Spec: core.PodSpec{ + Containers: []core.Container{{ + Resources: core.ResourceRequirements{ + Requests: core.ResourceList{ + cpu: resource.MustParse("0"), + }, + }, + }}, + }, + }, + namespace: "default", + recommendResources: []vpa_api_util.ContainerResources{ + { + Requests: core.ResourceList{ + cpu: resource.MustParse("1"), + }, + }, + }, + recommendAnnotations: vpa_api_util.ContainerToAnnotationsMap{}, + expectPatches: []resource_admission.PatchRecord{ + addResourceRequestPatch(0, cpu, "1"), + addAnnotationRequest([][]string{{cpu}}, request), + }, + }, + { + name: "two containers", + pod: &core.Pod{ + Spec: core.PodSpec{ + Containers: []core.Container{{ + Resources: core.ResourceRequirements{ + Requests: core.ResourceList{ + cpu: resource.MustParse("0"), + }, + }, + }, {}}, + }, + }, + namespace: "default", + recommendResources: []vpa_api_util.ContainerResources{ + { + Requests: core.ResourceList{ + cpu: resource.MustParse("1"), + }, + }, + { + Requests: core.ResourceList{ + cpu: resource.MustParse("2"), + }, + }, + }, + recommendAnnotations: vpa_api_util.ContainerToAnnotationsMap{}, + expectPatches: []resource_admission.PatchRecord{ + addResourceRequestPatch(0, cpu, "1"), + addResourcesPatch(1), + addRequestsPatch(1), + addResourceRequestPatch(1, cpu, "2"), + addAnnotationRequest([][]string{{cpu}, {cpu}}, request), + }, + }, + { + name: "new cpu limit", + pod: &core.Pod{ + Spec: core.PodSpec{ + Containers: []core.Container{{}}, + }, + }, + namespace: "default", + recommendResources: []vpa_api_util.ContainerResources{ + { + Limits: core.ResourceList{ + cpu: resource.MustParse("1"), + }, + }, + }, + recommendAnnotations: vpa_api_util.ContainerToAnnotationsMap{}, + expectPatches: []resource_admission.PatchRecord{ + addResourcesPatch(0), + addLimitsPatch(0), + addResourceLimitPatch(0, cpu, "1"), + addAnnotationRequest([][]string{{cpu}}, limit), + }, + }, + { + name: "replacement cpu limit", + pod: &core.Pod{ + Spec: core.PodSpec{ + Containers: []core.Container{{ + Resources: core.ResourceRequirements{ + Limits: core.ResourceList{ + cpu: resource.MustParse("0"), + }, + }, + }}, + }, + }, + namespace: "default", + recommendResources: []vpa_api_util.ContainerResources{ + { + Limits: core.ResourceList{ + cpu: resource.MustParse("1"), + }, + }, + }, + recommendAnnotations: vpa_api_util.ContainerToAnnotationsMap{}, + expectPatches: []resource_admission.PatchRecord{ + addResourceLimitPatch(0, cpu, "1"), + addAnnotationRequest([][]string{{cpu}}, limit), + }, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + frp := fakeRecommendationProvider{tc.recommendResources, tc.recommendAnnotations, tc.recommendError} + c := NewResourceUpdatesCalculator(&frp) + patches, err := c.CalculatePatches(tc.pod, test.MultidimPodAutoscaler().WithContainer("test").WithName("name").Get()) + if tc.expectError == nil { + assert.NoError(t, err) + } else { + if assert.Error(t, err) { + assert.Equal(t, tc.expectError.Error(), err.Error()) + } + } + if assert.Len(t, patches, len(tc.expectPatches), fmt.Sprintf("got %+v, want %+v", patches, tc.expectPatches)) { + for i, gotPatch := range patches { + if !EqPatch(gotPatch, tc.expectPatches[i]) { + t.Errorf("Expected patch at position %d to be %+v, got %+v", i, tc.expectPatches[i], gotPatch) + } + } + } + }) + } +} + +func TestGetPatches_TwoReplacementResources(t *testing.T) { + recommendResources := []vpa_api_util.ContainerResources{ + { + Requests: core.ResourceList{ + cpu: resource.MustParse("1"), + unobtanium: resource.MustParse("2"), + }, + }, + } + pod := &core.Pod{ + Spec: core.PodSpec{ + Containers: []core.Container{{ + Resources: core.ResourceRequirements{ + Requests: core.ResourceList{ + cpu: resource.MustParse("0"), + }, + }, + }}, + }, + } + recommendAnnotations := vpa_api_util.ContainerToAnnotationsMap{} + frp := fakeRecommendationProvider{recommendResources, recommendAnnotations, nil} + c := NewResourceUpdatesCalculator(&frp) + patches, err := c.CalculatePatches(pod, test.MultidimPodAutoscaler().WithName("name").WithContainer("test").Get()) + assert.NoError(t, err) + // Order of updates for cpu and unobtanium depends on order of iterating a map, both possible results are valid. + if assert.Len(t, patches, 3, "unexpected number of patches") { + cpuUpdate := addResourceRequestPatch(0, cpu, "1") + unobtaniumUpdate := addResourceRequestPatch(0, unobtanium, "2") + AssertPatchOneOf(t, patches[0], []resource_admission.PatchRecord{cpuUpdate, unobtaniumUpdate}) + AssertPatchOneOf(t, patches[1], []resource_admission.PatchRecord{cpuUpdate, unobtaniumUpdate}) + assert.False(t, EqPatch(patches[0], patches[1])) + cpuFirstUnobtaniumSecond := addAnnotationRequest([][]string{{cpu, unobtanium}}, request) + unobtaniumFirstCpuSecond := addAnnotationRequest([][]string{{unobtanium, cpu}}, request) + AssertPatchOneOf(t, patches[2], []resource_admission.PatchRecord{cpuFirstUnobtaniumSecond, unobtaniumFirstCpuSecond}) + } +} diff --git a/multidimensional-pod-autoscaler/pkg/admission-controller/resource/pod/patch/test_util.go b/multidimensional-pod-autoscaler/pkg/admission-controller/resource/pod/patch/test_util.go new file mode 100644 index 000000000000..881b8db95b28 --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/admission-controller/resource/pod/patch/test_util.go @@ -0,0 +1,51 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package patch + +import ( + "encoding/json" + "fmt" + "testing" + + resource_admission "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/admission-controller/resource" + + "github.com/stretchr/testify/assert" +) + +// EqPatch returns true if patches are equal by comparing their +// marshalling result. +func EqPatch(a, b resource_admission.PatchRecord) bool { + aJson, aErr := json.Marshal(a) + bJson, bErr := json.Marshal(b) + return string(aJson) == string(bJson) && aErr == bErr +} + +// AssertEqPatch asserts patches are equal. +func AssertEqPatch(t *testing.T, got, want resource_admission.PatchRecord) { + assert.True(t, EqPatch(got, want), "got %+v, want: %+v", got, want) +} + +// AssertPatchOneOf asserts patch is one of possible expected patches. +func AssertPatchOneOf(t *testing.T, got resource_admission.PatchRecord, want []resource_admission.PatchRecord) { + for _, wanted := range want { + if EqPatch(got, wanted) { + return + } + } + msg := fmt.Sprintf("got: %+v, expected one of %+v", got, want) + assert.Fail(t, msg) +} diff --git a/multidimensional-pod-autoscaler/pkg/admission-controller/resource/pod/patch/util.go b/multidimensional-pod-autoscaler/pkg/admission-controller/resource/pod/patch/util.go new file mode 100644 index 000000000000..985619f5bc85 --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/admission-controller/resource/pod/patch/util.go @@ -0,0 +1,41 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package patch + +import ( + "fmt" + + resource_admission "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/admission-controller/resource" +) + +// GetAddEmptyAnnotationsPatch returns a patch initializing empty annotations. +func GetAddEmptyAnnotationsPatch() resource_admission.PatchRecord { + return resource_admission.PatchRecord{ + Op: "add", + Path: "/metadata/annotations", + Value: map[string]string{}, + } +} + +// GetAddAnnotationPatch returns a patch for an annotation. +func GetAddAnnotationPatch(annotationName, annotationValue string) resource_admission.PatchRecord { + return resource_admission.PatchRecord{ + Op: "add", + Path: fmt.Sprintf("/metadata/annotations/%s", annotationName), + Value: annotationValue, + } +} diff --git a/multidimensional-pod-autoscaler/pkg/admission-controller/resource/pod/pre_processor.go b/multidimensional-pod-autoscaler/pkg/admission-controller/resource/pod/pre_processor.go new file mode 100644 index 000000000000..baa6323433b6 --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/admission-controller/resource/pod/pre_processor.go @@ -0,0 +1,39 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package pod + +import ( + apiv1 "k8s.io/api/core/v1" +) + +// PreProcessor processes the pods before building patches. +type PreProcessor interface { + Process(apiv1.Pod) (apiv1.Pod, error) +} + +// NoopPreProcessor leaves pods unchanged when processing +type NoopPreProcessor struct{} + +// Process leaves the pod unchanged +func (p *NoopPreProcessor) Process(pod apiv1.Pod) (apiv1.Pod, error) { + return pod, nil +} + +// NewDefaultPreProcessor creates a default PreProcessor +func NewDefaultPreProcessor() PreProcessor { + return &NoopPreProcessor{} +} diff --git a/multidimensional-pod-autoscaler/pkg/admission-controller/resource/pod/recommendation/recommendation_provider.go b/multidimensional-pod-autoscaler/pkg/admission-controller/resource/pod/recommendation/recommendation_provider.go new file mode 100644 index 000000000000..8b663470a4ec --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/admission-controller/resource/pod/recommendation/recommendation_provider.go @@ -0,0 +1,115 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package recommendation + +import ( + "fmt" + + core "k8s.io/api/core/v1" + mpa_types "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1alpha1" + mpa_api_util "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/utils/mpa" + vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/limitrange" + vpa_api_util "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/vpa" + "k8s.io/klog/v2" +) + +// Provider gets current recommendation, annotations and vpaName for the given pod. +type Provider interface { + GetContainersResourcesForPod(pod *core.Pod, mpa *mpa_types.MultidimPodAutoscaler) ([]vpa_api_util.ContainerResources, vpa_api_util.ContainerToAnnotationsMap, error) +} + +type recommendationProvider struct { + limitsRangeCalculator limitrange.LimitRangeCalculator + recommendationProcessor mpa_api_util.RecommendationProcessor +} + +// NewProvider constructs the recommendation provider that can be used to determine recommendations for pods. +func NewProvider(calculator limitrange.LimitRangeCalculator, + recommendationProcessor mpa_api_util.RecommendationProcessor) Provider { + return &recommendationProvider{ + limitsRangeCalculator: calculator, + recommendationProcessor: recommendationProcessor, + } +} + +// GetContainersResources returns the recommended resources for each container in the given pod in the same order they are specified in the pod.Spec. +// If addAll is set to true, containers w/o a recommendation are also added to the list, otherwise they're skipped (default behaviour). +func GetContainersResources(pod *core.Pod, vpaResourcePolicy *vpa_types.PodResourcePolicy, podRecommendation vpa_types.RecommendedPodResources, limitRange *core.LimitRangeItem, + addAll bool, annotations vpa_api_util.ContainerToAnnotationsMap) []vpa_api_util.ContainerResources { + resources := make([]vpa_api_util.ContainerResources, len(pod.Spec.Containers)) + for i, container := range pod.Spec.Containers { + recommendation := vpa_api_util.GetRecommendationForContainer(container.Name, &podRecommendation) + if recommendation == nil { + if !addAll { + klog.V(2).Infof("no matching recommendation found for container %s, skipping", container.Name) + continue + } + klog.V(2).Infof("no matching recommendation found for container %s, using Pod request", container.Name) + resources[i].Requests = container.Resources.Requests + } else { + resources[i].Requests = recommendation.Target + } + defaultLimit := core.ResourceList{} + if limitRange != nil { + defaultLimit = limitRange.Default + } + containerControlledValues := vpa_api_util.GetContainerControlledValues(container.Name, vpaResourcePolicy) + if containerControlledValues == vpa_types.ContainerControlledValuesRequestsAndLimits { + proportionalLimits, limitAnnotations := vpa_api_util.GetProportionalLimit(container.Resources.Limits, container.Resources.Requests, resources[i].Requests, defaultLimit) + if proportionalLimits != nil { + resources[i].Limits = proportionalLimits + if len(limitAnnotations) > 0 { + annotations[container.Name] = append(annotations[container.Name], limitAnnotations...) + } + } + } + } + return resources +} + +// GetContainersResourcesForPod returns recommended request for a given pod and associated annotations. +// The returned slice corresponds 1-1 to containers in the Pod. +func (p *recommendationProvider) GetContainersResourcesForPod(pod *core.Pod, mpa *mpa_types.MultidimPodAutoscaler) ([]vpa_api_util.ContainerResources, vpa_api_util.ContainerToAnnotationsMap, error) { + if mpa == nil || pod == nil { + klog.V(2).Infof("can't calculate recommendations, one of vpa(%+v), pod(%+v) is nil", mpa, pod) + return nil, nil, nil + } + klog.V(2).Infof("updating requirements for pod %s.", pod.Name) + + var annotations vpa_api_util.ContainerToAnnotationsMap + recommendedPodResources := &vpa_types.RecommendedPodResources{} + + if mpa.Status.Recommendation != nil { + var err error + recommendedPodResources, annotations, err = p.recommendationProcessor.Apply(mpa.Status.Recommendation, mpa.Spec.ResourcePolicy, mpa.Status.Conditions, pod) + if err != nil { + klog.V(2).Infof("cannot process recommendation for pod %s", pod.Name) + return nil, annotations, err + } + } + containerLimitRange, err := p.limitsRangeCalculator.GetContainerLimitRangeItem(pod.Namespace) + if err != nil { + return nil, nil, fmt.Errorf("error getting containerLimitRange: %s", err) + } + var resourcePolicy *vpa_types.PodResourcePolicy + if mpa.Spec.Policy == nil || mpa.Spec.Policy.UpdateMode == nil || *mpa.Spec.Policy.UpdateMode != vpa_types.UpdateModeOff { + resourcePolicy = mpa.Spec.ResourcePolicy + } + containerResources := GetContainersResources(pod, resourcePolicy, *recommendedPodResources, containerLimitRange, false, annotations) + return containerResources, annotations, nil +} diff --git a/multidimensional-pod-autoscaler/pkg/admission-controller/resource/pod/recommendation/recommendation_provider_test.go b/multidimensional-pod-autoscaler/pkg/admission-controller/resource/pod/recommendation/recommendation_provider_test.go new file mode 100644 index 000000000000..eb4f35b690dc --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/admission-controller/resource/pod/recommendation/recommendation_provider_test.go @@ -0,0 +1,361 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package recommendation + +import ( + "fmt" + "math" + "testing" + + apiv1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + mpa_types "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1alpha1" + mpa_api_util "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/utils/mpa" + "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/utils/test" + vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/limitrange" + vpa_test "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/test" + vpa_api_util "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/vpa" + + "github.com/stretchr/testify/assert" +) + +func mustParseResourcePointer(val string) *resource.Quantity { + q := resource.MustParse(val) + return &q +} + +type fakeLimitRangeCalculator struct { + containerLimitRange *apiv1.LimitRangeItem + containerErr error + podLimitRange *apiv1.LimitRangeItem + podErr error +} + +func (nlrc *fakeLimitRangeCalculator) GetContainerLimitRangeItem(namespace string) (*apiv1.LimitRangeItem, error) { + return nlrc.containerLimitRange, nlrc.containerErr +} + +func (nlrc *fakeLimitRangeCalculator) GetPodLimitRangeItem(namespace string) (*apiv1.LimitRangeItem, error) { + return nlrc.podLimitRange, nlrc.podErr +} + +func TestUpdateResourceRequests(t *testing.T) { + containerName := "container1" + vpaName := "vpa1" + labels := map[string]string{"app": "testingApp"} + mpaBuilder := test.MultidimPodAutoscaler(). + WithName(vpaName). + WithContainer(containerName). + WithTarget("2", "200Mi"). + WithMinAllowed(containerName, "1", "100Mi"). + WithMaxAllowed(containerName, "3", "1Gi") + mpa := mpaBuilder.Get() + + uninitialized := vpa_test.Pod().WithName("test_uninitialized"). + AddContainer(test.Container().WithName(containerName).Get()). + WithLabels(labels).Get() + + initializedContainer := test.Container().WithName(containerName). + WithCPURequest(resource.MustParse("2")).WithMemRequest(resource.MustParse("100Mi")).Get() + initialized := vpa_test.Pod().WithName("test_initialized"). + AddContainer(initializedContainer).WithLabels(labels).Get() + + limitsMatchRequestsContainer := test.Container().WithName(containerName). + WithCPURequest(resource.MustParse("2")).WithCPULimit(resource.MustParse("2")). + WithMemRequest(resource.MustParse("200Mi")).WithMemLimit(resource.MustParse("200Mi")).Get() + limitsMatchRequestsPod := vpa_test.Pod().WithName("test_initialized"). + AddContainer(limitsMatchRequestsContainer).WithLabels(labels).Get() + + containerWithDoubleLimit := test.Container().WithName(containerName). + WithCPURequest(resource.MustParse("1")).WithCPULimit(resource.MustParse("2")). + WithMemRequest(resource.MustParse("100Mi")).WithMemLimit(resource.MustParse("200Mi")).Get() + podWithDoubleLimit := vpa_test.Pod().WithName("test_initialized"). + AddContainer(containerWithDoubleLimit).WithLabels(labels).Get() + + containerWithTenfoldLimit := test.Container().WithName(containerName). + WithCPURequest(resource.MustParse("1")).WithCPULimit(resource.MustParse("10")). + WithMemRequest(resource.MustParse("100Mi")).WithMemLimit(resource.MustParse("1000Mi")).Get() + podWithTenfoldLimit := vpa_test.Pod().WithName("test_initialized"). + AddContainer(containerWithTenfoldLimit).WithLabels(labels).Get() + + limitsNoRequestsContainer := test.Container().WithName(containerName). + WithCPULimit(resource.MustParse("2")).WithMemLimit(resource.MustParse("200Mi")).Get() + limitsNoRequestsPod := vpa_test.Pod().WithName("test_initialized"). + AddContainer(limitsNoRequestsContainer).WithLabels(labels).Get() + + targetBelowMinVPA := mpaBuilder.WithTarget("3", "150Mi").WithMinAllowed(containerName, "4", "300Mi").WithMaxAllowed(containerName, "5", "1Gi").Get() + targetAboveMaxVPA := mpaBuilder.WithTarget("7", "2Gi").WithMinAllowed(containerName, "4", "300Mi").WithMaxAllowed(containerName, "5", "1Gi").Get() + mpaWithHighMemory := mpaBuilder.WithTarget("2", "1000Mi").WithMaxAllowed(containerName, "3", "3Gi").Get() + mpaWithExabyteRecommendation := mpaBuilder.WithTarget("1Ei", "1Ei").WithMaxAllowed(containerName, "1Ei", "1Ei").Get() + + resourceRequestsAndLimitsVPA := mpaBuilder.WithControlledValues(containerName, vpa_types.ContainerControlledValuesRequestsAndLimits).Get() + resourceRequestsOnlyVPA := mpaBuilder.WithControlledValues(containerName, vpa_types.ContainerControlledValuesRequestsOnly).Get() + resourceRequestsOnlyVPAHighTarget := mpaBuilder.WithControlledValues(containerName, vpa_types.ContainerControlledValuesRequestsOnly). + WithTarget("3", "500Mi").WithMaxAllowed(containerName, "5", "1Gi").Get() + + vpaWithEmptyRecommendation := mpaBuilder.Get() + vpaWithEmptyRecommendation.Status.Recommendation = &vpa_types.RecommendedPodResources{} + vpaWithNilRecommendation := mpaBuilder.Get() + vpaWithNilRecommendation.Status.Recommendation = nil + + testCases := []struct { + name string + pod *apiv1.Pod + mpa *mpa_types.MultidimPodAutoscaler + expectedAction bool + expectedError error + expectedMem resource.Quantity + expectedCPU resource.Quantity + expectedCPULimit *resource.Quantity + expectedMemLimit *resource.Quantity + limitRange *apiv1.LimitRangeItem + limitRangeCalcErr error + annotations vpa_api_util.ContainerToAnnotationsMap + }{ + { + name: "uninitialized pod", + pod: uninitialized, + mpa: mpa, + expectedAction: true, + expectedMem: resource.MustParse("200Mi"), + expectedCPU: resource.MustParse("2"), + }, + { + name: "target below min", + pod: uninitialized, + mpa: targetBelowMinVPA, + expectedAction: true, + expectedMem: resource.MustParse("300Mi"), // MinMemory is expected to be used + expectedCPU: resource.MustParse("4"), // MinCpu is expected to be used + annotations: vpa_api_util.ContainerToAnnotationsMap{ + containerName: []string{"cpu capped to minAllowed", "memory capped to minAllowed"}, + }, + }, + { + name: "target above max", + pod: uninitialized, + mpa: targetAboveMaxVPA, + expectedAction: true, + expectedMem: resource.MustParse("1Gi"), // MaxMemory is expected to be used + expectedCPU: resource.MustParse("5"), // MaxCpu is expected to be used + annotations: vpa_api_util.ContainerToAnnotationsMap{ + containerName: []string{"cpu capped to maxAllowed", "memory capped to maxAllowed"}, + }, + }, + { + name: "initialized pod", + pod: initialized, + mpa: mpa, + expectedAction: true, + expectedMem: resource.MustParse("200Mi"), + expectedCPU: resource.MustParse("2"), + }, + { + name: "high memory", + pod: initialized, + mpa: mpaWithHighMemory, + expectedAction: true, + expectedMem: resource.MustParse("1000Mi"), + expectedCPU: resource.MustParse("2"), + }, + { + name: "empty recommendation", + pod: initialized, + mpa: vpaWithEmptyRecommendation, + expectedAction: true, + expectedMem: resource.MustParse("0"), + expectedCPU: resource.MustParse("0"), + }, + { + name: "nil recommendation", + pod: initialized, + mpa: vpaWithNilRecommendation, + expectedAction: true, + expectedMem: resource.MustParse("0"), + expectedCPU: resource.MustParse("0"), + }, + { + name: "guaranteed resources", + pod: limitsMatchRequestsPod, + mpa: mpa, + expectedAction: true, + expectedMem: resource.MustParse("200Mi"), + expectedCPU: resource.MustParse("2"), + expectedCPULimit: mustParseResourcePointer("2"), + expectedMemLimit: mustParseResourcePointer("200Mi"), + }, + { + name: "guaranteed resources - no request", + pod: limitsNoRequestsPod, + mpa: mpa, + expectedAction: true, + expectedMem: resource.MustParse("200Mi"), + expectedCPU: resource.MustParse("2"), + expectedCPULimit: mustParseResourcePointer("2"), + expectedMemLimit: mustParseResourcePointer("200Mi"), + }, + { + name: "proportional limit - as default", + pod: podWithDoubleLimit, + mpa: mpa, + expectedAction: true, + expectedCPU: resource.MustParse("2"), + expectedMem: resource.MustParse("200Mi"), + expectedCPULimit: mustParseResourcePointer("4"), + expectedMemLimit: mustParseResourcePointer("400Mi"), + }, + { + name: "proportional limit - set explicit", + pod: podWithDoubleLimit, + mpa: resourceRequestsAndLimitsVPA, + expectedAction: true, + expectedCPU: resource.MustParse("2"), + expectedMem: resource.MustParse("200Mi"), + expectedCPULimit: mustParseResourcePointer("4"), + expectedMemLimit: mustParseResourcePointer("400Mi"), + }, + { + name: "disabled limit scaling", + pod: podWithDoubleLimit, + mpa: resourceRequestsOnlyVPA, + expectedAction: true, + expectedCPU: resource.MustParse("2"), + expectedMem: resource.MustParse("200Mi"), + }, + { + name: "disabled limit scaling - requests capped at limit", + pod: podWithDoubleLimit, + mpa: resourceRequestsOnlyVPAHighTarget, + expectedAction: true, + expectedCPU: resource.MustParse("2"), + expectedMem: resource.MustParse("200Mi"), + annotations: vpa_api_util.ContainerToAnnotationsMap{ + containerName: []string{ + "cpu capped to container limit", + "memory capped to container limit", + }, + }, + }, + { + name: "limit over int64", + pod: podWithTenfoldLimit, + mpa: mpaWithExabyteRecommendation, + expectedAction: true, + expectedCPU: resource.MustParse("1Ei"), + expectedMem: resource.MustParse("1Ei"), + expectedCPULimit: resource.NewMilliQuantity(math.MaxInt64, resource.DecimalExponent), + expectedMemLimit: resource.NewQuantity(math.MaxInt64, resource.DecimalExponent), + annotations: vpa_api_util.ContainerToAnnotationsMap{ + containerName: []string{ + "cpu: failed to keep limit to request ratio; capping limit to int64", + "memory: failed to keep limit to request ratio; capping limit to int64", + }, + }, + }, + { + name: "limit range calculation error", + pod: initialized, + mpa: mpa, + limitRangeCalcErr: fmt.Errorf("oh no"), + expectedAction: false, + expectedError: fmt.Errorf("error getting containerLimitRange: oh no"), + }, + { + name: "proportional limit from default", + pod: initialized, + mpa: mpa, + expectedAction: true, + expectedCPU: resource.MustParse("2"), + expectedMem: resource.MustParse("200Mi"), + expectedCPULimit: mustParseResourcePointer("2"), + expectedMemLimit: mustParseResourcePointer("200Mi"), + limitRange: &apiv1.LimitRangeItem{ + Type: apiv1.LimitTypeContainer, + Default: apiv1.ResourceList{ + apiv1.ResourceCPU: resource.MustParse("2"), + apiv1.ResourceMemory: resource.MustParse("100Mi"), + }, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + recommendationProvider := &recommendationProvider{ + recommendationProcessor: mpa_api_util.NewCappingRecommendationProcessor(limitrange.NewNoopLimitsCalculator()), + limitsRangeCalculator: &fakeLimitRangeCalculator{ + containerLimitRange: tc.limitRange, + containerErr: tc.limitRangeCalcErr, + }, + } + + resources, annotations, err := recommendationProvider.GetContainersResourcesForPod(tc.pod, tc.mpa) + + if tc.expectedAction { + assert.Nil(t, err) + if !assert.Equal(t, len(resources), 1) { + return + } + + cpuRequest := resources[0].Requests[apiv1.ResourceCPU] + assert.Equal(t, tc.expectedCPU.Value(), cpuRequest.Value(), "cpu request doesn't match") + + memoryRequest := resources[0].Requests[apiv1.ResourceMemory] + assert.Equal(t, tc.expectedMem.Value(), memoryRequest.Value(), "memory request doesn't match") + + cpuLimit, cpuLimitPresent := resources[0].Limits[apiv1.ResourceCPU] + if tc.expectedCPULimit == nil { + assert.False(t, cpuLimitPresent, "expected no cpu limit, got %s", cpuLimit.String()) + } else { + if assert.True(t, cpuLimitPresent, "expected cpu limit, but it's missing") { + assert.Equal(t, tc.expectedCPULimit.MilliValue(), cpuLimit.MilliValue(), "cpu limit doesn't match") + } + } + + memLimit, memLimitPresent := resources[0].Limits[apiv1.ResourceMemory] + if tc.expectedMemLimit == nil { + assert.False(t, memLimitPresent, "expected no memory limit, got %s", memLimit.String()) + } else { + if assert.True(t, memLimitPresent, "expected memory limit, but it's missing") { + assert.Equal(t, tc.expectedMemLimit.MilliValue(), memLimit.MilliValue(), "memory limit doesn't match") + } + } + + assert.Len(t, annotations, len(tc.annotations)) + if len(tc.annotations) > 0 { + for annotationKey, annotationValues := range tc.annotations { + assert.Len(t, annotations[annotationKey], len(annotationValues)) + for _, annotation := range annotationValues { + assert.Contains(t, annotations[annotationKey], annotation) + } + } + } + } else { + assert.Empty(t, resources) + if tc.expectedError != nil { + assert.Error(t, err) + assert.Equal(t, tc.expectedError.Error(), err.Error()) + } else { + assert.NoError(t, err) + } + } + + }) + + } +} diff --git a/multidimensional-pod-autoscaler/pkg/admission-controller/rmcerts.sh b/multidimensional-pod-autoscaler/pkg/admission-controller/rmcerts.sh new file mode 100755 index 000000000000..42894ea642e4 --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/admission-controller/rmcerts.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# Copyright 2024 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Generates the a CA cert, a server key, and a server cert signed by the CA. +# reference: +# https://github.com/kubernetes/kubernetes/blob/master/plugin/pkg/admission/webhook/gencerts.sh +set -e + +echo "Deleting MPA Admission Controller certs." +kubectl delete secret --namespace=kube-system mpa-tls-certs diff --git a/multidimensional-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1alpha1/doc.go b/multidimensional-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1alpha1/doc.go new file mode 100644 index 000000000000..7d5f1d511eb8 --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1alpha1/doc.go @@ -0,0 +1,22 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// +k8s:deepcopy-gen=package,register + +// Package v1alpha1 contains definitions of Multi-Dimensional Pod Autoscaler related objects. +// +groupName=autoscaling.k8s.io +// +kubebuilder:object:generate=true +package v1alpha1 diff --git a/multidimensional-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1alpha1/register.go b/multidimensional-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1alpha1/register.go new file mode 100644 index 000000000000..d123aae9c687 --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1alpha1/register.go @@ -0,0 +1,62 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +// SchemeGroupVersion is group version used to register these objects +var SchemeGroupVersion = schema.GroupVersion{ + Group: "autoscaling.k8s.io", + Version: "v1alpha1", +} + +var ( + // SchemeBuilder is the scheme builder for ProvisioningRequest. + SchemeBuilder runtime.SchemeBuilder + localSchemeBuilder = &SchemeBuilder + // AddToScheme is the func that applies all the stored functions to the scheme. + AddToScheme = SchemeBuilder.AddToScheme +) + +// Resource takes an unqualified resource and returns a Group qualified GroupResource +func Resource(resource string) schema.GroupResource { + return SchemeGroupVersion.WithResource(resource).GroupResource() +} + +func init() { + // We only register manually written functions here. The registration of the generated + // functions takes place in the generated files. The separation makes the code compile even + // when the generated files are missing. + localSchemeBuilder.Register(addKnownTypes) +} + +// Adds the list of known types to api.Scheme. +func addKnownTypes(scheme *runtime.Scheme) error { + scheme.AddKnownTypes(SchemeGroupVersion, + &MultidimPodAutoscaler{}, + &MultidimPodAutoscalerList{}, + &MultidimPodAutoscalerCheckpoint{}, + &MultidimPodAutoscalerCheckpointList{}, + ) + + metav1.AddToGroupVersion(scheme, SchemeGroupVersion) + return nil +} diff --git a/multidimensional-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1alpha1/types.go b/multidimensional-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1alpha1/types.go new file mode 100644 index 000000000000..b2f8d8e31849 --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1alpha1/types.go @@ -0,0 +1,304 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + autoscaling "k8s.io/api/autoscaling/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + vpa "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" +) + +// +genclient +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +kubebuilder:storageversion +// +kubebuilder:resource:shortName=mpa +// +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="Mode",type="string",JSONPath=".spec.updatePolicy.updateMode" +// +kubebuilder:printcolumn:name="CPU",type="string",JSONPath=".status.recommendation.containerRecommendations[0].target.cpu" +// +kubebuilder:printcolumn:name="Mem",type="string",JSONPath=".status.recommendation.containerRecommendations[0].target.memory" +// +kubebuilder:printcolumn:name="Provided",type="string",JSONPath=".status.conditions[?(@.type=='RecommendationProvided')].status" +// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" +// +kubebuilder:metadata:annotations="api-approved.kubernetes.io=https://github.com/kubernetes/kubernetes/pull/63797" + +// MultidimPodAutoscaler is the configuration for a multidimensional pod autoscaler, +// which automatically manages pod resources and number of replicas based on historical and +// real-time resource utilization as well as workload performance. +type MultidimPodAutoscaler struct { + metav1.TypeMeta `json:",inline"` + // Standard object metadata. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata + // +optional + metav1.ObjectMeta `json:"metadata,omitempty"` + + // Specification of the behavior of the autoscaler. + // More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#spec-and-status. + Spec MultidimPodAutoscalerSpec `json:"spec"` + + // Current information about the autoscaler. + // +optional + Status MultidimPodAutoscalerStatus `json:"status,omitempty"` +} + +// MultidimPodAutoscalerSpec is the specification of the behavior of the autoscaler. +type MultidimPodAutoscalerSpec struct { + // ScaleTargetRef points to the controller managing the set of pods for the autoscaler to + // control, e.g., Deployment, StatefulSet. MultidimPodAutoscaler can be targeted at controller + // implementing scale subresource (the pod set is retrieved from the controller's ScaleStatus + // or some well known controllers (e.g., for DaemonSet the pod set is read from the + // controller's spec). If MultidimPodAutoscaler cannot use specified target it will report + // the ConfigUnsupported condition. + ScaleTargetRef *autoscaling.CrossVersionObjectReference `json:"scaleTargetRef"` + + // Describes the rules on how changes are applied to the pods. + // If not specified, all fields in the `PodUpdatePolicy` are set to their default values. + // +optional + Policy *PodUpdatePolicy `json:"policy,omitempty"` + + // Describes the goals for autoscaling + Goals *Goals `json:"goals,omitempty"` + + // Describes the constraints for horizontal and vertical scaling. + Constraints *ScalingConstraints `json:"constraints,omitempty"` + + // Controls how the VPA autoscaler computes recommended resources. + // The resource policy is also used to set constraints on the recommendations for individual + // containers. If not specified, the autoscaler computes recommended resources for all + // containers in the pod, without additional constraints. + // +optional + ResourcePolicy *vpa.PodResourcePolicy `json:"resourcePolicy,omitempty"` + + // Recommender responsible for generating recommendation for the set of pods and the deployment. + // List should be empty (then the default recommender will be used) or contain exactly one + // recommender. + // +optional + Recommenders []*MultidimPodAutoscalerRecommenderSelector `json:"recommenders,omitempty"` +} + +// MultidimPodAutoscalerStatus describes the current status of a multidimensional pod autoscaler +type MultidimPodAutoscalerStatus struct { + // Last time the MultidimPodAutoscaler scaled the number of pods and resizes containers; + // Used by the autoscaler to control how often scaling operations are performed. + // +optional + LastScaleTime *metav1.Time `json:"lastScaleTime,omitempty"` + + // Current number of replicas of pods managed by this autoscaler. + CurrentReplicas int32 `json:"currentReplicas"` + + // Desired number of replicas of pods managed by this autoscaler. + DesiredReplicas int32 `json:"desiredReplicas"` + + // The most recently computed amount of resources for each controlled pod recommended by the + // autoscaler. + // +optional + Recommendation *vpa.RecommendedPodResources `json:"recommendation,omitempty"` + + // The last read state of the metrics used by this autoscaler. + // +listType=atomic + // +optional + CurrentMetrics []autoscalingv2.MetricStatus `json:"currentMetrics"` + + // Conditions is the set of conditions required for this autoscaler to scale its target, and + // indicates whether or not those conditions are met. + // +optional + // +patchMergeKey=type + // +patchStrategy=merge + Conditions []MultidimPodAutoscalerCondition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type"` +} + +// PodUpdatePolicy describes the rules on how changes are applied to the pods. +type PodUpdatePolicy struct { + // Controls when autoscaler applies changes to the pod resources. + // The default is 'Auto'. + // +optional + UpdateMode *vpa.UpdateMode `json:"updateMode,omitempty"` +} + +// Goals describe the scaling goals +type Goals struct { + // Contains the specifications about the metric type and target in terms of resource + // utilization or workload performance. See the individual metric source types for + // more information about how each type of metric must respond. + // +listType=atomic + // +optional + Metrics []autoscalingv2.MetricSpec `json:"metrics,omitempty"` +} + +// ScalingConstraints describe the scaling constraints +type ScalingConstraints struct { + Global *HorizontalScalingConstraints `json:"global,omitempty"` + + // Defines controlled resources. + // If not specified, the default of [cpu, memory] will be used. + ContainerControlledResources []*v1.ResourceName `json:"containerControlledResources,omitempty"` + + // Per-container resource policies. + // +optional + // +patchMergeKey=containerName + // +patchStrategy=merge + Container []*VerticalScalingConstraints `json:"container,omitempty"` +} + +// HorizontalScalingConstraints describes the constraints for horizontal scaling. +type HorizontalScalingConstraints struct { + // Lower limit for the number of pods that can be set by the autoscaler, default 1. + // +optional + MinReplicas *int32 `json:"minReplicas,omitempty"` + // Upper limit for the number of pods that can be set by the autoscaler; cannot be smaller than + // MinReplicas. + MaxReplicas *int32 `json:"maxReplicas"` + // Behavior configures the scaling behavior of the target in both Up and Down direction + // (scaleUp and scaleDown fields respectively). + // +optional + Behavior *autoscalingv2.HorizontalPodAutoscalerBehavior `json:"behavior,omitempty"` +} + +// VerticalScalingConstraints describes the constraints for vertical scaling. +type VerticalScalingConstraints struct { + // Name of the container. + Name string `json:"name,omitempty"` + + // Whether autoscaler is enabled for the container. The default is "Auto". + // +optional + Mode *vpa.ContainerScalingMode `json:"mode,omitempty"` + + // Describes the vertical scaling limits. + Requests *MinMaxVerticalScalingLimits `json:"requests,omitempty"` +} + +// MinMaxVerticalScalingLimits describes the min and max vertical scaling limits for the container. +type MinMaxVerticalScalingLimits struct { + // Specifies the minimal amount of resources that will be recommended + // for the container. The default is no minimum. + // +optional + MinAllowed v1.ResourceList `json:"minAllowed,omitempty"` + // Specifies the maximum amount of resources that will be recommended + // for the container. The default is no maximum. + // +optional + MaxAllowed v1.ResourceList `json:"maxAllowed,omitempty"` +} + +// MultidimPodAutoscalerRecommenderSelector points to a specific Multidimensional Pod Autoscaler +// recommender. +// In the future it might pass parameters to the recommender. +type MultidimPodAutoscalerRecommenderSelector struct { + // Name of the recommender responsible for generating recommendation for this object. + Name string `json:"name"` +} + +// MultidimPodAutoscalerCondition describes the state of a MultidimPodAutoscaler at a certain point. +type MultidimPodAutoscalerCondition struct { + // type describes the current condition + Type MultidimPodAutoscalerConditionType `json:"type"` + // status is the status of the condition (True, False, Unknown) + Status v1.ConditionStatus `json:"status"` + // lastTransitionTime is the last time the condition transitioned from one status to another + // +optional + LastTransitionTime metav1.Time `json:"lastTransitionTime,omitempty"` + // reason is the reason for the condition's last transition. + // +optional + Reason string `json:"reason,omitempty"` + // message is a human-readable explanation containing details about the transition + // +optional + Message string `json:"message,omitempty"` +} + +// +genclient +// +genclient:noStatus +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +kubebuilder:storageversion +// +kubebuilder:resource:shortName=mpacheckpoint +// +kubebuilder:metadata:annotations="api-approved.kubernetes.io=https://github.com/kubernetes/kubernetes/pull/63797" + +// MultidimPodAutoscalerCheckpoint is the checkpoint of the internal state of VPA that +// is used for recovery after recommender's restart. +type MultidimPodAutoscalerCheckpoint struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"` + + // Specification of the checkpoint. + // More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status. + // +optional + Spec MultidimPodAutoscalerCheckpointSpec `json:"spec,omitempty" protobuf:"bytes,2,opt,name=spec"` + + // Data of the checkpoint. + // +optional + Status vpa.VerticalPodAutoscalerCheckpointStatus `json:"status,omitempty" protobuf:"bytes,3,opt,name=status"` +} + +// MultidimPodAutoscalerCheckpointSpec is the specification of the checkpoint object. +type MultidimPodAutoscalerCheckpointSpec struct { + // Name of the MPA object that stored MultidimPodAutoscalerCheckpoint object. + MPAObjectName string `json:"mpaObjectName,omitempty" protobuf:"bytes,1,opt,name=mpaObjectName"` + + // Name of the checkpointed container. + ContainerName string `json:"containerName,omitempty" protobuf:"bytes,2,opt,name=containerName"` +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// MultidimPodAutoscalerCheckpointList is a list of MultidimPodAutoscalerCheckpoint objects. +type MultidimPodAutoscalerCheckpointList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata"` + Items []MultidimPodAutoscalerCheckpoint `json:"items"` +} + +// MultidimPodAutoscalerConditionType are the valid conditions of a MultidimPodAutoscaler. +type MultidimPodAutoscalerConditionType string + +var ( + // RecommendationProvided indicates whether the MPA recommender was able to give a + // recommendation. + RecommendationProvided MultidimPodAutoscalerConditionType = "RecommendationProvided" + // LowConfidence indicates whether the MPA recommender has low confidence in the recommendation + // for some of containers. + LowConfidence MultidimPodAutoscalerConditionType = "LowConfidence" + // NoPodsMatched indicates that label selector used with MPA object didn't match any pods. + NoPodsMatched MultidimPodAutoscalerConditionType = "NoPodsMatched" + // FetchingHistory indicates that MPA recommender is in the process of loading additional + // history samples. + FetchingHistory MultidimPodAutoscalerConditionType = "FetchingHistory" + // ConfigDeprecated indicates that this MPA configuration is deprecated and will stop being + // supported soon. + ConfigDeprecated MultidimPodAutoscalerConditionType = "ConfigDeprecated" + // ConfigUnsupported indicates that this MPA configuration is unsupported and recommendations + // will not be provided for it. + ConfigUnsupported MultidimPodAutoscalerConditionType = "ConfigUnsupported" + // ScalingActive indicates that the MPA controller is able to scale if necessary, i.e., + // it is correctly configured, can fetch the desired metrics, and isn't disabled. + ScalingActive MultidimPodAutoscalerConditionType = "ScalingActive" + // AbleToScale indicates a lack of transient issues which prevent scaling from occurring, + // such as being in a backoff window, or being unable to access/update the target scale. + AbleToScale MultidimPodAutoscalerConditionType = "AbleToScale" + // ScalingLimited indicates that the calculated scale based on metrics would be above or + // below the range for the MPA, and has thus been capped. + ScalingLimited MultidimPodAutoscalerConditionType = "ScalingLimited" +) + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// MultidimPodAutoscalerList is a list of MultidimPodAutoscaler objects. +type MultidimPodAutoscalerList struct { + metav1.TypeMeta `json:",inline"` + // metadata is the standard list metadata. + // +optional + metav1.ListMeta `json:"metadata"` + + // items is the list of Multidimensional Pod Autoscaler objects. + Items []MultidimPodAutoscaler `json:"items"` +} diff --git a/multidimensional-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1alpha1/zz_generated.deepcopy.go b/multidimensional-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1alpha1/zz_generated.deepcopy.go new file mode 100644 index 000000000000..b5d3d73a235e --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1alpha1/zz_generated.deepcopy.go @@ -0,0 +1,466 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by deepcopy-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + autoscalingv1 "k8s.io/api/autoscaling/v1" + v2 "k8s.io/api/autoscaling/v2" + v1 "k8s.io/api/core/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + autoscalingk8siov1 "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Goals) DeepCopyInto(out *Goals) { + *out = *in + if in.Metrics != nil { + in, out := &in.Metrics, &out.Metrics + *out = make([]v2.MetricSpec, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Goals. +func (in *Goals) DeepCopy() *Goals { + if in == nil { + return nil + } + out := new(Goals) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *HorizontalScalingConstraints) DeepCopyInto(out *HorizontalScalingConstraints) { + *out = *in + if in.MinReplicas != nil { + in, out := &in.MinReplicas, &out.MinReplicas + *out = new(int32) + **out = **in + } + if in.MaxReplicas != nil { + in, out := &in.MaxReplicas, &out.MaxReplicas + *out = new(int32) + **out = **in + } + if in.Behavior != nil { + in, out := &in.Behavior, &out.Behavior + *out = new(v2.HorizontalPodAutoscalerBehavior) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HorizontalScalingConstraints. +func (in *HorizontalScalingConstraints) DeepCopy() *HorizontalScalingConstraints { + if in == nil { + return nil + } + out := new(HorizontalScalingConstraints) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MinMaxVerticalScalingLimits) DeepCopyInto(out *MinMaxVerticalScalingLimits) { + *out = *in + if in.MinAllowed != nil { + in, out := &in.MinAllowed, &out.MinAllowed + *out = make(v1.ResourceList, len(*in)) + for key, val := range *in { + (*out)[key] = val.DeepCopy() + } + } + if in.MaxAllowed != nil { + in, out := &in.MaxAllowed, &out.MaxAllowed + *out = make(v1.ResourceList, len(*in)) + for key, val := range *in { + (*out)[key] = val.DeepCopy() + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MinMaxVerticalScalingLimits. +func (in *MinMaxVerticalScalingLimits) DeepCopy() *MinMaxVerticalScalingLimits { + if in == nil { + return nil + } + out := new(MinMaxVerticalScalingLimits) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MultidimPodAutoscaler) DeepCopyInto(out *MultidimPodAutoscaler) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MultidimPodAutoscaler. +func (in *MultidimPodAutoscaler) DeepCopy() *MultidimPodAutoscaler { + if in == nil { + return nil + } + out := new(MultidimPodAutoscaler) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *MultidimPodAutoscaler) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MultidimPodAutoscalerCheckpoint) DeepCopyInto(out *MultidimPodAutoscalerCheckpoint) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + in.Status.DeepCopyInto(&out.Status) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MultidimPodAutoscalerCheckpoint. +func (in *MultidimPodAutoscalerCheckpoint) DeepCopy() *MultidimPodAutoscalerCheckpoint { + if in == nil { + return nil + } + out := new(MultidimPodAutoscalerCheckpoint) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *MultidimPodAutoscalerCheckpoint) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MultidimPodAutoscalerCheckpointList) DeepCopyInto(out *MultidimPodAutoscalerCheckpointList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]MultidimPodAutoscalerCheckpoint, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MultidimPodAutoscalerCheckpointList. +func (in *MultidimPodAutoscalerCheckpointList) DeepCopy() *MultidimPodAutoscalerCheckpointList { + if in == nil { + return nil + } + out := new(MultidimPodAutoscalerCheckpointList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *MultidimPodAutoscalerCheckpointList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MultidimPodAutoscalerCheckpointSpec) DeepCopyInto(out *MultidimPodAutoscalerCheckpointSpec) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MultidimPodAutoscalerCheckpointSpec. +func (in *MultidimPodAutoscalerCheckpointSpec) DeepCopy() *MultidimPodAutoscalerCheckpointSpec { + if in == nil { + return nil + } + out := new(MultidimPodAutoscalerCheckpointSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MultidimPodAutoscalerCondition) DeepCopyInto(out *MultidimPodAutoscalerCondition) { + *out = *in + in.LastTransitionTime.DeepCopyInto(&out.LastTransitionTime) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MultidimPodAutoscalerCondition. +func (in *MultidimPodAutoscalerCondition) DeepCopy() *MultidimPodAutoscalerCondition { + if in == nil { + return nil + } + out := new(MultidimPodAutoscalerCondition) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MultidimPodAutoscalerList) DeepCopyInto(out *MultidimPodAutoscalerList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]MultidimPodAutoscaler, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MultidimPodAutoscalerList. +func (in *MultidimPodAutoscalerList) DeepCopy() *MultidimPodAutoscalerList { + if in == nil { + return nil + } + out := new(MultidimPodAutoscalerList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *MultidimPodAutoscalerList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MultidimPodAutoscalerRecommenderSelector) DeepCopyInto(out *MultidimPodAutoscalerRecommenderSelector) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MultidimPodAutoscalerRecommenderSelector. +func (in *MultidimPodAutoscalerRecommenderSelector) DeepCopy() *MultidimPodAutoscalerRecommenderSelector { + if in == nil { + return nil + } + out := new(MultidimPodAutoscalerRecommenderSelector) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MultidimPodAutoscalerSpec) DeepCopyInto(out *MultidimPodAutoscalerSpec) { + *out = *in + if in.ScaleTargetRef != nil { + in, out := &in.ScaleTargetRef, &out.ScaleTargetRef + *out = new(autoscalingv1.CrossVersionObjectReference) + **out = **in + } + if in.Policy != nil { + in, out := &in.Policy, &out.Policy + *out = new(PodUpdatePolicy) + (*in).DeepCopyInto(*out) + } + if in.Goals != nil { + in, out := &in.Goals, &out.Goals + *out = new(Goals) + (*in).DeepCopyInto(*out) + } + if in.Constraints != nil { + in, out := &in.Constraints, &out.Constraints + *out = new(ScalingConstraints) + (*in).DeepCopyInto(*out) + } + if in.ResourcePolicy != nil { + in, out := &in.ResourcePolicy, &out.ResourcePolicy + *out = new(autoscalingk8siov1.PodResourcePolicy) + (*in).DeepCopyInto(*out) + } + if in.Recommenders != nil { + in, out := &in.Recommenders, &out.Recommenders + *out = make([]*MultidimPodAutoscalerRecommenderSelector, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(MultidimPodAutoscalerRecommenderSelector) + **out = **in + } + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MultidimPodAutoscalerSpec. +func (in *MultidimPodAutoscalerSpec) DeepCopy() *MultidimPodAutoscalerSpec { + if in == nil { + return nil + } + out := new(MultidimPodAutoscalerSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MultidimPodAutoscalerStatus) DeepCopyInto(out *MultidimPodAutoscalerStatus) { + *out = *in + if in.LastScaleTime != nil { + in, out := &in.LastScaleTime, &out.LastScaleTime + *out = (*in).DeepCopy() + } + if in.Recommendation != nil { + in, out := &in.Recommendation, &out.Recommendation + *out = new(autoscalingk8siov1.RecommendedPodResources) + (*in).DeepCopyInto(*out) + } + if in.CurrentMetrics != nil { + in, out := &in.CurrentMetrics, &out.CurrentMetrics + *out = make([]v2.MetricStatus, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]MultidimPodAutoscalerCondition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MultidimPodAutoscalerStatus. +func (in *MultidimPodAutoscalerStatus) DeepCopy() *MultidimPodAutoscalerStatus { + if in == nil { + return nil + } + out := new(MultidimPodAutoscalerStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PodUpdatePolicy) DeepCopyInto(out *PodUpdatePolicy) { + *out = *in + if in.UpdateMode != nil { + in, out := &in.UpdateMode, &out.UpdateMode + *out = new(autoscalingk8siov1.UpdateMode) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodUpdatePolicy. +func (in *PodUpdatePolicy) DeepCopy() *PodUpdatePolicy { + if in == nil { + return nil + } + out := new(PodUpdatePolicy) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ScalingConstraints) DeepCopyInto(out *ScalingConstraints) { + *out = *in + if in.Global != nil { + in, out := &in.Global, &out.Global + *out = new(HorizontalScalingConstraints) + (*in).DeepCopyInto(*out) + } + if in.ContainerControlledResources != nil { + in, out := &in.ContainerControlledResources, &out.ContainerControlledResources + *out = make([]*v1.ResourceName, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(v1.ResourceName) + **out = **in + } + } + } + if in.Container != nil { + in, out := &in.Container, &out.Container + *out = make([]*VerticalScalingConstraints, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(VerticalScalingConstraints) + (*in).DeepCopyInto(*out) + } + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ScalingConstraints. +func (in *ScalingConstraints) DeepCopy() *ScalingConstraints { + if in == nil { + return nil + } + out := new(ScalingConstraints) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VerticalScalingConstraints) DeepCopyInto(out *VerticalScalingConstraints) { + *out = *in + if in.Mode != nil { + in, out := &in.Mode, &out.Mode + *out = new(autoscalingk8siov1.ContainerScalingMode) + **out = **in + } + if in.Requests != nil { + in, out := &in.Requests, &out.Requests + *out = new(MinMaxVerticalScalingLimits) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VerticalScalingConstraints. +func (in *VerticalScalingConstraints) DeepCopy() *VerticalScalingConstraints { + if in == nil { + return nil + } + out := new(VerticalScalingConstraints) + in.DeepCopyInto(out) + return out +} diff --git a/multidimensional-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1alpha1/zz_generated.defaults.go b/multidimensional-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1alpha1/zz_generated.defaults.go new file mode 100644 index 000000000000..5070cb91b90f --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1alpha1/zz_generated.defaults.go @@ -0,0 +1,33 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by defaulter-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// RegisterDefaults adds defaulters functions to the given scheme. +// Public to allow building arbitrary schemes. +// All generated defaulters are covering - they call all nested defaulters. +func RegisterDefaults(scheme *runtime.Scheme) error { + return nil +} diff --git a/multidimensional-pod-autoscaler/pkg/client/clientset/versioned/clientset.go b/multidimensional-pod-autoscaler/pkg/client/clientset/versioned/clientset.go new file mode 100644 index 000000000000..4995c0d004e4 --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/client/clientset/versioned/clientset.go @@ -0,0 +1,120 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package versioned + +import ( + fmt "fmt" + http "net/http" + + autoscalingv1alpha1 "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/client/clientset/versioned/typed/autoscaling.k8s.io/v1alpha1" + discovery "k8s.io/client-go/discovery" + rest "k8s.io/client-go/rest" + flowcontrol "k8s.io/client-go/util/flowcontrol" +) + +type Interface interface { + Discovery() discovery.DiscoveryInterface + AutoscalingV1alpha1() autoscalingv1alpha1.AutoscalingV1alpha1Interface +} + +// Clientset contains the clients for groups. +type Clientset struct { + *discovery.DiscoveryClient + autoscalingV1alpha1 *autoscalingv1alpha1.AutoscalingV1alpha1Client +} + +// AutoscalingV1alpha1 retrieves the AutoscalingV1alpha1Client +func (c *Clientset) AutoscalingV1alpha1() autoscalingv1alpha1.AutoscalingV1alpha1Interface { + return c.autoscalingV1alpha1 +} + +// Discovery retrieves the DiscoveryClient +func (c *Clientset) Discovery() discovery.DiscoveryInterface { + if c == nil { + return nil + } + return c.DiscoveryClient +} + +// NewForConfig creates a new Clientset for the given config. +// If config's RateLimiter is not set and QPS and Burst are acceptable, +// NewForConfig will generate a rate-limiter in configShallowCopy. +// NewForConfig is equivalent to NewForConfigAndClient(c, httpClient), +// where httpClient was generated with rest.HTTPClientFor(c). +func NewForConfig(c *rest.Config) (*Clientset, error) { + configShallowCopy := *c + + if configShallowCopy.UserAgent == "" { + configShallowCopy.UserAgent = rest.DefaultKubernetesUserAgent() + } + + // share the transport between all clients + httpClient, err := rest.HTTPClientFor(&configShallowCopy) + if err != nil { + return nil, err + } + + return NewForConfigAndClient(&configShallowCopy, httpClient) +} + +// NewForConfigAndClient creates a new Clientset for the given config and http client. +// Note the http client provided takes precedence over the configured transport values. +// If config's RateLimiter is not set and QPS and Burst are acceptable, +// NewForConfigAndClient will generate a rate-limiter in configShallowCopy. +func NewForConfigAndClient(c *rest.Config, httpClient *http.Client) (*Clientset, error) { + configShallowCopy := *c + if configShallowCopy.RateLimiter == nil && configShallowCopy.QPS > 0 { + if configShallowCopy.Burst <= 0 { + return nil, fmt.Errorf("burst is required to be greater than 0 when RateLimiter is not set and QPS is set to greater than 0") + } + configShallowCopy.RateLimiter = flowcontrol.NewTokenBucketRateLimiter(configShallowCopy.QPS, configShallowCopy.Burst) + } + + var cs Clientset + var err error + cs.autoscalingV1alpha1, err = autoscalingv1alpha1.NewForConfigAndClient(&configShallowCopy, httpClient) + if err != nil { + return nil, err + } + + cs.DiscoveryClient, err = discovery.NewDiscoveryClientForConfigAndClient(&configShallowCopy, httpClient) + if err != nil { + return nil, err + } + return &cs, nil +} + +// NewForConfigOrDie creates a new Clientset for the given config and +// panics if there is an error in the config. +func NewForConfigOrDie(c *rest.Config) *Clientset { + cs, err := NewForConfig(c) + if err != nil { + panic(err) + } + return cs +} + +// New creates a new Clientset for the given RESTClient. +func New(c rest.Interface) *Clientset { + var cs Clientset + cs.autoscalingV1alpha1 = autoscalingv1alpha1.New(c) + + cs.DiscoveryClient = discovery.NewDiscoveryClient(c) + return &cs +} diff --git a/multidimensional-pod-autoscaler/pkg/client/clientset/versioned/fake/clientset_generated.go b/multidimensional-pod-autoscaler/pkg/client/clientset/versioned/fake/clientset_generated.go new file mode 100644 index 000000000000..5f35d183ced9 --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/client/clientset/versioned/fake/clientset_generated.go @@ -0,0 +1,89 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/watch" + clientset "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/client/clientset/versioned" + autoscalingv1alpha1 "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/client/clientset/versioned/typed/autoscaling.k8s.io/v1alpha1" + fakeautoscalingv1alpha1 "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/client/clientset/versioned/typed/autoscaling.k8s.io/v1alpha1/fake" + "k8s.io/client-go/discovery" + fakediscovery "k8s.io/client-go/discovery/fake" + "k8s.io/client-go/testing" +) + +// NewSimpleClientset returns a clientset that will respond with the provided objects. +// It's backed by a very simple object tracker that processes creates, updates and deletions as-is, +// without applying any field management, validations and/or defaults. It shouldn't be considered a replacement +// for a real clientset and is mostly useful in simple unit tests. +// +// DEPRECATED: NewClientset replaces this with support for field management, which significantly improves +// server side apply testing. NewClientset is only available when apply configurations are generated (e.g. +// via --with-applyconfig). +func NewSimpleClientset(objects ...runtime.Object) *Clientset { + o := testing.NewObjectTracker(scheme, codecs.UniversalDecoder()) + for _, obj := range objects { + if err := o.Add(obj); err != nil { + panic(err) + } + } + + cs := &Clientset{tracker: o} + cs.discovery = &fakediscovery.FakeDiscovery{Fake: &cs.Fake} + cs.AddReactor("*", "*", testing.ObjectReaction(o)) + cs.AddWatchReactor("*", func(action testing.Action) (handled bool, ret watch.Interface, err error) { + gvr := action.GetResource() + ns := action.GetNamespace() + watch, err := o.Watch(gvr, ns) + if err != nil { + return false, nil, err + } + return true, watch, nil + }) + + return cs +} + +// Clientset implements clientset.Interface. Meant to be embedded into a +// struct to get a default implementation. This makes faking out just the method +// you want to test easier. +type Clientset struct { + testing.Fake + discovery *fakediscovery.FakeDiscovery + tracker testing.ObjectTracker +} + +func (c *Clientset) Discovery() discovery.DiscoveryInterface { + return c.discovery +} + +func (c *Clientset) Tracker() testing.ObjectTracker { + return c.tracker +} + +var ( + _ clientset.Interface = &Clientset{} + _ testing.FakeClient = &Clientset{} +) + +// AutoscalingV1alpha1 retrieves the AutoscalingV1alpha1Client +func (c *Clientset) AutoscalingV1alpha1() autoscalingv1alpha1.AutoscalingV1alpha1Interface { + return &fakeautoscalingv1alpha1.FakeAutoscalingV1alpha1{Fake: &c.Fake} +} diff --git a/multidimensional-pod-autoscaler/pkg/client/clientset/versioned/fake/doc.go b/multidimensional-pod-autoscaler/pkg/client/clientset/versioned/fake/doc.go new file mode 100644 index 000000000000..9b99e7167091 --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/client/clientset/versioned/fake/doc.go @@ -0,0 +1,20 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +// This package has the automatically generated fake clientset. +package fake diff --git a/multidimensional-pod-autoscaler/pkg/client/clientset/versioned/fake/register.go b/multidimensional-pod-autoscaler/pkg/client/clientset/versioned/fake/register.go new file mode 100644 index 000000000000..10b1c3f66dcb --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/client/clientset/versioned/fake/register.go @@ -0,0 +1,56 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + schema "k8s.io/apimachinery/pkg/runtime/schema" + serializer "k8s.io/apimachinery/pkg/runtime/serializer" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + autoscalingv1alpha1 "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1alpha1" +) + +var scheme = runtime.NewScheme() +var codecs = serializer.NewCodecFactory(scheme) + +var localSchemeBuilder = runtime.SchemeBuilder{ + autoscalingv1alpha1.AddToScheme, +} + +// AddToScheme adds all types of this clientset into the given scheme. This allows composition +// of clientsets, like in: +// +// import ( +// "k8s.io/client-go/kubernetes" +// clientsetscheme "k8s.io/client-go/kubernetes/scheme" +// aggregatorclientsetscheme "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme" +// ) +// +// kclientset, _ := kubernetes.NewForConfig(c) +// _ = aggregatorclientsetscheme.AddToScheme(clientsetscheme.Scheme) +// +// After this, RawExtensions in Kubernetes types will serialize kube-aggregator types +// correctly. +var AddToScheme = localSchemeBuilder.AddToScheme + +func init() { + v1.AddToGroupVersion(scheme, schema.GroupVersion{Version: "v1"}) + utilruntime.Must(AddToScheme(scheme)) +} diff --git a/multidimensional-pod-autoscaler/pkg/client/clientset/versioned/scheme/doc.go b/multidimensional-pod-autoscaler/pkg/client/clientset/versioned/scheme/doc.go new file mode 100644 index 000000000000..7dc3756168fa --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/client/clientset/versioned/scheme/doc.go @@ -0,0 +1,20 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +// This package contains the scheme of the automatically generated clientset. +package scheme diff --git a/multidimensional-pod-autoscaler/pkg/client/clientset/versioned/scheme/register.go b/multidimensional-pod-autoscaler/pkg/client/clientset/versioned/scheme/register.go new file mode 100644 index 000000000000..729d68c431d7 --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/client/clientset/versioned/scheme/register.go @@ -0,0 +1,56 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package scheme + +import ( + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + schema "k8s.io/apimachinery/pkg/runtime/schema" + serializer "k8s.io/apimachinery/pkg/runtime/serializer" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + autoscalingv1alpha1 "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1alpha1" +) + +var Scheme = runtime.NewScheme() +var Codecs = serializer.NewCodecFactory(Scheme) +var ParameterCodec = runtime.NewParameterCodec(Scheme) +var localSchemeBuilder = runtime.SchemeBuilder{ + autoscalingv1alpha1.AddToScheme, +} + +// AddToScheme adds all types of this clientset into the given scheme. This allows composition +// of clientsets, like in: +// +// import ( +// "k8s.io/client-go/kubernetes" +// clientsetscheme "k8s.io/client-go/kubernetes/scheme" +// aggregatorclientsetscheme "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme" +// ) +// +// kclientset, _ := kubernetes.NewForConfig(c) +// _ = aggregatorclientsetscheme.AddToScheme(clientsetscheme.Scheme) +// +// After this, RawExtensions in Kubernetes types will serialize kube-aggregator types +// correctly. +var AddToScheme = localSchemeBuilder.AddToScheme + +func init() { + v1.AddToGroupVersion(Scheme, schema.GroupVersion{Version: "v1"}) + utilruntime.Must(AddToScheme(Scheme)) +} diff --git a/multidimensional-pod-autoscaler/pkg/client/clientset/versioned/typed/autoscaling.k8s.io/v1alpha1/autoscaling.k8s.io_client.go b/multidimensional-pod-autoscaler/pkg/client/clientset/versioned/typed/autoscaling.k8s.io/v1alpha1/autoscaling.k8s.io_client.go new file mode 100644 index 000000000000..6dec63fe6284 --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/client/clientset/versioned/typed/autoscaling.k8s.io/v1alpha1/autoscaling.k8s.io_client.go @@ -0,0 +1,112 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + http "net/http" + + autoscalingk8siov1alpha1 "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1alpha1" + scheme "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/client/clientset/versioned/scheme" + rest "k8s.io/client-go/rest" +) + +type AutoscalingV1alpha1Interface interface { + RESTClient() rest.Interface + MultidimPodAutoscalersGetter + MultidimPodAutoscalerCheckpointsGetter +} + +// AutoscalingV1alpha1Client is used to interact with features provided by the autoscaling.k8s.io group. +type AutoscalingV1alpha1Client struct { + restClient rest.Interface +} + +func (c *AutoscalingV1alpha1Client) MultidimPodAutoscalers(namespace string) MultidimPodAutoscalerInterface { + return newMultidimPodAutoscalers(c, namespace) +} + +func (c *AutoscalingV1alpha1Client) MultidimPodAutoscalerCheckpoints(namespace string) MultidimPodAutoscalerCheckpointInterface { + return newMultidimPodAutoscalerCheckpoints(c, namespace) +} + +// NewForConfig creates a new AutoscalingV1alpha1Client for the given config. +// NewForConfig is equivalent to NewForConfigAndClient(c, httpClient), +// where httpClient was generated with rest.HTTPClientFor(c). +func NewForConfig(c *rest.Config) (*AutoscalingV1alpha1Client, error) { + config := *c + if err := setConfigDefaults(&config); err != nil { + return nil, err + } + httpClient, err := rest.HTTPClientFor(&config) + if err != nil { + return nil, err + } + return NewForConfigAndClient(&config, httpClient) +} + +// NewForConfigAndClient creates a new AutoscalingV1alpha1Client for the given config and http client. +// Note the http client provided takes precedence over the configured transport values. +func NewForConfigAndClient(c *rest.Config, h *http.Client) (*AutoscalingV1alpha1Client, error) { + config := *c + if err := setConfigDefaults(&config); err != nil { + return nil, err + } + client, err := rest.RESTClientForConfigAndClient(&config, h) + if err != nil { + return nil, err + } + return &AutoscalingV1alpha1Client{client}, nil +} + +// NewForConfigOrDie creates a new AutoscalingV1alpha1Client for the given config and +// panics if there is an error in the config. +func NewForConfigOrDie(c *rest.Config) *AutoscalingV1alpha1Client { + client, err := NewForConfig(c) + if err != nil { + panic(err) + } + return client +} + +// New creates a new AutoscalingV1alpha1Client for the given RESTClient. +func New(c rest.Interface) *AutoscalingV1alpha1Client { + return &AutoscalingV1alpha1Client{c} +} + +func setConfigDefaults(config *rest.Config) error { + gv := autoscalingk8siov1alpha1.SchemeGroupVersion + config.GroupVersion = &gv + config.APIPath = "/apis" + config.NegotiatedSerializer = rest.CodecFactoryForGeneratedClient(scheme.Scheme, scheme.Codecs).WithoutConversion() + + if config.UserAgent == "" { + config.UserAgent = rest.DefaultKubernetesUserAgent() + } + + return nil +} + +// RESTClient returns a RESTClient that is used to communicate +// with API server by this client implementation. +func (c *AutoscalingV1alpha1Client) RESTClient() rest.Interface { + if c == nil { + return nil + } + return c.restClient +} diff --git a/multidimensional-pod-autoscaler/pkg/client/clientset/versioned/typed/autoscaling.k8s.io/v1alpha1/doc.go b/multidimensional-pod-autoscaler/pkg/client/clientset/versioned/typed/autoscaling.k8s.io/v1alpha1/doc.go new file mode 100644 index 000000000000..df51baa4d4c1 --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/client/clientset/versioned/typed/autoscaling.k8s.io/v1alpha1/doc.go @@ -0,0 +1,20 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +// This package has the automatically generated typed clients. +package v1alpha1 diff --git a/multidimensional-pod-autoscaler/pkg/client/clientset/versioned/typed/autoscaling.k8s.io/v1alpha1/fake/doc.go b/multidimensional-pod-autoscaler/pkg/client/clientset/versioned/typed/autoscaling.k8s.io/v1alpha1/fake/doc.go new file mode 100644 index 000000000000..16f44399065e --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/client/clientset/versioned/typed/autoscaling.k8s.io/v1alpha1/fake/doc.go @@ -0,0 +1,20 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +// Package fake has the automatically generated clients. +package fake diff --git a/multidimensional-pod-autoscaler/pkg/client/clientset/versioned/typed/autoscaling.k8s.io/v1alpha1/fake/fake_autoscaling.k8s.io_client.go b/multidimensional-pod-autoscaler/pkg/client/clientset/versioned/typed/autoscaling.k8s.io/v1alpha1/fake/fake_autoscaling.k8s.io_client.go new file mode 100644 index 000000000000..886dfec5f075 --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/client/clientset/versioned/typed/autoscaling.k8s.io/v1alpha1/fake/fake_autoscaling.k8s.io_client.go @@ -0,0 +1,44 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + v1alpha1 "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/client/clientset/versioned/typed/autoscaling.k8s.io/v1alpha1" + rest "k8s.io/client-go/rest" + testing "k8s.io/client-go/testing" +) + +type FakeAutoscalingV1alpha1 struct { + *testing.Fake +} + +func (c *FakeAutoscalingV1alpha1) MultidimPodAutoscalers(namespace string) v1alpha1.MultidimPodAutoscalerInterface { + return newFakeMultidimPodAutoscalers(c, namespace) +} + +func (c *FakeAutoscalingV1alpha1) MultidimPodAutoscalerCheckpoints(namespace string) v1alpha1.MultidimPodAutoscalerCheckpointInterface { + return newFakeMultidimPodAutoscalerCheckpoints(c, namespace) +} + +// RESTClient returns a RESTClient that is used to communicate +// with API server by this client implementation. +func (c *FakeAutoscalingV1alpha1) RESTClient() rest.Interface { + var ret *rest.RESTClient + return ret +} diff --git a/multidimensional-pod-autoscaler/pkg/client/clientset/versioned/typed/autoscaling.k8s.io/v1alpha1/fake/fake_multidimpodautoscaler.go b/multidimensional-pod-autoscaler/pkg/client/clientset/versioned/typed/autoscaling.k8s.io/v1alpha1/fake/fake_multidimpodautoscaler.go new file mode 100644 index 000000000000..d290b95d73e2 --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/client/clientset/versioned/typed/autoscaling.k8s.io/v1alpha1/fake/fake_multidimpodautoscaler.go @@ -0,0 +1,52 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + v1alpha1 "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1alpha1" + autoscalingk8siov1alpha1 "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/client/clientset/versioned/typed/autoscaling.k8s.io/v1alpha1" + gentype "k8s.io/client-go/gentype" +) + +// fakeMultidimPodAutoscalers implements MultidimPodAutoscalerInterface +type fakeMultidimPodAutoscalers struct { + *gentype.FakeClientWithList[*v1alpha1.MultidimPodAutoscaler, *v1alpha1.MultidimPodAutoscalerList] + Fake *FakeAutoscalingV1alpha1 +} + +func newFakeMultidimPodAutoscalers(fake *FakeAutoscalingV1alpha1, namespace string) autoscalingk8siov1alpha1.MultidimPodAutoscalerInterface { + return &fakeMultidimPodAutoscalers{ + gentype.NewFakeClientWithList[*v1alpha1.MultidimPodAutoscaler, *v1alpha1.MultidimPodAutoscalerList]( + fake.Fake, + namespace, + v1alpha1.SchemeGroupVersion.WithResource("multidimpodautoscalers"), + v1alpha1.SchemeGroupVersion.WithKind("MultidimPodAutoscaler"), + func() *v1alpha1.MultidimPodAutoscaler { return &v1alpha1.MultidimPodAutoscaler{} }, + func() *v1alpha1.MultidimPodAutoscalerList { return &v1alpha1.MultidimPodAutoscalerList{} }, + func(dst, src *v1alpha1.MultidimPodAutoscalerList) { dst.ListMeta = src.ListMeta }, + func(list *v1alpha1.MultidimPodAutoscalerList) []*v1alpha1.MultidimPodAutoscaler { + return gentype.ToPointerSlice(list.Items) + }, + func(list *v1alpha1.MultidimPodAutoscalerList, items []*v1alpha1.MultidimPodAutoscaler) { + list.Items = gentype.FromPointerSlice(items) + }, + ), + fake, + } +} diff --git a/multidimensional-pod-autoscaler/pkg/client/clientset/versioned/typed/autoscaling.k8s.io/v1alpha1/fake/fake_multidimpodautoscalercheckpoint.go b/multidimensional-pod-autoscaler/pkg/client/clientset/versioned/typed/autoscaling.k8s.io/v1alpha1/fake/fake_multidimpodautoscalercheckpoint.go new file mode 100644 index 000000000000..ca58c799adcf --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/client/clientset/versioned/typed/autoscaling.k8s.io/v1alpha1/fake/fake_multidimpodautoscalercheckpoint.go @@ -0,0 +1,54 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + v1alpha1 "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1alpha1" + autoscalingk8siov1alpha1 "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/client/clientset/versioned/typed/autoscaling.k8s.io/v1alpha1" + gentype "k8s.io/client-go/gentype" +) + +// fakeMultidimPodAutoscalerCheckpoints implements MultidimPodAutoscalerCheckpointInterface +type fakeMultidimPodAutoscalerCheckpoints struct { + *gentype.FakeClientWithList[*v1alpha1.MultidimPodAutoscalerCheckpoint, *v1alpha1.MultidimPodAutoscalerCheckpointList] + Fake *FakeAutoscalingV1alpha1 +} + +func newFakeMultidimPodAutoscalerCheckpoints(fake *FakeAutoscalingV1alpha1, namespace string) autoscalingk8siov1alpha1.MultidimPodAutoscalerCheckpointInterface { + return &fakeMultidimPodAutoscalerCheckpoints{ + gentype.NewFakeClientWithList[*v1alpha1.MultidimPodAutoscalerCheckpoint, *v1alpha1.MultidimPodAutoscalerCheckpointList]( + fake.Fake, + namespace, + v1alpha1.SchemeGroupVersion.WithResource("multidimpodautoscalercheckpoints"), + v1alpha1.SchemeGroupVersion.WithKind("MultidimPodAutoscalerCheckpoint"), + func() *v1alpha1.MultidimPodAutoscalerCheckpoint { return &v1alpha1.MultidimPodAutoscalerCheckpoint{} }, + func() *v1alpha1.MultidimPodAutoscalerCheckpointList { + return &v1alpha1.MultidimPodAutoscalerCheckpointList{} + }, + func(dst, src *v1alpha1.MultidimPodAutoscalerCheckpointList) { dst.ListMeta = src.ListMeta }, + func(list *v1alpha1.MultidimPodAutoscalerCheckpointList) []*v1alpha1.MultidimPodAutoscalerCheckpoint { + return gentype.ToPointerSlice(list.Items) + }, + func(list *v1alpha1.MultidimPodAutoscalerCheckpointList, items []*v1alpha1.MultidimPodAutoscalerCheckpoint) { + list.Items = gentype.FromPointerSlice(items) + }, + ), + fake, + } +} diff --git a/multidimensional-pod-autoscaler/pkg/client/clientset/versioned/typed/autoscaling.k8s.io/v1alpha1/generated_expansion.go b/multidimensional-pod-autoscaler/pkg/client/clientset/versioned/typed/autoscaling.k8s.io/v1alpha1/generated_expansion.go new file mode 100644 index 000000000000..7dd2878a3ea5 --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/client/clientset/versioned/typed/autoscaling.k8s.io/v1alpha1/generated_expansion.go @@ -0,0 +1,23 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package v1alpha1 + +type MultidimPodAutoscalerExpansion interface{} + +type MultidimPodAutoscalerCheckpointExpansion interface{} diff --git a/multidimensional-pod-autoscaler/pkg/client/clientset/versioned/typed/autoscaling.k8s.io/v1alpha1/multidimpodautoscaler.go b/multidimensional-pod-autoscaler/pkg/client/clientset/versioned/typed/autoscaling.k8s.io/v1alpha1/multidimpodautoscaler.go new file mode 100644 index 000000000000..c9ecf4aa51b4 --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/client/clientset/versioned/typed/autoscaling.k8s.io/v1alpha1/multidimpodautoscaler.go @@ -0,0 +1,74 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + context "context" + + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + autoscalingk8siov1alpha1 "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1alpha1" + scheme "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/client/clientset/versioned/scheme" + gentype "k8s.io/client-go/gentype" +) + +// MultidimPodAutoscalersGetter has a method to return a MultidimPodAutoscalerInterface. +// A group's client should implement this interface. +type MultidimPodAutoscalersGetter interface { + MultidimPodAutoscalers(namespace string) MultidimPodAutoscalerInterface +} + +// MultidimPodAutoscalerInterface has methods to work with MultidimPodAutoscaler resources. +type MultidimPodAutoscalerInterface interface { + Create(ctx context.Context, multidimPodAutoscaler *autoscalingk8siov1alpha1.MultidimPodAutoscaler, opts v1.CreateOptions) (*autoscalingk8siov1alpha1.MultidimPodAutoscaler, error) + Update(ctx context.Context, multidimPodAutoscaler *autoscalingk8siov1alpha1.MultidimPodAutoscaler, opts v1.UpdateOptions) (*autoscalingk8siov1alpha1.MultidimPodAutoscaler, error) + // Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). + UpdateStatus(ctx context.Context, multidimPodAutoscaler *autoscalingk8siov1alpha1.MultidimPodAutoscaler, opts v1.UpdateOptions) (*autoscalingk8siov1alpha1.MultidimPodAutoscaler, error) + Delete(ctx context.Context, name string, opts v1.DeleteOptions) error + DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error + Get(ctx context.Context, name string, opts v1.GetOptions) (*autoscalingk8siov1alpha1.MultidimPodAutoscaler, error) + List(ctx context.Context, opts v1.ListOptions) (*autoscalingk8siov1alpha1.MultidimPodAutoscalerList, error) + Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) + Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *autoscalingk8siov1alpha1.MultidimPodAutoscaler, err error) + MultidimPodAutoscalerExpansion +} + +// multidimPodAutoscalers implements MultidimPodAutoscalerInterface +type multidimPodAutoscalers struct { + *gentype.ClientWithList[*autoscalingk8siov1alpha1.MultidimPodAutoscaler, *autoscalingk8siov1alpha1.MultidimPodAutoscalerList] +} + +// newMultidimPodAutoscalers returns a MultidimPodAutoscalers +func newMultidimPodAutoscalers(c *AutoscalingV1alpha1Client, namespace string) *multidimPodAutoscalers { + return &multidimPodAutoscalers{ + gentype.NewClientWithList[*autoscalingk8siov1alpha1.MultidimPodAutoscaler, *autoscalingk8siov1alpha1.MultidimPodAutoscalerList]( + "multidimpodautoscalers", + c.RESTClient(), + scheme.ParameterCodec, + namespace, + func() *autoscalingk8siov1alpha1.MultidimPodAutoscaler { + return &autoscalingk8siov1alpha1.MultidimPodAutoscaler{} + }, + func() *autoscalingk8siov1alpha1.MultidimPodAutoscalerList { + return &autoscalingk8siov1alpha1.MultidimPodAutoscalerList{} + }, + ), + } +} diff --git a/multidimensional-pod-autoscaler/pkg/client/clientset/versioned/typed/autoscaling.k8s.io/v1alpha1/multidimpodautoscalercheckpoint.go b/multidimensional-pod-autoscaler/pkg/client/clientset/versioned/typed/autoscaling.k8s.io/v1alpha1/multidimpodautoscalercheckpoint.go new file mode 100644 index 000000000000..d7b7e713452b --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/client/clientset/versioned/typed/autoscaling.k8s.io/v1alpha1/multidimpodautoscalercheckpoint.go @@ -0,0 +1,72 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + context "context" + + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + autoscalingk8siov1alpha1 "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1alpha1" + scheme "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/client/clientset/versioned/scheme" + gentype "k8s.io/client-go/gentype" +) + +// MultidimPodAutoscalerCheckpointsGetter has a method to return a MultidimPodAutoscalerCheckpointInterface. +// A group's client should implement this interface. +type MultidimPodAutoscalerCheckpointsGetter interface { + MultidimPodAutoscalerCheckpoints(namespace string) MultidimPodAutoscalerCheckpointInterface +} + +// MultidimPodAutoscalerCheckpointInterface has methods to work with MultidimPodAutoscalerCheckpoint resources. +type MultidimPodAutoscalerCheckpointInterface interface { + Create(ctx context.Context, multidimPodAutoscalerCheckpoint *autoscalingk8siov1alpha1.MultidimPodAutoscalerCheckpoint, opts v1.CreateOptions) (*autoscalingk8siov1alpha1.MultidimPodAutoscalerCheckpoint, error) + Update(ctx context.Context, multidimPodAutoscalerCheckpoint *autoscalingk8siov1alpha1.MultidimPodAutoscalerCheckpoint, opts v1.UpdateOptions) (*autoscalingk8siov1alpha1.MultidimPodAutoscalerCheckpoint, error) + Delete(ctx context.Context, name string, opts v1.DeleteOptions) error + DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error + Get(ctx context.Context, name string, opts v1.GetOptions) (*autoscalingk8siov1alpha1.MultidimPodAutoscalerCheckpoint, error) + List(ctx context.Context, opts v1.ListOptions) (*autoscalingk8siov1alpha1.MultidimPodAutoscalerCheckpointList, error) + Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) + Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *autoscalingk8siov1alpha1.MultidimPodAutoscalerCheckpoint, err error) + MultidimPodAutoscalerCheckpointExpansion +} + +// multidimPodAutoscalerCheckpoints implements MultidimPodAutoscalerCheckpointInterface +type multidimPodAutoscalerCheckpoints struct { + *gentype.ClientWithList[*autoscalingk8siov1alpha1.MultidimPodAutoscalerCheckpoint, *autoscalingk8siov1alpha1.MultidimPodAutoscalerCheckpointList] +} + +// newMultidimPodAutoscalerCheckpoints returns a MultidimPodAutoscalerCheckpoints +func newMultidimPodAutoscalerCheckpoints(c *AutoscalingV1alpha1Client, namespace string) *multidimPodAutoscalerCheckpoints { + return &multidimPodAutoscalerCheckpoints{ + gentype.NewClientWithList[*autoscalingk8siov1alpha1.MultidimPodAutoscalerCheckpoint, *autoscalingk8siov1alpha1.MultidimPodAutoscalerCheckpointList]( + "multidimpodautoscalercheckpoints", + c.RESTClient(), + scheme.ParameterCodec, + namespace, + func() *autoscalingk8siov1alpha1.MultidimPodAutoscalerCheckpoint { + return &autoscalingk8siov1alpha1.MultidimPodAutoscalerCheckpoint{} + }, + func() *autoscalingk8siov1alpha1.MultidimPodAutoscalerCheckpointList { + return &autoscalingk8siov1alpha1.MultidimPodAutoscalerCheckpointList{} + }, + ), + } +} diff --git a/multidimensional-pod-autoscaler/pkg/client/informers/externalversions/autoscaling.k8s.io/interface.go b/multidimensional-pod-autoscaler/pkg/client/informers/externalversions/autoscaling.k8s.io/interface.go new file mode 100644 index 000000000000..c0979758bff4 --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/client/informers/externalversions/autoscaling.k8s.io/interface.go @@ -0,0 +1,46 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by informer-gen. DO NOT EDIT. + +package autoscaling + +import ( + v1alpha1 "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/client/informers/externalversions/autoscaling.k8s.io/v1alpha1" + internalinterfaces "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/client/informers/externalversions/internalinterfaces" +) + +// Interface provides access to each of this group's versions. +type Interface interface { + // V1alpha1 provides access to shared informers for resources in V1alpha1. + V1alpha1() v1alpha1.Interface +} + +type group struct { + factory internalinterfaces.SharedInformerFactory + namespace string + tweakListOptions internalinterfaces.TweakListOptionsFunc +} + +// New returns a new Interface. +func New(f internalinterfaces.SharedInformerFactory, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) Interface { + return &group{factory: f, namespace: namespace, tweakListOptions: tweakListOptions} +} + +// V1alpha1 returns a new v1alpha1.Interface. +func (g *group) V1alpha1() v1alpha1.Interface { + return v1alpha1.New(g.factory, g.namespace, g.tweakListOptions) +} diff --git a/multidimensional-pod-autoscaler/pkg/client/informers/externalversions/autoscaling.k8s.io/v1alpha1/interface.go b/multidimensional-pod-autoscaler/pkg/client/informers/externalversions/autoscaling.k8s.io/v1alpha1/interface.go new file mode 100644 index 000000000000..752e79d0c604 --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/client/informers/externalversions/autoscaling.k8s.io/v1alpha1/interface.go @@ -0,0 +1,52 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by informer-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + internalinterfaces "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/client/informers/externalversions/internalinterfaces" +) + +// Interface provides access to all the informers in this group version. +type Interface interface { + // MultidimPodAutoscalers returns a MultidimPodAutoscalerInformer. + MultidimPodAutoscalers() MultidimPodAutoscalerInformer + // MultidimPodAutoscalerCheckpoints returns a MultidimPodAutoscalerCheckpointInformer. + MultidimPodAutoscalerCheckpoints() MultidimPodAutoscalerCheckpointInformer +} + +type version struct { + factory internalinterfaces.SharedInformerFactory + namespace string + tweakListOptions internalinterfaces.TweakListOptionsFunc +} + +// New returns a new Interface. +func New(f internalinterfaces.SharedInformerFactory, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) Interface { + return &version{factory: f, namespace: namespace, tweakListOptions: tweakListOptions} +} + +// MultidimPodAutoscalers returns a MultidimPodAutoscalerInformer. +func (v *version) MultidimPodAutoscalers() MultidimPodAutoscalerInformer { + return &multidimPodAutoscalerInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} +} + +// MultidimPodAutoscalerCheckpoints returns a MultidimPodAutoscalerCheckpointInformer. +func (v *version) MultidimPodAutoscalerCheckpoints() MultidimPodAutoscalerCheckpointInformer { + return &multidimPodAutoscalerCheckpointInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} +} diff --git a/multidimensional-pod-autoscaler/pkg/client/informers/externalversions/autoscaling.k8s.io/v1alpha1/multidimpodautoscaler.go b/multidimensional-pod-autoscaler/pkg/client/informers/externalversions/autoscaling.k8s.io/v1alpha1/multidimpodautoscaler.go new file mode 100644 index 000000000000..65585d5fe9bd --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/client/informers/externalversions/autoscaling.k8s.io/v1alpha1/multidimpodautoscaler.go @@ -0,0 +1,90 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by informer-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + context "context" + time "time" + + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + watch "k8s.io/apimachinery/pkg/watch" + apisautoscalingk8siov1alpha1 "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1alpha1" + versioned "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/client/clientset/versioned" + internalinterfaces "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/client/informers/externalversions/internalinterfaces" + autoscalingk8siov1alpha1 "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/client/listers/autoscaling.k8s.io/v1alpha1" + cache "k8s.io/client-go/tools/cache" +) + +// MultidimPodAutoscalerInformer provides access to a shared informer and lister for +// MultidimPodAutoscalers. +type MultidimPodAutoscalerInformer interface { + Informer() cache.SharedIndexInformer + Lister() autoscalingk8siov1alpha1.MultidimPodAutoscalerLister +} + +type multidimPodAutoscalerInformer struct { + factory internalinterfaces.SharedInformerFactory + tweakListOptions internalinterfaces.TweakListOptionsFunc + namespace string +} + +// NewMultidimPodAutoscalerInformer constructs a new informer for MultidimPodAutoscaler type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewMultidimPodAutoscalerInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer { + return NewFilteredMultidimPodAutoscalerInformer(client, namespace, resyncPeriod, indexers, nil) +} + +// NewFilteredMultidimPodAutoscalerInformer constructs a new informer for MultidimPodAutoscaler type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewFilteredMultidimPodAutoscalerInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { + return cache.NewSharedIndexInformer( + &cache.ListWatch{ + ListFunc: func(options v1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.AutoscalingV1alpha1().MultidimPodAutoscalers(namespace).List(context.TODO(), options) + }, + WatchFunc: func(options v1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.AutoscalingV1alpha1().MultidimPodAutoscalers(namespace).Watch(context.TODO(), options) + }, + }, + &apisautoscalingk8siov1alpha1.MultidimPodAutoscaler{}, + resyncPeriod, + indexers, + ) +} + +func (f *multidimPodAutoscalerInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { + return NewFilteredMultidimPodAutoscalerInformer(client, f.namespace, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions) +} + +func (f *multidimPodAutoscalerInformer) Informer() cache.SharedIndexInformer { + return f.factory.InformerFor(&apisautoscalingk8siov1alpha1.MultidimPodAutoscaler{}, f.defaultInformer) +} + +func (f *multidimPodAutoscalerInformer) Lister() autoscalingk8siov1alpha1.MultidimPodAutoscalerLister { + return autoscalingk8siov1alpha1.NewMultidimPodAutoscalerLister(f.Informer().GetIndexer()) +} diff --git a/multidimensional-pod-autoscaler/pkg/client/informers/externalversions/autoscaling.k8s.io/v1alpha1/multidimpodautoscalercheckpoint.go b/multidimensional-pod-autoscaler/pkg/client/informers/externalversions/autoscaling.k8s.io/v1alpha1/multidimpodautoscalercheckpoint.go new file mode 100644 index 000000000000..d237f7cf5821 --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/client/informers/externalversions/autoscaling.k8s.io/v1alpha1/multidimpodautoscalercheckpoint.go @@ -0,0 +1,90 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by informer-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + context "context" + time "time" + + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + watch "k8s.io/apimachinery/pkg/watch" + apisautoscalingk8siov1alpha1 "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1alpha1" + versioned "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/client/clientset/versioned" + internalinterfaces "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/client/informers/externalversions/internalinterfaces" + autoscalingk8siov1alpha1 "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/client/listers/autoscaling.k8s.io/v1alpha1" + cache "k8s.io/client-go/tools/cache" +) + +// MultidimPodAutoscalerCheckpointInformer provides access to a shared informer and lister for +// MultidimPodAutoscalerCheckpoints. +type MultidimPodAutoscalerCheckpointInformer interface { + Informer() cache.SharedIndexInformer + Lister() autoscalingk8siov1alpha1.MultidimPodAutoscalerCheckpointLister +} + +type multidimPodAutoscalerCheckpointInformer struct { + factory internalinterfaces.SharedInformerFactory + tweakListOptions internalinterfaces.TweakListOptionsFunc + namespace string +} + +// NewMultidimPodAutoscalerCheckpointInformer constructs a new informer for MultidimPodAutoscalerCheckpoint type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewMultidimPodAutoscalerCheckpointInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer { + return NewFilteredMultidimPodAutoscalerCheckpointInformer(client, namespace, resyncPeriod, indexers, nil) +} + +// NewFilteredMultidimPodAutoscalerCheckpointInformer constructs a new informer for MultidimPodAutoscalerCheckpoint type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewFilteredMultidimPodAutoscalerCheckpointInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { + return cache.NewSharedIndexInformer( + &cache.ListWatch{ + ListFunc: func(options v1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.AutoscalingV1alpha1().MultidimPodAutoscalerCheckpoints(namespace).List(context.TODO(), options) + }, + WatchFunc: func(options v1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.AutoscalingV1alpha1().MultidimPodAutoscalerCheckpoints(namespace).Watch(context.TODO(), options) + }, + }, + &apisautoscalingk8siov1alpha1.MultidimPodAutoscalerCheckpoint{}, + resyncPeriod, + indexers, + ) +} + +func (f *multidimPodAutoscalerCheckpointInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { + return NewFilteredMultidimPodAutoscalerCheckpointInformer(client, f.namespace, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions) +} + +func (f *multidimPodAutoscalerCheckpointInformer) Informer() cache.SharedIndexInformer { + return f.factory.InformerFor(&apisautoscalingk8siov1alpha1.MultidimPodAutoscalerCheckpoint{}, f.defaultInformer) +} + +func (f *multidimPodAutoscalerCheckpointInformer) Lister() autoscalingk8siov1alpha1.MultidimPodAutoscalerCheckpointLister { + return autoscalingk8siov1alpha1.NewMultidimPodAutoscalerCheckpointLister(f.Informer().GetIndexer()) +} diff --git a/multidimensional-pod-autoscaler/pkg/client/informers/externalversions/factory.go b/multidimensional-pod-autoscaler/pkg/client/informers/externalversions/factory.go new file mode 100644 index 000000000000..7e4114f303a1 --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/client/informers/externalversions/factory.go @@ -0,0 +1,262 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by informer-gen. DO NOT EDIT. + +package externalversions + +import ( + reflect "reflect" + sync "sync" + time "time" + + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + schema "k8s.io/apimachinery/pkg/runtime/schema" + versioned "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/client/clientset/versioned" + autoscalingk8sio "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/client/informers/externalversions/autoscaling.k8s.io" + internalinterfaces "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/client/informers/externalversions/internalinterfaces" + cache "k8s.io/client-go/tools/cache" +) + +// SharedInformerOption defines the functional option type for SharedInformerFactory. +type SharedInformerOption func(*sharedInformerFactory) *sharedInformerFactory + +type sharedInformerFactory struct { + client versioned.Interface + namespace string + tweakListOptions internalinterfaces.TweakListOptionsFunc + lock sync.Mutex + defaultResync time.Duration + customResync map[reflect.Type]time.Duration + transform cache.TransformFunc + + informers map[reflect.Type]cache.SharedIndexInformer + // startedInformers is used for tracking which informers have been started. + // This allows Start() to be called multiple times safely. + startedInformers map[reflect.Type]bool + // wg tracks how many goroutines were started. + wg sync.WaitGroup + // shuttingDown is true when Shutdown has been called. It may still be running + // because it needs to wait for goroutines. + shuttingDown bool +} + +// WithCustomResyncConfig sets a custom resync period for the specified informer types. +func WithCustomResyncConfig(resyncConfig map[v1.Object]time.Duration) SharedInformerOption { + return func(factory *sharedInformerFactory) *sharedInformerFactory { + for k, v := range resyncConfig { + factory.customResync[reflect.TypeOf(k)] = v + } + return factory + } +} + +// WithTweakListOptions sets a custom filter on all listers of the configured SharedInformerFactory. +func WithTweakListOptions(tweakListOptions internalinterfaces.TweakListOptionsFunc) SharedInformerOption { + return func(factory *sharedInformerFactory) *sharedInformerFactory { + factory.tweakListOptions = tweakListOptions + return factory + } +} + +// WithNamespace limits the SharedInformerFactory to the specified namespace. +func WithNamespace(namespace string) SharedInformerOption { + return func(factory *sharedInformerFactory) *sharedInformerFactory { + factory.namespace = namespace + return factory + } +} + +// WithTransform sets a transform on all informers. +func WithTransform(transform cache.TransformFunc) SharedInformerOption { + return func(factory *sharedInformerFactory) *sharedInformerFactory { + factory.transform = transform + return factory + } +} + +// NewSharedInformerFactory constructs a new instance of sharedInformerFactory for all namespaces. +func NewSharedInformerFactory(client versioned.Interface, defaultResync time.Duration) SharedInformerFactory { + return NewSharedInformerFactoryWithOptions(client, defaultResync) +} + +// NewFilteredSharedInformerFactory constructs a new instance of sharedInformerFactory. +// Listers obtained via this SharedInformerFactory will be subject to the same filters +// as specified here. +// Deprecated: Please use NewSharedInformerFactoryWithOptions instead +func NewFilteredSharedInformerFactory(client versioned.Interface, defaultResync time.Duration, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) SharedInformerFactory { + return NewSharedInformerFactoryWithOptions(client, defaultResync, WithNamespace(namespace), WithTweakListOptions(tweakListOptions)) +} + +// NewSharedInformerFactoryWithOptions constructs a new instance of a SharedInformerFactory with additional options. +func NewSharedInformerFactoryWithOptions(client versioned.Interface, defaultResync time.Duration, options ...SharedInformerOption) SharedInformerFactory { + factory := &sharedInformerFactory{ + client: client, + namespace: v1.NamespaceAll, + defaultResync: defaultResync, + informers: make(map[reflect.Type]cache.SharedIndexInformer), + startedInformers: make(map[reflect.Type]bool), + customResync: make(map[reflect.Type]time.Duration), + } + + // Apply all options + for _, opt := range options { + factory = opt(factory) + } + + return factory +} + +func (f *sharedInformerFactory) Start(stopCh <-chan struct{}) { + f.lock.Lock() + defer f.lock.Unlock() + + if f.shuttingDown { + return + } + + for informerType, informer := range f.informers { + if !f.startedInformers[informerType] { + f.wg.Add(1) + // We need a new variable in each loop iteration, + // otherwise the goroutine would use the loop variable + // and that keeps changing. + informer := informer + go func() { + defer f.wg.Done() + informer.Run(stopCh) + }() + f.startedInformers[informerType] = true + } + } +} + +func (f *sharedInformerFactory) Shutdown() { + f.lock.Lock() + f.shuttingDown = true + f.lock.Unlock() + + // Will return immediately if there is nothing to wait for. + f.wg.Wait() +} + +func (f *sharedInformerFactory) WaitForCacheSync(stopCh <-chan struct{}) map[reflect.Type]bool { + informers := func() map[reflect.Type]cache.SharedIndexInformer { + f.lock.Lock() + defer f.lock.Unlock() + + informers := map[reflect.Type]cache.SharedIndexInformer{} + for informerType, informer := range f.informers { + if f.startedInformers[informerType] { + informers[informerType] = informer + } + } + return informers + }() + + res := map[reflect.Type]bool{} + for informType, informer := range informers { + res[informType] = cache.WaitForCacheSync(stopCh, informer.HasSynced) + } + return res +} + +// InformerFor returns the SharedIndexInformer for obj using an internal +// client. +func (f *sharedInformerFactory) InformerFor(obj runtime.Object, newFunc internalinterfaces.NewInformerFunc) cache.SharedIndexInformer { + f.lock.Lock() + defer f.lock.Unlock() + + informerType := reflect.TypeOf(obj) + informer, exists := f.informers[informerType] + if exists { + return informer + } + + resyncPeriod, exists := f.customResync[informerType] + if !exists { + resyncPeriod = f.defaultResync + } + + informer = newFunc(f.client, resyncPeriod) + informer.SetTransform(f.transform) + f.informers[informerType] = informer + + return informer +} + +// SharedInformerFactory provides shared informers for resources in all known +// API group versions. +// +// It is typically used like this: +// +// ctx, cancel := context.Background() +// defer cancel() +// factory := NewSharedInformerFactory(client, resyncPeriod) +// defer factory.WaitForStop() // Returns immediately if nothing was started. +// genericInformer := factory.ForResource(resource) +// typedInformer := factory.SomeAPIGroup().V1().SomeType() +// factory.Start(ctx.Done()) // Start processing these informers. +// synced := factory.WaitForCacheSync(ctx.Done()) +// for v, ok := range synced { +// if !ok { +// fmt.Fprintf(os.Stderr, "caches failed to sync: %v", v) +// return +// } +// } +// +// // Creating informers can also be created after Start, but then +// // Start must be called again: +// anotherGenericInformer := factory.ForResource(resource) +// factory.Start(ctx.Done()) +type SharedInformerFactory interface { + internalinterfaces.SharedInformerFactory + + // Start initializes all requested informers. They are handled in goroutines + // which run until the stop channel gets closed. + // Warning: Start does not block. When run in a go-routine, it will race with a later WaitForCacheSync. + Start(stopCh <-chan struct{}) + + // Shutdown marks a factory as shutting down. At that point no new + // informers can be started anymore and Start will return without + // doing anything. + // + // In addition, Shutdown blocks until all goroutines have terminated. For that + // to happen, the close channel(s) that they were started with must be closed, + // either before Shutdown gets called or while it is waiting. + // + // Shutdown may be called multiple times, even concurrently. All such calls will + // block until all goroutines have terminated. + Shutdown() + + // WaitForCacheSync blocks until all started informers' caches were synced + // or the stop channel gets closed. + WaitForCacheSync(stopCh <-chan struct{}) map[reflect.Type]bool + + // ForResource gives generic access to a shared informer of the matching type. + ForResource(resource schema.GroupVersionResource) (GenericInformer, error) + + // InformerFor returns the SharedIndexInformer for obj using an internal + // client. + InformerFor(obj runtime.Object, newFunc internalinterfaces.NewInformerFunc) cache.SharedIndexInformer + + Autoscaling() autoscalingk8sio.Interface +} + +func (f *sharedInformerFactory) Autoscaling() autoscalingk8sio.Interface { + return autoscalingk8sio.New(f, f.namespace, f.tweakListOptions) +} diff --git a/multidimensional-pod-autoscaler/pkg/client/informers/externalversions/generic.go b/multidimensional-pod-autoscaler/pkg/client/informers/externalversions/generic.go new file mode 100644 index 000000000000..b41a168e7a89 --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/client/informers/externalversions/generic.go @@ -0,0 +1,64 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by informer-gen. DO NOT EDIT. + +package externalversions + +import ( + fmt "fmt" + + schema "k8s.io/apimachinery/pkg/runtime/schema" + v1alpha1 "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1alpha1" + cache "k8s.io/client-go/tools/cache" +) + +// GenericInformer is type of SharedIndexInformer which will locate and delegate to other +// sharedInformers based on type +type GenericInformer interface { + Informer() cache.SharedIndexInformer + Lister() cache.GenericLister +} + +type genericInformer struct { + informer cache.SharedIndexInformer + resource schema.GroupResource +} + +// Informer returns the SharedIndexInformer. +func (f *genericInformer) Informer() cache.SharedIndexInformer { + return f.informer +} + +// Lister returns the GenericLister. +func (f *genericInformer) Lister() cache.GenericLister { + return cache.NewGenericLister(f.Informer().GetIndexer(), f.resource) +} + +// ForResource gives generic access to a shared informer of the matching type +// TODO extend this to unknown resources with a client pool +func (f *sharedInformerFactory) ForResource(resource schema.GroupVersionResource) (GenericInformer, error) { + switch resource { + // Group=autoscaling.k8s.io, Version=v1alpha1 + case v1alpha1.SchemeGroupVersion.WithResource("multidimpodautoscalers"): + return &genericInformer{resource: resource.GroupResource(), informer: f.Autoscaling().V1alpha1().MultidimPodAutoscalers().Informer()}, nil + case v1alpha1.SchemeGroupVersion.WithResource("multidimpodautoscalercheckpoints"): + return &genericInformer{resource: resource.GroupResource(), informer: f.Autoscaling().V1alpha1().MultidimPodAutoscalerCheckpoints().Informer()}, nil + + } + + return nil, fmt.Errorf("no informer found for %v", resource) +} diff --git a/multidimensional-pod-autoscaler/pkg/client/informers/externalversions/internalinterfaces/factory_interfaces.go b/multidimensional-pod-autoscaler/pkg/client/informers/externalversions/internalinterfaces/factory_interfaces.go new file mode 100644 index 000000000000..b71e691ca225 --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/client/informers/externalversions/internalinterfaces/factory_interfaces.go @@ -0,0 +1,40 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by informer-gen. DO NOT EDIT. + +package internalinterfaces + +import ( + time "time" + + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + versioned "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/client/clientset/versioned" + cache "k8s.io/client-go/tools/cache" +) + +// NewInformerFunc takes versioned.Interface and time.Duration to return a SharedIndexInformer. +type NewInformerFunc func(versioned.Interface, time.Duration) cache.SharedIndexInformer + +// SharedInformerFactory a small interface to allow for adding an informer without an import cycle +type SharedInformerFactory interface { + Start(stopCh <-chan struct{}) + InformerFor(obj runtime.Object, newFunc NewInformerFunc) cache.SharedIndexInformer +} + +// TweakListOptionsFunc is a function that transforms a v1.ListOptions. +type TweakListOptionsFunc func(*v1.ListOptions) diff --git a/multidimensional-pod-autoscaler/pkg/client/listers/autoscaling.k8s.io/v1alpha1/expansion_generated.go b/multidimensional-pod-autoscaler/pkg/client/listers/autoscaling.k8s.io/v1alpha1/expansion_generated.go new file mode 100644 index 000000000000..6e1c7c50b895 --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/client/listers/autoscaling.k8s.io/v1alpha1/expansion_generated.go @@ -0,0 +1,35 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by lister-gen. DO NOT EDIT. + +package v1alpha1 + +// MultidimPodAutoscalerListerExpansion allows custom methods to be added to +// MultidimPodAutoscalerLister. +type MultidimPodAutoscalerListerExpansion interface{} + +// MultidimPodAutoscalerNamespaceListerExpansion allows custom methods to be added to +// MultidimPodAutoscalerNamespaceLister. +type MultidimPodAutoscalerNamespaceListerExpansion interface{} + +// MultidimPodAutoscalerCheckpointListerExpansion allows custom methods to be added to +// MultidimPodAutoscalerCheckpointLister. +type MultidimPodAutoscalerCheckpointListerExpansion interface{} + +// MultidimPodAutoscalerCheckpointNamespaceListerExpansion allows custom methods to be added to +// MultidimPodAutoscalerCheckpointNamespaceLister. +type MultidimPodAutoscalerCheckpointNamespaceListerExpansion interface{} diff --git a/multidimensional-pod-autoscaler/pkg/client/listers/autoscaling.k8s.io/v1alpha1/multidimpodautoscaler.go b/multidimensional-pod-autoscaler/pkg/client/listers/autoscaling.k8s.io/v1alpha1/multidimpodautoscaler.go new file mode 100644 index 000000000000..04c3250c0592 --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/client/listers/autoscaling.k8s.io/v1alpha1/multidimpodautoscaler.go @@ -0,0 +1,70 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by lister-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + labels "k8s.io/apimachinery/pkg/labels" + autoscalingk8siov1alpha1 "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1alpha1" + listers "k8s.io/client-go/listers" + cache "k8s.io/client-go/tools/cache" +) + +// MultidimPodAutoscalerLister helps list MultidimPodAutoscalers. +// All objects returned here must be treated as read-only. +type MultidimPodAutoscalerLister interface { + // List lists all MultidimPodAutoscalers in the indexer. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*autoscalingk8siov1alpha1.MultidimPodAutoscaler, err error) + // MultidimPodAutoscalers returns an object that can list and get MultidimPodAutoscalers. + MultidimPodAutoscalers(namespace string) MultidimPodAutoscalerNamespaceLister + MultidimPodAutoscalerListerExpansion +} + +// multidimPodAutoscalerLister implements the MultidimPodAutoscalerLister interface. +type multidimPodAutoscalerLister struct { + listers.ResourceIndexer[*autoscalingk8siov1alpha1.MultidimPodAutoscaler] +} + +// NewMultidimPodAutoscalerLister returns a new MultidimPodAutoscalerLister. +func NewMultidimPodAutoscalerLister(indexer cache.Indexer) MultidimPodAutoscalerLister { + return &multidimPodAutoscalerLister{listers.New[*autoscalingk8siov1alpha1.MultidimPodAutoscaler](indexer, autoscalingk8siov1alpha1.Resource("multidimpodautoscaler"))} +} + +// MultidimPodAutoscalers returns an object that can list and get MultidimPodAutoscalers. +func (s *multidimPodAutoscalerLister) MultidimPodAutoscalers(namespace string) MultidimPodAutoscalerNamespaceLister { + return multidimPodAutoscalerNamespaceLister{listers.NewNamespaced[*autoscalingk8siov1alpha1.MultidimPodAutoscaler](s.ResourceIndexer, namespace)} +} + +// MultidimPodAutoscalerNamespaceLister helps list and get MultidimPodAutoscalers. +// All objects returned here must be treated as read-only. +type MultidimPodAutoscalerNamespaceLister interface { + // List lists all MultidimPodAutoscalers in the indexer for a given namespace. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*autoscalingk8siov1alpha1.MultidimPodAutoscaler, err error) + // Get retrieves the MultidimPodAutoscaler from the indexer for a given namespace and name. + // Objects returned here must be treated as read-only. + Get(name string) (*autoscalingk8siov1alpha1.MultidimPodAutoscaler, error) + MultidimPodAutoscalerNamespaceListerExpansion +} + +// multidimPodAutoscalerNamespaceLister implements the MultidimPodAutoscalerNamespaceLister +// interface. +type multidimPodAutoscalerNamespaceLister struct { + listers.ResourceIndexer[*autoscalingk8siov1alpha1.MultidimPodAutoscaler] +} diff --git a/multidimensional-pod-autoscaler/pkg/client/listers/autoscaling.k8s.io/v1alpha1/multidimpodautoscalercheckpoint.go b/multidimensional-pod-autoscaler/pkg/client/listers/autoscaling.k8s.io/v1alpha1/multidimpodautoscalercheckpoint.go new file mode 100644 index 000000000000..12d459c8e7ad --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/client/listers/autoscaling.k8s.io/v1alpha1/multidimpodautoscalercheckpoint.go @@ -0,0 +1,70 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by lister-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + labels "k8s.io/apimachinery/pkg/labels" + autoscalingk8siov1alpha1 "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1alpha1" + listers "k8s.io/client-go/listers" + cache "k8s.io/client-go/tools/cache" +) + +// MultidimPodAutoscalerCheckpointLister helps list MultidimPodAutoscalerCheckpoints. +// All objects returned here must be treated as read-only. +type MultidimPodAutoscalerCheckpointLister interface { + // List lists all MultidimPodAutoscalerCheckpoints in the indexer. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*autoscalingk8siov1alpha1.MultidimPodAutoscalerCheckpoint, err error) + // MultidimPodAutoscalerCheckpoints returns an object that can list and get MultidimPodAutoscalerCheckpoints. + MultidimPodAutoscalerCheckpoints(namespace string) MultidimPodAutoscalerCheckpointNamespaceLister + MultidimPodAutoscalerCheckpointListerExpansion +} + +// multidimPodAutoscalerCheckpointLister implements the MultidimPodAutoscalerCheckpointLister interface. +type multidimPodAutoscalerCheckpointLister struct { + listers.ResourceIndexer[*autoscalingk8siov1alpha1.MultidimPodAutoscalerCheckpoint] +} + +// NewMultidimPodAutoscalerCheckpointLister returns a new MultidimPodAutoscalerCheckpointLister. +func NewMultidimPodAutoscalerCheckpointLister(indexer cache.Indexer) MultidimPodAutoscalerCheckpointLister { + return &multidimPodAutoscalerCheckpointLister{listers.New[*autoscalingk8siov1alpha1.MultidimPodAutoscalerCheckpoint](indexer, autoscalingk8siov1alpha1.Resource("multidimpodautoscalercheckpoint"))} +} + +// MultidimPodAutoscalerCheckpoints returns an object that can list and get MultidimPodAutoscalerCheckpoints. +func (s *multidimPodAutoscalerCheckpointLister) MultidimPodAutoscalerCheckpoints(namespace string) MultidimPodAutoscalerCheckpointNamespaceLister { + return multidimPodAutoscalerCheckpointNamespaceLister{listers.NewNamespaced[*autoscalingk8siov1alpha1.MultidimPodAutoscalerCheckpoint](s.ResourceIndexer, namespace)} +} + +// MultidimPodAutoscalerCheckpointNamespaceLister helps list and get MultidimPodAutoscalerCheckpoints. +// All objects returned here must be treated as read-only. +type MultidimPodAutoscalerCheckpointNamespaceLister interface { + // List lists all MultidimPodAutoscalerCheckpoints in the indexer for a given namespace. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*autoscalingk8siov1alpha1.MultidimPodAutoscalerCheckpoint, err error) + // Get retrieves the MultidimPodAutoscalerCheckpoint from the indexer for a given namespace and name. + // Objects returned here must be treated as read-only. + Get(name string) (*autoscalingk8siov1alpha1.MultidimPodAutoscalerCheckpoint, error) + MultidimPodAutoscalerCheckpointNamespaceListerExpansion +} + +// multidimPodAutoscalerCheckpointNamespaceLister implements the MultidimPodAutoscalerCheckpointNamespaceLister +// interface. +type multidimPodAutoscalerCheckpointNamespaceLister struct { + listers.ResourceIndexer[*autoscalingk8siov1alpha1.MultidimPodAutoscalerCheckpoint] +} diff --git a/multidimensional-pod-autoscaler/pkg/recommender/.gitignore b/multidimensional-pod-autoscaler/pkg/recommender/.gitignore new file mode 100644 index 000000000000..fe7fcf3d0850 --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/recommender/.gitignore @@ -0,0 +1,7 @@ +# Recommender binary +recommender +recommender-amd64 +recommender-arm64 +recommender-arm +recommender-ppc64le +recommender-s390x diff --git a/multidimensional-pod-autoscaler/pkg/recommender/Dockerfile b/multidimensional-pod-autoscaler/pkg/recommender/Dockerfile new file mode 100644 index 000000000000..fd713de15255 --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/recommender/Dockerfile @@ -0,0 +1,40 @@ +# Copyright 2017 The Kubernetes Authors. All rights reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM --platform=$BUILDPLATFORM golang:1.23.4 AS builder + +WORKDIR /workspace + +# Copy the Go Modules manifests +COPY multidimensional-pod-autoscaler/go.mod go.mod +COPY multidimensional-pod-autoscaler/go.sum go.sum +# TODO: This is temporary until the VPA has cut a new release +COPY vertical-pod-autoscaler /vertical-pod-autoscaler + +RUN go mod download + +COPY multidimensional-pod-autoscaler/common common +COPY multidimensional-pod-autoscaler/pkg pkg + +ARG TARGETOS TARGETARCH + +RUN CGO_ENABLED=0 LD_FLAGS=-s GOARCH=$TARGETARCH GOOS=$TARGETOS go build -C pkg/recommender -o recommender-$TARGETARCH + +FROM gcr.io/distroless/static:nonroot + +ARG TARGETARCH + +COPY --from=builder /workspace/pkg/recommender/recommender-$TARGETARCH /recommender + +ENTRYPOINT ["/recommender"] diff --git a/multidimensional-pod-autoscaler/pkg/recommender/Makefile b/multidimensional-pod-autoscaler/pkg/recommender/Makefile new file mode 100644 index 000000000000..cf8e3d2a6580 --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/recommender/Makefile @@ -0,0 +1,92 @@ +all: build + +TAG?=dev +REGISTRY?=staging-k8s.gcr.io +FLAGS= +TEST_ENVVAR=LD_FLAGS=-s GO111MODULE=on +ENVVAR=CGO_ENABLED=0 $(TEST_ENVVAR) +GOOS?=linux +COMPONENT=recommender +FULL_COMPONENT=mpa-${COMPONENT} +# localhost registries need --insecure for some docker commands. +INSECURE=$(subst localhost,--insecure,$(findstring localhost,$(REGISTRY))) + +ALL_ARCHITECTURES?=amd64 arm arm64 ppc64le s390x +export DOCKER_CLI_EXPERIMENTAL=enabled + +build: clean + $(ENVVAR) GOOS=$(GOOS) go build ./... + $(ENVVAR) GOOS=$(GOOS) go build -o ${COMPONENT} + +build-binary: clean + $(ENVVAR) GOOS=$(GOOS) go build -o ${COMPONENT} + +test-unit: clean build + $(TEST_ENVVAR) go test --test.short -race ./... $(FLAGS) + +.PHONY: docker-build +docker-build: $(addprefix docker-build-,$(ALL_ARCHITECTURES)) + +.PHONY: docker-build-* +docker-build-%: +ifndef REGISTRY + ERR = $(error REGISTRY is undefined) + $(ERR) +endif +ifndef TAG + ERR = $(error TAG is undefined) + $(ERR) +endif + docker buildx build --pull --load --platform linux/$* -t ${REGISTRY}/${FULL_COMPONENT}-$*:${TAG} -f ./Dockerfile ../../../ + +.PHONY: docker-push +docker-push: $(addprefix do-push-,$(ALL_ARCHITECTURES)) push-multi-arch; + +.PHONY: do-push-* +do-push-%: +ifndef REGISTRY + ERR = $(error REGISTRY is undefined) + $(ERR) +endif +ifndef TAG + ERR = $(error TAG is undefined) + $(ERR) +endif + docker push ${REGISTRY}/${FULL_COMPONENT}-$*:${TAG} + +.PHONY: push-multi-arch +push-multi-arch: + docker manifest create --amend $(REGISTRY)/${FULL_COMPONENT}:$(TAG) $(shell echo $(ALL_ARCHITECTURES) | sed -e "s~[^ ]*~$(REGISTRY)/${FULL_COMPONENT}\-&:$(TAG)~g") + @for arch in $(ALL_ARCHITECTURES); do docker manifest annotate --arch $${arch} $(REGISTRY)/${FULL_COMPONENT}:$(TAG) $(REGISTRY)/${FULL_COMPONENT}-$${arch}:${TAG}; done + docker manifest push --purge $(REGISTRY)/${FULL_COMPONENT}:$(TAG) + +.PHONY: show-git-info +show-git-info: + echo '=============== local git status ===============' + git status + echo '=============== last commit ===============' + git log -1 + echo '=============== bulding from the above ===============' + +.PHONY: create-buildx-builder +create-buildx-builder: + BUILDER=$(shell docker buildx create --driver=docker-container --use) + +.PHONY: remove-buildx-builder +remove-buildx-builder: + docker buildx rm ${BUILDER} + +.PHONY: release +release: show-git-info create-buildx-builder docker-build remove-buildx-builder docker-push + @echo "Full in-docker release ${FULL_COMPONENT}:${TAG} completed" + +clean: $(addprefix clean-,$(ALL_ARCHITECTURES)) + +clean-%: + rm -f ${COMPONENT}-$* + +format: + test -z "$$(find . -path ./vendor -prune -type f -o -name '*.go' -exec gofmt -s -d {} + | tee /dev/stderr)" || \ + test -z "$$(find . -path ./vendor -prune -type f -o -name '*.go' -exec gofmt -s -w {} + | tee /dev/stderr)" + +.PHONY: all build test-unit clean format release diff --git a/multidimensional-pod-autoscaler/pkg/recommender/README.md b/multidimensional-pod-autoscaler/pkg/recommender/README.md new file mode 100644 index 000000000000..7c75c434a7ec --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/recommender/README.md @@ -0,0 +1,46 @@ +# MPA Recommender + +- [Intro](#intro) +- [Running](#running) +- [Implementation](#implementation) + +## Intro + +Recommender is the core binary of Multi-dimensiional Pod Autoscaler (MPA) system. +It consists of both vertical and horizontal scaling of resources: +- Vertical: It computes the recommended resource requests for pods based on historical and current usage of the resources. Like VPA, the current recommendations are put in status of the MPA object, where they can be inspected. +- Horizontal: It updates the number of replicas based on specified target metrics threshold according to the following formula: + +``` +desiredReplicas = ceil[currentReplicas * ( currentMetricValue / desiredMetricValue )] +``` + +- Combined: To be released. + - The current way of combining vertical and horizontal scaling is simple: Each dimension is alternatively being considered. In the future, we will design and implement prioritization (e.g., to prioritize horizontal scaling for CPU-instensive workloads) and conflict-resolving (e.g., scaling in and up simultaneuously) mechanisms. + +## Running + +* In order to have historical data pulled in by the recommender, install Prometheus in your cluster and pass its address through a flag. +* Create RBAC configuration from `../../deploy/mpa-rbac.yaml` if not yet. +* Create a deployment with the recommender pod from `../../deploy/recommender-deployment.yaml`. +* The recommender will start running and pushing its recommendations to MPA object statuses. + +## Implementation + +The recommender is based on a model of the cluster that it builds in its memory. +The model contains Kubernetes resources: *Pods*, *MultidimPodAutoscalers*, with their configuration (e.g. labels) as well as other information, e.g., usage data for each container. + +After starting the binary, the recommender reads the history of running pods and their usage from Prometheus into the model. +It then runs in a loop and at each step performs the following actions: + +* update model with recent information on resources (using listers based on watch), +* update model with fresh usage samples from Metrics API, +* compute new recommendation for each MPA, +* put any changed recommendations into the MPA objects. + +## Building the Docker Image + +``` +make build-binary-with-vendor-amd64 +make docker-build-amd64 +``` diff --git a/multidimensional-pod-autoscaler/pkg/recommender/checkpoint/checkpoint_writer.go b/multidimensional-pod-autoscaler/pkg/recommender/checkpoint/checkpoint_writer.go new file mode 100644 index 000000000000..3f1794d72ece --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/recommender/checkpoint/checkpoint_writer.go @@ -0,0 +1,151 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package checkpoint + +import ( + "context" + "fmt" + "sort" + "time" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + mpa_types "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1alpha1" + mpa_api "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/client/clientset/versioned/typed/autoscaling.k8s.io/v1alpha1" + "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/recommender/model" + api_util "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/utils/mpa" + vpa_model "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/model" + "k8s.io/klog/v2" +) + +// CheckpointWriter persistently stores aggregated historical usage of containers +// controlled by MPA objects. This state can be restored to initialize the model after restart. +type CheckpointWriter interface { + // StoreCheckpoints writes at least minCheckpoints if there are more checkpoints to write. + // Checkpoints are written until ctx permits or all checkpoints are written. + StoreCheckpoints(ctx context.Context, now time.Time, minCheckpoints int) error +} + +type checkpointWriter struct { + mpaCheckpointClient mpa_api.MultidimPodAutoscalerCheckpointsGetter + cluster *model.ClusterState +} + +// NewCheckpointWriter returns new instance of a CheckpointWriter +func NewCheckpointWriter(cluster *model.ClusterState, mpaCheckpointClient mpa_api.MultidimPodAutoscalerCheckpointsGetter) CheckpointWriter { + return &checkpointWriter{ + mpaCheckpointClient: mpaCheckpointClient, + cluster: cluster, + } +} + +func isFetchingHistory(mpa *model.Mpa) bool { + condition, found := mpa.Conditions[mpa_types.FetchingHistory] + if !found { + return false + } + return condition.Status == v1.ConditionTrue +} + +func getMpasToCheckpoint(clusterMpas map[model.MpaID]*model.Mpa) []*model.Mpa { + mpas := make([]*model.Mpa, 0, len(clusterMpas)) + for _, mpa := range clusterMpas { + if isFetchingHistory(mpa) { + klog.V(3).Infof("MPA %s/%s is loading history, skipping checkpoints", mpa.ID.Namespace, mpa.ID.MpaName) + continue + } + mpas = append(mpas, mpa) + } + sort.Slice(mpas, func(i, j int) bool { + return mpas[i].CheckpointWritten.Before(mpas[j].CheckpointWritten) + }) + return mpas +} + +func (writer *checkpointWriter) StoreCheckpoints(ctx context.Context, now time.Time, minCheckpoints int) error { + mpas := getMpasToCheckpoint(writer.cluster.Mpas) + for _, mpa := range mpas { + + // Draining ctx.Done() channel. ctx.Err() will be checked if timeout occurred, but minCheckpoints have + // to be written before return from this function. + select { + case <-ctx.Done(): + default: + } + + if ctx.Err() != nil && minCheckpoints <= 0 { + return ctx.Err() + } + + aggregateContainerStateMap := buildAggregateContainerStateMap(mpa, writer.cluster, now) + for container, aggregatedContainerState := range aggregateContainerStateMap { + containerCheckpoint, err := aggregatedContainerState.SaveToCheckpoint() + if err != nil { + klog.Errorf("Cannot serialize checkpoint for mpa %v container %v. Reason: %+v", mpa.ID.MpaName, container, err) + continue + } + checkpointName := fmt.Sprintf("%s-%s", mpa.ID.MpaName, container) + mpaCheckpoint := mpa_types.MultidimPodAutoscalerCheckpoint{ + ObjectMeta: metav1.ObjectMeta{Name: checkpointName}, + Spec: mpa_types.MultidimPodAutoscalerCheckpointSpec{ + ContainerName: container, + MPAObjectName: mpa.ID.MpaName, + }, + Status: *containerCheckpoint, + } + err = api_util.CreateOrUpdateMpaCheckpoint(writer.mpaCheckpointClient.MultidimPodAutoscalerCheckpoints(mpa.ID.Namespace), &mpaCheckpoint) + if err != nil { + klog.Errorf("Cannot save MPA %s/%s checkpoint for %s. Reason: %+v", + mpa.ID.Namespace, mpaCheckpoint.Spec.MPAObjectName, mpaCheckpoint.Spec.ContainerName, err) + } else { + klog.V(3).Infof("Saved MPA %s/%s checkpoint for %s", + mpa.ID.Namespace, mpaCheckpoint.Spec.MPAObjectName, mpaCheckpoint.Spec.ContainerName) + mpa.CheckpointWritten = now + } + minCheckpoints-- + } + } + return nil +} + +// Build the AggregateContainerState for the purpose of the checkpoint. This is an aggregation of state of all +// containers that belong to pods matched by the MPA. +// Note however that we exclude the most recent memory peak for each container (see below). +func buildAggregateContainerStateMap(mpa *model.Mpa, cluster *model.ClusterState, now time.Time) map[string]*vpa_model.AggregateContainerState { + aggregateContainerStateMap := mpa.AggregateStateByContainerName() + // Note: the memory peak from the current (ongoing) aggregation interval is not included in the + // checkpoint to avoid having multiple peaks in the same interval after the state is restored from + // the checkpoint. Therefore we are extracting the current peak from all containers. + // TODO: Avoid the nested loop over all containers for each MPA. + for _, pod := range cluster.Pods { + for containerName, container := range pod.Containers { + aggregateKey := cluster.MakeAggregateStateKey(pod, containerName) + if mpa.UsesAggregation(aggregateKey) { + if aggregateContainerState, exists := aggregateContainerStateMap[containerName]; exists { + subtractCurrentContainerMemoryPeak(aggregateContainerState, container, now) + } + } + } + } + return aggregateContainerStateMap +} + +func subtractCurrentContainerMemoryPeak(a *vpa_model.AggregateContainerState, container *model.ContainerState, now time.Time) { + if now.Before(container.WindowEnd) { + a.AggregateMemoryPeaks.SubtractSample(vpa_model.BytesFromMemoryAmount(container.GetMaxMemoryPeak()), 1.0, container.WindowEnd) + } +} diff --git a/multidimensional-pod-autoscaler/pkg/recommender/checkpoint/checkpoint_writer_test.go b/multidimensional-pod-autoscaler/pkg/recommender/checkpoint/checkpoint_writer_test.go new file mode 100644 index 000000000000..581f4cfe2c90 --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/recommender/checkpoint/checkpoint_writer_test.go @@ -0,0 +1,174 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package checkpoint + +import ( + "fmt" + "testing" + "time" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + mpa_types "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1alpha1" + "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/recommender/model" + vpa_model "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/model" + + "github.com/stretchr/testify/assert" +) + +// TODO: Extract these constants to a common test module. +var ( + testPodID1 = vpa_model.PodID{ + Namespace: "namespace-1", + PodName: "pod-1", + } + testContainerID1 = vpa_model.ContainerID{ + PodID: testPodID1, + ContainerName: "container-1", + } + testMpaID1 = model.MpaID{ + Namespace: "namespace-1", + MpaName: "mpa-1", + } + testLabels = map[string]string{"label-1": "value-1"} + testSelectorStr = "label-1 = value-1" + testRequest = vpa_model.Resources{ + vpa_model.ResourceCPU: vpa_model.CPUAmountFromCores(3.14), + vpa_model.ResourceMemory: vpa_model.MemoryAmountFromBytes(3.14e9), + } +) + +const testGcPeriod = time.Minute + +func addMpa(t *testing.T, cluster *model.ClusterState, mpaID model.MpaID, selector string) *model.Mpa { + var apiObject mpa_types.MultidimPodAutoscaler + apiObject.Namespace = mpaID.Namespace + apiObject.Name = mpaID.MpaName + labelSelector, _ := metav1.ParseToLabelSelector(selector) + parsedSelector, _ := metav1.LabelSelectorAsSelector(labelSelector) + err := cluster.AddOrUpdateMpa(&apiObject, parsedSelector) + if err != nil { + t.Fatalf("AddOrUpdateMpa() failed: %v", err) + } + return cluster.Mpas[mpaID] +} + +func TestMergeContainerStateForCheckpointDropsRecentMemoryPeak(t *testing.T) { + cluster := model.NewClusterState(testGcPeriod) + cluster.AddOrUpdatePod(testPodID1, testLabels, v1.PodRunning) + assert.NoError(t, cluster.AddOrUpdateContainer(testContainerID1, testRequest)) + container := cluster.GetContainer(testContainerID1) + + timeNow := time.Unix(1, 0) + container.AddSample(&vpa_model.ContainerUsageSample{ + MeasureStart: timeNow, + Usage: vpa_model.MemoryAmountFromBytes(1024 * 1024 * 1024), + Request: testRequest[vpa_model.ResourceMemory], + Resource: vpa_model.ResourceMemory, + }) + mpa := addMpa(t, cluster, testMpaID1, testSelectorStr) + + // Verify that the current peak is excluded from the aggregation. + aggregateContainerStateMap := buildAggregateContainerStateMap(mpa, cluster, timeNow) + if assert.Contains(t, aggregateContainerStateMap, "container-1") { + assert.True(t, aggregateContainerStateMap["container-1"].AggregateMemoryPeaks.IsEmpty(), + "Current peak was not excluded from the aggregation.") + } + // Verify that an old peak is not excluded from the aggregation. + timeNow = timeNow.Add(vpa_model.GetAggregationsConfig().MemoryAggregationInterval) + aggregateContainerStateMap = buildAggregateContainerStateMap(mpa, cluster, timeNow) + if assert.Contains(t, aggregateContainerStateMap, "container-1") { + assert.False(t, aggregateContainerStateMap["container-1"].AggregateMemoryPeaks.IsEmpty(), + "Old peak should not be excluded from the aggregation.") + } +} + +func TestIsFetchingHistory(t *testing.T) { + + testCases := []struct { + mpa model.Mpa + isFetchingHistory bool + }{ + { + mpa: model.Mpa{}, + isFetchingHistory: false, + }, + { + mpa: model.Mpa{ + PodSelector: nil, + Conditions: map[mpa_types.MultidimPodAutoscalerConditionType]mpa_types.MultidimPodAutoscalerCondition{ + mpa_types.FetchingHistory: { + Type: mpa_types.FetchingHistory, + Status: v1.ConditionFalse, + }, + }, + }, + isFetchingHistory: false, + }, + { + mpa: model.Mpa{ + PodSelector: nil, + Conditions: map[mpa_types.MultidimPodAutoscalerConditionType]mpa_types.MultidimPodAutoscalerCondition{ + mpa_types.FetchingHistory: { + Type: mpa_types.FetchingHistory, + Status: v1.ConditionTrue, + }, + }, + }, + isFetchingHistory: true, + }, + } + + for _, tc := range testCases { + assert.Equalf(t, tc.isFetchingHistory, isFetchingHistory(&tc.mpa), "%+v should have %v as isFetchingHistoryResult", tc.mpa, tc.isFetchingHistory) + } +} + +func TestGetMpasToCheckpointSorts(t *testing.T) { + + time1 := time.Unix(10000, 0) + time2 := time.Unix(20000, 0) + + genMpaID := func(index int) model.MpaID { + return model.MpaID{ + MpaName: fmt.Sprintf("mpa-%d", index), + } + } + mpa0 := &model.Mpa{ + ID: genMpaID(0), + } + mpa1 := &model.Mpa{ + ID: genMpaID(1), + CheckpointWritten: time1, + } + mpa2 := &model.Mpa{ + ID: genMpaID(2), + CheckpointWritten: time2, + } + mpas := make(map[model.MpaID]*model.Mpa) + addMpa := func(mpa *model.Mpa) { + mpas[mpa.ID] = mpa + } + addMpa(mpa2) + addMpa(mpa0) + addMpa(mpa1) + result := getMpasToCheckpoint(mpas) + assert.Equal(t, genMpaID(0), result[0].ID) + assert.Equal(t, genMpaID(1), result[1].ID) + assert.Equal(t, genMpaID(2), result[2].ID) + +} diff --git a/multidimensional-pod-autoscaler/pkg/recommender/input/cluster_feeder.go b/multidimensional-pod-autoscaler/pkg/recommender/input/cluster_feeder.go new file mode 100644 index 000000000000..faf8dd3b55b4 --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/recommender/input/cluster_feeder.go @@ -0,0 +1,590 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package input + +import ( + "context" + "fmt" + "time" + + apiv1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/apimachinery/pkg/watch" + mpa_types "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1alpha1" + mpa_clientset "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/client/clientset/versioned" + mpa_api "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/client/clientset/versioned/typed/autoscaling.k8s.io/v1alpha1" + mpa_lister "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/client/listers/autoscaling.k8s.io/v1alpha1" + "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/recommender/input/metrics" + "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/recommender/model" + "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/target" + mpa_api_util "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/utils/mpa" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/input/history" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/input/oom" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/input/spec" + vpa_model "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/model" + controllerfetcher "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/target/controller_fetcher" + metrics_recommender "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/metrics/recommender" + "k8s.io/client-go/informers" + kube_client "k8s.io/client-go/kubernetes" + corev1 "k8s.io/client-go/kubernetes/typed/core/v1" + v1lister "k8s.io/client-go/listers/core/v1" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/cache" + klog "k8s.io/klog/v2" + resourceclient "k8s.io/metrics/pkg/client/clientset/versioned/typed/metrics/v1beta1" +) + +const ( + evictionWatchRetryWait = 10 * time.Second + evictionWatchJitterFactor = 0.5 + scaleCacheLoopPeriod = 7 * time.Second + scaleCacheEntryLifetime = time.Hour + scaleCacheEntryFreshnessTime = 10 * time.Minute + scaleCacheEntryJitterFactor float64 = 1. + defaultResyncPeriod = 10 * time.Minute + // DefaultRecommenderName designates the recommender that will handle MPA objects which don't specify + // recommender name explicitly (and so implicitly specify that the default recommender should handle them) + DefaultRecommenderName = "default" +) + +// ClusterStateFeeder can update state of ClusterState object. +type ClusterStateFeeder interface { + // InitFromHistoryProvider loads historical pod spec into clusterState. + InitFromHistoryProvider(historyProvider history.HistoryProvider) + + // InitFromCheckpoints loads historical checkpoints into clusterState. + InitFromCheckpoints() + + // LoadMPAs updates clusterState with current state of MPAs. + LoadMPAs(ctx context.Context) + + // LoadPods updates clusterState with current specification of Pods and their Containers. + LoadPods() + + // LoadRealTimeMetrics updates clusterState with current usage metrics of containers. + LoadRealTimeMetrics() + + // GarbageCollectCheckpoints removes historical checkpoints that don't have a matching MPA. + GarbageCollectCheckpoints() + + // Get the PodLister (for HPA). + GetPodLister() v1lister.PodLister +} + +// ClusterStateFeederFactory makes instances of ClusterStateFeeder. +type ClusterStateFeederFactory struct { + ClusterState *model.ClusterState + KubeClient kube_client.Interface + MetricsClient metrics.MetricsClient + MpaCheckpointClient mpa_api.MultidimPodAutoscalerCheckpointsGetter + MpaLister mpa_lister.MultidimPodAutoscalerLister + PodLister v1lister.PodLister + OOMObserver oom.Observer + SelectorFetcher target.MpaTargetSelectorFetcher + MemorySaveMode bool + ControllerFetcher controllerfetcher.ControllerFetcher + RecommenderName string + IgnoredNamespaces []string +} + +// Make creates new ClusterStateFeeder with internal data providers, based on kube client. +func (m ClusterStateFeederFactory) Make() *clusterStateFeeder { + return &clusterStateFeeder{ + coreClient: m.KubeClient.CoreV1(), + metricsClient: m.MetricsClient, + oomChan: m.OOMObserver.GetObservedOomsChannel(), + mpaCheckpointClient: m.MpaCheckpointClient, + mpaLister: m.MpaLister, + clusterState: m.ClusterState, + specClient: spec.NewSpecClient(m.PodLister), + PodLister: m.PodLister, + selectorFetcher: m.SelectorFetcher, + memorySaveMode: m.MemorySaveMode, + controllerFetcher: m.ControllerFetcher, + recommenderName: m.RecommenderName, + ignoredNamespaces: m.IgnoredNamespaces, + } +} + +// NewClusterStateFeeder creates new ClusterStateFeeder with internal data providers, based on kube client config. +// Deprecated; Use ClusterStateFeederFactory instead. +func NewClusterStateFeeder(config *rest.Config, clusterState *model.ClusterState, memorySave bool, namespace, metricsClientName string, recommenderName string) ClusterStateFeeder { + kubeClient := kube_client.NewForConfigOrDie(config) + podLister, oomObserver := NewPodListerAndOOMObserver(kubeClient, namespace) + factory := informers.NewSharedInformerFactoryWithOptions(kubeClient, defaultResyncPeriod, informers.WithNamespace(namespace)) + controllerFetcher := controllerfetcher.NewControllerFetcher(config, kubeClient, factory, scaleCacheEntryFreshnessTime, scaleCacheEntryLifetime, scaleCacheEntryJitterFactor) + controllerFetcher.Start(context.TODO(), scaleCacheLoopPeriod) + return ClusterStateFeederFactory{ + PodLister: podLister, + OOMObserver: oomObserver, + KubeClient: kubeClient, + MetricsClient: newMetricsClient(config, namespace, metricsClientName), + MpaCheckpointClient: mpa_clientset.NewForConfigOrDie(config).AutoscalingV1alpha1(), + MpaLister: mpa_api_util.NewMpasLister(mpa_clientset.NewForConfigOrDie(config), make(chan struct{}), namespace), + ClusterState: clusterState, + SelectorFetcher: target.NewMpaTargetSelectorFetcher(config, kubeClient, factory), + MemorySaveMode: memorySave, + ControllerFetcher: controllerFetcher, + RecommenderName: recommenderName, + }.Make() +} + +func newMetricsClient(config *rest.Config, namespace, clientName string) metrics.MetricsClient { + metricsGetter := resourceclient.NewForConfigOrDie(config) + return metrics.NewMetricsClient(metrics.NewPodMetricsesSource(metricsGetter), namespace, clientName) +} + +// WatchEvictionEventsWithRetries watches new Events with reason=Evicted and passes them to the observer. +func WatchEvictionEventsWithRetries(kubeClient kube_client.Interface, observer oom.Observer, namespace string) { + go func() { + options := metav1.ListOptions{ + FieldSelector: "reason=Evicted", + } + + watchEvictionEventsOnce := func() { + watchInterface, err := kubeClient.CoreV1().Events(namespace).Watch(context.TODO(), options) + if err != nil { + klog.Errorf("Cannot initialize watching events. Reason %v", err) + return + } + watchEvictionEvents(watchInterface.ResultChan(), observer) + } + for { + watchEvictionEventsOnce() + // Wait between attempts, retrying too often breaks API server. + waitTime := wait.Jitter(evictionWatchRetryWait, evictionWatchJitterFactor) + klog.V(1).Infof("An attempt to watch eviction events finished. Waiting %v before the next one.", waitTime) + time.Sleep(waitTime) + } + }() +} + +func watchEvictionEvents(evictedEventChan <-chan watch.Event, observer oom.Observer) { + for { + evictedEvent, ok := <-evictedEventChan + if !ok { + klog.V(3).Infof("Eviction event chan closed") + return + } + if evictedEvent.Type == watch.Added { + evictedEvent, ok := evictedEvent.Object.(*apiv1.Event) + if !ok { + continue + } + observer.OnEvent(evictedEvent) + } + } +} + +// Creates clients watching pods: PodLister (listing only not terminated pods). +func newPodClients(kubeClient kube_client.Interface, resourceEventHandler cache.ResourceEventHandler, namespace string) v1lister.PodLister { + // We are interested in pods which are Running or Unknown (in case the pod is + // running but there are some transient errors we don't want to delete it from + // our model). + // We don't want to watch Pending pods because they didn't generate any usage + // yet. + // Succeeded and Failed failed pods don't generate any usage anymore but we + // don't necessarily want to immediately delete them. + selector := fields.ParseSelectorOrDie("status.phase!=" + string(apiv1.PodPending)) + podListWatch := cache.NewListWatchFromClient(kubeClient.CoreV1().RESTClient(), "pods", namespace, selector) + indexer, controller := cache.NewIndexerInformer( + podListWatch, + &apiv1.Pod{}, + time.Hour, + resourceEventHandler, + cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, + ) + podLister := v1lister.NewPodLister(indexer) + stopCh := make(chan struct{}) + go controller.Run(stopCh) + return podLister +} + +// NewPodListerAndOOMObserver creates pair of pod lister and OOM observer. +func NewPodListerAndOOMObserver(kubeClient kube_client.Interface, namespace string) (v1lister.PodLister, oom.Observer) { + oomObserver := oom.NewObserver() + podLister := newPodClients(kubeClient, oomObserver, namespace) + WatchEvictionEventsWithRetries(kubeClient, oomObserver, namespace) + return podLister, oomObserver +} + +type clusterStateFeeder struct { + coreClient corev1.CoreV1Interface + specClient spec.SpecClient + metricsClient metrics.MetricsClient + oomChan <-chan oom.OomInfo + mpaCheckpointClient mpa_api.MultidimPodAutoscalerCheckpointsGetter + mpaLister mpa_lister.MultidimPodAutoscalerLister + clusterState *model.ClusterState + selectorFetcher target.MpaTargetSelectorFetcher + memorySaveMode bool + controllerFetcher controllerfetcher.ControllerFetcher + recommenderName string + ignoredNamespaces []string + PodLister v1lister.PodLister // For HPA. +} + +func (feeder *clusterStateFeeder) InitFromHistoryProvider(historyProvider history.HistoryProvider) { + klog.V(3).Info("Initializing MPA from history provider") + clusterHistory, err := historyProvider.GetClusterHistory() + if err != nil { + klog.Errorf("Cannot get cluster history: %v", err) + } + for podID, podHistory := range clusterHistory { + klog.V(4).Infof("Adding pod %v with labels %v", podID, podHistory.LastLabels) + feeder.clusterState.AddOrUpdatePod(podID, podHistory.LastLabels, apiv1.PodUnknown) + for containerName, sampleList := range podHistory.Samples { + containerID := vpa_model.ContainerID{ + PodID: podID, + ContainerName: containerName, + } + if err = feeder.clusterState.AddOrUpdateContainer(containerID, nil); err != nil { + klog.Warningf("Failed to add container %+v. Reason: %+v", containerID, err) + } + klog.V(4).Infof("Adding %d samples for container %v", len(sampleList), containerID) + for _, sample := range sampleList { + if err := feeder.clusterState.AddSample( + &model.ContainerUsageSampleWithKey{ + ContainerUsageSample: sample, + Container: containerID, + }); err != nil { + klog.Warningf("Error adding metric sample for container %v: %v", containerID, err) + } + } + } + } +} + +func (feeder *clusterStateFeeder) setMpaCheckpoint(checkpoint *mpa_types.MultidimPodAutoscalerCheckpoint) error { + mpaID := model.MpaID{Namespace: checkpoint.Namespace, MpaName: checkpoint.Spec.MPAObjectName} + mpa, exists := feeder.clusterState.Mpas[mpaID] + if !exists { + return fmt.Errorf("cannot load checkpoint to missing MPA object %+v", mpaID) + } + + cs := vpa_model.NewAggregateContainerState() + err := cs.LoadFromCheckpoint(&checkpoint.Status) + if err != nil { + return fmt.Errorf("cannot load checkpoint for MPA %+v. Reason: %v", mpa.ID, err) + } + mpa.ContainersInitialAggregateState[checkpoint.Spec.ContainerName] = cs + return nil +} + +func (feeder *clusterStateFeeder) InitFromCheckpoints() { + klog.V(3).Info("Initializing MPA from checkpoints") + feeder.LoadMPAs(context.TODO()) + + namespaces := make(map[string]bool) + for _, v := range feeder.clusterState.Mpas { + namespaces[v.ID.Namespace] = true + } + + for namespace := range namespaces { + klog.V(3).Infof("Fetching checkpoints from namespace %s", namespace) + checkpointList, err := feeder.mpaCheckpointClient.MultidimPodAutoscalerCheckpoints(namespace).List(context.TODO(), metav1.ListOptions{}) + if err != nil { + klog.Errorf("Cannot list MPA checkpoints from namespace %v. Reason: %+v", namespace, err) + } + for _, checkpoint := range checkpointList.Items { + + klog.V(3).Infof("Loading MPA %s/%s checkpoint for %s", checkpoint.ObjectMeta.Namespace, checkpoint.Spec.MPAObjectName, checkpoint.Spec.ContainerName) + err = feeder.setMpaCheckpoint(&checkpoint) + if err != nil { + klog.Errorf("Error while loading checkpoint. Reason: %+v", err) + } + + } + } +} + +func (feeder *clusterStateFeeder) GetPodLister() v1lister.PodLister { + return feeder.PodLister +} + +func (feeder *clusterStateFeeder) GarbageCollectCheckpoints() { + klog.V(3).Info("Starting garbage collection of checkpoints") + feeder.LoadMPAs(context.TODO()) + + namspaceList, err := feeder.coreClient.Namespaces().List(context.TODO(), metav1.ListOptions{}) + if err != nil { + klog.Errorf("Cannot list namespaces. Reason: %+v", err) + return + } + + for _, namespaceItem := range namspaceList.Items { + namespace := namespaceItem.Name + checkpointList, err := feeder.mpaCheckpointClient.MultidimPodAutoscalerCheckpoints(namespace).List(context.TODO(), metav1.ListOptions{}) + if err != nil { + klog.Errorf("Cannot list MPA checkpoints from namespace %v. Reason: %+v", namespace, err) + } + for _, checkpoint := range checkpointList.Items { + mpaID := model.MpaID{Namespace: checkpoint.Namespace, MpaName: checkpoint.Spec.MPAObjectName} + _, exists := feeder.clusterState.Mpas[mpaID] + if !exists { + err = feeder.mpaCheckpointClient.MultidimPodAutoscalerCheckpoints(namespace).Delete(context.TODO(), checkpoint.Name, metav1.DeleteOptions{}) + if err == nil { + klog.V(3).Infof("Orphaned MPA checkpoint cleanup - deleting %v/%v.", namespace, checkpoint.Name) + } else { + klog.Errorf("Cannot delete MPA checkpoint %v/%v. Reason: %+v", namespace, checkpoint.Name, err) + } + } + } + } +} + +func implicitDefaultRecommender(selectors []*mpa_types.MultidimPodAutoscalerRecommenderSelector) bool { + return len(selectors) == 0 +} + +func selectsRecommender(selectors []*mpa_types.MultidimPodAutoscalerRecommenderSelector, name *string) bool { + for _, s := range selectors { + if s.Name == *name { + return true + } + } + return false +} + +// Filter MPA objects whose specified recommender names are not default +func filterMPAs(feeder *clusterStateFeeder, allMpaCRDs []*mpa_types.MultidimPodAutoscaler) []*mpa_types.MultidimPodAutoscaler { + klog.V(3).Infof("Start selecting the mpaCRDs.") + var mpaCRDs []*mpa_types.MultidimPodAutoscaler + for _, mpaCRD := range allMpaCRDs { + if feeder.recommenderName == DefaultRecommenderName { + if !implicitDefaultRecommender(mpaCRD.Spec.Recommenders) && !selectsRecommender(mpaCRD.Spec.Recommenders, &feeder.recommenderName) { + klog.V(6).Infof("Ignoring mpaCRD %s in namespace %s as current recommender's name %v doesn't appear among its recommenders", mpaCRD.Name, mpaCRD.Namespace, feeder.recommenderName) + continue + } + } else { + if implicitDefaultRecommender(mpaCRD.Spec.Recommenders) { + klog.V(6).Infof("Ignoring mpaCRD %s in namespace %s as %v recommender doesn't process CRDs implicitly destined to %v recommender", mpaCRD.Name, mpaCRD.Namespace, feeder.recommenderName, DefaultRecommenderName) + continue + } + if !selectsRecommender(mpaCRD.Spec.Recommenders, &feeder.recommenderName) { + klog.V(6).Infof("Ignoring mpaCRD %s in namespace %s as current recommender's name %v doesn't appear among its recommenders", mpaCRD.Name, mpaCRD.Namespace, feeder.recommenderName) + continue + } + } + mpaCRDs = append(mpaCRDs, mpaCRD) + } + return mpaCRDs +} + +// Fetch MPA objects and load them into the cluster state. +func (feeder *clusterStateFeeder) LoadMPAs(ctx context.Context) { + // List MPA API objects. + allMpaCRDs, err := feeder.mpaLister.List(labels.Everything()) + if err != nil { + klog.Errorf("Cannot list MPAs. Reason: %+v", err) + return + } + + // Filter out MPAs that specified recommenders with names not equal to "default" + mpaCRDs := filterMPAs(feeder, allMpaCRDs) + + klog.V(3).Infof("Fetched %d MPAs.", len(mpaCRDs)) + // Add or update existing MPAs in the model. + mpaKeys := make(map[model.MpaID]bool) + for _, mpaCRD := range mpaCRDs { + mpaID := model.MpaID{ + Namespace: mpaCRD.Namespace, + MpaName: mpaCRD.Name, + } + + selector, conditions := feeder.getSelector(ctx, mpaCRD) + klog.V(4).Infof("Using selector %s for MPA %s/%s", selector.String(), mpaCRD.Namespace, mpaCRD.Name) + + if feeder.clusterState.AddOrUpdateMpa(mpaCRD, selector) == nil { + // Successfully added MPA to the model. + mpaKeys[mpaID] = true + klog.V(4).Infof("Added MPA %v to cluster state.", mpaID) + + for _, condition := range conditions { + if condition.delete { + delete(feeder.clusterState.Mpas[mpaID].Conditions, condition.conditionType) + } else { + feeder.clusterState.Mpas[mpaID].Conditions.Set(condition.conditionType, true, "", condition.message) + } + } + } + } + // Delete non-existent MPAs from the model. + for mpaID := range feeder.clusterState.Mpas { + if _, exists := mpaKeys[mpaID]; !exists { + klog.V(3).Infof("Deleting MPA %v", mpaID) + if err := feeder.clusterState.DeleteMpa(mpaID); err != nil { + klog.Errorf("Deleting MPA %v failed: %v", mpaID, err) + } + } + } + feeder.clusterState.ObservedMpas = mpaCRDs +} + +// Load pod into the cluster state. +func (feeder *clusterStateFeeder) LoadPods() { + podSpecs, err := feeder.specClient.GetPodSpecs() + if err != nil { + klog.Errorf("Cannot get SimplePodSpecs. Reason: %+v", err) + } + pods := make(map[vpa_model.PodID]*spec.BasicPodSpec) + for _, spec := range podSpecs { + pods[spec.ID] = spec + } + for key := range feeder.clusterState.Pods { + if _, exists := pods[key]; !exists { + klog.V(3).Infof("Deleting Pod %v", key) + feeder.clusterState.DeletePod(key) + } + } + for _, pod := range pods { + if feeder.memorySaveMode && !feeder.matchesMPA(pod) { + continue + } + feeder.clusterState.AddOrUpdatePod(pod.ID, pod.PodLabels, pod.Phase) + for _, container := range pod.Containers { + if err = feeder.clusterState.AddOrUpdateContainer(container.ID, container.Request); err != nil { + klog.Warningf("Failed to add container %+v. Reason: %+v", container.ID, err) + } + } + } +} + +func (feeder *clusterStateFeeder) LoadRealTimeMetrics() { + containersMetrics, err := feeder.metricsClient.GetContainersMetrics() + if err != nil { + klog.Errorf("Cannot get ContainerMetricsSnapshot from MetricsClient. Reason: %+v", err) + } + + sampleCount := 0 + droppedSampleCount := 0 + for _, containerMetrics := range containersMetrics { + for _, sample := range newContainerUsageSamplesWithKey(containerMetrics) { + if err := feeder.clusterState.AddSample(sample); err != nil { + // Not all pod states are tracked in memory saver mode + if _, isKeyError := err.(vpa_model.KeyError); isKeyError && feeder.memorySaveMode { + continue + } + klog.Warningf("Error adding metric sample for container %v: %v", sample.Container, err) + droppedSampleCount++ + } else { + sampleCount++ + } + } + } + klog.V(3).Infof("ClusterSpec fed with #%v ContainerUsageSamples for #%v containers. Dropped #%v samples.", sampleCount, len(containersMetrics), droppedSampleCount) +Loop: + for { + select { + case oomInfo := <-feeder.oomChan: + klog.V(3).Infof("OOM detected %+v", oomInfo) + if err = feeder.clusterState.RecordOOM(oomInfo.ContainerID, oomInfo.Timestamp, oomInfo.Memory); err != nil { + klog.Warningf("Failed to record OOM %+v. Reason: %+v", oomInfo, err) + } + default: + break Loop + } + } + metrics_recommender.RecordAggregateContainerStatesCount(feeder.clusterState.StateMapSize()) +} + +func (feeder *clusterStateFeeder) matchesMPA(pod *spec.BasicPodSpec) bool { + for mpaKey, mpa := range feeder.clusterState.Mpas { + podLabels := labels.Set(pod.PodLabels) + if mpaKey.Namespace == pod.ID.Namespace && mpa.PodSelector.Matches(podLabels) { + return true + } + } + return false +} + +func newContainerUsageSamplesWithKey(metrics *metrics.ContainerMetricsSnapshot) []*model.ContainerUsageSampleWithKey { + var samples []*model.ContainerUsageSampleWithKey + + for metricName, resourceAmount := range metrics.Usage { + sample := &model.ContainerUsageSampleWithKey{ + Container: metrics.ID, + ContainerUsageSample: vpa_model.ContainerUsageSample{ + MeasureStart: metrics.SnapshotTime, + Resource: metricName, + Usage: resourceAmount, + }, + } + samples = append(samples, sample) + } + return samples +} + +type condition struct { + conditionType mpa_types.MultidimPodAutoscalerConditionType + delete bool + message string +} + +func (feeder *clusterStateFeeder) validateTargetRef(ctx context.Context, mpa *mpa_types.MultidimPodAutoscaler) (bool, condition) { + if mpa.Spec.ScaleTargetRef == nil { + return false, condition{} + } + k := controllerfetcher.ControllerKeyWithAPIVersion{ + ControllerKey: controllerfetcher.ControllerKey{ + Namespace: mpa.Namespace, + Kind: mpa.Spec.ScaleTargetRef.Kind, + Name: mpa.Spec.ScaleTargetRef.Name, + }, + ApiVersion: mpa.Spec.ScaleTargetRef.APIVersion, + } + top, err := feeder.controllerFetcher.FindTopMostWellKnownOrScalable(ctx, &k) + if err != nil { + return false, condition{conditionType: mpa_types.ConfigUnsupported, delete: false, message: fmt.Sprintf("Error checking if target is a topmost well-known or scalable controller: %s", err)} + } + if top == nil { + return false, condition{conditionType: mpa_types.ConfigUnsupported, delete: false, message: fmt.Sprintf("Unknown error during checking if target is a topmost well-known or scalable controller: %s", err)} + } + if *top != k { + return false, condition{conditionType: mpa_types.ConfigUnsupported, delete: false, message: "The scaleTargetRef controller has a parent but it should point to a topmost well-known or scalable controller"} + } + return true, condition{} +} + +func (feeder *clusterStateFeeder) getSelector(ctx context.Context, mpa *mpa_types.MultidimPodAutoscaler) (labels.Selector, []condition) { + selector, fetchErr := feeder.selectorFetcher.Fetch(ctx, mpa) + if selector != nil { + validTargetRef, unsupportedCondition := feeder.validateTargetRef(ctx, mpa) + if !validTargetRef { + return labels.Nothing(), []condition{ + unsupportedCondition, + {conditionType: mpa_types.ConfigDeprecated, delete: true}, + } + } + return selector, []condition{ + {conditionType: mpa_types.ConfigUnsupported, delete: true}, + {conditionType: mpa_types.ConfigDeprecated, delete: true}, + } + } + msg := "Cannot read scaleTargetRef" + if fetchErr != nil { + klog.Errorf("Cannot get target selector from MPA's scaleTargetRef. Reason: %+v", fetchErr) + msg = fmt.Sprintf("Cannot read scaleTargetRef. Reason: %s", fetchErr.Error()) + } + return labels.Nothing(), []condition{ + {conditionType: mpa_types.ConfigUnsupported, delete: false, message: msg}, + {conditionType: mpa_types.ConfigDeprecated, delete: true}, + } +} diff --git a/multidimensional-pod-autoscaler/pkg/recommender/input/cluster_feeder_test.go b/multidimensional-pod-autoscaler/pkg/recommender/input/cluster_feeder_test.go new file mode 100644 index 000000000000..41b348eb472b --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/recommender/input/cluster_feeder_test.go @@ -0,0 +1,540 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package input + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/assert" + + autoscalingv1 "k8s.io/api/autoscaling/v1" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime/schema" + mpa_types "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1alpha1" + "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/recommender/model" + target_mock "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/target/mock" + "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/utils/test" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/input/history" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/input/spec" + vpa_model "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/model" + controllerfetcher "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/target/controller_fetcher" + "k8s.io/client-go/restmapper" + "k8s.io/client-go/scale" +) + +type fakeControllerFetcher struct { + key *controllerfetcher.ControllerKeyWithAPIVersion + err error + mapper restmapper.DeferredDiscoveryRESTMapper + scaleNamespacer scale.ScalesGetter +} + +func (f *fakeControllerFetcher) FindTopMostWellKnownOrScalable(_ context.Context, _ *controllerfetcher.ControllerKeyWithAPIVersion) (*controllerfetcher.ControllerKeyWithAPIVersion, error) { + return f.key, f.err +} + +func (f *fakeControllerFetcher) GetRESTMappings(groupKind schema.GroupKind) ([]*apimeta.RESTMapping, error) { + return f.mapper.RESTMappings(groupKind) +} + +func (f *fakeControllerFetcher) Scales(namespace string) scale.ScaleInterface { + return f.scaleNamespacer.Scales(namespace) +} + +func parseLabelSelector(selector string) labels.Selector { + labelSelector, _ := metav1.ParseToLabelSelector(selector) + parsedSelector, _ := metav1.LabelSelectorAsSelector(labelSelector) + return parsedSelector +} + +var ( + recommenderName = "name" + empty = "" + unsupportedConditionTextFromFetcher = "Cannot read scaleTargetRef. Reason: scaleTargetRef not defined" + unsupportedConditionNoExtraText = "Cannot read scaleTargetRef" + unsupportedConditionNoTargetRef = "Cannot read scaleTargetRef" + unsupportedConditionMudaMudaMuda = "Error checking if target is a topmost well-known or scalable controller: muda muda muda" + unsupportedTargetRefHasParent = "The scaleTargetRef controller has a parent but it should point to a topmost well-known or scalable controller" +) + +const ( + kind = "dodokind" + name1 = "dotaro" + name2 = "doseph" + namespace = "testNamespace" + apiVersion = "stardust" + testGcPeriod = time.Minute +) + +func TestLoadPods(t *testing.T) { + + type testCase struct { + name string + selector labels.Selector + fetchSelectorError error + scaleTargetRef *autoscalingv1.CrossVersionObjectReference + topMostWellKnownOrScalableKey *controllerfetcher.ControllerKeyWithAPIVersion + findTopMostWellKnownOrScalableError error + expectedSelector labels.Selector + expectedConfigUnsupported *string + expectedConfigDeprecated *string + expectedMpaFetch bool + recommenderName *string + recommender string + } + + testCases := []testCase{ + { + name: "no selector", + selector: nil, + fetchSelectorError: fmt.Errorf("scaleTargetRef not defined"), + expectedSelector: labels.Nothing(), + expectedConfigUnsupported: &unsupportedConditionTextFromFetcher, + expectedConfigDeprecated: nil, + expectedMpaFetch: true, + }, + { + name: "also no selector but no error", + selector: nil, + fetchSelectorError: nil, + expectedSelector: labels.Nothing(), + expectedConfigUnsupported: &unsupportedConditionNoExtraText, + expectedConfigDeprecated: nil, + expectedMpaFetch: true, + }, + { + name: "scaleTargetRef selector", + selector: parseLabelSelector("app = test"), + fetchSelectorError: nil, + scaleTargetRef: &autoscalingv1.CrossVersionObjectReference{ + Kind: kind, + Name: name1, + APIVersion: apiVersion, + }, + topMostWellKnownOrScalableKey: &controllerfetcher.ControllerKeyWithAPIVersion{ + ControllerKey: controllerfetcher.ControllerKey{ + Kind: kind, + Name: name1, + Namespace: namespace, + }, + ApiVersion: apiVersion, + }, + expectedSelector: parseLabelSelector("app = test"), + expectedConfigUnsupported: nil, + expectedConfigDeprecated: nil, + expectedMpaFetch: true, + }, + { + name: "no scaleTargetRef", + selector: parseLabelSelector("app = test"), + fetchSelectorError: nil, + expectedSelector: labels.Nothing(), + expectedConfigUnsupported: nil, + expectedConfigDeprecated: nil, + expectedMpaFetch: true, + }, + { + name: "can't decide if top-level-ref", + selector: nil, + fetchSelectorError: nil, + expectedSelector: labels.Nothing(), + scaleTargetRef: &autoscalingv1.CrossVersionObjectReference{ + Kind: kind, + Name: name1, + APIVersion: apiVersion, + }, + expectedConfigUnsupported: &unsupportedConditionNoTargetRef, + expectedMpaFetch: true, + }, + { + name: "non-top-level scaleTargetRef", + selector: parseLabelSelector("app = test"), + fetchSelectorError: nil, + expectedSelector: labels.Nothing(), + scaleTargetRef: &autoscalingv1.CrossVersionObjectReference{ + Kind: kind, + Name: name1, + APIVersion: apiVersion, + }, + topMostWellKnownOrScalableKey: &controllerfetcher.ControllerKeyWithAPIVersion{ + ControllerKey: controllerfetcher.ControllerKey{ + Kind: kind, + Name: name2, + Namespace: namespace, + }, + ApiVersion: apiVersion, + }, + expectedConfigUnsupported: &unsupportedTargetRefHasParent, + expectedMpaFetch: true, + }, + { + name: "error checking if top-level-ref", + selector: parseLabelSelector("app = test"), + fetchSelectorError: nil, + expectedSelector: labels.Nothing(), + scaleTargetRef: &autoscalingv1.CrossVersionObjectReference{ + Kind: "doestar", + Name: "doseph-doestar", + APIVersion: "taxonomy", + }, + expectedConfigUnsupported: &unsupportedConditionMudaMudaMuda, + expectedMpaFetch: true, + findTopMostWellKnownOrScalableError: fmt.Errorf("muda muda muda"), + }, + { + name: "top-level target ref", + selector: parseLabelSelector("app = test"), + fetchSelectorError: nil, + expectedSelector: parseLabelSelector("app = test"), + scaleTargetRef: &autoscalingv1.CrossVersionObjectReference{ + Kind: kind, + Name: name1, + APIVersion: apiVersion, + }, + topMostWellKnownOrScalableKey: &controllerfetcher.ControllerKeyWithAPIVersion{ + ControllerKey: controllerfetcher.ControllerKey{ + Kind: kind, + Name: name1, + Namespace: namespace, + }, + ApiVersion: apiVersion, + }, + expectedConfigUnsupported: nil, + expectedMpaFetch: true, + }, + { + name: "no recommenderName", + selector: parseLabelSelector("app = test"), + fetchSelectorError: nil, + scaleTargetRef: &autoscalingv1.CrossVersionObjectReference{ + Kind: kind, + Name: name1, + APIVersion: apiVersion, + }, + topMostWellKnownOrScalableKey: &controllerfetcher.ControllerKeyWithAPIVersion{ + ControllerKey: controllerfetcher.ControllerKey{ + Kind: kind, + Name: name1, + Namespace: namespace, + }, + ApiVersion: apiVersion, + }, + expectedSelector: parseLabelSelector("app = test"), + expectedConfigUnsupported: nil, + expectedConfigDeprecated: nil, + expectedMpaFetch: false, + recommenderName: &empty, + }, + { + name: "recommenderName doesn't match recommender", + selector: parseLabelSelector("app = test"), + fetchSelectorError: nil, + scaleTargetRef: &autoscalingv1.CrossVersionObjectReference{ + Kind: kind, + Name: name1, + APIVersion: apiVersion, + }, + topMostWellKnownOrScalableKey: &controllerfetcher.ControllerKeyWithAPIVersion{ + ControllerKey: controllerfetcher.ControllerKey{ + Kind: kind, + Name: name1, + Namespace: namespace, + }, + ApiVersion: apiVersion, + }, + expectedSelector: parseLabelSelector("app = test"), + expectedConfigUnsupported: nil, + expectedConfigDeprecated: nil, + expectedMpaFetch: false, + recommenderName: &recommenderName, + recommender: "other", + }, + { + name: "recommenderName matches recommender", + selector: parseLabelSelector("app = test"), + fetchSelectorError: nil, + scaleTargetRef: &autoscalingv1.CrossVersionObjectReference{ + Kind: kind, + Name: name1, + APIVersion: apiVersion, + }, + topMostWellKnownOrScalableKey: &controllerfetcher.ControllerKeyWithAPIVersion{ + ControllerKey: controllerfetcher.ControllerKey{ + Kind: kind, + Name: name1, + Namespace: namespace, + }, + ApiVersion: apiVersion, + }, + expectedSelector: parseLabelSelector("app = test"), + expectedConfigUnsupported: nil, + expectedConfigDeprecated: nil, + expectedMpaFetch: true, + recommenderName: &recommenderName, + recommender: recommenderName, + }, + } + + for _, tc := range testCases { + + t.Run(tc.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mpaBuilder := test.MultidimPodAutoscaler().WithName("testMpa").WithContainer("container").WithNamespace("testNamespace").WithScaleTargetRef(tc.scaleTargetRef) + if tc.recommender != "" { + mpaBuilder = mpaBuilder.WithRecommender(tc.recommender) + } + mpa := mpaBuilder.Get() + mpaLister := &test.MultidimPodAutoscalerListerMock{} + mpaLister.On("List").Return([]*mpa_types.MultidimPodAutoscaler{mpa}, nil) + + targetSelectorFetcher := target_mock.NewMockMpaTargetSelectorFetcher(ctrl) + clusterState := model.NewClusterState(testGcPeriod) + + clusterStateFeeder := clusterStateFeeder{ + mpaLister: mpaLister, + clusterState: clusterState, + selectorFetcher: targetSelectorFetcher, + controllerFetcher: &fakeControllerFetcher{ + key: tc.topMostWellKnownOrScalableKey, + err: tc.findTopMostWellKnownOrScalableError, + }, + } + if tc.recommenderName == nil { + clusterStateFeeder.recommenderName = DefaultRecommenderName + } else { + clusterStateFeeder.recommenderName = *tc.recommenderName + } + + if tc.expectedMpaFetch { + targetSelectorFetcher.EXPECT().Fetch(mpa).Return(tc.selector, tc.fetchSelectorError) + } + clusterStateFeeder.LoadMPAs(context.Background()) + + mpaID := model.MpaID{ + Namespace: mpa.Namespace, + MpaName: mpa.Name, + } + + if !tc.expectedMpaFetch { + assert.NotContains(t, clusterState.Mpas, mpaID) + return + } + assert.Contains(t, clusterState.Mpas, mpaID) + storedMpa := clusterState.Mpas[mpaID] + if tc.expectedSelector != nil { + assert.NotNil(t, storedMpa.PodSelector) + assert.Equal(t, tc.expectedSelector.String(), storedMpa.PodSelector.String()) + } else { + assert.Nil(t, storedMpa.PodSelector) + } + + if tc.expectedConfigDeprecated != nil { + assert.Contains(t, storedMpa.Conditions, mpa_types.ConfigDeprecated) + assert.Equal(t, *tc.expectedConfigDeprecated, storedMpa.Conditions[mpa_types.ConfigDeprecated].Message) + } else { + assert.NotContains(t, storedMpa.Conditions, mpa_types.ConfigDeprecated) + } + + if tc.expectedConfigUnsupported != nil { + assert.Contains(t, storedMpa.Conditions, mpa_types.ConfigUnsupported) + assert.Equal(t, *tc.expectedConfigUnsupported, storedMpa.Conditions[mpa_types.ConfigUnsupported].Message) + } else { + assert.NotContains(t, storedMpa.Conditions, mpa_types.ConfigUnsupported) + } + + }) + } +} + +type testSpecClient struct { + pods []*spec.BasicPodSpec +} + +func (c *testSpecClient) GetPodSpecs() ([]*spec.BasicPodSpec, error) { + return c.pods, nil +} + +func makeTestSpecClient(podLabels []map[string]string) spec.SpecClient { + pods := make([]*spec.BasicPodSpec, len(podLabels)) + for i, l := range podLabels { + pods[i] = &spec.BasicPodSpec{ + ID: vpa_model.PodID{Namespace: "default", PodName: fmt.Sprintf("pod-%d", i)}, + PodLabels: l, + } + } + return &testSpecClient{ + pods: pods, + } +} + +func TestClusterStateFeeder_LoadPods(t *testing.T) { + for _, tc := range []struct { + Name string + MPALabelSelectors []string + PodLabels []map[string]string + TrackedPods int + }{ + { + Name: "simple", + MPALabelSelectors: []string{"name=mpa-pod"}, + PodLabels: []map[string]string{ + {"name": "mpa-pod"}, + {"type": "stateful"}, + }, + TrackedPods: 1, + }, + { + Name: "multiple", + MPALabelSelectors: []string{"name=mpa-pod,type=stateful"}, + PodLabels: []map[string]string{ + {"name": "mpa-pod", "type": "stateful"}, + {"type": "stateful"}, + {"name": "mpa-pod"}, + }, + TrackedPods: 1, + }, + { + Name: "no matches", + MPALabelSelectors: []string{"name=mpa-pod"}, + PodLabels: []map[string]string{ + {"name": "non-mpa-pod", "type": "stateful"}, + }, + TrackedPods: 0, + }, + { + Name: "set based", + MPALabelSelectors: []string{"environment in (staging, qa),name=mpa-pod"}, + PodLabels: []map[string]string{ + {"name": "mpa-pod", "environment": "staging"}, + {"name": "mpa-pod", "environment": "production"}, + {"name": "non-mpa-pod", "environment": "staging"}, + {"name": "non-mpa-pod", "environment": "production"}, + }, + TrackedPods: 1, + }, + } { + t.Run(tc.Name, func(t *testing.T) { + clusterState := model.NewClusterState(testGcPeriod) + for i, selector := range tc.MPALabelSelectors { + mpaLabel, err := labels.Parse(selector) + assert.NoError(t, err) + clusterState.Mpas = map[model.MpaID]*model.Mpa{ + {MpaName: fmt.Sprintf("test-mpa-%d", i), Namespace: "default"}: {PodSelector: mpaLabel}, + } + } + + feeder := clusterStateFeeder{ + specClient: makeTestSpecClient(tc.PodLabels), + memorySaveMode: true, + clusterState: clusterState, + } + + feeder.LoadPods() + assert.Len(t, feeder.clusterState.Pods, tc.TrackedPods, "number of pods is not %d", tc.TrackedPods) + + feeder = clusterStateFeeder{ + specClient: makeTestSpecClient(tc.PodLabels), + memorySaveMode: false, + clusterState: clusterState, + } + + feeder.LoadPods() + assert.Len(t, feeder.clusterState.Pods, len(tc.PodLabels), "number of pods is not %d", len(tc.PodLabels)) + }) + } +} + +type fakeHistoryProvider struct { + history map[vpa_model.PodID]*history.PodHistory + err error +} + +func (fhp *fakeHistoryProvider) GetClusterHistory() (map[vpa_model.PodID]*history.PodHistory, error) { + return fhp.history, fhp.err +} + +func TestClusterStateFeeder_InitFromHistoryProvider(t *testing.T) { + pod1 := vpa_model.PodID{ + Namespace: "ns", + PodName: "a-pod", + } + memAmount := vpa_model.ResourceAmount(128 * 1024 * 1024) + t0 := time.Date(2021, time.August, 30, 10, 21, 0, 0, time.UTC) + containerCpu := "containerCpu" + containerMem := "containerMem" + pod1History := history.PodHistory{ + LastLabels: map[string]string{}, + LastSeen: t0, + Samples: map[string][]vpa_model.ContainerUsageSample{ + containerCpu: { + { + MeasureStart: t0, + Usage: 10, + Request: 101, + Resource: vpa_model.ResourceCPU, + }, + }, + containerMem: { + { + MeasureStart: t0, + Usage: memAmount, + Request: 1024 * 1024 * 1024, + Resource: vpa_model.ResourceMemory, + }, + }, + }, + } + provider := fakeHistoryProvider{ + history: map[vpa_model.PodID]*history.PodHistory{ + pod1: &pod1History, + }, + } + + clusterState := model.NewClusterState(testGcPeriod) + feeder := clusterStateFeeder{ + clusterState: clusterState, + } + feeder.InitFromHistoryProvider(&provider) + if !assert.Contains(t, feeder.clusterState.Pods, pod1) { + return + } + pod1State := feeder.clusterState.Pods[pod1] + if !assert.Contains(t, pod1State.Containers, containerCpu) { + return + } + containerState := pod1State.Containers[containerCpu] + if !assert.NotNil(t, containerState) { + return + } + assert.Equal(t, t0, containerState.LastCPUSampleStart) + if !assert.Contains(t, pod1State.Containers, containerMem) { + return + } + containerState = pod1State.Containers[containerMem] + if !assert.NotNil(t, containerState) { + return + } + assert.Equal(t, memAmount, containerState.GetMaxMemoryPeak()) +} diff --git a/multidimensional-pod-autoscaler/pkg/recommender/input/metrics/metrics_client.go b/multidimensional-pod-autoscaler/pkg/recommender/input/metrics/metrics_client.go new file mode 100644 index 000000000000..6e0de6a2039a --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/recommender/input/metrics/metrics_client.go @@ -0,0 +1,118 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metrics + +import ( + "context" + "time" + + k8sapiv1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + recommender_metrics "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/utils/metrics/recommender" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/model" + "k8s.io/klog/v2" + "k8s.io/metrics/pkg/apis/metrics/v1beta1" +) + +// ContainerMetricsSnapshot contains information about usage of certain container within defined time window. +type ContainerMetricsSnapshot struct { + // ID identifies a specific container those metrics are coming from. + ID model.ContainerID + // End time of the measurement interval. + SnapshotTime time.Time + // Duration of the measurement interval, which is [SnapshotTime - SnapshotWindow, SnapshotTime]. + SnapshotWindow time.Duration + // Actual usage of the resources over the measurement interval. + Usage model.Resources +} + +// MetricsClient provides simple metrics on resources usage on container level. +type MetricsClient interface { + // GetContainersMetrics returns an array of ContainerMetricsSnapshots, + // representing resource usage for every running container in the cluster + GetContainersMetrics() ([]*ContainerMetricsSnapshot, error) +} + +type metricsClient struct { + source PodMetricsLister + namespace string + clientName string +} + +// NewMetricsClient creates new instance of MetricsClient, which is used by recommender. +// namespace limits queries to particular namespace, use k8sapiv1.NamespaceAll to select all namespaces. +func NewMetricsClient(source PodMetricsLister, namespace, clientName string) MetricsClient { + return &metricsClient{ + source: source, + namespace: namespace, + clientName: clientName, + } +} + +func (c *metricsClient) GetContainersMetrics() ([]*ContainerMetricsSnapshot, error) { + var metricsSnapshots []*ContainerMetricsSnapshot + + podMetricsList, err := c.source.List(context.TODO(), c.namespace, metav1.ListOptions{}) + recommender_metrics.RecordMetricsServerResponse(err, c.clientName) + if err != nil { + return nil, err + } + klog.V(3).InfoS("podMetrics retrieved for all namespaces", "podMetrics", len(podMetricsList.Items)) + for _, podMetrics := range podMetricsList.Items { + metricsSnapshotsForPod := createContainerMetricsSnapshots(podMetrics) + metricsSnapshots = append(metricsSnapshots, metricsSnapshotsForPod...) + } + return metricsSnapshots, nil +} + +func createContainerMetricsSnapshots(podMetrics v1beta1.PodMetrics) []*ContainerMetricsSnapshot { + snapshots := make([]*ContainerMetricsSnapshot, len(podMetrics.Containers)) + for i, containerMetrics := range podMetrics.Containers { + snapshots[i] = newContainerMetricsSnapshot(containerMetrics, podMetrics) + } + return snapshots +} + +func newContainerMetricsSnapshot(containerMetrics v1beta1.ContainerMetrics, podMetrics v1beta1.PodMetrics) *ContainerMetricsSnapshot { + usage := calculateUsage(containerMetrics.Usage) + + return &ContainerMetricsSnapshot{ + ID: model.ContainerID{ + ContainerName: containerMetrics.Name, + PodID: model.PodID{ + Namespace: podMetrics.Namespace, + PodName: podMetrics.Name, + }, + }, + Usage: usage, + SnapshotTime: podMetrics.Timestamp.Time, + SnapshotWindow: podMetrics.Window.Duration, + } +} + +func calculateUsage(containerUsage k8sapiv1.ResourceList) model.Resources { + cpuQuantity := containerUsage[k8sapiv1.ResourceCPU] + cpuMillicores := cpuQuantity.MilliValue() + + memoryQuantity := containerUsage[k8sapiv1.ResourceMemory] + memoryBytes := memoryQuantity.Value() + + return model.Resources{ + model.ResourceCPU: model.ResourceAmount(cpuMillicores), + model.ResourceMemory: model.ResourceAmount(memoryBytes), + } +} diff --git a/multidimensional-pod-autoscaler/pkg/recommender/input/metrics/metrics_client_test.go b/multidimensional-pod-autoscaler/pkg/recommender/input/metrics/metrics_client_test.go new file mode 100644 index 000000000000..3e29284fadb6 --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/recommender/input/metrics/metrics_client_test.go @@ -0,0 +1,46 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metrics + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGetContainersMetricsReturnsEmptyList(t *testing.T) { + tc := newEmptyMetricsClientTestCase() + emptyMetricsClient := tc.createFakeMetricsClient() + + containerMetricsSnapshots, err := emptyMetricsClient.GetContainersMetrics() + + assert.NoError(t, err) + assert.Empty(t, containerMetricsSnapshots, "should be empty for empty MetricsGetter") +} + +func TestGetContainersMetricsReturnsResults(t *testing.T) { + tc := newMetricsClientTestCase() + fakeMetricsClient := tc.createFakeMetricsClient() + + snapshots, err := fakeMetricsClient.GetContainersMetrics() + + assert.NoError(t, err) + assert.Len(t, snapshots, len(tc.getAllSnaps()), "It should return right number of snapshots") + for _, snap := range snapshots { + assert.Contains(t, tc.getAllSnaps(), snap, "One of returned ContainerMetricsSnapshot is different then expected ") + } +} diff --git a/multidimensional-pod-autoscaler/pkg/recommender/input/metrics/metrics_client_test_util.go b/multidimensional-pod-autoscaler/pkg/recommender/input/metrics/metrics_client_test_util.go new file mode 100644 index 000000000000..5f967ae741aa --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/recommender/input/metrics/metrics_client_test_util.go @@ -0,0 +1,138 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metrics + +import ( + "math/big" + "time" + + k8sapiv1 "k8s.io/api/core/v1" + + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "k8s.io/apimachinery/pkg/runtime" + core "k8s.io/client-go/testing" + + metricsapi "k8s.io/metrics/pkg/apis/metrics/v1beta1" + "k8s.io/metrics/pkg/client/clientset/versioned/fake" + + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/model" +) + +type metricsClientTestCase struct { + snapshotTimestamp time.Time + snapshotWindow time.Duration + namespace *v1.Namespace + pod1Snaps, pod2Snaps []*ContainerMetricsSnapshot +} + +func newMetricsClientTestCase() *metricsClientTestCase { + namespaceName := "test-namespace" + + testCase := &metricsClientTestCase{ + snapshotTimestamp: time.Now(), + snapshotWindow: time.Duration(1234), + namespace: &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespaceName}}, + } + + id1 := model.ContainerID{PodID: model.PodID{Namespace: namespaceName, PodName: "Pod1"}, ContainerName: "Name1"} + id2 := model.ContainerID{PodID: model.PodID{Namespace: namespaceName, PodName: "Pod1"}, ContainerName: "Name2"} + id3 := model.ContainerID{PodID: model.PodID{Namespace: namespaceName, PodName: "Pod2"}, ContainerName: "Name1"} + id4 := model.ContainerID{PodID: model.PodID{Namespace: namespaceName, PodName: "Pod2"}, ContainerName: "Name2"} + + testCase.pod1Snaps = append(testCase.pod1Snaps, testCase.newContainerMetricsSnapshot(id1, 400, 333)) + testCase.pod1Snaps = append(testCase.pod1Snaps, testCase.newContainerMetricsSnapshot(id2, 800, 666)) + testCase.pod2Snaps = append(testCase.pod2Snaps, testCase.newContainerMetricsSnapshot(id3, 401, 334)) + testCase.pod2Snaps = append(testCase.pod2Snaps, testCase.newContainerMetricsSnapshot(id4, 801, 667)) + + return testCase +} + +func newEmptyMetricsClientTestCase() *metricsClientTestCase { + return &metricsClientTestCase{} +} + +func (tc *metricsClientTestCase) newContainerMetricsSnapshot(id model.ContainerID, cpuUsage int64, memUsage int64) *ContainerMetricsSnapshot { + return &ContainerMetricsSnapshot{ + ID: id, + SnapshotTime: tc.snapshotTimestamp, + SnapshotWindow: tc.snapshotWindow, + Usage: model.Resources{ + model.ResourceCPU: model.ResourceAmount(cpuUsage), + model.ResourceMemory: model.ResourceAmount(memUsage), + }, + } +} + +func (tc *metricsClientTestCase) createFakeMetricsClient() MetricsClient { + fakeMetricsGetter := &fake.Clientset{} + fakeMetricsGetter.AddReactor("list", "pods", func(action core.Action) (handled bool, ret runtime.Object, err error) { + return true, tc.getFakePodMetricsList(), nil + }) + return NewMetricsClient(NewPodMetricsesSource(fakeMetricsGetter.MetricsV1beta1()), "", "fake") +} + +func (tc *metricsClientTestCase) getFakePodMetricsList() *metricsapi.PodMetricsList { + metrics := &metricsapi.PodMetricsList{} + if tc.pod1Snaps != nil && tc.pod2Snaps != nil { + metrics.Items = append(metrics.Items, makePodMetrics(tc.pod1Snaps)) + metrics.Items = append(metrics.Items, makePodMetrics(tc.pod2Snaps)) + } + return metrics +} + +func makePodMetrics(snaps []*ContainerMetricsSnapshot) metricsapi.PodMetrics { + firstSnap := snaps[0] + podMetrics := metricsapi.PodMetrics{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: firstSnap.ID.Namespace, + Name: firstSnap.ID.PodName, + }, + Timestamp: metav1.Time{Time: firstSnap.SnapshotTime}, + Window: metav1.Duration{Duration: firstSnap.SnapshotWindow}, + Containers: make([]metricsapi.ContainerMetrics, len(snaps)), + } + + for i, snap := range snaps { + resourceList := calculateResourceList(snap.Usage) + podMetrics.Containers[i] = metricsapi.ContainerMetrics{ + Name: snap.ID.ContainerName, + Usage: resourceList, + } + } + return podMetrics +} + +func calculateResourceList(usage model.Resources) k8sapiv1.ResourceList { + cpuCores := big.NewRat(int64(usage[model.ResourceCPU]), 1000) + cpuQuantityString := cpuCores.FloatString(3) + + memoryBytes := big.NewInt(int64(usage[model.ResourceMemory])) + memoryQuantityString := memoryBytes.String() + + resourceMap := map[k8sapiv1.ResourceName]resource.Quantity{ + k8sapiv1.ResourceCPU: resource.MustParse(cpuQuantityString), + k8sapiv1.ResourceMemory: resource.MustParse(memoryQuantityString), + } + return k8sapiv1.ResourceList(resourceMap) +} + +func (tc *metricsClientTestCase) getAllSnaps() []*ContainerMetricsSnapshot { + return append(tc.pod1Snaps, tc.pod2Snaps...) +} diff --git a/multidimensional-pod-autoscaler/pkg/recommender/input/metrics/metrics_source.go b/multidimensional-pod-autoscaler/pkg/recommender/input/metrics/metrics_source.go new file mode 100644 index 000000000000..1df8bf76a2fe --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/recommender/input/metrics/metrics_source.go @@ -0,0 +1,145 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metrics + +import ( + "context" + "time" + + k8sapiv1 "k8s.io/api/core/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/selection" + "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/recommender/model" + "k8s.io/client-go/rest" + "k8s.io/klog/v2" + "k8s.io/metrics/pkg/apis/metrics/v1beta1" + resourceclient "k8s.io/metrics/pkg/client/clientset/versioned/typed/metrics/v1beta1" + "k8s.io/metrics/pkg/client/external_metrics" +) + +// PodMetricsLister wraps both metrics-client and External Metrics +type PodMetricsLister interface { + List(ctx context.Context, namespace string, opts v1.ListOptions) (*v1beta1.PodMetricsList, error) +} + +// podMetricsSource is the metrics-client source of metrics. +type podMetricsSource struct { + metricsGetter resourceclient.PodMetricsesGetter +} + +// NewPodMetricsesSource Returns a Source-wrapper around PodMetricsesGetter. +func NewPodMetricsesSource(source resourceclient.PodMetricsesGetter) PodMetricsLister { + return podMetricsSource{metricsGetter: source} +} + +func (s podMetricsSource) List(ctx context.Context, namespace string, opts v1.ListOptions) (*v1beta1.PodMetricsList, error) { + podMetricsInterface := s.metricsGetter.PodMetricses(namespace) + return podMetricsInterface.List(ctx, opts) +} + +// externalMetricsClient is the External Metrics source of metrics. +type externalMetricsClient struct { + externalClient external_metrics.ExternalMetricsClient + options ExternalClientOptions + clusterState *model.ClusterState +} + +// ExternalClientOptions specifies parameters for using an External Metrics Client. +type ExternalClientOptions struct { + ResourceMetrics map[k8sapiv1.ResourceName]string + // Label to use for the container name. + ContainerNameLabel string +} + +// NewExternalClient returns a Source for an External Metrics Client. +func NewExternalClient(c *rest.Config, clusterState *model.ClusterState, options ExternalClientOptions) PodMetricsLister { + extClient, err := external_metrics.NewForConfig(c) + if err != nil { + klog.Fatalf("Failed initializing external metrics client: %v", err) + } + return &externalMetricsClient{ + externalClient: extClient, + options: options, + clusterState: clusterState, + } +} + +func (s *externalMetricsClient) List(ctx context.Context, namespace string, opts v1.ListOptions) (*v1beta1.PodMetricsList, error) { + result := v1beta1.PodMetricsList{} + + for _, mpa := range s.clusterState.Mpas { + if mpa.PodCount == 0 { + continue + } + + if namespace != "" && mpa.ID.Namespace != namespace { + continue + } + + nsClient := s.externalClient.NamespacedMetrics(mpa.ID.Namespace) + pods := s.clusterState.GetMatchingPods(mpa) + + for _, pod := range pods { + podNameReq, err := labels.NewRequirement("pod", selection.Equals, []string{pod.PodName}) + if err != nil { + return nil, err + } + selector := mpa.PodSelector.Add(*podNameReq) + podMets := v1beta1.PodMetrics{ + TypeMeta: v1.TypeMeta{}, + ObjectMeta: v1.ObjectMeta{Namespace: mpa.ID.Namespace, Name: pod.PodName}, + Window: v1.Duration{}, + Containers: make([]v1beta1.ContainerMetrics, 0), + } + // Query each resource in turn, then assemble back to a single []ContainerMetrics. + containerMetrics := make(map[string]k8sapiv1.ResourceList) + for resourceName, metricName := range s.options.ResourceMetrics { + m, err := nsClient.List(metricName, selector) + if err != nil { + return nil, err + } + if m == nil || len(m.Items) == 0 { + klog.V(4).InfoS("External Metrics Query for MPA: No items", "mpa", klog.KRef(mpa.ID.Namespace, mpa.ID.MpaName), "resource", resourceName, "metric", metricName) + continue + } + klog.V(4).InfoS("External Metrics Query for MPA", "mpa", klog.KRef(mpa.ID.Namespace, mpa.ID.MpaName), "resource", resourceName, "metric", metricName, "itemCount", len(m.Items), "firstItem", m.Items[0]) + podMets.Timestamp = m.Items[0].Timestamp + if m.Items[0].WindowSeconds != nil { + podMets.Window = v1.Duration{Duration: time.Duration(*m.Items[0].WindowSeconds) * time.Second} + } + for _, val := range m.Items { + ctrName, hasCtrName := val.MetricLabels[s.options.ContainerNameLabel] + if !hasCtrName { + continue + } + if containerMetrics[ctrName] == nil { + containerMetrics[ctrName] = make(k8sapiv1.ResourceList) + } + containerMetrics[ctrName][resourceName] = val.Value + } + + } + for cname, res := range containerMetrics { + podMets.Containers = append(podMets.Containers, v1beta1.ContainerMetrics{Name: cname, Usage: res}) + } + result.Items = append(result.Items, podMets) + + } + } + return &result, nil +} diff --git a/multidimensional-pod-autoscaler/pkg/recommender/main.go b/multidimensional-pod-autoscaler/pkg/recommender/main.go new file mode 100644 index 000000000000..32d631849c3c --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/recommender/main.go @@ -0,0 +1,394 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + "flag" + "os" + "strings" + "time" + + "github.com/spf13/pflag" + "k8s.io/apimachinery/pkg/util/uuid" + "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/recommender/checkpoint" + "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/recommender/input" + "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/recommender/model" + "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/recommender/routines" + "k8s.io/metrics/pkg/client/custom_metrics" + "k8s.io/metrics/pkg/client/external_metrics" + + apiv1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/autoscaler/multidimensional-pod-autoscaler/common" + + mpa_clientset "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/client/clientset/versioned" + input_metrics "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/recommender/input/metrics" + "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/target" + metrics_recommender "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/utils/metrics/recommender" + mpa_api_util "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/utils/mpa" + vpa_common "k8s.io/autoscaler/vertical-pod-autoscaler/common" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/input/history" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/logic" + vpa_model "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/model" + controllerfetcher "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/target/controller_fetcher" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/metrics" + metrics_quality "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/metrics/quality" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/server" + "k8s.io/client-go/discovery" + cacheddiscovery "k8s.io/client-go/discovery/cached" + "k8s.io/client-go/informers" + kube_client "k8s.io/client-go/kubernetes" + "k8s.io/client-go/restmapper" + "k8s.io/client-go/tools/leaderelection" + "k8s.io/client-go/tools/leaderelection/resourcelock" + kube_flag "k8s.io/component-base/cli/flag" + componentbaseconfig "k8s.io/component-base/config" + componentbaseoptions "k8s.io/component-base/config/options" + klog "k8s.io/klog/v2" + hpa_metrics "k8s.io/kubernetes/pkg/controller/podautoscaler/metrics" + resourceclient "k8s.io/metrics/pkg/client/clientset/versioned/typed/metrics/v1beta1" +) + +var ( + recommenderName = flag.String("recommender-name", input.DefaultRecommenderName, "Set the recommender name. Recommender will generate recommendations for MPAs that configure the same recommender name. If the recommender name is left as default it will also generate recommendations that don't explicitly specify recommender. You shouldn't run two recommenders with the same name in a cluster.") + metricsFetcherInterval = flag.Duration("recommender-interval", 1*time.Minute, `How often metrics should be fetched`) + checkpointsGCInterval = flag.Duration("checkpoints-gc-interval", 10*time.Minute, `How often orphaned checkpoints should be garbage collected`) + address = flag.String("address", ":8942", "The address to expose Prometheus metrics.") + storage = flag.String("storage", "", `Specifies storage mode. Supported values: prometheus, checkpoint (default)`) + memorySaver = flag.Bool("memory-saver", false, `If true, only track pods which have an associated MPA`) + mpaObjectNamespace = flag.String("mpa-object-namespace", apiv1.NamespaceAll, "Namespace to search for MPA objects and pod stats. Empty means all namespaces will be used.") + ignoredMpaObjectNamespaces = flag.String("ignored-mpa-object-namespaces", "", "Comma separated list of namespaces to ignore when searching for MPA objects. Empty means no namespaces will be ignored.") +) + +// prometheus history provider configs +var ( + prometheusAddress = flag.String("prometheus-address", "", `Where to reach for Prometheus metrics`) + prometheusJobName = flag.String("prometheus-cadvisor-job-name", "kubernetes-cadvisor", `Name of the prometheus job name which scrapes the cAdvisor metrics`) + historyLength = flag.String("history-length", "8d", `How much time back prometheus have to be queried to get historical metrics`) + historyResolution = flag.String("history-resolution", "1h", `Resolution at which Prometheus is queried for historical metrics`) + queryTimeout = flag.String("prometheus-query-timeout", "5m", `How long to wait before killing long queries`) + podLabelPrefix = flag.String("pod-label-prefix", "pod_label_", `Which prefix to look for pod labels in metrics`) + podLabelsMetricName = flag.String("metric-for-pod-labels", "up{job=\"kubernetes-pods\"}", `Which metric to look for pod labels in metrics`) + podNamespaceLabel = flag.String("pod-namespace-label", "kubernetes_namespace", `Label name to look for pod namespaces`) + podNameLabel = flag.String("pod-name-label", "kubernetes_pod_name", `Label name to look for pod names`) + ctrNamespaceLabel = flag.String("container-namespace-label", "namespace", `Label name to look for container namespaces`) + ctrPodNameLabel = flag.String("container-pod-name-label", "pod_name", `Label name to look for container pod names`) + ctrNameLabel = flag.String("container-name-label", "name", `Label name to look for container names`) + username = flag.String("username", "", "The username used in the prometheus server basic auth") + password = flag.String("password", "", "The password used in the prometheus server basic auth") +) + +// External metrics provider flags +var ( + useExternalMetrics = flag.Bool("use-external-metrics", false, "ALPHA. Use an external metrics provider instead of metrics_server.") + externalCpuMetric = flag.String("external-metrics-cpu-metric", "", "ALPHA. Metric to use with external metrics provider for CPU usage.") + externalMemoryMetric = flag.String("external-metrics-memory-metric", "", "ALPHA. Metric to use with external metrics provider for memory usage.") +) + +// Aggregation configuration flags +var ( + memoryAggregationInterval = flag.Duration("memory-aggregation-interval", vpa_model.DefaultMemoryAggregationInterval, `The length of a single interval, for which the peak memory usage is computed. Memory usage peaks are aggregated in multiples of this interval. In other words there is one memory usage sample per interval (the maximum usage over that interval)`) + memoryAggregationIntervalCount = flag.Int64("memory-aggregation-interval-count", vpa_model.DefaultMemoryAggregationIntervalCount, `The number of consecutive memory-aggregation-intervals which make up the MemoryAggregationWindowLength which in turn is the period for memory usage aggregation by VPA. In other words, MemoryAggregationWindowLength = memory-aggregation-interval * memory-aggregation-interval-count.`) + memoryHistogramDecayHalfLife = flag.Duration("memory-histogram-decay-half-life", vpa_model.DefaultMemoryHistogramDecayHalfLife, `The amount of time it takes a historical memory usage sample to lose half of its weight. In other words, a fresh usage sample is twice as 'important' as one with age equal to the half life period.`) + cpuHistogramDecayHalfLife = flag.Duration("cpu-histogram-decay-half-life", vpa_model.DefaultCPUHistogramDecayHalfLife, `The amount of time it takes a historical CPU usage sample to lose half of its weight.`) + oomBumpUpRatio = flag.Float64("oom-bump-up-ratio", vpa_model.DefaultOOMBumpUpRatio, `Specifies the memory bump up ratio when OOM occurred.`) + oomMinBumpUp = flag.Float64("oom-min-bump-up", vpa_model.DefaultOOMMinBumpUp, `Specifies the minimal increase of memory when OOM occurred in bytes..`) +) + +// HPA-related flags +var ( + // horizontalPodAutoscalerSyncPeriod is the period for syncing the number of pods in MPA. + hpaSyncPeriod = flag.Duration("hpa-sync-period", 15*time.Second, `The period for syncing the number of pods in horizontal pod autoscaler.`) + // horizontalPodAutoscalerUpscaleForbiddenWindow is a period after which next upscale allowed. + hpaUpscaleForbiddenWindow = flag.Duration("hpa-upscale-forbidden-window", 3*time.Minute, `The period after which next upscale allowed.`) + // horizontalPodAutoscalerDownscaleForbiddenWindow is a period after which next downscale allowed. + hpaDownscaleForbiddenWindow = flag.Duration("hpa-downscale-forbidden-window", 5*time.Minute, `The period after which next downscale allowed.`) + // HorizontalPodAutoscalerDowncaleStabilizationWindow is a period for which autoscaler will look + // backwards and not scale down below any recommendation it made during that period. + hpaDownscaleStabilizationWindow = flag.Duration("hpa-downscale-stabilization-window", 5*time.Minute, `The period for which autoscaler will look backwards and not scale down below any recommendation it made during that period.`) + // horizontalPodAutoscalerTolerance is the tolerance for when resource usage suggests upscaling/downscaling + hpaTolerance = flag.Float64("hpa-tolerance", 0.1, `The tolerance for when resource usage suggests horizontally upscaling/downscaling.`) + // HorizontalPodAutoscalerCPUInitializationPeriod is the period after pod start when CPU samples + // might be skipped. + hpaCPUInitializationPeriod = flag.Duration("hpa-cpu-initialization-period", 5*time.Minute, `The period after pod start when CPU samples might be skipped.`) + // HorizontalPodAutoscalerInitialReadinessDelay is period after pod start during which readiness + // changes are treated as readiness being set for the first time. The only effect of this is + // that HPA will disregard CPU samples from unready pods that had last readiness change during + // that period. + hpaInitialReadinessDelay = flag.Duration("hpa-initial-readiness-delay", 30*time.Second, `The period after pod start during which readiness changes are treated as readiness being set for the first time.`) + concurrentHPASyncs = flag.Int64("concurrent-hpa-syncs", 5, `The number of horizontal pod autoscaler objects that are allowed to sync concurrently. Larger number = more responsive MPA objects processing, but more CPU (and network) load.`) +) + +// Post processors flags +var ( + // CPU as integer to benefit for CPU management Static Policy ( https://kubernetes.io/docs/tasks/administer-cluster/cpu-management-policies/#static-policy ) + postProcessorCPUasInteger = flag.Bool("cpu-integer-post-processor-enabled", false, "Enable the cpu-integer recommendation post processor. The post processor will round up CPU recommendations to a whole CPU for pods which were opted in by setting an appropriate label on VPA object (experimental)") +) + +const ( + // aggregateContainerStateGCInterval defines how often expired AggregateContainerStates are garbage collected. + aggregateContainerStateGCInterval = 1 * time.Hour + scaleCacheEntryLifetime time.Duration = time.Hour + scaleCacheEntryFreshnessTime time.Duration = 10 * time.Minute + scaleCacheEntryJitterFactor float64 = 1. + scaleCacheLoopPeriod = 7 * time.Second + defaultResyncPeriod time.Duration = 10 * time.Minute + discoveryResetPeriod time.Duration = 5 * time.Minute +) + +func main() { + commonFlags := vpa_common.InitCommonFlags() + klog.InitFlags(nil) + vpa_common.InitLoggingFlags() + + leaderElection := defaultLeaderElectionConfiguration() + componentbaseoptions.BindLeaderElectionFlags(&leaderElection, pflag.CommandLine) + + kube_flag.InitFlags() + klog.V(1).Infof("Multi-dimensional Pod Autoscaler %s Recommender: %v", common.MultidimPodAutoscalerVersion, recommenderName) + + if len(*mpaObjectNamespace) > 0 && len(*ignoredMpaObjectNamespaces) > 0 { + klog.Fatalf("--mpa-object-namespace and --ignored-mpa-object-namespaces are mutually exclusive and can't be set together.") + } + + healthCheck := metrics.NewHealthCheck(*metricsFetcherInterval * 5) + metrics_recommender.Register() + metrics_quality.Register() + server.Initialize(&commonFlags.EnableProfiling, healthCheck, address) + + if !leaderElection.LeaderElect { + run(healthCheck, commonFlags) + } else { + id, err := os.Hostname() + if err != nil { + klog.Fatalf("Unable to get hostname: %v", err) + } + id = id + "_" + string(uuid.NewUUID()) + + config := common.CreateKubeConfigOrDie(commonFlags.KubeConfig, float32(commonFlags.KubeApiQps), int(commonFlags.KubeApiBurst)) + kubeClient := kube_client.NewForConfigOrDie(config) + + lock, err := resourcelock.New( + leaderElection.ResourceLock, + leaderElection.ResourceNamespace, + leaderElection.ResourceName, + kubeClient.CoreV1(), + kubeClient.CoordinationV1(), + resourcelock.ResourceLockConfig{ + Identity: id, + }, + ) + if err != nil { + klog.Fatalf("Unable to create leader election lock: %v", err) + } + + leaderelection.RunOrDie(context.TODO(), leaderelection.LeaderElectionConfig{ + Lock: lock, + LeaseDuration: leaderElection.LeaseDuration.Duration, + RenewDeadline: leaderElection.RenewDeadline.Duration, + RetryPeriod: leaderElection.RetryPeriod.Duration, + ReleaseOnCancel: true, + Callbacks: leaderelection.LeaderCallbacks{ + OnStartedLeading: func(_ context.Context) { + run(healthCheck, commonFlags) + }, + OnStoppedLeading: func() { + klog.Fatal("lost master") + }, + }, + }) + } +} + +const ( + defaultLeaseDuration = 15 * time.Second + defaultRenewDeadline = 10 * time.Second + defaultRetryPeriod = 2 * time.Second +) + +func defaultLeaderElectionConfiguration() componentbaseconfig.LeaderElectionConfiguration { + return componentbaseconfig.LeaderElectionConfiguration{ + LeaderElect: false, + LeaseDuration: metav1.Duration{Duration: defaultLeaseDuration}, + RenewDeadline: metav1.Duration{Duration: defaultRenewDeadline}, + RetryPeriod: metav1.Duration{Duration: defaultRetryPeriod}, + ResourceLock: resourcelock.LeasesResourceLock, + // This was changed from "vpa-recommender" to avoid conflicts with managed VPA deployments. + ResourceName: "mpa-recommender-lease", + ResourceNamespace: metav1.NamespaceSystem, + } +} + +func run(healthCheck *metrics.HealthCheck, commonFlag *vpa_common.CommonFlags) { + config := common.CreateKubeConfigOrDie(commonFlag.KubeConfig, float32(commonFlag.KubeApiQps), int(commonFlag.KubeApiBurst)) + kubeClient := kube_client.NewForConfigOrDie(config) + clusterState := model.NewClusterState(aggregateContainerStateGCInterval) + factory := informers.NewSharedInformerFactoryWithOptions(kubeClient, defaultResyncPeriod, informers.WithNamespace(*ignoredMpaObjectNamespaces)) + controllerFetcher := controllerfetcher.NewControllerFetcher(config, kubeClient, factory, scaleCacheEntryFreshnessTime, scaleCacheEntryLifetime, scaleCacheEntryJitterFactor) + podLister, oomObserver := input.NewPodListerAndOOMObserver(kubeClient, commonFlag.IgnoredVpaObjectNamespaces) + + vpa_model.InitializeAggregationsConfig(vpa_model.NewAggregationsConfig(*memoryAggregationInterval, *memoryAggregationIntervalCount, *memoryHistogramDecayHalfLife, *cpuHistogramDecayHalfLife, *oomBumpUpRatio, *oomMinBumpUp)) + + useCheckpoints := *storage != "prometheus" + + var postProcessors []routines.RecommendationPostProcessor + if *postProcessorCPUasInteger { + postProcessors = append(postProcessors, &routines.IntegerCPUPostProcessor{}) + } + + // CappingPostProcessor, should always come in the last position for post-processing + postProcessors = append(postProcessors, &routines.CappingPostProcessor{}) + var source input_metrics.PodMetricsLister + if *useExternalMetrics { + resourceMetrics := map[apiv1.ResourceName]string{} + if externalCpuMetric != nil && *externalCpuMetric != "" { + resourceMetrics[apiv1.ResourceCPU] = *externalCpuMetric + } + if externalMemoryMetric != nil && *externalMemoryMetric != "" { + resourceMetrics[apiv1.ResourceMemory] = *externalMemoryMetric + } + externalClientOptions := &input_metrics.ExternalClientOptions{ResourceMetrics: resourceMetrics, ContainerNameLabel: *ctrNameLabel} + klog.V(1).InfoS("Using External Metrics", "options", externalClientOptions) + source = input_metrics.NewExternalClient(config, clusterState, *externalClientOptions) + } else { + klog.V(1).InfoS("Using Metrics Server") + source = input_metrics.NewPodMetricsesSource(resourceclient.NewForConfigOrDie(config)) + } + + ignoredNamespaces := strings.Split(*ignoredMpaObjectNamespaces, ",") + + clusterStateFeeder := input.ClusterStateFeederFactory{ + PodLister: podLister, + OOMObserver: oomObserver, + KubeClient: kubeClient, + MetricsClient: input_metrics.NewMetricsClient(source, *mpaObjectNamespace, "default-metrics-client"), + MpaCheckpointClient: mpa_clientset.NewForConfigOrDie(config).AutoscalingV1alpha1(), + MpaLister: mpa_api_util.NewMpasLister(mpa_clientset.NewForConfigOrDie(config), make(chan struct{}), *mpaObjectNamespace), + ClusterState: clusterState, + SelectorFetcher: target.NewMpaTargetSelectorFetcher(config, kubeClient, factory), + MemorySaveMode: *memorySaver, + ControllerFetcher: controllerFetcher, + RecommenderName: *recommenderName, + IgnoredNamespaces: ignoredNamespaces, + }.Make() + controllerFetcher.Start(context.Background(), scaleCacheLoopPeriod) + + // For HPA. + // Use a discovery client capable of being refreshed. + discoveryClient, err := discovery.NewDiscoveryClientForConfig(config) + if err != nil { + klog.Fatalf("Could not create discoveryClient: %v", err) + } + cachedDiscoveryClient := cacheddiscovery.NewMemCacheClient(discoveryClient) + mapper := restmapper.NewDeferredDiscoveryRESTMapper(cachedDiscoveryClient) + go wait.Until(func() { + mapper.Reset() + }, discoveryResetPeriod, make(chan struct{})) + mpaClient := mpa_clientset.NewForConfigOrDie(config) + apiVersionsGetter := custom_metrics.NewAvailableAPIsGetter(mpaClient.Discovery()) + ctx := context.Background() // TODO: Add a deadline to this ctx? + // invalidate the discovery information roughly once per resync interval our API + // information is *at most* two resync intervals old. + go custom_metrics.PeriodicallyInvalidate( + apiVersionsGetter, + *hpaSyncPeriod, + ctx.Done()) + + metricsClient := hpa_metrics.NewRESTMetricsClient( + resourceclient.NewForConfigOrDie(config), + custom_metrics.NewForConfig(config, mapper, apiVersionsGetter), + external_metrics.NewForConfigOrDie(config), + ) + + recommender := routines.RecommenderFactory{ + ClusterState: clusterState, + ClusterStateFeeder: clusterStateFeeder, + ControllerFetcher: controllerFetcher, + CheckpointWriter: checkpoint.NewCheckpointWriter(clusterState, mpa_clientset.NewForConfigOrDie(config).AutoscalingV1alpha1()), + MpaClient: mpa_clientset.NewForConfigOrDie(config).AutoscalingV1alpha1(), + SelectorFetcher: target.NewMpaTargetSelectorFetcher(config, kubeClient, factory), + PodResourceRecommender: logic.CreatePodResourceRecommender(), + RecommendationPostProcessors: postProcessors, + CheckpointsGCInterval: *checkpointsGCInterval, + UseCheckpoints: useCheckpoints, + + // HPA-related flags + EvtNamespacer: kubeClient.CoreV1(), + PodInformer: factory.Core().V1().Pods(), + MetricsClient: metricsClient, + ResyncPeriod: *hpaSyncPeriod, + DownscaleStabilisationWindow: *hpaDownscaleStabilizationWindow, + Tolerance: *hpaTolerance, + CpuInitializationPeriod: *hpaCPUInitializationPeriod, + DelayOfInitialReadinessStatus: *hpaInitialReadinessDelay, + }.Make() + + klog.Infof("MPA Recommender created!") + + promQueryTimeout, err := time.ParseDuration(*queryTimeout) + if err != nil { + klog.Fatalf("Could not parse --prometheus-query-timeout as a time.Duration: %v", err) + } + + if useCheckpoints { + recommender.GetClusterStateFeeder().InitFromCheckpoints() + } else { + klog.Info("Creating Prometheus history provider...") + config := history.PrometheusHistoryProviderConfig{ + Address: *prometheusAddress, + QueryTimeout: promQueryTimeout, + HistoryLength: *historyLength, + HistoryResolution: *historyResolution, + PodLabelPrefix: *podLabelPrefix, + PodLabelsMetricName: *podLabelsMetricName, + PodNamespaceLabel: *podNamespaceLabel, + PodNameLabel: *podNameLabel, + CtrNamespaceLabel: *ctrNamespaceLabel, + CtrPodNameLabel: *ctrPodNameLabel, + CtrNameLabel: *ctrNameLabel, + CadvisorMetricsJobName: *prometheusJobName, + Namespace: *mpaObjectNamespace, + PrometheusBasicAuthTransport: history.PrometheusBasicAuthTransport{ + Username: *username, + Password: *password, + }, + } + provider, err := history.NewPrometheusHistoryProvider(config) + if err != nil { + klog.Fatalf("Could not initialize history provider: %v", err) + } + klog.Info("History provider initialized!") + recommender.GetClusterStateFeeder().InitFromHistoryProvider(provider) + klog.Info("Recommender initialized!") + } + + ticker := time.Tick(*metricsFetcherInterval) + klog.Info("Start running MPA Recommender...") + var vpaOrHpa = "vpa" + for range ticker { + recommender.RunOnce(int(*concurrentHPASyncs), vpaOrHpa) + healthCheck.UpdateLastActivity() + klog.Info("Health check completed.") + if vpaOrHpa == "vpa" { + vpaOrHpa = "hpa" + } else if vpaOrHpa == "hpa" { + vpaOrHpa = "vpa" + } + } +} diff --git a/multidimensional-pod-autoscaler/pkg/recommender/model/aggregate_container_state.go b/multidimensional-pod-autoscaler/pkg/recommender/model/aggregate_container_state.go new file mode 100644 index 000000000000..ad679e722f03 --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/recommender/model/aggregate_container_state.go @@ -0,0 +1,125 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// VPA collects CPU and memory usage measurements from all containers running in +// the cluster and aggregates them in memory in structures called +// AggregateContainerState. +// During aggregation the usage samples are grouped together by the key called +// AggregateStateKey and stored in structures such as histograms of CPU and +// memory usage, that are parts of the AggregateContainerState. +// +// The AggregateStateKey consists of the container name, the namespace and the +// set of labels on the pod the container belongs to. In other words, whenever +// two samples come from containers with the same name, in the same namespace +// and with the same pod labels, they end up in the same histogram. +// +// Recall that VPA produces one recommendation for all containers with a given +// name and namespace, having pod labels that match a given selector. Therefore +// for each VPA object and container name the recommender has to take all +// matching AggregateContainerStates and further aggregate them together, in +// order to obtain the final aggregation that is the input to the recommender +// function. + +package model + +import ( + "time" + + corev1 "k8s.io/api/core/v1" + + vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" + vpa_model "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/model" +) + +// The isStateExpired and isStateEmpty functions are from VPA lib (because isExpired and isEmpty +// are private functions so not callable). +func isStateExpired(a *vpa_model.AggregateContainerState, now time.Time) bool { + if isStateEmpty(a) { + return now.Sub(a.CreationTime) >= vpa_model.GetAggregationsConfig().GetMemoryAggregationWindowLength() + } + return now.Sub(a.LastSampleStart) >= vpa_model.GetAggregationsConfig().GetMemoryAggregationWindowLength() +} + +func isStateEmpty(a *vpa_model.AggregateContainerState) bool { + return a.TotalSamplesCount == 0 +} + +// AggregateStateByContainerName takes a set of AggregateContainerStates and merge them +// grouping by the container name. The result is a map from the container name to the aggregation +// from all input containers with the given name. +func AggregateStateByContainerName(aggregateContainerStateMap aggregateContainerStatesMap) vpa_model.ContainerNameToAggregateStateMap { + containerNameToAggregateStateMap := make(vpa_model.ContainerNameToAggregateStateMap) + for aggregationKey, aggregation := range aggregateContainerStateMap { + containerName := aggregationKey.ContainerName() + aggregateContainerState, isInitialized := containerNameToAggregateStateMap[containerName] + if !isInitialized { + aggregateContainerState = vpa_model.NewAggregateContainerState() + containerNameToAggregateStateMap[containerName] = aggregateContainerState + } + aggregateContainerState.MergeContainerState(aggregation) + } + return containerNameToAggregateStateMap +} + +// ContainerStateAggregatorProxy is a wrapper for ContainerStateAggregator +// that creates ContainerStateAgregator for container if it is no longer +// present in the cluster state. +type ContainerStateAggregatorProxy struct { + containerID vpa_model.ContainerID + cluster *ClusterState +} + +// NewContainerStateAggregatorProxy creates a ContainerStateAggregatorProxy +// pointing to the cluster state. +func NewContainerStateAggregatorProxy(cluster *ClusterState, containerID vpa_model.ContainerID) vpa_model.ContainerStateAggregator { + return &ContainerStateAggregatorProxy{containerID, cluster} +} + +// AddSample adds a container sample to the aggregator. +func (p *ContainerStateAggregatorProxy) AddSample(sample *vpa_model.ContainerUsageSample) { + aggregator := p.cluster.findOrCreateAggregateContainerState(p.containerID) + aggregator.AddSample(sample) +} + +// SubtractSample subtracts a container sample from the aggregator. +func (p *ContainerStateAggregatorProxy) SubtractSample(sample *vpa_model.ContainerUsageSample) { + aggregator := p.cluster.findOrCreateAggregateContainerState(p.containerID) + aggregator.SubtractSample(sample) +} + +// GetLastRecommendation returns last recorded recommendation. +func (p *ContainerStateAggregatorProxy) GetLastRecommendation() corev1.ResourceList { + aggregator := p.cluster.findOrCreateAggregateContainerState(p.containerID) + return aggregator.GetLastRecommendation() +} + +// NeedsRecommendation returns true if the aggregator should have recommendation calculated. +func (p *ContainerStateAggregatorProxy) NeedsRecommendation() bool { + aggregator := p.cluster.findOrCreateAggregateContainerState(p.containerID) + return aggregator.NeedsRecommendation() +} + +// GetUpdateMode returns update mode of VPA controlling the aggregator. +func (p *ContainerStateAggregatorProxy) GetUpdateMode() *vpa_types.UpdateMode { + aggregator := p.cluster.findOrCreateAggregateContainerState(p.containerID) + return aggregator.GetUpdateMode() +} + +// GetScalingMode returns scaling mode of container represented by the aggregator. +func (p *ContainerStateAggregatorProxy) GetScalingMode() *vpa_types.ContainerScalingMode { + aggregator := p.cluster.findOrCreateAggregateContainerState(p.containerID) + return aggregator.GetScalingMode() +} diff --git a/multidimensional-pod-autoscaler/pkg/recommender/model/aggregate_container_state_test.go b/multidimensional-pod-autoscaler/pkg/recommender/model/aggregate_container_state_test.go new file mode 100644 index 000000000000..509fbf8e5a32 --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/recommender/model/aggregate_container_state_test.go @@ -0,0 +1,297 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package model + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + apiv1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + labels "k8s.io/apimachinery/pkg/labels" + vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" + vpa_model "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/model" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/util" +) + +var ( + testPodID1 = vpa_model.PodID{Namespace: "namespace-1", PodName: "pod-1"} + testPodID2 = vpa_model.PodID{Namespace: "namespace-1", PodName: "pod-2"} + testRequest = vpa_model.Resources{ + vpa_model.ResourceCPU: vpa_model.CPUAmountFromCores(3.14), + vpa_model.ResourceMemory: vpa_model.MemoryAmountFromBytes(3.14e9), + } +) + +func addTestCPUSample(cluster *ClusterState, container vpa_model.ContainerID, cpuCores float64) error { + sample := ContainerUsageSampleWithKey{ + Container: container, + ContainerUsageSample: vpa_model.ContainerUsageSample{ + MeasureStart: testTimestamp, + Usage: vpa_model.CPUAmountFromCores(cpuCores), + Request: testRequest[vpa_model.ResourceCPU], + Resource: vpa_model.ResourceCPU, + }, + } + return cluster.AddSample(&sample) +} + +func addTestMemorySample(cluster *ClusterState, container vpa_model.ContainerID, memoryBytes float64) error { + sample := ContainerUsageSampleWithKey{ + Container: container, + ContainerUsageSample: vpa_model.ContainerUsageSample{ + MeasureStart: testTimestamp, + Usage: vpa_model.MemoryAmountFromBytes(memoryBytes), + Request: testRequest[vpa_model.ResourceMemory], + Resource: vpa_model.ResourceMemory, + }, + } + return cluster.AddSample(&sample) +} + +// Creates two pods, each having two containers: +// +// testPodID1: { 'app-A', 'app-B' } +// testPodID2: { 'app-A', 'app-C' } +// +// Adds a few usage samples to the containers. +// Verifies that AggregateStateByContainerName() properly aggregates +// container CPU and memory peak histograms, grouping the two containers +// with the same name ('app-A') together. +func TestAggregateStateByContainerName(t *testing.T) { + cluster := NewClusterState(testGcPeriod) + cluster.AddOrUpdatePod(testPodID1, testLabels, apiv1.PodRunning) + otherLabels := labels.Set{"label-2": "value-2"} + cluster.AddOrUpdatePod(testPodID2, otherLabels, apiv1.PodRunning) + + // Create 4 containers: 2 with the same name and 2 with different names. + containers := []vpa_model.ContainerID{ + {PodID: testPodID1, ContainerName: "app-A"}, + {PodID: testPodID1, ContainerName: "app-B"}, + {PodID: testPodID2, ContainerName: "app-A"}, + {PodID: testPodID2, ContainerName: "app-C"}, + } + for _, c := range containers { + assert.NoError(t, cluster.AddOrUpdateContainer(c, testRequest)) + } + + // Add CPU usage samples to all containers. + assert.NoError(t, addTestCPUSample(cluster, containers[0], 1.0)) // app-A + assert.NoError(t, addTestCPUSample(cluster, containers[1], 5.0)) // app-B + assert.NoError(t, addTestCPUSample(cluster, containers[2], 3.0)) // app-A + assert.NoError(t, addTestCPUSample(cluster, containers[3], 5.0)) // app-C + // Add Memory usage samples to all containers. + assert.NoError(t, addTestMemorySample(cluster, containers[0], 2e9)) // app-A + assert.NoError(t, addTestMemorySample(cluster, containers[1], 10e9)) // app-B + assert.NoError(t, addTestMemorySample(cluster, containers[2], 4e9)) // app-A + assert.NoError(t, addTestMemorySample(cluster, containers[3], 10e9)) // app-C + + // Build the AggregateContainerStateMap. + aggregateResources := AggregateStateByContainerName(cluster.aggregateStateMap) + assert.Contains(t, aggregateResources, "app-A") + assert.Contains(t, aggregateResources, "app-B") + assert.Contains(t, aggregateResources, "app-C") + + // Expect samples from all containers to be grouped by the container name. + assert.Equal(t, 2, aggregateResources["app-A"].TotalSamplesCount) + assert.Equal(t, 1, aggregateResources["app-B"].TotalSamplesCount) + assert.Equal(t, 1, aggregateResources["app-C"].TotalSamplesCount) + + config := vpa_model.GetAggregationsConfig() + // Compute the expected histograms for the "app-A" containers. + expectedCPUHistogram := util.NewDecayingHistogram(config.CPUHistogramOptions, config.CPUHistogramDecayHalfLife) + expectedCPUHistogram.Merge(cluster.findOrCreateAggregateContainerState(containers[0]).AggregateCPUUsage) + expectedCPUHistogram.Merge(cluster.findOrCreateAggregateContainerState(containers[2]).AggregateCPUUsage) + actualCPUHistogram := aggregateResources["app-A"].AggregateCPUUsage + + expectedMemoryHistogram := util.NewDecayingHistogram(config.MemoryHistogramOptions, config.MemoryHistogramDecayHalfLife) + expectedMemoryHistogram.AddSample(2e9, 1.0, cluster.GetContainer(containers[0]).WindowEnd) + expectedMemoryHistogram.AddSample(4e9, 1.0, cluster.GetContainer(containers[2]).WindowEnd) + actualMemoryHistogram := aggregateResources["app-A"].AggregateMemoryPeaks + + assert.True(t, expectedCPUHistogram.Equals(actualCPUHistogram), "Expected:\n%s\nActual:\n%s", expectedCPUHistogram, actualCPUHistogram) + assert.True(t, expectedMemoryHistogram.Equals(actualMemoryHistogram), "Expected:\n%s\nActual:\n%s", expectedMemoryHistogram, actualMemoryHistogram) +} + +func TestAggregateContainerStateSaveToCheckpoint(t *testing.T) { + location, _ := time.LoadLocation("UTC") + cs := vpa_model.NewAggregateContainerState() + t1, t2 := time.Date(2018, time.January, 1, 2, 3, 4, 0, location), time.Date(2018, time.February, 1, 2, 3, 4, 0, location) + cs.FirstSampleStart = t1 + cs.LastSampleStart = t2 + cs.TotalSamplesCount = 10 + + cs.AggregateCPUUsage.AddSample(1, 33, t2) + cs.AggregateMemoryPeaks.AddSample(1, 55, t1) + cs.AggregateMemoryPeaks.AddSample(10000000, 55, t1) + checkpoint, err := cs.SaveToCheckpoint() + + assert.NoError(t, err) + + assert.True(t, time.Since(checkpoint.LastUpdateTime.Time) < 10*time.Second) + assert.Equal(t, t1, checkpoint.FirstSampleStart.Time) + assert.Equal(t, t2, checkpoint.LastSampleStart.Time) + assert.Equal(t, 10, checkpoint.TotalSamplesCount) + + assert.Equal(t, vpa_model.SupportedCheckpointVersion, checkpoint.Version) + + // Basic check that serialization of histograms happened. + // Full tests are part of the Histogram. + assert.Len(t, checkpoint.CPUHistogram.BucketWeights, 1) + assert.Len(t, checkpoint.MemoryHistogram.BucketWeights, 2) +} + +func TestAggregateContainerStateLoadFromCheckpointFailsForVersionMismatch(t *testing.T) { + checkpoint := vpa_types.VerticalPodAutoscalerCheckpointStatus{ + Version: "foo", + } + cs := vpa_model.NewAggregateContainerState() + err := cs.LoadFromCheckpoint(&checkpoint) + assert.Error(t, err) +} + +func TestAggregateContainerStateLoadFromCheckpoint(t *testing.T) { + location, _ := time.LoadLocation("UTC") + t1, t2 := time.Date(2018, time.January, 1, 2, 3, 4, 0, location), time.Date(2018, time.February, 1, 2, 3, 4, 0, location) + + checkpoint := vpa_types.VerticalPodAutoscalerCheckpointStatus{ + Version: vpa_model.SupportedCheckpointVersion, + LastUpdateTime: metav1.NewTime(time.Now()), + FirstSampleStart: metav1.NewTime(t1), + LastSampleStart: metav1.NewTime(t2), + TotalSamplesCount: 20, + MemoryHistogram: vpa_types.HistogramCheckpoint{ + BucketWeights: map[int]uint32{ + 0: 10, + }, + TotalWeight: 33.0, + }, + CPUHistogram: vpa_types.HistogramCheckpoint{ + BucketWeights: map[int]uint32{ + 0: 10, + }, + TotalWeight: 44.0, + }, + } + + cs := vpa_model.NewAggregateContainerState() + err := cs.LoadFromCheckpoint(&checkpoint) + assert.NoError(t, err) + + assert.Equal(t, t1, cs.FirstSampleStart) + assert.Equal(t, t2, cs.LastSampleStart) + assert.Equal(t, 20, cs.TotalSamplesCount) + assert.False(t, cs.AggregateCPUUsage.IsEmpty()) + assert.False(t, cs.AggregateMemoryPeaks.IsEmpty()) +} + +func TestAggregateContainerStateIsExpired(t *testing.T) { + cs := vpa_model.NewAggregateContainerState() + cs.LastSampleStart = testTimestamp + cs.TotalSamplesCount = 1 + assert.False(t, isStateExpired(cs, testTimestamp.Add(7*24*time.Hour))) + assert.True(t, isStateExpired(cs, testTimestamp.Add(8*24*time.Hour))) + + csEmpty := vpa_model.NewAggregateContainerState() + csEmpty.TotalSamplesCount = 0 + csEmpty.CreationTime = testTimestamp + assert.False(t, isStateExpired(csEmpty, testTimestamp.Add(7*24*time.Hour))) + assert.True(t, isStateExpired(csEmpty, testTimestamp.Add(8*24*time.Hour))) +} + +func TestUpdateFromPolicyScalingMode(t *testing.T) { + scalingModeAuto := vpa_types.ContainerScalingModeAuto + scalingModeOff := vpa_types.ContainerScalingModeOff + testCases := []struct { + name string + policy *vpa_types.ContainerResourcePolicy + expected *vpa_types.ContainerScalingMode + }{ + { + name: "Explicit auto scaling mode", + policy: &vpa_types.ContainerResourcePolicy{ + Mode: &scalingModeAuto, + }, + expected: &scalingModeAuto, + }, { + name: "Off scaling mode", + policy: &vpa_types.ContainerResourcePolicy{ + Mode: &scalingModeOff, + }, + expected: &scalingModeOff, + }, { + name: "No mode specified - default to Auto", + policy: &vpa_types.ContainerResourcePolicy{}, + expected: &scalingModeAuto, + }, { + name: "Nil policy - default to Auto", + policy: nil, + expected: &scalingModeAuto, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + cs := vpa_model.NewAggregateContainerState() + cs.UpdateFromPolicy(tc.policy) + assert.Equal(t, tc.expected, cs.GetScalingMode()) + }) + } +} + +func TestUpdateFromPolicyControlledResources(t *testing.T) { + testCases := []struct { + name string + policy *vpa_types.ContainerResourcePolicy + expected []vpa_model.ResourceName + }{ + { + name: "Explicit ControlledResources", + policy: &vpa_types.ContainerResourcePolicy{ + ControlledResources: &[]apiv1.ResourceName{apiv1.ResourceCPU, apiv1.ResourceMemory}, + }, + expected: []vpa_model.ResourceName{vpa_model.ResourceCPU, vpa_model.ResourceMemory}, + }, { + name: "Empty ControlledResources", + policy: &vpa_types.ContainerResourcePolicy{ + ControlledResources: &[]apiv1.ResourceName{}, + }, + expected: []vpa_model.ResourceName{}, + }, { + name: "ControlledResources with one resource", + policy: &vpa_types.ContainerResourcePolicy{ + ControlledResources: &[]apiv1.ResourceName{apiv1.ResourceMemory}, + }, + expected: []vpa_model.ResourceName{vpa_model.ResourceMemory}, + }, { + name: "No ControlledResources specified - used default", + policy: &vpa_types.ContainerResourcePolicy{}, + expected: []vpa_model.ResourceName{vpa_model.ResourceCPU, vpa_model.ResourceMemory}, + }, { + name: "Nil policy - use default", + policy: nil, + expected: []vpa_model.ResourceName{vpa_model.ResourceCPU, vpa_model.ResourceMemory}, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + cs := vpa_model.NewAggregateContainerState() + cs.UpdateFromPolicy(tc.policy) + assert.Equal(t, tc.expected, cs.GetControlledResources()) + }) + } +} diff --git a/multidimensional-pod-autoscaler/pkg/recommender/model/cluster.go b/multidimensional-pod-autoscaler/pkg/recommender/model/cluster.go new file mode 100644 index 000000000000..67661a59c728 --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/recommender/model/cluster.go @@ -0,0 +1,507 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package model + +import ( + "context" + "fmt" + "time" + + apiv1 "k8s.io/api/core/v1" + labels "k8s.io/apimachinery/pkg/labels" + mpa_types "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1alpha1" + vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" + vpa_model "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/model" + controllerfetcher "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/target/controller_fetcher" + vpa_utils "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/vpa" + "k8s.io/klog/v2" +) + +const ( + // RecommendationMissingMaxDuration is maximum time that we accept the recommendation can be missing. + RecommendationMissingMaxDuration = 30 * time.Minute +) + +// ClusterState holds all runtime information about the cluster required for the +// VPA operations, i.e. configuration of resources (pods, containers, +// VPA objects), aggregated utilization of compute resources (CPU, memory) and +// events (container OOMs). +// All input to the VPA Recommender algorithm lives in this structure. +type ClusterState struct { + // Pods in the cluster. + Pods map[vpa_model.PodID]*PodState + + // MPA objects in the cluster. + Mpas map[MpaID]*Mpa + // MPA objects in the cluster that have no recommendation mapped to the first + // time we've noticed the recommendation missing or last time we logged + // a warning about it. + EmptyMPAs map[MpaID]time.Time + // Observed MPAs. Used to check if there are updates needed. + ObservedMpas []*mpa_types.MultidimPodAutoscaler + + // All container aggregations where the usage samples are stored. + aggregateStateMap aggregateContainerStatesMap + // Map with all label sets used by the aggregations. It serves as a cache + // that allows to quickly access labels.Set corresponding to a labelSetKey. + labelSetMap labelSetMap + + lastAggregateContainerStateGC time.Time + gcInterval time.Duration +} + +// StateMapSize is the number of pods being tracked by the VPA +func (cluster *ClusterState) StateMapSize() int { + return len(cluster.aggregateStateMap) +} + +// String representation of the labels.LabelSet. This is the value returned by +// labelSet.String(). As opposed to the LabelSet object, it can be used as a map key. +type labelSetKey string + +// Map of label sets keyed by their string representation. +type labelSetMap map[labelSetKey]labels.Set + +// AggregateContainerStatesMap is a map from AggregateStateKey to AggregateContainerState. +type aggregateContainerStatesMap map[vpa_model.AggregateStateKey]*vpa_model.AggregateContainerState + +// PodState holds runtime information about a single Pod. +type PodState struct { + // Unique id of the Pod. + ID vpa_model.PodID + // Set of labels attached to the Pod. + labelSetKey labelSetKey + // Containers that belong to the Pod, keyed by the container name. + Containers map[string]*ContainerState + // PodPhase describing current life cycle phase of the Pod. + Phase apiv1.PodPhase +} + +// NewClusterState returns a new ClusterState with no pods. +func NewClusterState(gcInterval time.Duration) *ClusterState { + return &ClusterState{ + Pods: make(map[vpa_model.PodID]*PodState), + Mpas: make(map[MpaID]*Mpa), + EmptyMPAs: make(map[MpaID]time.Time), + aggregateStateMap: make(aggregateContainerStatesMap), + labelSetMap: make(labelSetMap), + lastAggregateContainerStateGC: time.Unix(0, 0), + gcInterval: gcInterval, + } +} + +// ContainerUsageSampleWithKey holds a ContainerUsageSample together with the +// ID of the container it belongs to. +type ContainerUsageSampleWithKey struct { + vpa_model.ContainerUsageSample + Container vpa_model.ContainerID +} + +// AddOrUpdatePod updates the state of the pod with a given PodID, if it is +// present in the cluster object. Otherwise a new pod is created and added to +// the Cluster object. +// If the labels of the pod have changed, it updates the links between the containers +// and the aggregations. +func (cluster *ClusterState) AddOrUpdatePod(podID vpa_model.PodID, newLabels labels.Set, phase apiv1.PodPhase) { + pod, podExists := cluster.Pods[podID] + if !podExists { + pod = newPod(podID) + cluster.Pods[podID] = pod + } + + newlabelSetKey := cluster.getLabelSetKey(newLabels) + if podExists && pod.labelSetKey != newlabelSetKey { + // This Pod is already counted in the old MPA, remove the link. + cluster.removePodFromItsMpa(pod) + } + if !podExists || pod.labelSetKey != newlabelSetKey { + pod.labelSetKey = newlabelSetKey + // Set the links between the containers and aggregations based on the current pod labels. + for containerName, container := range pod.Containers { + containerID := vpa_model.ContainerID{PodID: podID, ContainerName: containerName} + container.aggregator = cluster.findOrCreateAggregateContainerState(containerID) + } + + cluster.addPodToItsMpa(pod) + } + pod.Phase = phase +} + +// addPodToItsVpa increases the count of Pods associated with a MPA object. +// Does a scan similar to findOrCreateAggregateContainerState so could be optimized if needed. +func (cluster *ClusterState) addPodToItsMpa(pod *PodState) { + for _, mpa := range cluster.Mpas { + if vpa_utils.PodLabelsMatchVPA(pod.ID.Namespace, cluster.labelSetMap[pod.labelSetKey], mpa.ID.Namespace, mpa.PodSelector) { + mpa.PodCount++ + } + } +} + +// removePodFromItsVpa decreases the count of Pods associated with a VPA object. +func (cluster *ClusterState) removePodFromItsMpa(pod *PodState) { + for _, mpa := range cluster.Mpas { + if vpa_utils.PodLabelsMatchVPA(pod.ID.Namespace, cluster.labelSetMap[pod.labelSetKey], mpa.ID.Namespace, mpa.PodSelector) { + mpa.PodCount-- + } + } +} + +// GetContainer returns the ContainerState object for a given ContainerID or +// null if it's not present in the model. +func (cluster *ClusterState) GetContainer(containerID vpa_model.ContainerID) *ContainerState { + pod, podExists := cluster.Pods[containerID.PodID] + if podExists { + container, containerExists := pod.Containers[containerID.ContainerName] + if containerExists { + return container + } + } + return nil +} + +// DeletePod removes an existing pod from the cluster. +func (cluster *ClusterState) DeletePod(podID vpa_model.PodID) { + pod, found := cluster.Pods[podID] + if found { + cluster.removePodFromItsMpa(pod) + } + delete(cluster.Pods, podID) +} + +// AddOrUpdateContainer creates a new container with the given ContainerID and +// adds it to the parent pod in the ClusterState object, if not yet present. +// Requires the pod to be added to the ClusterState first. Otherwise an error is +// returned. +func (cluster *ClusterState) AddOrUpdateContainer(containerID vpa_model.ContainerID, request vpa_model.Resources) error { + pod, podExists := cluster.Pods[containerID.PodID] + if !podExists { + return vpa_model.NewKeyError(containerID.PodID) + } + if container, containerExists := pod.Containers[containerID.ContainerName]; !containerExists { + cluster.findOrCreateAggregateContainerState(containerID) + pod.Containers[containerID.ContainerName] = NewContainerState(request, NewContainerStateAggregatorProxy(cluster, containerID)) + } else { + // Container aleady exists. Possibly update the request. + container.Request = request + } + return nil +} + +// AddSample adds a new usage sample to the proper container in the ClusterState +// object. Requires the container as well as the parent pod to be added to the +// ClusterState first. Otherwise an error is returned. +func (cluster *ClusterState) AddSample(sample *ContainerUsageSampleWithKey) error { + pod, podExists := cluster.Pods[sample.Container.PodID] + if !podExists { + return vpa_model.NewKeyError(sample.Container.PodID) + } + containerState, containerExists := pod.Containers[sample.Container.ContainerName] + if !containerExists { + return vpa_model.NewKeyError(sample.Container) + } + if !containerState.AddSample(&sample.ContainerUsageSample) { + return fmt.Errorf("sample discarded (invalid or out of order)") + } + return nil +} + +// RecordOOM adds info regarding OOM event in the model as an artificial memory sample. +func (cluster *ClusterState) RecordOOM(containerID vpa_model.ContainerID, timestamp time.Time, requestedMemory vpa_model.ResourceAmount) error { + pod, podExists := cluster.Pods[containerID.PodID] + if !podExists { + return vpa_model.NewKeyError(containerID.PodID) + } + containerState, containerExists := pod.Containers[containerID.ContainerName] + if !containerExists { + return vpa_model.NewKeyError(containerID.ContainerName) + } + err := containerState.RecordOOM(timestamp, requestedMemory) + if err != nil { + return fmt.Errorf("error while recording OOM for %v, Reason: %v", containerID, err) + } + return nil +} + +// AddOrUpdateMpa adds a new MPA with a given ID to the ClusterState if it +// didn't yet exist. If the MPA already existed but had a different pod +// selector, the pod selector is updated. Updates the links between the MPA and +// all aggregations it matches. +func (cluster *ClusterState) AddOrUpdateMpa(apiObject *mpa_types.MultidimPodAutoscaler, selector labels.Selector) error { + mpaID := MpaID{Namespace: apiObject.Namespace, MpaName: apiObject.Name} + annotationsMap := apiObject.Annotations + conditionsMap := make(mpaConditionsMap) + for _, condition := range apiObject.Status.Conditions { + conditionsMap[condition.Type] = condition + } + var currentRecommendation *vpa_types.RecommendedPodResources + if conditionsMap[mpa_types.RecommendationProvided].Status == apiv1.ConditionTrue { + currentRecommendation = apiObject.Status.Recommendation + } + + mpa, mpaExists := cluster.Mpas[mpaID] + if mpaExists && (mpa.PodSelector.String() != selector.String()) { + // Pod selector was changed. Delete the MPA object and recreate + // it with the new selector. + if err := cluster.DeleteMpa(mpaID); err != nil { + return err + } + mpaExists = false + } + if !mpaExists { + mpa = NewMpa(mpaID, selector, apiObject.CreationTimestamp.Time) + cluster.Mpas[mpaID] = mpa + for aggregationKey, aggregation := range cluster.aggregateStateMap { + mpa.UseAggregationIfMatching(aggregationKey, aggregation) + } + mpa.PodCount = len(cluster.GetMatchingPods(mpa)) + } + mpa.ScaleTargetRef = apiObject.Spec.ScaleTargetRef + mpa.Annotations = annotationsMap + mpa.Conditions = conditionsMap + mpa.Recommendation = currentRecommendation + mpa.SetUpdateMode(apiObject.Spec.Policy) + mpa.SetResourcePolicy(apiObject.Spec.ResourcePolicy) + + // For HPA-related fields. + // mpa.SetHPAConstraints(apiObject.Spec.Metrics, *apiObject.Spec.Constraints.MinReplicas, *apiObject.Spec.Constraints.MaxReplicas, apiObject.Spec.Constraints.Behavior) + return nil +} + +// DeleteMpa removes a MPA with the given ID from the ClusterState. +func (cluster *ClusterState) DeleteMpa(mpaID MpaID) error { + mpa, mpaExists := cluster.Mpas[mpaID] + if !mpaExists { + return vpa_model.NewKeyError(mpaID) + } + for _, state := range mpa.aggregateContainerStates { + state.MarkNotAutoscaled() + } + delete(cluster.Mpas, mpaID) + delete(cluster.EmptyMPAs, mpaID) + return nil +} + +func newPod(id vpa_model.PodID) *PodState { + return &PodState{ + ID: id, + Containers: make(map[string]*ContainerState), + } +} + +// getLabelSetKey puts the given labelSet in the global labelSet map and returns a +// corresponding labelSetKey. +func (cluster *ClusterState) getLabelSetKey(labelSet labels.Set) labelSetKey { + labelSetKey := labelSetKey(labelSet.String()) + cluster.labelSetMap[labelSetKey] = labelSet + return labelSetKey +} + +// MakeAggregateStateKey returns the AggregateStateKey that should be used +// to aggregate usage samples from a container with the given name in a given pod. +func (cluster *ClusterState) MakeAggregateStateKey(pod *PodState, containerName string) vpa_model.AggregateStateKey { + return aggregateStateKey{ + namespace: pod.ID.Namespace, + containerName: containerName, + labelSetKey: pod.labelSetKey, + labelSetMap: &cluster.labelSetMap, + } +} + +// aggregateStateKeyForContainerID returns the AggregateStateKey for the ContainerID. +// The pod with the corresponding PodID must already be present in the ClusterState. +func (cluster *ClusterState) aggregateStateKeyForContainerID(containerID vpa_model.ContainerID) vpa_model.AggregateStateKey { + pod, podExists := cluster.Pods[containerID.PodID] + if !podExists { + panic(fmt.Sprintf("Pod not present in the ClusterState: %v", containerID.PodID)) + } + return cluster.MakeAggregateStateKey(pod, containerID.ContainerName) +} + +// findOrCreateAggregateContainerState returns (possibly newly created) AggregateContainerState +// that should be used to aggregate usage samples from container with a given ID. +// The pod with the corresponding PodID must already be present in the ClusterState. +func (cluster *ClusterState) findOrCreateAggregateContainerState(containerID vpa_model.ContainerID) *vpa_model.AggregateContainerState { + aggregateStateKey := cluster.aggregateStateKeyForContainerID(containerID) + aggregateContainerState, aggregateStateExists := cluster.aggregateStateMap[aggregateStateKey] + if !aggregateStateExists { + aggregateContainerState = vpa_model.NewAggregateContainerState() + cluster.aggregateStateMap[aggregateStateKey] = aggregateContainerState + // Link the new aggregation to the existing VPAs. + for _, mpa := range cluster.Mpas { + mpa.UseAggregationIfMatching(aggregateStateKey, aggregateContainerState) + } + } + return aggregateContainerState +} + +// garbageCollectAggregateCollectionStates removes obsolete AggregateCollectionStates from the ClusterState. +// AggregateCollectionState is obsolete in following situations: +// 1) It has no samples and there are no more contributive pods - a pod is contributive in any of following situations: +// +// a) It is in an active state - i.e. not PodSucceeded nor PodFailed. +// b) Its associated controller (e.g. Deployment) still exists. +// +// 2) The last sample is too old to give meaningful recommendation (>8 days), +// 3) There are no samples and the aggregate state was created >8 days ago. +func (cluster *ClusterState) garbageCollectAggregateCollectionStates(ctx context.Context, now time.Time, controllerFetcher controllerfetcher.ControllerFetcher) { + klog.V(1).Info("Garbage collection of AggregateCollectionStates triggered") + keysToDelete := make([]vpa_model.AggregateStateKey, 0) + contributiveKeys := cluster.getContributiveAggregateStateKeys(ctx, controllerFetcher) + for key, aggregateContainerState := range cluster.aggregateStateMap { + isKeyContributive := contributiveKeys[key] + if !isKeyContributive && isStateEmpty(aggregateContainerState) { + keysToDelete = append(keysToDelete, key) + klog.V(1).Infof("Removing empty and not contributive AggregateCollectionState for %+v", key) + continue + } + if isStateExpired(aggregateContainerState, now) { + keysToDelete = append(keysToDelete, key) + klog.V(1).Infof("Removing expired AggregateCollectionState for %+v", key) + } + } + for _, key := range keysToDelete { + delete(cluster.aggregateStateMap, key) + for _, mpa := range cluster.Mpas { + mpa.DeleteAggregation(key) + } + } +} + +// RateLimitedGarbageCollectAggregateCollectionStates removes obsolete AggregateCollectionStates from the ClusterState. +// It performs clean up only if more than `gcInterval` passed since the last time it performed a clean up. +// AggregateCollectionState is obsolete in following situations: +// 1) It has no samples and there are no more contributive pods - a pod is contributive in any of following situations: +// +// a) It is in an active state - i.e. not PodSucceeded nor PodFailed. +// b) Its associated controller (e.g. Deployment) still exists. +// +// 2) The last sample is too old to give meaningful recommendation (>8 days), +// 3) There are no samples and the aggregate state was created >8 days ago. +func (cluster *ClusterState) RateLimitedGarbageCollectAggregateCollectionStates(ctx context.Context, now time.Time, controllerFetcher controllerfetcher.ControllerFetcher) { + if now.Sub(cluster.lastAggregateContainerStateGC) < cluster.gcInterval { + return + } + cluster.garbageCollectAggregateCollectionStates(ctx, now, controllerFetcher) + cluster.lastAggregateContainerStateGC = now +} + +func (cluster *ClusterState) getContributiveAggregateStateKeys(ctx context.Context, controllerFetcher controllerfetcher.ControllerFetcher) map[vpa_model.AggregateStateKey]bool { + contributiveKeys := map[vpa_model.AggregateStateKey]bool{} + for _, pod := range cluster.Pods { + // Pod is considered contributive in any of following situations: + // 1) It is in active state - i.e. not PodSucceeded nor PodFailed. + // 2) Its associated controller (e.g. Deployment) still exists. + podControllerExists := cluster.GetControllerForPodUnderVPA(ctx, pod, controllerFetcher) != nil + podActive := pod.Phase != apiv1.PodSucceeded && pod.Phase != apiv1.PodFailed + if podActive || podControllerExists { + for container := range pod.Containers { + contributiveKeys[cluster.MakeAggregateStateKey(pod, container)] = true + } + } + } + return contributiveKeys +} + +// RecordRecommendation marks the state of recommendation in the cluster. We +// keep track of empty recommendations and log information about them +// periodically. +func (cluster *ClusterState) RecordRecommendation(mpa *Mpa, now time.Time) error { + if mpa.Recommendation != nil && len(mpa.Recommendation.ContainerRecommendations) > 0 { + delete(cluster.EmptyMPAs, mpa.ID) + return nil + } + lastLogged, ok := cluster.EmptyMPAs[mpa.ID] + if !ok { + cluster.EmptyMPAs[mpa.ID] = now + } else { + if lastLogged.Add(RecommendationMissingMaxDuration).Before(now) { + cluster.EmptyMPAs[mpa.ID] = now + return fmt.Errorf("MPA %v/%v is missing recommendation for more than %v", mpa.ID.Namespace, mpa.ID.MpaName, RecommendationMissingMaxDuration) + } + } + return nil +} + +// GetMatchingPods returns a list of currently active pods that match the +// given MPA. Traverses through all pods in the cluster - use sparingly. +func (cluster *ClusterState) GetMatchingPods(mpa *Mpa) []vpa_model.PodID { + matchingPods := []vpa_model.PodID{} + for podID, pod := range cluster.Pods { + if vpa_utils.PodLabelsMatchVPA(podID.Namespace, cluster.labelSetMap[pod.labelSetKey], + mpa.ID.Namespace, mpa.PodSelector) { + matchingPods = append(matchingPods, podID) + } + } + return matchingPods +} + +// GetControllerForPodUnderVPA returns controller associated with given Pod. Returns nil if Pod is not controlled by a VPA object. +func (cluster *ClusterState) GetControllerForPodUnderVPA(ctx context.Context, pod *PodState, controllerFetcher controllerfetcher.ControllerFetcher) *controllerfetcher.ControllerKeyWithAPIVersion { + controllingMPA := cluster.GetControllingMPA(pod) + if controllingMPA != nil { + controller := &controllerfetcher.ControllerKeyWithAPIVersion{ + ControllerKey: controllerfetcher.ControllerKey{ + Namespace: controllingMPA.ID.Namespace, + Kind: controllingMPA.ScaleTargetRef.Kind, + Name: controllingMPA.ScaleTargetRef.Name, + }, + ApiVersion: controllingMPA.ScaleTargetRef.APIVersion, + } + topLevelController, _ := controllerFetcher.FindTopMostWellKnownOrScalable(ctx, controller) + return topLevelController + } + return nil +} + +// GetControllingMPA returns a VPA object controlling given Pod. +func (cluster *ClusterState) GetControllingMPA(pod *PodState) *Mpa { + for _, mpa := range cluster.Mpas { + if vpa_utils.PodLabelsMatchVPA(pod.ID.Namespace, cluster.labelSetMap[pod.labelSetKey], + mpa.ID.Namespace, mpa.PodSelector) { + return mpa + } + } + return nil +} + +// Implementation of the AggregateStateKey interface. It can be used as a map key. +type aggregateStateKey struct { + namespace string + containerName string + labelSetKey labelSetKey + // Pointer to the global map from labelSetKey to labels.Set. + // Note: a pointer is used so that two copies of the same key are equal. + labelSetMap *labelSetMap +} + +// Labels returns the namespace for the aggregateStateKey. +func (k aggregateStateKey) Namespace() string { + return k.namespace +} + +// ContainerName returns the name of the container for the aggregateStateKey. +func (k aggregateStateKey) ContainerName() string { + return k.containerName +} + +// Labels returns the set of labels for the aggregateStateKey. +func (k aggregateStateKey) Labels() labels.Labels { + if k.labelSetMap == nil { + return labels.Set{} + } + return (*k.labelSetMap)[k.labelSetKey] +} diff --git a/multidimensional-pod-autoscaler/pkg/recommender/model/cluster_test.go b/multidimensional-pod-autoscaler/pkg/recommender/model/cluster_test.go new file mode 100644 index 000000000000..81140b6dd3a9 --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/recommender/model/cluster_test.go @@ -0,0 +1,913 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package model + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/assert" + autoscaling "k8s.io/api/autoscaling/v1" + apiv1 "k8s.io/api/core/v1" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + labels "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime/schema" + mpa_types "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1alpha1" + "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/utils/test" + vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" + vpa_model "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/model" + controllerfetcher "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/target/controller_fetcher" + "k8s.io/client-go/restmapper" + "k8s.io/client-go/scale" + "k8s.io/klog/v2" +) + +var ( + testPodID = vpa_model.PodID{Namespace: "namespace-1", PodName: "pod-1"} + testPodID3 = vpa_model.PodID{Namespace: "namespace-1", PodName: "pod-3"} + testPodID4 = vpa_model.PodID{Namespace: "namespace-1", PodName: "pod-4"} + testContainerID = vpa_model.ContainerID{PodID: testPodID, ContainerName: "container-1"} + testMpaID = MpaID{"namespace-1", "mpa-1"} + testAnnotations = mpaAnnotationsMap{"key-1": "value-1"} + testLabels = map[string]string{"label-1": "value-1"} + emptyLabels = map[string]string{} + testSelectorStr = "label-1 = value-1" + testTargetRef = &autoscaling.CrossVersionObjectReference{ + Kind: "kind-1", + Name: "name-1", + APIVersion: "apiVersion-1", + } + testControllerKey = &controllerfetcher.ControllerKeyWithAPIVersion{ + ControllerKey: controllerfetcher.ControllerKey{ + Kind: "kind-1", + Name: "name-1", + Namespace: "namespace-1", + }, + ApiVersion: "apiVersion-1", + } + testControllerFetcher = &fakeControllerFetcher{ + key: testControllerKey, + err: nil, + } +) + +type fakeControllerFetcher struct { + key *controllerfetcher.ControllerKeyWithAPIVersion + err error + mapper restmapper.DeferredDiscoveryRESTMapper + scaleNamespacer scale.ScalesGetter +} + +func (f *fakeControllerFetcher) GetRESTMappings(groupKind schema.GroupKind) ([]*apimeta.RESTMapping, error) { + return f.mapper.RESTMappings(groupKind) +} + +func (f *fakeControllerFetcher) Scales(namespace string) scale.ScaleInterface { + return f.scaleNamespacer.Scales(namespace) +} + +func (f *fakeControllerFetcher) FindTopMostWellKnownOrScalable(_ context.Context, controller *controllerfetcher.ControllerKeyWithAPIVersion) (*controllerfetcher.ControllerKeyWithAPIVersion, error) { + return f.key, f.err +} + +const testGcPeriod = time.Minute + +func makeTestUsageSample() *ContainerUsageSampleWithKey { + return &ContainerUsageSampleWithKey{vpa_model.ContainerUsageSample{ + MeasureStart: testTimestamp, + Usage: 1.0, + Request: testRequest[vpa_model.ResourceCPU], + Resource: vpa_model.ResourceCPU}, + testContainerID} +} + +func TestClusterAddSample(t *testing.T) { + // Create a pod with a single container. + cluster := NewClusterState(testGcPeriod) + cluster.AddOrUpdatePod(testPodID, testLabels, apiv1.PodRunning) + assert.NoError(t, cluster.AddOrUpdateContainer(testContainerID, testRequest)) + + // Add a usage sample to the container. + assert.NoError(t, cluster.AddSample(makeTestUsageSample())) + + // Verify that the sample was aggregated into the container stats. + containerStats := cluster.Pods[testPodID].Containers["container-1"] + assert.Equal(t, testTimestamp, containerStats.LastCPUSampleStart) +} + +func TestClusterGCAggregateContainerStateDeletesOld(t *testing.T) { + ctx := context.Background() + + // Create a pod with a single container. + cluster := NewClusterState(testGcPeriod) + mpa := addTestMpa(cluster) + addTestPod(cluster) + + assert.NoError(t, cluster.AddOrUpdateContainer(testContainerID, testRequest)) + usageSample := makeTestUsageSample() + + // Add a usage sample to the container. + assert.NoError(t, cluster.AddSample(usageSample)) + + assert.NotEmpty(t, cluster.aggregateStateMap) + assert.NotEmpty(t, mpa.aggregateContainerStates) + + // AggegateContainerState are valid for 8 days since last sample + cluster.garbageCollectAggregateCollectionStates(ctx, usageSample.MeasureStart.Add(9*24*time.Hour), testControllerFetcher) + + // AggegateContainerState should be deleted from both cluster and mpa + assert.Empty(t, cluster.aggregateStateMap) + assert.Empty(t, mpa.aggregateContainerStates) +} + +func TestClusterGCAggregateContainerStateDeletesOldEmpty(t *testing.T) { + ctx := context.Background() + + // Create a pod with a single container. + cluster := NewClusterState(testGcPeriod) + mpa := addTestMpa(cluster) + addTestPod(cluster) + + assert.NoError(t, cluster.AddOrUpdateContainer(testContainerID, testRequest)) + // No usage samples added. + + assert.NotEmpty(t, cluster.aggregateStateMap) + assert.NotEmpty(t, mpa.aggregateContainerStates) + + assert.Len(t, cluster.aggregateStateMap, 1) + var creationTime time.Time + for _, aggregateState := range cluster.aggregateStateMap { + creationTime = aggregateState.CreationTime + } + + // Verify empty aggregate states are not removed right away. + cluster.garbageCollectAggregateCollectionStates(ctx, creationTime.Add(1*time.Minute), testControllerFetcher) // AggegateContainerState should be deleted from both cluster and mpa + assert.NotEmpty(t, cluster.aggregateStateMap) + assert.NotEmpty(t, mpa.aggregateContainerStates) + + // AggegateContainerState are valid for 8 days since creation + cluster.garbageCollectAggregateCollectionStates(ctx, creationTime.Add(9*24*time.Hour), testControllerFetcher) + + // AggegateContainerState should be deleted from both cluster and mpa + assert.Empty(t, cluster.aggregateStateMap) + assert.Empty(t, mpa.aggregateContainerStates) +} + +func TestClusterGCAggregateContainerStateDeletesEmptyInactiveWithoutController(t *testing.T) { + ctx := context.Background() + + // Create a pod with a single container. + cluster := NewClusterState(testGcPeriod) + mpa := addTestMpa(cluster) + pod := addTestPod(cluster) + // Controller Fetcher returns nil, meaning that there is no corresponding controller alive. + controller := &fakeControllerFetcher{ + key: nil, + err: nil, + } + + assert.NoError(t, cluster.AddOrUpdateContainer(testContainerID, testRequest)) + // No usage samples added. + + assert.NotEmpty(t, cluster.aggregateStateMap) + assert.NotEmpty(t, mpa.aggregateContainerStates) + + cluster.garbageCollectAggregateCollectionStates(ctx, testTimestamp, controller) + + // AggegateContainerState should not be deleted as the pod is still active. + assert.NotEmpty(t, cluster.aggregateStateMap) + assert.NotEmpty(t, mpa.aggregateContainerStates) + + cluster.Pods[pod.ID].Phase = apiv1.PodSucceeded + cluster.garbageCollectAggregateCollectionStates(ctx, testTimestamp, controller) + + // AggegateContainerState should be empty as the pod is no longer active, controller is not alive + // and there are no usage samples. + assert.Empty(t, cluster.aggregateStateMap) + assert.Empty(t, mpa.aggregateContainerStates) +} + +func TestClusterGCAggregateContainerStateLeavesEmptyInactiveWithController(t *testing.T) { + ctx := context.Background() + + // Create a pod with a single container. + cluster := NewClusterState(testGcPeriod) + mpa := addTestMpa(cluster) + pod := addTestPod(cluster) + // Controller Fetcher returns existing controller, meaning that there is a corresponding controller alive. + controller := testControllerFetcher + + assert.NoError(t, cluster.AddOrUpdateContainer(testContainerID, testRequest)) + // No usage samples added. + + assert.NotEmpty(t, cluster.aggregateStateMap) + assert.NotEmpty(t, mpa.aggregateContainerStates) + + cluster.garbageCollectAggregateCollectionStates(ctx, testTimestamp, controller) + + // AggegateContainerState should not be deleted as the pod is still active. + assert.NotEmpty(t, cluster.aggregateStateMap) + assert.NotEmpty(t, mpa.aggregateContainerStates) + + cluster.Pods[pod.ID].Phase = apiv1.PodSucceeded + cluster.garbageCollectAggregateCollectionStates(ctx, testTimestamp, controller) + + // AggegateContainerState should not be delated as the controller is still alive. + assert.NotEmpty(t, cluster.aggregateStateMap) + assert.NotEmpty(t, mpa.aggregateContainerStates) +} + +func TestClusterGCAggregateContainerStateLeavesValid(t *testing.T) { + ctx := context.Background() + + // Create a pod with a single container. + cluster := NewClusterState(testGcPeriod) + mpa := addTestMpa(cluster) + addTestPod(cluster) + + assert.NoError(t, cluster.AddOrUpdateContainer(testContainerID, testRequest)) + usageSample := makeTestUsageSample() + + // Add a usage sample to the container. + assert.NoError(t, cluster.AddSample(usageSample)) + + assert.NotEmpty(t, cluster.aggregateStateMap) + assert.NotEmpty(t, mpa.aggregateContainerStates) + + // AggegateContainerState are valid for 8 days since last sample + cluster.garbageCollectAggregateCollectionStates(ctx, usageSample.MeasureStart.Add(7*24*time.Hour), testControllerFetcher) + + assert.NotEmpty(t, cluster.aggregateStateMap) + assert.NotEmpty(t, mpa.aggregateContainerStates) +} + +func TestAddSampleAfterAggregateContainerStateGCed(t *testing.T) { + ctx := context.Background() + + // Create a pod with a single container. + cluster := NewClusterState(testGcPeriod) + mpa := addTestMpa(cluster) + pod := addTestPod(cluster) + addTestContainer(t, cluster) + + assert.NoError(t, cluster.AddOrUpdateContainer(testContainerID, testRequest)) + usageSample := makeTestUsageSample() + + // Add a usage sample to the container. + assert.NoError(t, cluster.AddSample(usageSample)) + + assert.NotEmpty(t, cluster.aggregateStateMap) + assert.NotEmpty(t, mpa.aggregateContainerStates) + + aggregateStateKey := cluster.aggregateStateKeyForContainerID(testContainerID) + assert.Contains(t, mpa.aggregateContainerStates, aggregateStateKey) + + // AggegateContainerState are invalid after 8 days since last sample + gcTimestamp := usageSample.MeasureStart.Add(10 * 24 * time.Hour) + cluster.garbageCollectAggregateCollectionStates(ctx, gcTimestamp, testControllerFetcher) + + assert.Empty(t, cluster.aggregateStateMap) + assert.Empty(t, mpa.aggregateContainerStates) + assert.Contains(t, pod.Containers, testContainerID.ContainerName) + + newUsageSample := &ContainerUsageSampleWithKey{vpa_model.ContainerUsageSample{ + MeasureStart: gcTimestamp.Add(1 * time.Hour), + Usage: usageSample.Usage, + Request: usageSample.Request, + Resource: usageSample.Resource}, + testContainerID} + // Add usage sample to the container again. + assert.NoError(t, cluster.AddSample(newUsageSample)) + + assert.Contains(t, mpa.aggregateContainerStates, aggregateStateKey) +} + +func TestClusterGCRateLimiting(t *testing.T) { + ctx := context.Background() + + // Create a pod with a single container. + cluster := NewClusterState(testGcPeriod) + usageSample := makeTestUsageSample() + sampleExpireTime := usageSample.MeasureStart.Add(9 * 24 * time.Hour) + // AggegateContainerState are valid for 8 days since last sample but this run + // doesn't remove the sample, because we didn't add it yet. + cluster.RateLimitedGarbageCollectAggregateCollectionStates(ctx, sampleExpireTime, testControllerFetcher) + mpa := addTestMpa(cluster) + addTestPod(cluster) + assert.NoError(t, cluster.AddOrUpdateContainer(testContainerID, testRequest)) + + // Add a usage sample to the container. + assert.NoError(t, cluster.AddSample(usageSample)) + assert.NotEmpty(t, cluster.aggregateStateMap) + assert.NotEmpty(t, mpa.aggregateContainerStates) + + // Sample is expired but this run doesn't remove it yet, because less than testGcPeriod + // elapsed since the previous run. + cluster.RateLimitedGarbageCollectAggregateCollectionStates(ctx, sampleExpireTime.Add(testGcPeriod/2), testControllerFetcher) + assert.NotEmpty(t, cluster.aggregateStateMap) + assert.NotEmpty(t, mpa.aggregateContainerStates) + + // AggegateContainerState should be deleted from both cluster and mpa + cluster.RateLimitedGarbageCollectAggregateCollectionStates(ctx, sampleExpireTime.Add(2*testGcPeriod), testControllerFetcher) + assert.Empty(t, cluster.aggregateStateMap) + assert.Empty(t, mpa.aggregateContainerStates) +} + +func TestClusterRecordOOM(t *testing.T) { + // Create a pod with a single container. + cluster := NewClusterState(testGcPeriod) + cluster.AddOrUpdatePod(testPodID, testLabels, apiv1.PodRunning) + assert.NoError(t, cluster.AddOrUpdateContainer(testContainerID, testRequest)) + + // RecordOOM + assert.NoError(t, cluster.RecordOOM(testContainerID, time.Unix(0, 0), vpa_model.ResourceAmount(10))) + + // Verify that OOM was aggregated into the aggregated stats. + aggregation := cluster.findOrCreateAggregateContainerState(testContainerID) + assert.NotEmpty(t, aggregation.AggregateMemoryPeaks) +} + +// Verifies that AddSample and AddOrUpdateContainer methods return a proper +// KeyError when referring to a non-existent pod. +func TestMissingKeys(t *testing.T) { + cluster := NewClusterState(testGcPeriod) + err := cluster.AddSample(makeTestUsageSample()) + assert.EqualError(t, err, "KeyError: {namespace-1 pod-1}") + + err = cluster.RecordOOM(testContainerID, time.Unix(0, 0), vpa_model.ResourceAmount(10)) + assert.EqualError(t, err, "KeyError: {namespace-1 pod-1}") + + err = cluster.AddOrUpdateContainer(testContainerID, testRequest) + assert.EqualError(t, err, "KeyError: {namespace-1 pod-1}") +} + +func addMpa(cluster *ClusterState, id MpaID, annotations mpaAnnotationsMap, selector string, scaleTargetRef *autoscaling.CrossVersionObjectReference) *Mpa { + apiObject := test.MultidimPodAutoscaler().WithNamespace(id.Namespace). + WithName(id.MpaName).WithContainer(testContainerID.ContainerName).WithAnnotations(annotations).WithScaleTargetRef(scaleTargetRef).Get() + return addMpaObject(cluster, id, apiObject, selector) +} + +func addMpaObject(cluster *ClusterState, id MpaID, mpa *mpa_types.MultidimPodAutoscaler, selector string) *Mpa { + labelSelector, _ := metav1.ParseToLabelSelector(selector) + parsedSelector, _ := metav1.LabelSelectorAsSelector(labelSelector) + err := cluster.AddOrUpdateMpa(mpa, parsedSelector) + if err != nil { + klog.Fatalf("AddOrUpdateMpa() failed: %v", err) + } + return cluster.Mpas[id] +} + +func addTestMpa(cluster *ClusterState) *Mpa { + return addMpa(cluster, testMpaID, testAnnotations, testSelectorStr, testTargetRef) +} + +func addTestPod(cluster *ClusterState) *PodState { + cluster.AddOrUpdatePod(testPodID, testLabels, apiv1.PodRunning) + return cluster.Pods[testPodID] +} + +func addTestContainer(t *testing.T, cluster *ClusterState) *ContainerState { + err := cluster.AddOrUpdateContainer(testContainerID, testRequest) + assert.NoError(t, err) + return cluster.GetContainer(testContainerID) +} + +// Creates a MPA followed by a matching pod. Verifies that the links between +// MPA, the container and the aggregation are set correctly. +func TestAddMpaThenAddPod(t *testing.T) { + cluster := NewClusterState(testGcPeriod) + mpa := addTestMpa(cluster) + assert.Empty(t, mpa.aggregateContainerStates) + addTestPod(cluster) + addTestContainer(t, cluster) + aggregateStateKey := cluster.aggregateStateKeyForContainerID(testContainerID) + assert.Contains(t, mpa.aggregateContainerStates, aggregateStateKey) +} + +// Creates a pod followed by a matching MPA. Verifies that the links between +// MPA, the container and the aggregation are set correctly. +func TestAddPodThenAddMpa(t *testing.T) { + cluster := NewClusterState(testGcPeriod) + addTestPod(cluster) + addTestContainer(t, cluster) + mpa := addTestMpa(cluster) + aggregateStateKey := cluster.aggregateStateKeyForContainerID(testContainerID) + assert.Contains(t, mpa.aggregateContainerStates, aggregateStateKey) +} + +// Creates a MPA and a matching pod, then change the pod labels such that it is +// no longer matched by the MPA. Verifies that the links between the pod and the +// MPA are removed. +func TestChangePodLabels(t *testing.T) { + cluster := NewClusterState(testGcPeriod) + mpa := addTestMpa(cluster) + addTestPod(cluster) + addTestContainer(t, cluster) + aggregateStateKey := cluster.aggregateStateKeyForContainerID(testContainerID) + assert.Contains(t, mpa.aggregateContainerStates, aggregateStateKey) + // Update Pod labels to no longer match the MPA. + cluster.AddOrUpdatePod(testPodID, emptyLabels, apiv1.PodRunning) + aggregateStateKey = cluster.aggregateStateKeyForContainerID(testContainerID) + assert.NotContains(t, mpa.aggregateContainerStates, aggregateStateKey) +} + +// Creates a MPA and verifies that annotation updates work properly. +func TestUpdateAnnotations(t *testing.T) { + cluster := NewClusterState(testGcPeriod) + mpa := addTestMpa(cluster) + // Verify that the annotations match the test annotations. + assert.Equal(t, mpa.Annotations, testAnnotations) + // Update the annotations (non-empty). + annotations := mpaAnnotationsMap{"key-2": "value-2"} + mpa = addMpa(cluster, testMpaID, annotations, testSelectorStr, testTargetRef) + assert.Equal(t, mpa.Annotations, annotations) + // Update the annotations (empty). + annotations = mpaAnnotationsMap{} + mpa = addMpa(cluster, testMpaID, annotations, testSelectorStr, testTargetRef) + assert.Equal(t, mpa.Annotations, annotations) +} + +// Creates a MPA and a matching pod, then change the MPA pod selector 3 times: +// first such that it still matches the pod, then such that it no longer matches +// the pod, finally such that it matches the pod again. Verifies that the links +// between the pod and the MPA are updated correctly each time. +func TestUpdatePodSelector(t *testing.T) { + cluster := NewClusterState(testGcPeriod) + addTestPod(cluster) + addTestContainer(t, cluster) + + // Update the MPA selector such that it still matches the Pod. + mpa := addMpa(cluster, testMpaID, testAnnotations, "label-1 in (value-1,value-2)", testTargetRef) + assert.Contains(t, mpa.aggregateContainerStates, cluster.aggregateStateKeyForContainerID(testContainerID)) + + // Update the MPA selector to no longer match the Pod. + mpa = addMpa(cluster, testMpaID, testAnnotations, "label-1 = value-2", testTargetRef) + assert.NotContains(t, mpa.aggregateContainerStates, cluster.aggregateStateKeyForContainerID(testContainerID)) + + // Update the MPA selector to match the Pod again. + mpa = addMpa(cluster, testMpaID, testAnnotations, "label-1 = value-1", testTargetRef) + assert.Contains(t, mpa.aggregateContainerStates, cluster.aggregateStateKeyForContainerID(testContainerID)) +} + +// Test setting ResourcePolicy and UpdatePolicy on adding or updating MPA object +func TestAddOrUpdateMPAPolicies(t *testing.T) { + testMpaBuilder := test.MultidimPodAutoscaler().WithName(testMpaID.MpaName). + WithNamespace(testMpaID.Namespace).WithContainer(testContainerID.ContainerName) + updateModeAuto := vpa_types.UpdateModeAuto + updateModeOff := vpa_types.UpdateModeOff + scalingModeAuto := vpa_types.ContainerScalingModeAuto + scalingModeOff := vpa_types.ContainerScalingModeOff + cases := []struct { + name string + oldMpa *mpa_types.MultidimPodAutoscaler + newMpa *mpa_types.MultidimPodAutoscaler + resourcePolicy *vpa_types.PodResourcePolicy + expectedScalingMode *vpa_types.ContainerScalingMode + expectedUpdateMode *vpa_types.UpdateMode + }{ + { + name: "Defaults to auto", + oldMpa: nil, + newMpa: testMpaBuilder.WithUpdateMode(vpa_types.UpdateModeOff).Get(), + // Container scaling mode is a separate concept from update mode in the MPA object, + // hence the UpdateModeOff does not influence container scaling mode here. + expectedScalingMode: &scalingModeAuto, + expectedUpdateMode: &updateModeOff, + }, { + name: "Default scaling mode set to Off", + oldMpa: nil, + newMpa: testMpaBuilder.WithUpdateMode(vpa_types.UpdateModeAuto).Get(), + resourcePolicy: &vpa_types.PodResourcePolicy{ + ContainerPolicies: []vpa_types.ContainerResourcePolicy{ + { + ContainerName: "*", + Mode: &scalingModeOff, + }, + }, + }, + expectedScalingMode: &scalingModeOff, + expectedUpdateMode: &updateModeAuto, + }, { + name: "Explicit scaling mode set to Off", + oldMpa: nil, + newMpa: testMpaBuilder.WithUpdateMode(vpa_types.UpdateModeAuto).Get(), + resourcePolicy: &vpa_types.PodResourcePolicy{ + ContainerPolicies: []vpa_types.ContainerResourcePolicy{ + { + ContainerName: testContainerID.ContainerName, + Mode: &scalingModeOff, + }, + }, + }, + expectedScalingMode: &scalingModeOff, + expectedUpdateMode: &updateModeAuto, + }, { + name: "Other container has explicit scaling mode Off", + oldMpa: nil, + newMpa: testMpaBuilder.WithUpdateMode(vpa_types.UpdateModeAuto).Get(), + resourcePolicy: &vpa_types.PodResourcePolicy{ + ContainerPolicies: []vpa_types.ContainerResourcePolicy{ + { + ContainerName: "other-container", + Mode: &scalingModeOff, + }, + }, + }, + expectedScalingMode: &scalingModeAuto, + expectedUpdateMode: &updateModeAuto, + }, { + name: "Scaling mode to default Off", + oldMpa: testMpaBuilder.WithUpdateMode(vpa_types.UpdateModeAuto).Get(), + newMpa: testMpaBuilder.WithUpdateMode(vpa_types.UpdateModeAuto).Get(), + resourcePolicy: &vpa_types.PodResourcePolicy{ + ContainerPolicies: []vpa_types.ContainerResourcePolicy{ + { + ContainerName: "*", + Mode: &scalingModeOff, + }, + }, + }, + expectedScalingMode: &scalingModeOff, + expectedUpdateMode: &updateModeAuto, + }, { + name: "Scaling mode to explicit Off", + oldMpa: testMpaBuilder.WithUpdateMode(vpa_types.UpdateModeAuto).Get(), + newMpa: testMpaBuilder.WithUpdateMode(vpa_types.UpdateModeAuto).Get(), + resourcePolicy: &vpa_types.PodResourcePolicy{ + ContainerPolicies: []vpa_types.ContainerResourcePolicy{ + { + ContainerName: testContainerID.ContainerName, + Mode: &scalingModeOff, + }, + }, + }, + expectedScalingMode: &scalingModeOff, + expectedUpdateMode: &updateModeAuto, + }, + // Tests checking changes to UpdateMode. + { + name: "UpdateMode from Off to Auto", + oldMpa: testMpaBuilder.WithUpdateMode(vpa_types.UpdateModeOff).Get(), + newMpa: testMpaBuilder.WithUpdateMode(vpa_types.UpdateModeAuto).Get(), + expectedScalingMode: &scalingModeAuto, + expectedUpdateMode: &updateModeAuto, + }, { + name: "UpdateMode from Auto to Off", + oldMpa: testMpaBuilder.WithUpdateMode(vpa_types.UpdateModeAuto).Get(), + newMpa: testMpaBuilder.WithUpdateMode(vpa_types.UpdateModeOff).Get(), + expectedScalingMode: &scalingModeAuto, + expectedUpdateMode: &updateModeOff, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + cluster := NewClusterState(testGcPeriod) + addTestPod(cluster) + addTestContainer(t, cluster) + if tc.oldMpa != nil { + oldMpa := addMpaObject(cluster, testMpaID, tc.oldMpa, testSelectorStr) + if !assert.Contains(t, cluster.Mpas, testMpaID) { + t.FailNow() + } + assert.Len(t, oldMpa.aggregateContainerStates, 1, "Expected one container aggregation in MPA %v", testMpaID) + for containerName, aggregation := range oldMpa.aggregateContainerStates { + assert.Equal(t, &scalingModeAuto, aggregation.GetScalingMode(), "Unexpected scaling mode for container %s", containerName) + } + } + tc.newMpa.Spec.ResourcePolicy = tc.resourcePolicy + addMpaObject(cluster, testMpaID, tc.newMpa, testSelectorStr) + mpa, found := cluster.Mpas[testMpaID] + if !assert.True(t, found, "MPA %+v not found in cluster state.", testMpaID) { + t.FailNow() + } + assert.Equal(t, tc.expectedUpdateMode, mpa.UpdateMode) + assert.Len(t, mpa.aggregateContainerStates, 1, "Expected one container aggregation in MPA %v", testMpaID) + for containerName, aggregation := range mpa.aggregateContainerStates { + assert.Equal(t, tc.expectedUpdateMode, aggregation.UpdateMode, "Unexpected update mode for container %s", containerName) + assert.Equal(t, tc.expectedScalingMode, aggregation.GetScalingMode(), "Unexpected scaling mode for container %s", containerName) + } + }) + } +} + +// Verify that two copies of the same AggregateStateKey are equal. +func TestEqualAggregateStateKey(t *testing.T) { + cluster := NewClusterState(testGcPeriod) + pod := addTestPod(cluster) + key1 := cluster.MakeAggregateStateKey(pod, "container-1") + key2 := cluster.MakeAggregateStateKey(pod, "container-1") + assert.True(t, key1 == key2) +} + +// Verify that two containers with the same name, living in two pods with the same namespace and labels +// (although different pod names) are aggregated together. +func TestTwoPodsWithSameLabels(t *testing.T) { + podID1 := vpa_model.PodID{Namespace: "namespace-1", PodName: "pod-1"} + podID2 := vpa_model.PodID{Namespace: "namespace-1", PodName: "pod-2"} + containerID1 := vpa_model.ContainerID{PodID: podID1, ContainerName: "foo-container"} + containerID2 := vpa_model.ContainerID{PodID: podID2, ContainerName: "foo-container"} + + cluster := NewClusterState(testGcPeriod) + cluster.AddOrUpdatePod(podID1, testLabels, apiv1.PodRunning) + cluster.AddOrUpdatePod(podID2, testLabels, apiv1.PodRunning) + err := cluster.AddOrUpdateContainer(containerID1, testRequest) + assert.NoError(t, err) + err = cluster.AddOrUpdateContainer(containerID2, testRequest) + assert.NoError(t, err) + + // Expect only one aggregation to be created. + assert.Equal(t, 1, len(cluster.aggregateStateMap)) +} + +// Verify that two identical containers in different namespaces are not aggregated together. +func TestTwoPodsWithDifferentNamespaces(t *testing.T) { + podID1 := vpa_model.PodID{Namespace: "namespace-1", PodName: "foo-pod"} + podID2 := vpa_model.PodID{Namespace: "namespace-2", PodName: "foo-pod"} + containerID1 := vpa_model.ContainerID{PodID: podID1, ContainerName: "foo-container"} + containerID2 := vpa_model.ContainerID{PodID: podID2, ContainerName: "foo-container"} + + cluster := NewClusterState(testGcPeriod) + cluster.AddOrUpdatePod(podID1, testLabels, apiv1.PodRunning) + cluster.AddOrUpdatePod(podID2, testLabels, apiv1.PodRunning) + err := cluster.AddOrUpdateContainer(containerID1, testRequest) + assert.NoError(t, err) + err = cluster.AddOrUpdateContainer(containerID2, testRequest) + assert.NoError(t, err) + + // Expect two separate aggregations to be created. + assert.Equal(t, 2, len(cluster.aggregateStateMap)) + // Expect only one entry to be present in the labels set map. + assert.Equal(t, 1, len(cluster.labelSetMap)) +} + +// Verifies that a MPA with an empty selector (matching all pods) matches a pod +// with labels as well as a pod with no labels. +func TestEmptySelector(t *testing.T) { + cluster := NewClusterState(testGcPeriod) + // Create a MPA with an empty selector (matching all pods). + mpa := addMpa(cluster, testMpaID, testAnnotations, "", testTargetRef) + // Create a pod with labels. Add a container. + cluster.AddOrUpdatePod(testPodID, testLabels, apiv1.PodRunning) + containerID1 := vpa_model.ContainerID{PodID: testPodID, ContainerName: "foo"} + assert.NoError(t, cluster.AddOrUpdateContainer(containerID1, testRequest)) + + // Create a pod without labels. Add a container. + anotherPodID := vpa_model.PodID{Namespace: "namespace-1", PodName: "pod-2"} + cluster.AddOrUpdatePod(anotherPodID, emptyLabels, apiv1.PodRunning) + containerID2 := vpa_model.ContainerID{PodID: anotherPodID, ContainerName: "foo"} + assert.NoError(t, cluster.AddOrUpdateContainer(containerID2, testRequest)) + + // Both pods should be matched by the MPA. + assert.Contains(t, mpa.aggregateContainerStates, cluster.aggregateStateKeyForContainerID(containerID1)) + assert.Contains(t, mpa.aggregateContainerStates, cluster.aggregateStateKeyForContainerID(containerID2)) +} + +func TestRecordRecommendation(t *testing.T) { + cases := []struct { + name string + recommendation *vpa_types.RecommendedPodResources + lastLogged time.Time + now time.Time + expectedEmpty bool + expectedLastLogged time.Time + expectedError error + }{ + { + name: "MPA has recommendation", + recommendation: test.Recommendation().WithContainer("test").WithTarget("100m", "200G").Get(), + now: testTimestamp, + expectedEmpty: false, + expectedError: nil, + }, { + name: "MPA recommendation appears", + recommendation: test.Recommendation().WithContainer("test").WithTarget("100m", "200G").Get(), + lastLogged: testTimestamp.Add(-10 * time.Minute), + now: testTimestamp, + expectedEmpty: false, + expectedError: nil, + }, { + name: "MPA recommendation missing", + recommendation: &vpa_types.RecommendedPodResources{}, + lastLogged: testTimestamp.Add(-10 * time.Minute), + now: testTimestamp, + expectedEmpty: true, + expectedLastLogged: testTimestamp.Add(-10 * time.Minute), + expectedError: nil, + }, { + name: "MPA recommendation missing and needs logging", + recommendation: &vpa_types.RecommendedPodResources{}, + lastLogged: testTimestamp.Add(-40 * time.Minute), + now: testTimestamp, + expectedEmpty: true, + expectedLastLogged: testTimestamp, + expectedError: fmt.Errorf("MPA namespace-1/mpa-1 is missing recommendation for more than %v", RecommendationMissingMaxDuration), + }, { + name: "MPA recommendation disappears", + recommendation: &vpa_types.RecommendedPodResources{}, + now: testTimestamp, + expectedEmpty: true, + expectedLastLogged: testTimestamp, + expectedError: nil, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + cluster := NewClusterState(testGcPeriod) + mpa := addMpa(cluster, testMpaID, testAnnotations, testSelectorStr, testTargetRef) + cluster.Mpas[testMpaID].Recommendation = tc.recommendation + if !tc.lastLogged.IsZero() { + cluster.EmptyMPAs[testMpaID] = tc.lastLogged + } + + err := cluster.RecordRecommendation(mpa, tc.now) + if tc.expectedError != nil { + assert.Equal(t, tc.expectedError, err) + } else { + assert.NoError(t, err) + if tc.expectedEmpty { + assert.Contains(t, cluster.EmptyMPAs, testMpaID) + assert.Equal(t, cluster.EmptyMPAs[testMpaID], tc.expectedLastLogged) + } else { + assert.NotContains(t, cluster.EmptyMPAs, testMpaID) + } + } + }) + } +} + +type podDesc struct { + id vpa_model.PodID + labels labels.Set + phase apiv1.PodPhase +} + +func TestGetActiveMatchingPods(t *testing.T) { + cases := []struct { + name string + mpaSelector string + pods []podDesc + expectedPods []vpa_model.PodID + }{ + { + name: "No pods", + mpaSelector: testSelectorStr, + pods: []podDesc{}, + expectedPods: []vpa_model.PodID{}, + }, { + name: "Matching pod", + mpaSelector: testSelectorStr, + pods: []podDesc{ + { + id: testPodID, + labels: testLabels, + phase: apiv1.PodRunning, + }, + }, + expectedPods: []vpa_model.PodID{testPodID}, + }, { + name: "Matching pod is inactive", + mpaSelector: testSelectorStr, + pods: []podDesc{ + { + id: testPodID, + labels: testLabels, + phase: apiv1.PodFailed, + }, + }, + expectedPods: []vpa_model.PodID{testPodID}, + }, { + name: "No matching pods", + mpaSelector: testSelectorStr, + pods: []podDesc{ + { + id: testPodID, + labels: emptyLabels, + phase: apiv1.PodRunning, + }, { + id: vpa_model.PodID{Namespace: "different-than-mpa", PodName: "pod-1"}, + labels: testLabels, + phase: apiv1.PodRunning, + }, + }, + expectedPods: []vpa_model.PodID{}, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + cluster := NewClusterState(testGcPeriod) + mpa := addMpa(cluster, testMpaID, testAnnotations, tc.mpaSelector, testTargetRef) + for _, pod := range tc.pods { + cluster.AddOrUpdatePod(pod.id, pod.labels, pod.phase) + } + pods := cluster.GetMatchingPods(mpa) + assert.ElementsMatch(t, tc.expectedPods, pods) + }) + } +} + +func TestMPAWithMatchingPods(t *testing.T) { + cases := []struct { + name string + mpaSelector string + pods []podDesc + expectedMatch int + }{ + { + name: "No pods", + mpaSelector: testSelectorStr, + pods: []podDesc{}, + expectedMatch: 0, + }, + { + name: "MPA with matching pod", + mpaSelector: testSelectorStr, + pods: []podDesc{ + { + testPodID, + testLabels, + apiv1.PodRunning, + }, + }, + expectedMatch: 1, + }, + { + name: "No matching pod", + mpaSelector: testSelectorStr, + pods: []podDesc{ + { + testPodID, + emptyLabels, + apiv1.PodRunning, + }, + }, + expectedMatch: 0, + }, + { + name: "MPA with 2 matching pods, 1 not matching", + mpaSelector: testSelectorStr, + pods: []podDesc{ + { + testPodID, + emptyLabels, // does not match MPA + apiv1.PodRunning, + }, + { + testPodID3, + testLabels, + apiv1.PodRunning, + }, + { + testPodID4, + testLabels, + apiv1.PodRunning, + }, + }, + expectedMatch: 2, + }, + } + // Run with adding MPA first + for _, tc := range cases { + t.Run(tc.name+", MPA first", func(t *testing.T) { + cluster := NewClusterState(testGcPeriod) + mpa := addMpa(cluster, testMpaID, testAnnotations, tc.mpaSelector, testTargetRef) + for _, podDesc := range tc.pods { + cluster.AddOrUpdatePod(podDesc.id, podDesc.labels, podDesc.phase) + containerID := vpa_model.ContainerID{PodID: testPodID, ContainerName: "foo"} + assert.NoError(t, cluster.AddOrUpdateContainer(containerID, testRequest)) + } + assert.Equal(t, tc.expectedMatch, cluster.Mpas[mpa.ID].PodCount) + }) + } + // Run with adding Pods first + for _, tc := range cases { + t.Run(tc.name+", Pods first", func(t *testing.T) { + cluster := NewClusterState(testGcPeriod) + for _, podDesc := range tc.pods { + cluster.AddOrUpdatePod(podDesc.id, podDesc.labels, podDesc.phase) + containerID := vpa_model.ContainerID{PodID: testPodID, ContainerName: "foo"} + assert.NoError(t, cluster.AddOrUpdateContainer(containerID, testRequest)) + } + mpa := addMpa(cluster, testMpaID, testAnnotations, tc.mpaSelector, testTargetRef) + assert.Equal(t, tc.expectedMatch, cluster.Mpas[mpa.ID].PodCount) + }) + } +} diff --git a/multidimensional-pod-autoscaler/pkg/recommender/model/container.go b/multidimensional-pod-autoscaler/pkg/recommender/model/container.go new file mode 100644 index 000000000000..5a19252e6418 --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/recommender/model/container.go @@ -0,0 +1,220 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package model + +import ( + "fmt" + "time" + + corev1 "k8s.io/api/core/v1" + vpa_model "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/model" + metrics_quality "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/metrics/quality" + "k8s.io/klog/v2" +) + +// ContainerState stores information about a single container instance. +// Each ContainerState has a pointer to the aggregation that is used for +// aggregating its usage samples. +// It holds the recent history of CPU and memory utilization. +// +// Note: samples are added to intervals based on their start timestamps. +type ContainerState struct { + // Current request. + Request vpa_model.Resources + // Start of the latest CPU usage sample that was aggregated. + LastCPUSampleStart time.Time + // Max memory usage observed in the current aggregation interval. + memoryPeak vpa_model.ResourceAmount + // Max memory usage estimated from an OOM event in the current aggregation interval. + oomPeak vpa_model.ResourceAmount + // End time of the current memory aggregation interval (not inclusive). + WindowEnd time.Time + // Start of the latest memory usage sample that was aggregated. + lastMemorySampleStart time.Time + // Aggregation to add usage samples to. + aggregator vpa_model.ContainerStateAggregator +} + +// NewContainerState returns a new ContainerState. +func NewContainerState(request vpa_model.Resources, aggregator vpa_model.ContainerStateAggregator) *ContainerState { + return &ContainerState{ + Request: request, + LastCPUSampleStart: time.Time{}, + WindowEnd: time.Time{}, + lastMemorySampleStart: time.Time{}, + aggregator: aggregator, + } +} + +func (container *ContainerState) addCPUSample(sample *vpa_model.ContainerUsageSample) bool { + // Order should not matter for the histogram, other than deduplication. + if !(sample.Usage >= 0 && sample.Resource == vpa_model.ResourceCPU) || !sample.MeasureStart.After(container.LastCPUSampleStart) { + return false // Discard invalid, duplicate or out-of-order samples. + } + container.observeQualityMetrics(sample.Usage, false, corev1.ResourceCPU) + container.aggregator.AddSample(sample) + container.LastCPUSampleStart = sample.MeasureStart + return true +} + +func (container *ContainerState) observeQualityMetrics(usage vpa_model.ResourceAmount, isOOM bool, resource corev1.ResourceName) { + if !container.aggregator.NeedsRecommendation() { + return + } + updateMode := container.aggregator.GetUpdateMode() + var usageValue float64 + switch resource { + case corev1.ResourceCPU: + usageValue = vpa_model.CoresFromCPUAmount(usage) + case corev1.ResourceMemory: + usageValue = vpa_model.BytesFromMemoryAmount(usage) + } + if container.aggregator.GetLastRecommendation() == nil { + metrics_quality.ObserveQualityMetricsRecommendationMissing(usageValue, isOOM, resource, updateMode) + return + } + recommendation := container.aggregator.GetLastRecommendation()[resource] + if recommendation.IsZero() { + metrics_quality.ObserveQualityMetricsRecommendationMissing(usageValue, isOOM, resource, updateMode) + return + } + var recommendationValue float64 + switch resource { + case corev1.ResourceCPU: + recommendationValue = float64(recommendation.MilliValue()) / 1000.0 + case corev1.ResourceMemory: + recommendationValue = float64(recommendation.Value()) + default: + klog.Warningf("Unknown resource: %v", resource) + return + } + metrics_quality.ObserveQualityMetrics(usageValue, recommendationValue, isOOM, resource, updateMode) +} + +// GetMaxMemoryPeak returns maximum memory usage in the sample, possibly estimated from OOM +func (container *ContainerState) GetMaxMemoryPeak() vpa_model.ResourceAmount { + return vpa_model.ResourceAmountMax(container.memoryPeak, container.oomPeak) +} + +func (container *ContainerState) addMemorySample(sample *vpa_model.ContainerUsageSample, isOOM bool) bool { + ts := sample.MeasureStart + // We always process OOM samples. + if !(sample.Usage >= 0 && sample.Resource == vpa_model.ResourceMemory) || + (!isOOM && ts.Before(container.lastMemorySampleStart)) { + return false // Discard invalid or outdated samples. + } + container.lastMemorySampleStart = ts + if container.WindowEnd.IsZero() { // This is the first sample. + container.WindowEnd = ts + } + + // Each container aggregates one peak per aggregation interval. If the timestamp of the + // current sample is earlier than the end of the current interval (WindowEnd) and is larger + // than the current peak, the peak is updated in the aggregation by subtracting the old value + // and adding the new value. + addNewPeak := false + if ts.Before(container.WindowEnd) { + oldMaxMem := container.GetMaxMemoryPeak() + if oldMaxMem != 0 && sample.Usage > oldMaxMem { + // Remove the old peak. + oldPeak := vpa_model.ContainerUsageSample{ + MeasureStart: container.WindowEnd, + Usage: oldMaxMem, + Request: sample.Request, + Resource: vpa_model.ResourceMemory, + } + container.aggregator.SubtractSample(&oldPeak) + addNewPeak = true + } + } else { + // Shift the memory aggregation window to the next interval. + memoryAggregationInterval := vpa_model.GetAggregationsConfig().MemoryAggregationInterval + shift := truncate(ts.Sub(container.WindowEnd), memoryAggregationInterval) + memoryAggregationInterval + container.WindowEnd = container.WindowEnd.Add(shift) + container.memoryPeak = 0 + container.oomPeak = 0 + addNewPeak = true + } + container.observeQualityMetrics(sample.Usage, isOOM, corev1.ResourceMemory) + if addNewPeak { + newPeak := vpa_model.ContainerUsageSample{ + MeasureStart: container.WindowEnd, + Usage: sample.Usage, + Request: sample.Request, + Resource: vpa_model.ResourceMemory, + } + container.aggregator.AddSample(&newPeak) + if isOOM { + container.oomPeak = sample.Usage + } else { + container.memoryPeak = sample.Usage + } + } + return true +} + +// RecordOOM adds info regarding OOM event in the model as an artificial memory sample. +func (container *ContainerState) RecordOOM(timestamp time.Time, requestedMemory vpa_model.ResourceAmount) error { + // Discard old OOM + if timestamp.Before(container.WindowEnd.Add(-1 * vpa_model.GetAggregationsConfig().MemoryAggregationInterval)) { + return fmt.Errorf("OOM event will be discarded - it is too old (%v)", timestamp) + } + // Get max of the request and the recent usage-based memory peak. + // Omitting oomPeak here to protect against recommendation running too high on subsequent OOMs. + memoryUsed := vpa_model.ResourceAmountMax(requestedMemory, container.memoryPeak) + memoryNeeded := vpa_model.ResourceAmountMax(memoryUsed+vpa_model.MemoryAmountFromBytes(vpa_model.GetAggregationsConfig().OOMMinBumpUp), + vpa_model.ScaleResource(memoryUsed, vpa_model.GetAggregationsConfig().OOMBumpUpRatio)) + + oomMemorySample := vpa_model.ContainerUsageSample{ + MeasureStart: timestamp, + Usage: memoryNeeded, + Resource: vpa_model.ResourceMemory, + } + if !container.addMemorySample(&oomMemorySample, true) { + return fmt.Errorf("adding OOM sample failed") + } + return nil +} + +// AddSample adds a usage sample to the given ContainerState. Requires samples +// for a single resource to be passed in chronological order (i.e. in order of +// growing MeasureStart). Invalid samples (out of order or measure out of legal +// range) are discarded. Returns true if the sample was aggregated, false if it +// was discarded. +// Note: usage samples don't hold their end timestamp / duration. They are +// implicitly assumed to be disjoint when aggregating. +func (container *ContainerState) AddSample(sample *vpa_model.ContainerUsageSample) bool { + switch sample.Resource { + case vpa_model.ResourceCPU: + return container.addCPUSample(sample) + case vpa_model.ResourceMemory: + return container.addMemorySample(sample, false) + default: + return false + } +} + +// Truncate returns the result of rounding d toward zero to a multiple of m. +// If m <= 0, Truncate returns d unchanged. +// This helper function is introduced to support older implementations of the +// time package that don't provide Duration.Truncate function. +func truncate(d, m time.Duration) time.Duration { + if m <= 0 { + return d + } + return d - d%m +} diff --git a/multidimensional-pod-autoscaler/pkg/recommender/model/container_test.go b/multidimensional-pod-autoscaler/pkg/recommender/model/container_test.go new file mode 100644 index 000000000000..24bad9c26d0b --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/recommender/model/container_test.go @@ -0,0 +1,197 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package model + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + vpa_model "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/model" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/util" +) + +var ( + timeLayout = "2006-01-02 15:04:05" + testTimestamp, _ = time.Parse(timeLayout, "2017-04-18 17:35:05") + + TestRequest = vpa_model.Resources{ + vpa_model.ResourceCPU: vpa_model.CPUAmountFromCores(2.3), + vpa_model.ResourceMemory: vpa_model.MemoryAmountFromBytes(5e8), + } +) + +const ( + kb = 1024 + mb = 1024 * kb +) + +func newUsageSample(timestamp time.Time, usage int64, resource vpa_model.ResourceName) *vpa_model.ContainerUsageSample { + return &vpa_model.ContainerUsageSample{ + MeasureStart: timestamp, + Usage: vpa_model.ResourceAmount(usage), + Request: TestRequest[resource], + Resource: resource, + } +} + +type ContainerTest struct { + mockCPUHistogram *util.MockHistogram + mockMemoryHistogram *util.MockHistogram + aggregateContainerState *vpa_model.AggregateContainerState + container *ContainerState +} + +func newContainerTest() ContainerTest { + mockCPUHistogram := new(util.MockHistogram) + mockMemoryHistogram := new(util.MockHistogram) + aggregateContainerState := &vpa_model.AggregateContainerState{ + AggregateCPUUsage: mockCPUHistogram, + AggregateMemoryPeaks: mockMemoryHistogram, + } + container := &ContainerState{ + Request: TestRequest, + aggregator: aggregateContainerState, + } + return ContainerTest{ + mockCPUHistogram: mockCPUHistogram, + mockMemoryHistogram: mockMemoryHistogram, + aggregateContainerState: aggregateContainerState, + container: container, + } +} + +// Add 6 usage samples (3 valid, 3 invalid) to a container. Verifies that for +// valid samples the CPU measures are aggregated in the CPU histogram and +// the memory measures are aggregated in the memory peaks sliding window. +// Verifies that invalid samples (out-of-order or negative usage) are ignored. +func TestAggregateContainerUsageSamples(t *testing.T) { + test := newContainerTest() + c := test.container + memoryAggregationInterval := vpa_model.GetAggregationsConfig().MemoryAggregationInterval + // Verify that CPU measures are added to the CPU histogram. + // The weight should be equal to the current request. + timeStep := memoryAggregationInterval / 2 + test.mockCPUHistogram.On("AddSample", 3.14, 2.3, testTimestamp) + test.mockCPUHistogram.On("AddSample", 6.28, 2.3, testTimestamp.Add(timeStep)) + test.mockCPUHistogram.On("AddSample", 1.57, 2.3, testTimestamp.Add(2*timeStep)) + // Verify that memory peaks are added to the memory peaks histogram. + memoryAggregationWindowEnd := testTimestamp.Add(memoryAggregationInterval) + test.mockMemoryHistogram.On("AddSample", 5.0, 1.0, memoryAggregationWindowEnd) + test.mockMemoryHistogram.On("SubtractSample", 5.0, 1.0, memoryAggregationWindowEnd) + test.mockMemoryHistogram.On("AddSample", 10.0, 1.0, memoryAggregationWindowEnd) + memoryAggregationWindowEnd = memoryAggregationWindowEnd.Add(memoryAggregationInterval) + test.mockMemoryHistogram.On("AddSample", 2.0, 1.0, memoryAggregationWindowEnd) + + // Add three CPU and memory usage samples. + assert.True(t, c.AddSample(newUsageSample( + testTimestamp, 3140, vpa_model.ResourceCPU))) + assert.True(t, c.AddSample(newUsageSample( + testTimestamp, 5, vpa_model.ResourceMemory))) + + assert.True(t, c.AddSample(newUsageSample( + testTimestamp.Add(timeStep), 6280, vpa_model.ResourceCPU))) + assert.True(t, c.AddSample(newUsageSample( + testTimestamp.Add(timeStep), 10, vpa_model.ResourceMemory))) + + assert.True(t, c.AddSample(newUsageSample( + testTimestamp.Add(2*timeStep), 1570, vpa_model.ResourceCPU))) + assert.True(t, c.AddSample(newUsageSample( + testTimestamp.Add(2*timeStep), 2, vpa_model.ResourceMemory))) + + // Discard invalid samples. + assert.False(t, c.AddSample(newUsageSample( // Out of order sample. + testTimestamp.Add(2*timeStep), 1000, vpa_model.ResourceCPU))) + assert.False(t, c.AddSample(newUsageSample( // Negative CPU usage. + testTimestamp.Add(4*timeStep), -1000, vpa_model.ResourceCPU))) + assert.False(t, c.AddSample(newUsageSample( // Negative memory usage. + testTimestamp.Add(4*timeStep), -1000, vpa_model.ResourceMemory))) +} + +func TestRecordOOMIncreasedByBumpUp(t *testing.T) { + test := newContainerTest() + memoryAggregationWindowEnd := testTimestamp.Add(vpa_model.GetAggregationsConfig().MemoryAggregationInterval) + // Bump Up factor is 20%. + test.mockMemoryHistogram.On("AddSample", 1200.0*mb, 1.0, memoryAggregationWindowEnd) + + assert.NoError(t, test.container.RecordOOM(testTimestamp, vpa_model.ResourceAmount(1000*mb))) +} + +func TestRecordOOMDontRunAway(t *testing.T) { + test := newContainerTest() + memoryAggregationWindowEnd := testTimestamp.Add(vpa_model.GetAggregationsConfig().MemoryAggregationInterval) + + // Bump Up factor is 20%. + test.mockMemoryHistogram.On("AddSample", 1200.0*mb, 1.0, memoryAggregationWindowEnd) + assert.NoError(t, test.container.RecordOOM(testTimestamp, vpa_model.ResourceAmount(1000*mb))) + + // new smaller OOMs don't influence the sample value (oomPeak) + assert.NoError(t, test.container.RecordOOM(testTimestamp, vpa_model.ResourceAmount(999*mb))) + assert.NoError(t, test.container.RecordOOM(testTimestamp, vpa_model.ResourceAmount(999*mb))) + + test.mockMemoryHistogram.On("SubtractSample", 1200.0*mb, 1.0, memoryAggregationWindowEnd) + test.mockMemoryHistogram.On("AddSample", 2400.0*mb, 1.0, memoryAggregationWindowEnd) + // a larger OOM should increase the sample value + assert.NoError(t, test.container.RecordOOM(testTimestamp, vpa_model.ResourceAmount(2000*mb))) +} + +func TestRecordOOMIncreasedByMin(t *testing.T) { + test := newContainerTest() + memoryAggregationWindowEnd := testTimestamp.Add(vpa_model.GetAggregationsConfig().MemoryAggregationInterval) + // Min grow by 100Mb. + test.mockMemoryHistogram.On("AddSample", 101.0*mb, 1.0, memoryAggregationWindowEnd) + + assert.NoError(t, test.container.RecordOOM(testTimestamp, vpa_model.ResourceAmount(1*mb))) +} + +func TestRecordOOMMaxedWithKnownSample(t *testing.T) { + test := newContainerTest() + memoryAggregationWindowEnd := testTimestamp.Add(vpa_model.GetAggregationsConfig().MemoryAggregationInterval) + + test.mockMemoryHistogram.On("AddSample", 3000.0*mb, 1.0, memoryAggregationWindowEnd) + assert.True(t, test.container.AddSample(newUsageSample(testTimestamp, 3000*mb, vpa_model.ResourceMemory))) + + // Last known sample is higher than request, so it is taken. + test.mockMemoryHistogram.On("SubtractSample", 3000.0*mb, 1.0, memoryAggregationWindowEnd) + test.mockMemoryHistogram.On("AddSample", 3600.0*mb, 1.0, memoryAggregationWindowEnd) + + assert.NoError(t, test.container.RecordOOM(testTimestamp, vpa_model.ResourceAmount(1000*mb))) +} + +func TestRecordOOMDiscardsOldSample(t *testing.T) { + test := newContainerTest() + memoryAggregationWindowEnd := testTimestamp.Add(vpa_model.GetAggregationsConfig().MemoryAggregationInterval) + + test.mockMemoryHistogram.On("AddSample", 1000.0*mb, 1.0, memoryAggregationWindowEnd) + assert.True(t, test.container.AddSample(newUsageSample(testTimestamp, 1000*mb, vpa_model.ResourceMemory))) + + // OOM is stale, mem not changed. + assert.Error(t, test.container.RecordOOM(testTimestamp.Add(-30*time.Hour), vpa_model.ResourceAmount(1000*mb))) +} + +func TestRecordOOMInNewWindow(t *testing.T) { + test := newContainerTest() + memoryAggregationInterval := vpa_model.GetAggregationsConfig().MemoryAggregationInterval + memoryAggregationWindowEnd := testTimestamp.Add(memoryAggregationInterval) + + test.mockMemoryHistogram.On("AddSample", 2000.0*mb, 1.0, memoryAggregationWindowEnd) + assert.True(t, test.container.AddSample(newUsageSample(testTimestamp, 2000*mb, vpa_model.ResourceMemory))) + + memoryAggregationWindowEnd = memoryAggregationWindowEnd.Add(2 * memoryAggregationInterval) + test.mockMemoryHistogram.On("AddSample", 2400.0*mb, 1.0, memoryAggregationWindowEnd) + assert.NoError(t, test.container.RecordOOM(testTimestamp.Add(2*memoryAggregationInterval), vpa_model.ResourceAmount(1000*mb))) +} diff --git a/multidimensional-pod-autoscaler/pkg/recommender/model/mpa.go b/multidimensional-pod-autoscaler/pkg/recommender/model/mpa.go new file mode 100644 index 000000000000..f507281b99bb --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/recommender/model/mpa.go @@ -0,0 +1,310 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package model + +import ( + "sort" + "time" + + autoscaling "k8s.io/api/autoscaling/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" + apiv1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + mpa_types "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1alpha1" + vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" + vpa_model "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/model" + metrics_quality "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/metrics/quality" + vpa_api_util "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/vpa" +) + +// Map from MPA annotation key to value. +type mpaAnnotationsMap map[string]string + +// Map from MPA condition type to condition. +type mpaConditionsMap map[mpa_types.MultidimPodAutoscalerConditionType]mpa_types.MultidimPodAutoscalerCondition + +func (conditionsMap *mpaConditionsMap) Set( + conditionType mpa_types.MultidimPodAutoscalerConditionType, + status bool, reason string, message string) *mpaConditionsMap { + oldCondition, alreadyPresent := (*conditionsMap)[conditionType] + condition := mpa_types.MultidimPodAutoscalerCondition{ + Type: conditionType, + Reason: reason, + Message: message, + } + if status { + condition.Status = apiv1.ConditionTrue + } else { + condition.Status = apiv1.ConditionFalse + } + if alreadyPresent && oldCondition.Status == condition.Status { + condition.LastTransitionTime = oldCondition.LastTransitionTime + } else { + condition.LastTransitionTime = metav1.Now() + } + (*conditionsMap)[conditionType] = condition + return conditionsMap +} + +func (conditionsMap *mpaConditionsMap) AsList() []mpa_types.MultidimPodAutoscalerCondition { + conditions := make([]mpa_types.MultidimPodAutoscalerCondition, 0, len(*conditionsMap)) + for _, condition := range *conditionsMap { + conditions = append(conditions, condition) + } + + // Sort conditions by type to avoid elements floating on the list + sort.Slice(conditions, func(i, j int) bool { + return conditions[i].Type < conditions[j].Type + }) + + return conditions +} + +func (conditionsMap *mpaConditionsMap) ConditionActive(conditionType mpa_types.MultidimPodAutoscalerConditionType) bool { + condition, found := (*conditionsMap)[conditionType] + return found && condition.Status == apiv1.ConditionTrue +} + +// Mpa (Multidimensional Pod Autoscaler) object is responsible for horizontal and vertical scaling +// of Pods matching a given label selector. +type Mpa struct { + ID MpaID + // Labels selector that determines which Pods are controlled by this MPA + // object. Can be nil, in which case no Pod is matched. + PodSelector labels.Selector + // Map of the object annotations (key-value pairs). + Annotations mpaAnnotationsMap + // Map of the status conditions (keys are condition types). + Conditions mpaConditionsMap + // Most recently computed recommendation. Can be nil. + Recommendation *vpa_types.RecommendedPodResources + // All container aggregations that contribute to this MPA. + // TODO: Garbage collect old AggregateContainerStates. + aggregateContainerStates aggregateContainerStatesMap + // Pod Resource Policy provided in the MPA API object. Can be nil. + ResourcePolicy *vpa_types.PodResourcePolicy + // Initial checkpoints of AggregateContainerStates for containers. + // The key is container name. + ContainersInitialAggregateState vpa_model.ContainerNameToAggregateStateMap + // UpdateMode describes how recommendations will be applied to pods + UpdateMode *vpa_types.UpdateMode + // Created denotes timestamp of the original MPA object creation + Created time.Time + // CheckpointWritten indicates when last checkpoint for the MPA object was stored. + CheckpointWritten time.Time + // IsV1Beta1API is set to true if MPA object has labelSelector defined as in v1beta1 api. + IsV1Beta1API bool + // ScaleTargetRef points to the controller managing the set of pods. + ScaleTargetRef *autoscaling.CrossVersionObjectReference + // PodCount contains number of live Pods matching a given MPA object. + PodCount int + + // Added for HPA-related fields. + // TODO: Currently HPA-related logic is directly manipulating the MPA object but not the MPA + // model here. + Metrics []autoscalingv2.MetricSpec + MinReplicas int32 + MaxReplicas int32 + HorizontalScalingBehavior *autoscalingv2.HorizontalPodAutoscalerBehavior + DesiredReplicas int32 + CurrentMetrics []autoscalingv2.MetricStatus +} + +// NewMpa returns a new Mpa with a given ID and pod selector. Doesn't set the +// links to the matched aggregations. +func NewMpa(id MpaID, selector labels.Selector, created time.Time) *Mpa { + mpa := &Mpa{ + ID: id, + PodSelector: selector, + aggregateContainerStates: make(aggregateContainerStatesMap), + ContainersInitialAggregateState: make(vpa_model.ContainerNameToAggregateStateMap), + Created: created, + Annotations: make(mpaAnnotationsMap), + Conditions: make(mpaConditionsMap), + IsV1Beta1API: false, + PodCount: 0, + } + return mpa +} + +// UseAggregationIfMatching checks if the given aggregation matches (contributes to) this MPA +// and adds it to the set of MPA's aggregations if that is the case. +func (mpa *Mpa) UseAggregationIfMatching(aggregationKey vpa_model.AggregateStateKey, aggregation *vpa_model.AggregateContainerState) { + if mpa.UsesAggregation(aggregationKey) { + // Already linked, we can return quickly. + return + } + if mpa.matchesAggregation(aggregationKey) { + mpa.aggregateContainerStates[aggregationKey] = aggregation + aggregation.IsUnderVPA = true + aggregation.UpdateMode = mpa.UpdateMode + aggregation.UpdateFromPolicy(vpa_api_util.GetContainerResourcePolicy(aggregationKey.ContainerName(), mpa.ResourcePolicy)) + } +} + +// UsesAggregation returns true iff an aggregation with the given key contributes to the MPA. +func (mpa *Mpa) UsesAggregation(aggregationKey vpa_model.AggregateStateKey) bool { + _, exists := mpa.aggregateContainerStates[aggregationKey] + return exists +} + +// DeleteAggregation deletes aggregation used by this container +func (mpa *Mpa) DeleteAggregation(aggregationKey vpa_model.AggregateStateKey) { + state, ok := mpa.aggregateContainerStates[aggregationKey] + if !ok { + return + } + state.MarkNotAutoscaled() + delete(mpa.aggregateContainerStates, aggregationKey) +} + +// matchesAggregation returns true iff the MPA matches the given aggregation key. +func (mpa *Mpa) matchesAggregation(aggregationKey vpa_model.AggregateStateKey) bool { + if mpa.ID.Namespace != aggregationKey.Namespace() { + return false + } + return mpa.PodSelector != nil && mpa.PodSelector.Matches(aggregationKey.Labels()) +} + +// SetResourcePolicy updates the resource policy of the MPA and the scaling +// policies of aggregators under this MPA. +func (mpa *Mpa) SetResourcePolicy(resourcePolicy *vpa_types.PodResourcePolicy) { + if resourcePolicy == mpa.ResourcePolicy { + return + } + mpa.ResourcePolicy = resourcePolicy + for container, state := range mpa.aggregateContainerStates { + state.UpdateFromPolicy(vpa_api_util.GetContainerResourcePolicy(container.ContainerName(), mpa.ResourcePolicy)) + } +} + +// SetUpdateMode updates the update mode of the MPA and aggregators under this MPA. +func (mpa *Mpa) SetUpdateMode(updatePolicy *mpa_types.PodUpdatePolicy) { + if updatePolicy == nil { + mpa.UpdateMode = nil + } else { + if updatePolicy.UpdateMode == mpa.UpdateMode { + return + } + mpa.UpdateMode = updatePolicy.UpdateMode + } + for _, state := range mpa.aggregateContainerStates { + state.UpdateMode = mpa.UpdateMode + } +} + +// SetHPAConstraints sets HPA-related constraints. +func (mpa *Mpa) SetHPAConstraints(metrics []autoscalingv2.MetricSpec, minReplicas int32, maxReplicas int32, hpaBehavior *autoscalingv2.HorizontalPodAutoscalerBehavior) { + mpa.Metrics = metrics + mpa.MinReplicas = minReplicas + mpa.MaxReplicas = maxReplicas + mpa.HorizontalScalingBehavior = hpaBehavior +} + +// SetDesiredNumberOfReplicas sets the desired number of replicas. +func (mpa *Mpa) SetDesiredNumberOfReplicas(replicas int32) { + mpa.DesiredReplicas = replicas +} + +// SetCurrentMetrics sets the current metrics. +func (mpa *Mpa) SetCurrentMetrics(metrics []autoscalingv2.MetricStatus) { + mpa.CurrentMetrics = metrics +} + +// MergeCheckpointedState adds checkpointed MPA aggregations to the given aggregateStateMap. +func (mpa *Mpa) MergeCheckpointedState(aggregateContainerStateMap vpa_model.ContainerNameToAggregateStateMap) { + for containerName, aggregation := range mpa.ContainersInitialAggregateState { + aggregateContainerState, found := aggregateContainerStateMap[containerName] + if !found { + aggregateContainerState = vpa_model.NewAggregateContainerState() + aggregateContainerStateMap[containerName] = aggregateContainerState + } + aggregateContainerState.MergeContainerState(aggregation) + } +} + +// AggregateStateByContainerName returns a map from container name to the aggregated state +// of all containers with that name, belonging to pods matched by the MPA. +func (mpa *Mpa) AggregateStateByContainerName() vpa_model.ContainerNameToAggregateStateMap { + containerNameToAggregateStateMap := AggregateStateByContainerName(mpa.aggregateContainerStates) + mpa.MergeCheckpointedState(containerNameToAggregateStateMap) + return containerNameToAggregateStateMap +} + +// HasRecommendation returns if the MPA object contains any recommendation +func (mpa *Mpa) HasRecommendation() bool { + return (mpa.Recommendation != nil) && len(mpa.Recommendation.ContainerRecommendations) > 0 +} + +// UpdateRecommendation updates the recommended resources in the MPA and its +// aggregations with the given recommendation. +func (mpa *Mpa) UpdateRecommendation(recommendation *vpa_types.RecommendedPodResources) { + for _, containerRecommendation := range recommendation.ContainerRecommendations { + for container, state := range mpa.aggregateContainerStates { + if container.ContainerName() == containerRecommendation.ContainerName { + metrics_quality.ObserveRecommendationChange(state.LastRecommendation, containerRecommendation.UncappedTarget, mpa.UpdateMode, mpa.PodCount) + state.LastRecommendation = containerRecommendation.UncappedTarget + } + } + } + mpa.Recommendation = recommendation +} + +// UpdateConditions updates the conditions of MPA objects based on it's state. +// PodsMatched is passed to indicate if there are currently active pods in the +// cluster matching this MPA. +func (mpa *Mpa) UpdateConditions(podsMatched bool) { + reason := "" + msg := "" + if podsMatched { + delete(mpa.Conditions, mpa_types.NoPodsMatched) + } else { + reason = "NoPodsMatched" + msg = "No pods match this MPA object" + mpa.Conditions.Set(mpa_types.NoPodsMatched, true, reason, msg) + } + if mpa.HasRecommendation() { + mpa.Conditions.Set(mpa_types.RecommendationProvided, true, "", "") + } else { + mpa.Conditions.Set(mpa_types.RecommendationProvided, false, reason, msg) + } + +} + +// HasMatchedPods returns true if there are are currently active pods in the +// cluster matching this MPA, based on conditions. UpdateConditions should be +// called first. +func (mpa *Mpa) HasMatchedPods() bool { + noPodsMatched, found := mpa.Conditions[mpa_types.NoPodsMatched] + if found && noPodsMatched.Status == apiv1.ConditionTrue { + return false + } + return true +} + +// AsStatus returns this objects equivalent of MPA Status. UpdateConditions +// should be called first. +func (mpa *Mpa) AsStatus() *mpa_types.MultidimPodAutoscalerStatus { + status := &mpa_types.MultidimPodAutoscalerStatus{ + Conditions: mpa.Conditions.AsList(), + } + if mpa.Recommendation != nil { + status.Recommendation = mpa.Recommendation + } + return status +} diff --git a/multidimensional-pod-autoscaler/pkg/recommender/model/types.go b/multidimensional-pod-autoscaler/pkg/recommender/model/types.go new file mode 100644 index 000000000000..dd8830da0057 --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/recommender/model/types.go @@ -0,0 +1,23 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package model + +// MpaID contains information needed to identify a MPA API object within a cluster. +type MpaID struct { + Namespace string + MpaName string +} diff --git a/multidimensional-pod-autoscaler/pkg/recommender/routines/capping_post_processor.go b/multidimensional-pod-autoscaler/pkg/recommender/routines/capping_post_processor.go new file mode 100644 index 000000000000..4d7e7a1a3576 --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/recommender/routines/capping_post_processor.go @@ -0,0 +1,42 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package routines + +import ( + "k8s.io/klog/v2" + + mpa_types "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1alpha1" + vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" + vpa_utils "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/vpa" +) + +// CappingPostProcessor ensure that the policy is applied to recommendation +// it applies policy for fields: MinAllowed and MaxAllowed +type CappingPostProcessor struct{} + +var _ RecommendationPostProcessor = &CappingPostProcessor{} + +// Process apply the capping post-processing to the recommendation. (use to be function getCappedRecommendation) +func (c CappingPostProcessor) Process(mpa *mpa_types.MultidimPodAutoscaler, recommendation *vpa_types.RecommendedPodResources) *vpa_types.RecommendedPodResources { + // TODO: maybe rename the vpa_utils.ApplyVPAPolicy to something that mention that it is doing capping only + cappedRecommendation, err := vpa_utils.ApplyVPAPolicy(recommendation, mpa.Spec.ResourcePolicy) + if err != nil { + klog.ErrorS(err, "Failed to apply policy for VPA", "vpa", klog.KObj(mpa)) + return recommendation + } + return cappedRecommendation +} diff --git a/multidimensional-pod-autoscaler/pkg/recommender/routines/cpu_integer_post_processor.go b/multidimensional-pod-autoscaler/pkg/recommender/routines/cpu_integer_post_processor.go new file mode 100644 index 000000000000..1b34b9021235 --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/recommender/routines/cpu_integer_post_processor.go @@ -0,0 +1,89 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package routines + +import ( + "strings" + + apiv1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + + mpa_types "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1alpha1" + vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" +) + +// IntegerCPUPostProcessor ensures that the recommendation delivers an integer value for CPU +// This is need for users who want to use CPU Management with static policy: https://kubernetes.io/docs/tasks/administer-cluster/cpu-management-policies/#static-policy +type IntegerCPUPostProcessor struct{} + +const ( + // The user interface for that post processor is an annotation on the VPA object with the following format: + // vpa-post-processor.kubernetes.io/{containerName}_integerCPU=true + mpaPostProcessorPrefix = "mpa-post-processor.kubernetes.io/" + mpaPostProcessorIntegerCPUSuffix = "_integerCPU" + mpaPostProcessorIntegerCPUValue = "true" +) + +var _ RecommendationPostProcessor = &IntegerCPUPostProcessor{} + +// Process apply the capping post-processing to the recommendation. +// For this post processor the CPU value is rounded up to an integer +func (p *IntegerCPUPostProcessor) Process(mpa *mpa_types.MultidimPodAutoscaler, recommendation *vpa_types.RecommendedPodResources) *vpa_types.RecommendedPodResources { + + amendedRecommendation := recommendation.DeepCopy() + + for key, value := range mpa.Annotations { + containerName := extractContainerName(key, mpaPostProcessorPrefix, mpaPostProcessorIntegerCPUSuffix) + if containerName == "" || value != mpaPostProcessorIntegerCPUValue { + continue + } + + for _, r := range amendedRecommendation.ContainerRecommendations { + if r.ContainerName != containerName { + continue + } + setIntegerCPURecommendation(r.Target) + setIntegerCPURecommendation(r.LowerBound) + setIntegerCPURecommendation(r.UpperBound) + setIntegerCPURecommendation(r.UncappedTarget) + } + } + return amendedRecommendation +} + +func setIntegerCPURecommendation(recommendation apiv1.ResourceList) { + for resourceName, recommended := range recommendation { + if resourceName != apiv1.ResourceCPU { + continue + } + recommended.RoundUp(resource.Scale(0)) + recommendation[resourceName] = recommended + } +} + +// extractContainerName return the container name for the feature based on annotation key +// if the returned value is empty that means that the key does not match +func extractContainerName(key, prefix, suffix string) string { + if !strings.HasPrefix(key, prefix) { + return "" + } + if !strings.HasSuffix(key, suffix) { + return "" + } + + return key[len(prefix) : len(key)-len(suffix)] +} diff --git a/multidimensional-pod-autoscaler/pkg/recommender/routines/cpu_integer_post_processor_test.go b/multidimensional-pod-autoscaler/pkg/recommender/routines/cpu_integer_post_processor_test.go new file mode 100644 index 000000000000..8e1c32c0960d --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/recommender/routines/cpu_integer_post_processor_test.go @@ -0,0 +1,239 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package routines + +import ( + "testing" + + "github.com/stretchr/testify/assert" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + mpa_types "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1alpha1" + vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/test" +) + +func TestExtractContainerName(t *testing.T) { + tests := []struct { + name string + key string + prefix string + suffix string + want string + }{ + { + name: "empty", + key: "", + prefix: "", + suffix: "", + want: "", + }, + { + name: "no match", + key: "abc", + prefix: "z", + suffix: "x", + want: "", + }, + { + name: "match", + key: "abc", + prefix: "a", + suffix: "c", + want: "b", + }, + { + name: "real", + key: mpaPostProcessorPrefix + "kafka" + mpaPostProcessorIntegerCPUSuffix, + prefix: mpaPostProcessorPrefix, + suffix: mpaPostProcessorIntegerCPUSuffix, + want: "kafka", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equalf(t, tt.want, extractContainerName(tt.key, tt.prefix, tt.suffix), "extractContainerName(%v, %v, %v)", tt.key, tt.prefix, tt.suffix) + }) + } +} + +func TestIntegerCPUPostProcessor_Process(t *testing.T) { + tests := []struct { + name string + mpa *mpa_types.MultidimPodAutoscaler + recommendation *vpa_types.RecommendedPodResources + want *vpa_types.RecommendedPodResources + }{ + { + name: "No containers match", + mpa: &mpa_types.MultidimPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + mpaPostProcessorPrefix + "container-other" + mpaPostProcessorIntegerCPUSuffix: mpaPostProcessorIntegerCPUValue, + }}}, + recommendation: &vpa_types.RecommendedPodResources{ + ContainerRecommendations: []vpa_types.RecommendedContainerResources{ + test.Recommendation().WithContainer("container1").WithTarget("8.6", "200Mi").GetContainerResources(), + test.Recommendation().WithContainer("container2").WithTarget("8.2", "300Mi").GetContainerResources(), + }, + }, + want: &vpa_types.RecommendedPodResources{ + ContainerRecommendations: []vpa_types.RecommendedContainerResources{ + test.Recommendation().WithContainer("container1").WithTarget("8.6", "200Mi").GetContainerResources(), + test.Recommendation().WithContainer("container2").WithTarget("8.2", "300Mi").GetContainerResources(), + }, + }, + }, + { + name: "2 containers, 1 matching only", + mpa: &mpa_types.MultidimPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{ + mpaPostProcessorPrefix + "container1" + mpaPostProcessorIntegerCPUSuffix: mpaPostProcessorIntegerCPUValue, + }}}, + recommendation: &vpa_types.RecommendedPodResources{ + ContainerRecommendations: []vpa_types.RecommendedContainerResources{ + test.Recommendation().WithContainer("container1").WithTarget("8.6", "200Mi").GetContainerResources(), + test.Recommendation().WithContainer("container2").WithTarget("8.2", "300Mi").GetContainerResources(), + }, + }, + want: &vpa_types.RecommendedPodResources{ + ContainerRecommendations: []vpa_types.RecommendedContainerResources{ + test.Recommendation().WithContainer("container1").WithTarget("9", "200Mi").GetContainerResources(), + test.Recommendation().WithContainer("container2").WithTarget("8.2", "300Mi").GetContainerResources(), + }, + }, + }, + { + name: "2 containers, 2 matching", + mpa: &mpa_types.MultidimPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{ + mpaPostProcessorPrefix + "container1" + mpaPostProcessorIntegerCPUSuffix: mpaPostProcessorIntegerCPUValue, + mpaPostProcessorPrefix + "container2" + mpaPostProcessorIntegerCPUSuffix: mpaPostProcessorIntegerCPUValue, + }}}, + recommendation: &vpa_types.RecommendedPodResources{ + ContainerRecommendations: []vpa_types.RecommendedContainerResources{ + test.Recommendation().WithContainer("container1").WithTarget("8.6", "200Mi").GetContainerResources(), + test.Recommendation().WithContainer("container2").WithTarget("5.2", "300Mi").GetContainerResources(), + }, + }, + want: &vpa_types.RecommendedPodResources{ + ContainerRecommendations: []vpa_types.RecommendedContainerResources{ + test.Recommendation().WithContainer("container1").WithTarget("9", "200Mi").GetContainerResources(), + test.Recommendation().WithContainer("container2").WithTarget("6", "300Mi").GetContainerResources(), + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := IntegerCPUPostProcessor{} + got := c.Process(tt.mpa, tt.recommendation) + assert.True(t, equalRecommendedPodResources(tt.want, got), "Process(%v, %v, nil)", tt.mpa, tt.recommendation) + }) + } +} + +func equalRecommendedPodResources(a, b *vpa_types.RecommendedPodResources) bool { + if len(a.ContainerRecommendations) != len(b.ContainerRecommendations) { + return false + } + + for i := range a.ContainerRecommendations { + if !equalResourceList(a.ContainerRecommendations[i].LowerBound, b.ContainerRecommendations[i].LowerBound) { + return false + } + if !equalResourceList(a.ContainerRecommendations[i].Target, b.ContainerRecommendations[i].Target) { + return false + } + if !equalResourceList(a.ContainerRecommendations[i].UncappedTarget, b.ContainerRecommendations[i].UncappedTarget) { + return false + } + if !equalResourceList(a.ContainerRecommendations[i].UpperBound, b.ContainerRecommendations[i].UpperBound) { + return false + } + } + return true +} + +func equalResourceList(rla, rlb v1.ResourceList) bool { + if len(rla) != len(rlb) { + return false + } + for k := range rla { + q := rla[k] + if q.Cmp(rlb[k]) != 0 { + return false + } + } + for k := range rlb { + q := rlb[k] + if q.Cmp(rla[k]) != 0 { + return false + } + } + return true +} + +func TestSetIntegerCPURecommendation(t *testing.T) { + tests := []struct { + name string + recommendation v1.ResourceList + expectedRecommendation v1.ResourceList + }{ + { + name: "unchanged", + recommendation: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("8"), + v1.ResourceMemory: resource.MustParse("6Gi"), + }, + expectedRecommendation: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("8"), + v1.ResourceMemory: resource.MustParse("6Gi"), + }, + }, + { + name: "round up from 0.1", + recommendation: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("8.1"), + v1.ResourceMemory: resource.MustParse("6Gi"), + }, + expectedRecommendation: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("9"), + v1.ResourceMemory: resource.MustParse("6Gi"), + }, + }, + { + name: "round up from 0.9", + recommendation: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("8.9"), + v1.ResourceMemory: resource.MustParse("6Gi"), + }, + expectedRecommendation: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("9"), + v1.ResourceMemory: resource.MustParse("6Gi"), + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + setIntegerCPURecommendation(tt.recommendation) + assert.True(t, equalResourceList(tt.recommendation, tt.expectedRecommendation)) + }) + } +} diff --git a/multidimensional-pod-autoscaler/pkg/recommender/routines/hpa_logic.go b/multidimensional-pod-autoscaler/pkg/recommender/routines/hpa_logic.go new file mode 100644 index 000000000000..32d8b988e31b --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/recommender/routines/hpa_logic.go @@ -0,0 +1,1068 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package routines + +import ( + "context" + "encoding/json" + "fmt" + "math" + "time" + + autoscalingv1 "k8s.io/api/autoscaling/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" + v1 "k8s.io/api/core/v1" + apiequality "k8s.io/apimachinery/pkg/api/equality" + apimeta "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + mpa_types "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1alpha1" + mpa_api "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/client/clientset/versioned/typed/autoscaling.k8s.io/v1alpha1" + "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/recommender/model" + klog "k8s.io/klog/v2" +) + +// NormalizationArg is used to pass all needed information between functions as one structure +type NormalizationArg struct { + Key model.MpaID + ScaleUpBehavior *autoscalingv2.HPAScalingRules + ScaleDownBehavior *autoscalingv2.HPAScalingRules + MinReplicas int32 + MaxReplicas int32 + CurrentReplicas int32 + DesiredReplicas int32 +} + +func (r *recommender) ReconcileHorizontalAutoscaling(ctx context.Context, mpaShared *mpa_types.MultidimPodAutoscaler, key model.MpaID) error { + // make a copy so that we never mutate the shared informer cache (conversion can mutate the object) + mpa := mpaShared.DeepCopy() + mpaStatusOriginal := mpa.Status.DeepCopy() + + reference := fmt.Sprintf("%s/%s/%s", mpa.Spec.ScaleTargetRef.Kind, mpa.Namespace, mpa.Spec.ScaleTargetRef.Name) + + targetGV, err := schema.ParseGroupVersion(mpa.Spec.ScaleTargetRef.APIVersion) + if err != nil { + klog.Errorf("%s: FailedGetScale - error: %v", v1.EventTypeWarning, err.Error()) + r.eventRecorder.Event(mpa, v1.EventTypeWarning, "FailedGetScale", err.Error()) + setCondition(mpa, mpa_types.AbleToScale, v1.ConditionFalse, "FailedGetScale", "the MPA recommender was unable to get the target's current scale: %v", err) + if err := r.updateStatusIfNeeded(ctx, mpaStatusOriginal, mpa); err != nil { + klog.Errorf("Error updating MPA status: %v", err.Error()) + utilruntime.HandleError(err) + } + return fmt.Errorf("invalid API version in scale target reference: %v", err) + } + + targetGK := schema.GroupKind{ + Group: targetGV.Group, + Kind: mpa.Spec.ScaleTargetRef.Kind, + } + + mappings, err := r.selectorFetcher.GetRESTMappings(targetGK) + if err != nil { + klog.Errorf("%s: FailedGetScale - error: %v", v1.EventTypeWarning, err.Error()) + r.eventRecorder.Event(mpa, v1.EventTypeWarning, "FailedGetScale", err.Error()) + setCondition(mpa, mpa_types.AbleToScale, v1.ConditionFalse, "FailedGetScale", "the MPA recommender was unable to get the target's current scale: %v", err) + if err := r.updateStatusIfNeeded(ctx, mpaStatusOriginal, mpa); err != nil { + klog.Errorf("Error updating MPA status: %v", err.Error()) + utilruntime.HandleError(err) + } + return fmt.Errorf("unable to determine resource for scale target reference: %v", err) + } + + scale, targetGR, err := r.scaleForResourceMappings(ctx, mpa.Namespace, mpa.Spec.ScaleTargetRef.Name, mappings) + if err != nil { + klog.Errorf("%s: FailedGetScale - error: %v", v1.EventTypeWarning, err.Error()) + r.eventRecorder.Event(mpa, v1.EventTypeWarning, "FailedGetScale", err.Error()) + setCondition(mpa, mpa_types.AbleToScale, v1.ConditionFalse, "FailedGetScale", "the MPA recommender was unable to get the target's current scale: %v", err) + if err := r.updateStatusIfNeeded(ctx, mpaStatusOriginal, mpa); err != nil { + klog.Errorf("Error updating MPA status: %v", err.Error()) + utilruntime.HandleError(err) + } + return fmt.Errorf("failed to query scale subresource for %s: %v", reference, err) + } + setCondition(mpa, mpa_types.AbleToScale, v1.ConditionTrue, "SucceededGetScale", "the MPA recommender was able to get the target's current scale") + klog.V(4).Infof("MPA recommender was able to get the target's current scale = %d for targetGR %v", scale.Spec.Replicas, targetGR) + currentReplicas := scale.Spec.Replicas + r.recordInitialRecommendation(currentReplicas, key) + + var ( + metricStatuses []autoscalingv2.MetricStatus + metricDesiredReplicas int32 + metricName string + ) + + desiredReplicas := int32(0) + rescaleReason := "" + + var minReplicas int32 + + if mpa.Spec.Constraints.Global.MinReplicas != nil { + minReplicas = *mpa.Spec.Constraints.Global.MinReplicas + } else { + // Default value is 1. + minReplicas = 1 + } + + rescale := true + + if scale.Spec.Replicas == 0 && minReplicas != 0 { + // Autoscaling is disabled for this resource + desiredReplicas = 0 + rescale = false + setCondition(mpa, mpa_types.ScalingActive, v1.ConditionFalse, "ScalingDisabled", "scaling is disabled since the replica count of the target is zero") + klog.V(4).Infof("Scaling is disabled since the replica count of the target is zero.") + } else if currentReplicas > *mpa.Spec.Constraints.Global.MaxReplicas { + rescaleReason = "Current number of replicas above Spec.Constraints.MaxReplicas" + desiredReplicas = *mpa.Spec.Constraints.Global.MaxReplicas + klog.V(4).Infof("Current number of replicas above Spec.Constraints.MaxReplicas.") + } else if currentReplicas < minReplicas { + rescaleReason = "Current number of replicas below Spec.Constraints.MinReplicas" + desiredReplicas = minReplicas + klog.V(4).Infof("Current number of replicas below Spec.Constraints.MinReplicas.") + } else { + var metricTimestamp time.Time + metricDesiredReplicas, metricName, metricStatuses, metricTimestamp, err = r.computeReplicasForMetrics(ctx, mpa, scale, mpa.Spec.Goals.Metrics) + if err != nil { + r.setCurrentReplicasInStatus(mpa, currentReplicas) + if err := r.updateStatusIfNeeded(ctx, mpaStatusOriginal, mpa); err != nil { + klog.Errorf("Error updating MPA status: %v", err.Error()) + utilruntime.HandleError(err) + } + r.eventRecorder.Event(mpa, v1.EventTypeWarning, "FailedComputeMetricsReplicas", err.Error()) + klog.Errorf("%s: FailedComputeMetricsReplicas - error: %v", v1.EventTypeWarning, err.Error()) + return fmt.Errorf("failed to compute desired number of replicas based on listed metrics for %s: %v", reference, err) + } + + klog.V(4).Infof("metricDesiredReplicas = %d desired replicas (based on %s from %s) for %s", metricDesiredReplicas, metricName, metricTimestamp, reference) + + rescaleMetric := "" + if metricDesiredReplicas > desiredReplicas { + desiredReplicas = metricDesiredReplicas + rescaleMetric = metricName + } + if desiredReplicas > currentReplicas { + rescaleReason = fmt.Sprintf("%s above target", rescaleMetric) + } + if desiredReplicas < currentReplicas { + rescaleReason = "All metrics below target" + } + if mpa.Spec.Constraints.Global.Behavior == nil { + desiredReplicas = r.normalizeDesiredReplicas(mpa, key, currentReplicas, desiredReplicas, minReplicas) + } else { + desiredReplicas = r.normalizeDesiredReplicasWithBehaviors(mpa, key, currentReplicas, desiredReplicas, minReplicas) + } + rescale = desiredReplicas != currentReplicas + } + + if rescale { + // scale.Spec.Replicas = desiredReplicas + // klog.V(4).Infof("Updating the number of replicas to %d for MPA %v", desiredReplicas, key) + // _, err = r.controllerFetcher.Scales(mpa.Namespace).Update(ctx, targetGR, scale, metav1.UpdateOptions{}) + r.setCurrentReplicasInStatus(mpa, currentReplicas) + if err := r.updateStatusIfNeeded(ctx, mpaStatusOriginal, mpa); err != nil { + klog.Errorf("Error updating MPA status: %v", err.Error()) + utilruntime.HandleError(err) + } + // if err != nil { + // r.eventRecorder.Eventf(mpa, v1.EventTypeWarning, "FailedRescale", "New size: %d; reason: %s; error: %v", desiredReplicas, rescaleReason, err.Error()) + // klog.Errorf("%s: FailedRescale - New size: %d; reason: %s; error: %v", v1.EventTypeWarning, desiredReplicas, rescaleReason, err.Error()) + // setCondition(mpa, mpa_types.AbleToScale, v1.ConditionFalse, "FailedUpdateScale", "the MPA controller was unable to update the target scale: %v", err) + // r.setCurrentReplicasInStatus(mpa, currentReplicas) + // if err := r.updateStatusIfNeeded(ctx, mpaStatusOriginal, mpa); err != nil { + // klog.Errorf("Error updating MPA status: %v", err.Error()) + // utilruntime.HandleError(err) + // } + // return fmt.Errorf("failed to rescale %s: %v", reference, err) + // } + setCondition(mpa, mpa_types.AbleToScale, v1.ConditionTrue, "SucceededRescale", "the MPA controller was able to update the target scale to %d", desiredReplicas) + r.eventRecorder.Eventf(mpa, v1.EventTypeNormal, "SuccessfulRescale", "New size: %d; reason: %s", desiredReplicas, rescaleReason) + // klog.V(4).Infof("%s: Successfully rescaled the number of replicas to %d for MPA %v", v1.EventTypeNormal, desiredReplicas, key) + r.storeScaleEvent(mpa.Spec.Constraints.Global.Behavior, key, currentReplicas, desiredReplicas) + // klog.Infof("Successful rescaled of %s, old size: %d, new size: %d, reason: %s", mpa.Name, currentReplicas, desiredReplicas, rescaleReason) + } else { + klog.V(4).Infof("decided not to scale %s to %v (reason: %s) (the last scale time was %s)", reference, desiredReplicas, rescaleReason, mpa.Status.LastScaleTime) + desiredReplicas = currentReplicas + } + + r.setStatus(mpa, currentReplicas, desiredReplicas, metricStatuses, rescale) + return r.updateStatusIfNeeded(ctx, mpaStatusOriginal, mpa) +} + +// setCondition sets the specific condition type on the given MPA to the specified value with the +// given reason and message. The message and args are treated like a format string. The condition +// will be added if it is not present. +func setCondition(mpa *mpa_types.MultidimPodAutoscaler, conditionType mpa_types.MultidimPodAutoscalerConditionType, status v1.ConditionStatus, reason, message string, args ...interface{}) { + mpa.Status.Conditions = setConditionInList(mpa.Status.Conditions, conditionType, status, reason, message, args...) +} + +// setConditionInList sets the specific condition type on the given MPA to the specified value with +// the given reason and message. The message and args are treated like a format string. The +// condition will be added if it is not present. The new list will be returned. +func setConditionInList(inputList []mpa_types.MultidimPodAutoscalerCondition, conditionType mpa_types.MultidimPodAutoscalerConditionType, status v1.ConditionStatus, reason, message string, args ...interface{}) []mpa_types.MultidimPodAutoscalerCondition { + resList := inputList + var existingCond *mpa_types.MultidimPodAutoscalerCondition + for i, condition := range resList { + if condition.Type == conditionType { + // can't take a pointer to an iteration variable + existingCond = &resList[i] + break + } + } + + if existingCond == nil { + resList = append(resList, mpa_types.MultidimPodAutoscalerCondition{ + Type: conditionType, + }) + existingCond = &resList[len(resList)-1] + } + + if existingCond.Status != status { + existingCond.LastTransitionTime = metav1.Now() + } + + existingCond.Status = status + existingCond.Reason = reason + existingCond.Message = fmt.Sprintf(message, args...) + + return resList +} + +// updateStatusIfNeeded calls updateStatus only if the status of the new MPA is not the same as the +// old status +func (r *recommender) updateStatusIfNeeded(ctx context.Context, oldStatus *mpa_types.MultidimPodAutoscalerStatus, newMPA *mpa_types.MultidimPodAutoscaler) error { + // skip a write if we wouldn't need to update + if apiequality.Semantic.DeepEqual(oldStatus, &newMPA.Status) { + return nil + } + + patches := []patchRecord{{ + Op: "add", + Path: "/status", + Value: newMPA.Status, + }} + klog.V(4).Infof("Updating MPA status with the desired number of replicas = %d", newMPA.Status.DesiredReplicas) + + return patchMpa(r.mpaClient.MultidimPodAutoscalers(newMPA.Namespace), newMPA.Name, patches) +} + +type patchRecord struct { + Op string `json:"op,inline"` + Path string `json:"path,inline"` + Value interface{} `json:"value"` +} + +func patchMpa(mpaClient mpa_api.MultidimPodAutoscalerInterface, mpaName string, patches []patchRecord) (err error) { + bytes, err := json.Marshal(patches) + if err != nil { + klog.Errorf("Cannot marshal MPA status patches %+v. Reason: %+v", patches, err) + return + } + + updatedMPA, err := mpaClient.Patch(context.TODO(), mpaName, types.JSONPatchType, bytes, metav1.PatchOptions{}) + + klog.V(4).Infof("MPA %s status updated (desiredReplicas = %d)", updatedMPA.Name, updatedMPA.Status.DesiredReplicas) + + return err +} + +// updateStatus actually does the update request for the status of the given MPA +func (r *recommender) updateStatus(ctx context.Context, mpa *mpa_types.MultidimPodAutoscaler) error { + _, err := r.mpaClient.MultidimPodAutoscalers(mpa.Namespace).UpdateStatus(ctx, mpa, metav1.UpdateOptions{}) + if err != nil { + klog.Errorf("%s: FailedUpdateStatus - error updating status for MPA %s (namespace %s): %v", v1.EventTypeWarning, mpa.Name, mpa.Namespace, err.Error()) + r.eventRecorder.Event(mpa, v1.EventTypeWarning, "FailedUpdateStatus", err.Error()) + return fmt.Errorf("failed to update status for %s: %v", mpa.Name, err) + } + klog.V(2).Infof("Successfully updated status (HPA-related) for %s", mpa.Name) + return nil +} + +// scaleForResourceMappings attempts to fetch the scale for the resource with the given name and +// namespace, trying each RESTMapping in turn until a working one is found. If none work, the first +// error is returned. It returns both the scale, as well as the group-resource from the working +// mapping. +func (r *recommender) scaleForResourceMappings(ctx context.Context, namespace, name string, mappings []*apimeta.RESTMapping) (*autoscalingv1.Scale, schema.GroupResource, error) { + var firstErr error + for i, mapping := range mappings { + targetGR := mapping.Resource.GroupResource() + scale, err := r.selectorFetcher.Scales(namespace).Get(ctx, targetGR, name, metav1.GetOptions{}) + if err == nil { + return scale, targetGR, nil + } + + // if this is the first error, remember it, + // then go on and try other mappings until we find a good one + if i == 0 { + firstErr = err + } + } + + // make sure we handle an empty set of mappings + if firstErr == nil { + firstErr = fmt.Errorf("unrecognized resource") + } + + return nil, schema.GroupResource{}, firstErr +} + +func (r *recommender) recordInitialRecommendation(currentReplicas int32, key model.MpaID) { + r.recommendationsLock.Lock() + defer r.recommendationsLock.Unlock() + if r.recommendations[key] == nil { + r.recommendations[key] = []timestampedRecommendation{{currentReplicas, time.Now()}} + } +} + +// computeReplicasForMetrics computes the desired number of replicas for the metric specifications +// listed in the MPA, returning the maximum of the computed replica counts, a description of the +// associated metric, and the statuses of all metrics computed. +func (r *recommender) computeReplicasForMetrics(ctx context.Context, mpa *mpa_types.MultidimPodAutoscaler, scale *autoscalingv1.Scale, metricSpecs []autoscalingv2.MetricSpec) (replicas int32, metric string, statuses []autoscalingv2.MetricStatus, timestamp time.Time, err error) { + if scale.Status.Selector == "" { + errMsg := "selector is required" + r.eventRecorder.Event(mpa, v1.EventTypeWarning, "SelectorRequired", errMsg) + klog.Errorf("%s: SelectorRequired", v1.EventTypeWarning) + setCondition(mpa, mpa_types.ScalingActive, v1.ConditionFalse, "InvalidSelector", "the MPA target's scale is missing a selector") + return 0, "", nil, time.Time{}, fmt.Errorf(errMsg) + } + + selector, err := labels.Parse(scale.Status.Selector) + if err != nil { + errMsg := fmt.Sprintf("couldn't convert selector into a corresponding internal selector object: %v", err) + r.eventRecorder.Event(mpa, v1.EventTypeWarning, "InvalidSelector", errMsg) + klog.Errorf("%s: InvalidSelector - error: %v", v1.EventTypeWarning, errMsg) + setCondition(mpa, mpa_types.ScalingActive, v1.ConditionFalse, "InvalidSelector", errMsg) + return 0, "", nil, time.Time{}, fmt.Errorf(errMsg) + } + klog.V(4).Infof("Label Selector parsed as %v", selector) + + specReplicas := scale.Spec.Replicas + statusReplicas := scale.Status.Replicas + statuses = make([]autoscalingv2.MetricStatus, len(metricSpecs)) + + invalidMetricsCount := 0 + var invalidMetricError error + var invalidMetricCondition mpa_types.MultidimPodAutoscalerCondition + + for i, metricSpec := range metricSpecs { + replicaCountProposal, metricNameProposal, timestampProposal, condition, err := r.computeReplicasForMetric(ctx, mpa, metricSpec, specReplicas, statusReplicas, selector, &statuses[i]) + + if err != nil { + if invalidMetricsCount <= 0 { + invalidMetricCondition = condition + invalidMetricError = err + } + invalidMetricsCount++ + } + if err == nil && (replicas == 0 || replicaCountProposal > replicas) { + timestamp = timestampProposal + replicas = replicaCountProposal + metric = metricNameProposal + } + } + + // If all metrics are invalid or some are invalid and we would scale down, + // return an error and set the condition of the MPA based on the first invalid metric. + // Otherwise set the condition as scaling active as we're going to scale + if invalidMetricsCount >= len(metricSpecs) || (invalidMetricsCount > 0 && replicas < specReplicas) { + setCondition(mpa, mpa_types.AbleToScale, invalidMetricCondition.Status, invalidMetricCondition.Reason, invalidMetricCondition.Message) + return 0, "", statuses, time.Time{}, fmt.Errorf("invalid metrics (%v invalid out of %v), first error is: %v", invalidMetricsCount, len(metricSpecs), invalidMetricError) + } + setCondition(mpa, mpa_types.ScalingActive, v1.ConditionTrue, "ValidMetricFound", "the MPA was able to successfully calculate a replica count from %s", metric) + return replicas, metric, statuses, timestamp, nil +} + +// Computes the desired number of replicas for a specific MPA and metric specification, +// returning the metric status and a proposed condition to be set on the MPA object. +func (r *recommender) computeReplicasForMetric(ctx context.Context, mpa *mpa_types.MultidimPodAutoscaler, spec autoscalingv2.MetricSpec, specReplicas, statusReplicas int32, selector labels.Selector, status *autoscalingv2.MetricStatus) (replicaCountProposal int32, metricNameProposal string, timestampProposal time.Time, condition mpa_types.MultidimPodAutoscalerCondition, err error) { + switch spec.Type { + case autoscalingv2.ObjectMetricSourceType: + klog.V(4).Infof("Pulling metrics from the source of type ObjectMetricSourceType") + metricSelector, err := metav1.LabelSelectorAsSelector(spec.Object.Metric.Selector) + if err != nil { + condition := r.getUnableComputeReplicaCountCondition(mpa, "FailedGetObjectMetric", err) + return 0, "", time.Time{}, condition, fmt.Errorf("failed to get object metric value: %v", err) + } + replicaCountProposal, timestampProposal, metricNameProposal, condition, err = r.computeStatusForObjectMetric(specReplicas, statusReplicas, spec, mpa, selector, status, metricSelector) + if err != nil { + return 0, "", time.Time{}, condition, fmt.Errorf("failed to get object metric value: %v", err) + } + case autoscalingv2.PodsMetricSourceType: + klog.V(4).Infof("Pulling metrics from the source of type PodMetricSourceType") + metricSelector, err := metav1.LabelSelectorAsSelector(spec.Pods.Metric.Selector) + if err != nil { + condition := r.getUnableComputeReplicaCountCondition(mpa, "FailedGetPodsMetric", err) + return 0, "", time.Time{}, condition, fmt.Errorf("failed to get pods metric value: %v", err) + } + replicaCountProposal, timestampProposal, metricNameProposal, condition, err = r.computeStatusForPodsMetric(specReplicas, spec, mpa, selector, status, metricSelector) + if err != nil { + return 0, "", time.Time{}, condition, fmt.Errorf("failed to get pods metric value: %v", err) + } + case autoscalingv2.ResourceMetricSourceType: + klog.V(4).Infof("Pulling metrics from the source of type ResourceMetricSourceType") + replicaCountProposal, timestampProposal, metricNameProposal, condition, err = r.computeStatusForResourceMetric(ctx, specReplicas, spec, mpa, selector, status) + if err != nil { + return 0, "", time.Time{}, condition, fmt.Errorf("failed to get %s resource metric value: %v", spec.Resource.Name, err) + } + case autoscalingv2.ContainerResourceMetricSourceType: + klog.V(4).Infof("Pulling metrics from the source of type ContainerResourceMetricSourceType") + replicaCountProposal, timestampProposal, metricNameProposal, condition, err = r.computeStatusForContainerResourceMetric(ctx, specReplicas, spec, mpa, selector, status) + if err != nil { + return 0, "", time.Time{}, condition, fmt.Errorf("failed to get %s container metric value: %v", spec.ContainerResource.Container, err) + } + case autoscalingv2.ExternalMetricSourceType: + klog.V(4).Infof("Pulling metrics from the source of type ExternalMetricSourceType") + replicaCountProposal, timestampProposal, metricNameProposal, condition, err = r.computeStatusForExternalMetric(specReplicas, statusReplicas, spec, mpa, selector, status) + if err != nil { + return 0, "", time.Time{}, condition, fmt.Errorf("failed to get %s external metric value: %v", spec.External.Metric.Name, err) + } + default: + klog.Warningf("Unknown metric source type!") + errMsg := fmt.Sprintf("unknown metric source type %q", string(spec.Type)) + err = fmt.Errorf(errMsg) + condition := r.getUnableComputeReplicaCountCondition(mpa, "InvalidMetricSourceType", err) + return 0, "", time.Time{}, condition, err + } + return replicaCountProposal, metricNameProposal, timestampProposal, mpa_types.MultidimPodAutoscalerCondition{}, nil +} + +func (r *recommender) getUnableComputeReplicaCountCondition(mpa runtime.Object, reason string, err error) (condition mpa_types.MultidimPodAutoscalerCondition) { + r.eventRecorder.Event(mpa, v1.EventTypeWarning, reason, err.Error()) + klog.Errorf("%s: %s - error: %v", v1.EventTypeWarning, reason, err.Error()) + return mpa_types.MultidimPodAutoscalerCondition{ + Type: mpa_types.ScalingActive, + Status: v1.ConditionFalse, + Reason: reason, + Message: fmt.Sprintf("the MPA was unable to compute the replica count: %v", err), + } +} + +// computeStatusForObjectMetric computes the desired number of replicas for the specified metric of type ObjectMetricSourceType. +func (r *recommender) computeStatusForObjectMetric(specReplicas, statusReplicas int32, metricSpec autoscalingv2.MetricSpec, mpa *mpa_types.MultidimPodAutoscaler, selector labels.Selector, status *autoscalingv2.MetricStatus, metricSelector labels.Selector) (replicas int32, timestamp time.Time, metricName string, condition mpa_types.MultidimPodAutoscalerCondition, err error) { + if metricSpec.Object.Target.Type == autoscalingv2.ValueMetricType { + replicaCountProposal, usageProposal, timestampProposal, err := r.replicaCalc.GetObjectMetricReplicas(specReplicas, metricSpec.Object.Target.Value.MilliValue(), metricSpec.Object.Metric.Name, mpa.Namespace, &metricSpec.Object.DescribedObject, selector, metricSelector) + if err != nil { + condition := r.getUnableComputeReplicaCountCondition(mpa, "FailedGetObjectMetric", err) + return 0, timestampProposal, "", condition, err + } + *status = autoscalingv2.MetricStatus{ + Type: autoscalingv2.ObjectMetricSourceType, + Object: &autoscalingv2.ObjectMetricStatus{ + DescribedObject: metricSpec.Object.DescribedObject, + Metric: autoscalingv2.MetricIdentifier{ + Name: metricSpec.Object.Metric.Name, + Selector: metricSpec.Object.Metric.Selector, + }, + Current: autoscalingv2.MetricValueStatus{ + Value: resource.NewMilliQuantity(usageProposal, resource.DecimalSI), + }, + }, + } + return replicaCountProposal, timestampProposal, fmt.Sprintf("%s metric %s", metricSpec.Object.DescribedObject.Kind, metricSpec.Object.Metric.Name), mpa_types.MultidimPodAutoscalerCondition{}, nil + } else if metricSpec.Object.Target.Type == autoscalingv2.AverageValueMetricType { + replicaCountProposal, usageProposal, timestampProposal, err := r.replicaCalc.GetObjectPerPodMetricReplicas(statusReplicas, metricSpec.Object.Target.AverageValue.MilliValue(), metricSpec.Object.Metric.Name, mpa.Namespace, &metricSpec.Object.DescribedObject, metricSelector) + if err != nil { + condition := r.getUnableComputeReplicaCountCondition(mpa, "FailedGetObjectMetric", err) + return 0, time.Time{}, "", condition, fmt.Errorf("failed to get %s object metric: %v", metricSpec.Object.Metric.Name, err) + } + *status = autoscalingv2.MetricStatus{ + Type: autoscalingv2.ObjectMetricSourceType, + Object: &autoscalingv2.ObjectMetricStatus{ + Metric: autoscalingv2.MetricIdentifier{ + Name: metricSpec.Object.Metric.Name, + Selector: metricSpec.Object.Metric.Selector, + }, + Current: autoscalingv2.MetricValueStatus{ + AverageValue: resource.NewMilliQuantity(usageProposal, resource.DecimalSI), + }, + }, + } + return replicaCountProposal, timestampProposal, fmt.Sprintf("external metric %s(%+v)", metricSpec.Object.Metric.Name, metricSpec.Object.Metric.Selector), mpa_types.MultidimPodAutoscalerCondition{}, nil + } + errMsg := "invalid object metric source: neither a value target nor an average value target was set" + err = fmt.Errorf(errMsg) + condition = r.getUnableComputeReplicaCountCondition(mpa, "FailedGetObjectMetric", err) + return 0, time.Time{}, "", condition, err +} + +// computeStatusForPodsMetric computes the desired number of replicas for the specified metric of type PodsMetricSourceType. +func (r *recommender) computeStatusForPodsMetric(currentReplicas int32, metricSpec autoscalingv2.MetricSpec, mpa *mpa_types.MultidimPodAutoscaler, selector labels.Selector, status *autoscalingv2.MetricStatus, metricSelector labels.Selector) (replicaCountProposal int32, timestampProposal time.Time, metricNameProposal string, condition mpa_types.MultidimPodAutoscalerCondition, err error) { + replicaCountProposal, usageProposal, timestampProposal, err := r.replicaCalc.GetMetricReplicas(currentReplicas, metricSpec.Pods.Target.AverageValue.MilliValue(), metricSpec.Pods.Metric.Name, mpa.Namespace, selector, metricSelector) + if err != nil { + condition = r.getUnableComputeReplicaCountCondition(mpa, "FailedGetPodsMetric", err) + return 0, timestampProposal, "", condition, err + } + *status = autoscalingv2.MetricStatus{ + Type: autoscalingv2.PodsMetricSourceType, + Pods: &autoscalingv2.PodsMetricStatus{ + Metric: autoscalingv2.MetricIdentifier{ + Name: metricSpec.Pods.Metric.Name, + Selector: metricSpec.Pods.Metric.Selector, + }, + Current: autoscalingv2.MetricValueStatus{ + AverageValue: resource.NewMilliQuantity(usageProposal, resource.DecimalSI), + }, + }, + } + + return replicaCountProposal, timestampProposal, fmt.Sprintf("pods metric %s", metricSpec.Pods.Metric.Name), mpa_types.MultidimPodAutoscalerCondition{}, nil +} + +func (r *recommender) computeStatusForResourceMetricGeneric(ctx context.Context, currentReplicas int32, target autoscalingv2.MetricTarget, resourceName v1.ResourceName, namespace string, container string, selector labels.Selector) (replicaCountProposal int32, metricStatus *autoscalingv2.MetricValueStatus, timestampProposal time.Time, metricNameProposal string, condition mpa_types.MultidimPodAutoscalerCondition, err error) { + if target.AverageValue != nil { + var rawProposal int64 + replicaCountProposal, rawProposal, timestampProposal, err := r.replicaCalc.GetRawResourceReplicas(ctx, currentReplicas, target.AverageValue.MilliValue(), resourceName, namespace, selector, container) + if err != nil { + return 0, nil, time.Time{}, "", condition, fmt.Errorf("failed to get %s usage: %v", resourceName, err) + } + metricNameProposal = fmt.Sprintf("%s resource", resourceName.String()) + status := autoscalingv2.MetricValueStatus{ + AverageValue: resource.NewMilliQuantity(rawProposal, resource.DecimalSI), + } + return replicaCountProposal, &status, timestampProposal, metricNameProposal, mpa_types.MultidimPodAutoscalerCondition{}, nil + } + + if target.AverageUtilization == nil { + errMsg := "invalid resource metric source: neither an average utilization target nor an average value (usage) target was set" + return 0, nil, time.Time{}, "", condition, fmt.Errorf(errMsg) + } + + targetUtilization := *target.AverageUtilization + replicaCountProposal, percentageProposal, rawProposal, timestampProposal, err := r.replicaCalc.GetResourceReplicas(ctx, currentReplicas, targetUtilization, resourceName, namespace, selector, container) + if err != nil { + return 0, nil, time.Time{}, "", condition, fmt.Errorf("failed to get %s utilization: %v", resourceName, err) + } + + metricNameProposal = fmt.Sprintf("%s resource utilization (percentage of request)", resourceName) + status := autoscalingv2.MetricValueStatus{ + AverageUtilization: &percentageProposal, + AverageValue: resource.NewMilliQuantity(rawProposal, resource.DecimalSI), + } + klog.V(4).Infof("Current average utilization = %d average value = %v", percentageProposal, status.AverageValue) + return replicaCountProposal, &status, timestampProposal, metricNameProposal, mpa_types.MultidimPodAutoscalerCondition{}, nil +} + +// computeStatusForResourceMetric computes the desired number of replicas for the specified metric of type ResourceMetricSourceType. +func (r *recommender) computeStatusForResourceMetric(ctx context.Context, currentReplicas int32, metricSpec autoscalingv2.MetricSpec, mpa *mpa_types.MultidimPodAutoscaler, selector labels.Selector, status *autoscalingv2.MetricStatus) (replicaCountProposal int32, timestampProposal time.Time, metricNameProposal string, condition mpa_types.MultidimPodAutoscalerCondition, err error) { + replicaCountProposal, metricValueStatus, timestampProposal, metricNameProposal, condition, err := r.computeStatusForResourceMetricGeneric(ctx, currentReplicas, metricSpec.Resource.Target, metricSpec.Resource.Name, mpa.Namespace, "", selector) + if err != nil { + condition = r.getUnableComputeReplicaCountCondition(mpa, "FailedGetResourceMetric", err) + return replicaCountProposal, timestampProposal, metricNameProposal, condition, err + } + *status = autoscalingv2.MetricStatus{ + Type: autoscalingv2.ResourceMetricSourceType, + Resource: &autoscalingv2.ResourceMetricStatus{ + Name: metricSpec.Resource.Name, + Current: *metricValueStatus, + }, + } + return replicaCountProposal, timestampProposal, metricNameProposal, condition, nil +} + +// computeStatusForContainerResourceMetric computes the desired number of replicas for the specified metric of type ResourceMetricSourceType. +func (r *recommender) computeStatusForContainerResourceMetric(ctx context.Context, currentReplicas int32, metricSpec autoscalingv2.MetricSpec, mpa *mpa_types.MultidimPodAutoscaler, selector labels.Selector, status *autoscalingv2.MetricStatus) (replicaCountProposal int32, timestampProposal time.Time, metricNameProposal string, condition mpa_types.MultidimPodAutoscalerCondition, err error) { + replicaCountProposal, metricValueStatus, timestampProposal, metricNameProposal, condition, err := r.computeStatusForResourceMetricGeneric(ctx, currentReplicas, metricSpec.ContainerResource.Target, metricSpec.ContainerResource.Name, mpa.Namespace, metricSpec.ContainerResource.Container, selector) + if err != nil { + condition = r.getUnableComputeReplicaCountCondition(mpa, "FailedGetContainerResourceMetric", err) + return replicaCountProposal, timestampProposal, metricNameProposal, condition, err + } + *status = autoscalingv2.MetricStatus{ + Type: autoscalingv2.ContainerResourceMetricSourceType, + ContainerResource: &autoscalingv2.ContainerResourceMetricStatus{ + Name: metricSpec.ContainerResource.Name, + Container: metricSpec.ContainerResource.Container, + Current: *metricValueStatus, + }, + } + return replicaCountProposal, timestampProposal, metricNameProposal, condition, nil +} + +// computeStatusForExternalMetric computes the desired number of replicas for the specified metric of type ExternalMetricSourceType. +func (r *recommender) computeStatusForExternalMetric(specReplicas, statusReplicas int32, metricSpec autoscalingv2.MetricSpec, mpa *mpa_types.MultidimPodAutoscaler, selector labels.Selector, status *autoscalingv2.MetricStatus) (replicaCountProposal int32, timestampProposal time.Time, metricNameProposal string, condition mpa_types.MultidimPodAutoscalerCondition, err error) { + if metricSpec.External.Target.AverageValue != nil { + replicaCountProposal, usageProposal, timestampProposal, err := r.replicaCalc.GetExternalPerPodMetricReplicas(statusReplicas, metricSpec.External.Target.AverageValue.MilliValue(), metricSpec.External.Metric.Name, mpa.Namespace, metricSpec.External.Metric.Selector) + if err != nil { + condition = r.getUnableComputeReplicaCountCondition(mpa, "FailedGetExternalMetric", err) + return 0, time.Time{}, "", condition, fmt.Errorf("failed to get %s external metric: %v", metricSpec.External.Metric.Name, err) + } + *status = autoscalingv2.MetricStatus{ + Type: autoscalingv2.ExternalMetricSourceType, + External: &autoscalingv2.ExternalMetricStatus{ + Metric: autoscalingv2.MetricIdentifier{ + Name: metricSpec.External.Metric.Name, + Selector: metricSpec.External.Metric.Selector, + }, + Current: autoscalingv2.MetricValueStatus{ + AverageValue: resource.NewMilliQuantity(usageProposal, resource.DecimalSI), + }, + }, + } + return replicaCountProposal, timestampProposal, fmt.Sprintf("external metric %s(%+v)", metricSpec.External.Metric.Name, metricSpec.External.Metric.Selector), mpa_types.MultidimPodAutoscalerCondition{}, nil + } + if metricSpec.External.Target.Value != nil { + replicaCountProposal, usageProposal, timestampProposal, err := r.replicaCalc.GetExternalMetricReplicas(specReplicas, metricSpec.External.Target.Value.MilliValue(), metricSpec.External.Metric.Name, mpa.Namespace, metricSpec.External.Metric.Selector, selector) + if err != nil { + condition = r.getUnableComputeReplicaCountCondition(mpa, "FailedGetExternalMetric", err) + return 0, time.Time{}, "", condition, fmt.Errorf("failed to get external metric %s: %v", metricSpec.External.Metric.Name, err) + } + *status = autoscalingv2.MetricStatus{ + Type: autoscalingv2.ExternalMetricSourceType, + External: &autoscalingv2.ExternalMetricStatus{ + Metric: autoscalingv2.MetricIdentifier{ + Name: metricSpec.External.Metric.Name, + Selector: metricSpec.External.Metric.Selector, + }, + Current: autoscalingv2.MetricValueStatus{ + Value: resource.NewMilliQuantity(usageProposal, resource.DecimalSI), + }, + }, + } + return replicaCountProposal, timestampProposal, fmt.Sprintf("external metric %s(%+v)", metricSpec.External.Metric.Name, metricSpec.External.Metric.Selector), mpa_types.MultidimPodAutoscalerCondition{}, nil + } + errMsg := "invalid external metric source: neither a value target nor an average value target was set" + err = fmt.Errorf(errMsg) + condition = r.getUnableComputeReplicaCountCondition(mpa, "FailedGetExternalMetric", err) + return 0, time.Time{}, "", condition, fmt.Errorf(errMsg) +} + +// setCurrentReplicasInStatus sets the current replica count in the status of the MPA. +func (r *recommender) setCurrentReplicasInStatus(mpa *mpa_types.MultidimPodAutoscaler, currentReplicas int32) { + r.setStatus(mpa, currentReplicas, mpa.Status.DesiredReplicas, mpa.Status.CurrentMetrics, false) +} + +// setStatus recreates the status of the given MPA, updating the current and desired replicas, as +// well as the metric statuses +func (r *recommender) setStatus(mpa *mpa_types.MultidimPodAutoscaler, currentReplicas, desiredReplicas int32, metricStatuses []autoscalingv2.MetricStatus, rescale bool) { + mpa.Status = mpa_types.MultidimPodAutoscalerStatus{ + CurrentReplicas: currentReplicas, + DesiredReplicas: desiredReplicas, + LastScaleTime: mpa.Status.LastScaleTime, + CurrentMetrics: metricStatuses, + Conditions: mpa.Status.Conditions, + // Keep VPA-related untouched. + Recommendation: mpa.Status.Recommendation, + } + + if rescale { + now := metav1.NewTime(time.Now()) + mpa.Status.LastScaleTime = &now + } +} + +// normalizeDesiredReplicas takes the metrics desired replicas value and normalizes it based on the appropriate conditions (i.e., < maxReplicas, > minReplicas, etc...) +func (r *recommender) normalizeDesiredReplicas(mpa *mpa_types.MultidimPodAutoscaler, key model.MpaID, currentReplicas int32, prenormalizedDesiredReplicas int32, minReplicas int32) int32 { + stabilizedRecommendation := r.stabilizeRecommendation(key, prenormalizedDesiredReplicas) + if stabilizedRecommendation != prenormalizedDesiredReplicas { + setCondition(mpa, mpa_types.AbleToScale, v1.ConditionTrue, "ScaleDownStabilized", "recent recommendations were higher than current one, applying the highest recent recommendation") + } else { + setCondition(mpa, mpa_types.AbleToScale, v1.ConditionTrue, "ReadyForNewScale", "recommended size matches current size") + } + + desiredReplicas, condition, reason := convertDesiredReplicasWithRules(currentReplicas, stabilizedRecommendation, minReplicas, *mpa.Spec.Constraints.Global.MaxReplicas) + + if desiredReplicas == stabilizedRecommendation { + setCondition(mpa, mpa_types.ScalingLimited, v1.ConditionFalse, condition, reason) + } else { + setCondition(mpa, mpa_types.ScalingLimited, v1.ConditionTrue, condition, reason) + } + + return desiredReplicas +} + +// convertDesiredReplicas performs the actual normalization, without depending on `HorizontalController` or `HorizontalPodAutoscaler` +func convertDesiredReplicasWithRules(currentReplicas, desiredReplicas, hpaMinReplicas, hpaMaxReplicas int32) (int32, string, string) { + var minimumAllowedReplicas int32 + var maximumAllowedReplicas int32 + + var possibleLimitingCondition string + var possibleLimitingReason string + + minimumAllowedReplicas = hpaMinReplicas + + // Do not scaleup too much to prevent incorrect rapid increase of the number of master replicas + // caused by bogus CPU usage report from heapster/kubelet (like in issue #32304). + scaleUpLimit := calculateScaleUpLimit(currentReplicas) + + if hpaMaxReplicas > scaleUpLimit { + maximumAllowedReplicas = scaleUpLimit + possibleLimitingCondition = "ScaleUpLimit" + possibleLimitingReason = "the desired replica count is increasing faster than the maximum scale rate" + } else { + maximumAllowedReplicas = hpaMaxReplicas + possibleLimitingCondition = "TooManyReplicas" + possibleLimitingReason = "the desired replica count is more than the maximum replica count" + } + + if desiredReplicas < minimumAllowedReplicas { + possibleLimitingCondition = "TooFewReplicas" + possibleLimitingReason = "the desired replica count is less than the minimum replica count" + return minimumAllowedReplicas, possibleLimitingCondition, possibleLimitingReason + } else if desiredReplicas > maximumAllowedReplicas { + return maximumAllowedReplicas, possibleLimitingCondition, possibleLimitingReason + } + + return desiredReplicas, "DesiredWithinRange", "the desired count is within the acceptable range" +} + +func calculateScaleUpLimit(currentReplicas int32) int32 { + return int32(math.Max(scaleUpLimitFactor*float64(currentReplicas), scaleUpLimitMinimum)) +} + +// convertDesiredReplicasWithBehaviorRate performs the actual normalization given the constraint +// rate. It doesn't consider the stabilizationWindow, it is done separately. +func (r *recommender) convertDesiredReplicasWithBehaviorRate(args NormalizationArg) (int32, string, string) { + var possibleLimitingReason, possibleLimitingMessage string + + if args.DesiredReplicas > args.CurrentReplicas { + r.scaleUpEventsLock.RLock() + defer r.scaleUpEventsLock.RUnlock() + r.scaleDownEventsLock.RLock() + defer r.scaleDownEventsLock.RUnlock() + scaleUpLimit := calculateScaleUpLimitWithScalingRules(args.CurrentReplicas, r.scaleUpEvents[args.Key], r.scaleDownEvents[args.Key], args.ScaleUpBehavior) + + if scaleUpLimit < args.CurrentReplicas { + // We shouldn't scale up further until the scaleUpEvents will be cleaned up + scaleUpLimit = args.CurrentReplicas + } + maximumAllowedReplicas := args.MaxReplicas + if maximumAllowedReplicas > scaleUpLimit { + maximumAllowedReplicas = scaleUpLimit + possibleLimitingReason = "ScaleUpLimit" + possibleLimitingMessage = "the desired replica count is increasing faster than the maximum scale rate" + } else { + possibleLimitingReason = "TooManyReplicas" + possibleLimitingMessage = "the desired replica count is more than the maximum replica count" + } + if args.DesiredReplicas > maximumAllowedReplicas { + return maximumAllowedReplicas, possibleLimitingReason, possibleLimitingMessage + } + } else if args.DesiredReplicas < args.CurrentReplicas { + r.scaleUpEventsLock.RLock() + defer r.scaleUpEventsLock.RUnlock() + r.scaleDownEventsLock.RLock() + defer r.scaleDownEventsLock.RUnlock() + scaleDownLimit := calculateScaleDownLimitWithBehaviors(args.CurrentReplicas, r.scaleUpEvents[args.Key], r.scaleDownEvents[args.Key], args.ScaleDownBehavior) + + if scaleDownLimit > args.CurrentReplicas { + // We shouldn't scale down further until the scaleDownEvents will be cleaned up + scaleDownLimit = args.CurrentReplicas + } + minimumAllowedReplicas := args.MinReplicas + if minimumAllowedReplicas < scaleDownLimit { + minimumAllowedReplicas = scaleDownLimit + possibleLimitingReason = "ScaleDownLimit" + possibleLimitingMessage = "the desired replica count is decreasing faster than the maximum scale rate" + } else { + possibleLimitingMessage = "the desired replica count is less than the minimum replica count" + possibleLimitingReason = "TooFewReplicas" + } + if args.DesiredReplicas < minimumAllowedReplicas { + return minimumAllowedReplicas, possibleLimitingReason, possibleLimitingMessage + } + } + return args.DesiredReplicas, "DesiredWithinRange", "the desired count is within the acceptable range" +} + +// stabilizeRecommendation: +// - replaces old recommendation with the newest recommendation, +// - returns max of recommendations that are not older than downscaleStabilisationWindow. +func (r *recommender) stabilizeRecommendation(key model.MpaID, prenormalizedDesiredReplicas int32) int32 { + maxRecommendation := prenormalizedDesiredReplicas + foundOldSample := false + oldSampleIndex := 0 + cutoff := time.Now().Add(-r.downscaleStabilisationWindow) + + r.recommendationsLock.Lock() + defer r.recommendationsLock.Unlock() + for i, rec := range r.recommendations[key] { + if rec.timestamp.Before(cutoff) { + foundOldSample = true + oldSampleIndex = i + } else if rec.recommendation > maxRecommendation { + maxRecommendation = rec.recommendation + } + } + if foundOldSample { + r.recommendations[key][oldSampleIndex] = timestampedRecommendation{prenormalizedDesiredReplicas, time.Now()} + } else { + r.recommendations[key] = append(r.recommendations[key], timestampedRecommendation{prenormalizedDesiredReplicas, time.Now()}) + } + return maxRecommendation +} + +// normalizeDesiredReplicasWithBehaviors takes the metrics desired replicas value and normalizes it: +// 1. Apply the basic conditions (i.e. < maxReplicas, > minReplicas, etc...) +// 2. Apply the scale up/down limits from the mpaSpec.Constraints.Behaviors (i.e., add no more than 4 pods) +// 3. Apply the constraints period (i.e., add no more than 4 pods per minute) +// 4. Apply the stabilization (i.e., add no more than 4 pods per minute, and pick the smallest recommendation during last 5 minutes) +func (r *recommender) normalizeDesiredReplicasWithBehaviors(mpa *mpa_types.MultidimPodAutoscaler, key model.MpaID, currentReplicas, prenormalizedDesiredReplicas, minReplicas int32) int32 { + r.maybeInitScaleDownStabilizationWindow(mpa) + normalizationArg := NormalizationArg{ + Key: key, + ScaleUpBehavior: mpa.Spec.Constraints.Global.Behavior.ScaleUp, + ScaleDownBehavior: mpa.Spec.Constraints.Global.Behavior.ScaleDown, + MinReplicas: minReplicas, + MaxReplicas: *mpa.Spec.Constraints.Global.MaxReplicas, + CurrentReplicas: currentReplicas, + DesiredReplicas: prenormalizedDesiredReplicas} + stabilizedRecommendation, reason, message := r.stabilizeRecommendationWithBehaviors(normalizationArg) + normalizationArg.DesiredReplicas = stabilizedRecommendation + if stabilizedRecommendation != prenormalizedDesiredReplicas { + // "ScaleUpStabilized" || "ScaleDownStabilized" + setCondition(mpa, mpa_types.AbleToScale, v1.ConditionTrue, reason, message) + } else { + setCondition(mpa, mpa_types.AbleToScale, v1.ConditionTrue, "ReadyForNewScale", "recommended size matches current size") + } + desiredReplicas, reason, message := r.convertDesiredReplicasWithBehaviorRate(normalizationArg) + if desiredReplicas == stabilizedRecommendation { + setCondition(mpa, mpa_types.ScalingLimited, v1.ConditionFalse, reason, message) + } else { + setCondition(mpa, mpa_types.ScalingLimited, v1.ConditionTrue, reason, message) + } + + return desiredReplicas +} + +func (r *recommender) maybeInitScaleDownStabilizationWindow(mpa *mpa_types.MultidimPodAutoscaler) { + behavior := mpa.Spec.Constraints.Global.Behavior + if behavior != nil && behavior.ScaleDown != nil && behavior.ScaleDown.StabilizationWindowSeconds == nil { + stabilizationWindowSeconds := (int32)(r.downscaleStabilisationWindow.Seconds()) + mpa.Spec.Constraints.Global.Behavior.ScaleDown.StabilizationWindowSeconds = &stabilizationWindowSeconds + } +} + +// stabilizeRecommendationWithBehaviors: +// - replaces old recommendation with the newest recommendation, +// - returns {max,min} of recommendations that are not older than constraints.Scale{Up,Down}.DelaySeconds +func (r *recommender) stabilizeRecommendationWithBehaviors(args NormalizationArg) (int32, string, string) { + now := time.Now() + + foundOldSample := false + oldSampleIndex := 0 + + upRecommendation := args.DesiredReplicas + upDelaySeconds := *args.ScaleUpBehavior.StabilizationWindowSeconds + upCutoff := now.Add(-time.Second * time.Duration(upDelaySeconds)) + + downRecommendation := args.DesiredReplicas + downDelaySeconds := *args.ScaleDownBehavior.StabilizationWindowSeconds + downCutoff := now.Add(-time.Second * time.Duration(downDelaySeconds)) + + // Calculate the upper and lower stabilization limits. + r.recommendationsLock.Lock() + defer r.recommendationsLock.Unlock() + for i, rec := range r.recommendations[args.Key] { + if rec.timestamp.After(upCutoff) { + upRecommendation = min(rec.recommendation, upRecommendation) + } + if rec.timestamp.After(downCutoff) { + downRecommendation = max(rec.recommendation, downRecommendation) + } + if rec.timestamp.Before(upCutoff) && rec.timestamp.Before(downCutoff) { + foundOldSample = true + oldSampleIndex = i + } + } + + // Bring the recommendation to within the upper and lower limits (stabilize). + recommendation := args.CurrentReplicas + if recommendation < upRecommendation { + recommendation = upRecommendation + } + if recommendation > downRecommendation { + recommendation = downRecommendation + } + + // Record the unstabilized recommendation. + if foundOldSample { + r.recommendations[args.Key][oldSampleIndex] = timestampedRecommendation{args.DesiredReplicas, time.Now()} + } else { + r.recommendations[args.Key] = append(r.recommendations[args.Key], timestampedRecommendation{args.DesiredReplicas, time.Now()}) + } + + // Determine a human-friendly message. + var reason, message string + if args.DesiredReplicas >= args.CurrentReplicas { + reason = "ScaleUpStabilized" + message = "recent recommendations were lower than current one, applying the lowest recent recommendation" + } else { + reason = "ScaleDownStabilized" + message = "recent recommendations were higher than current one, applying the highest recent recommendation" + } + return recommendation, reason, message +} + +func max(a, b int32) int32 { + if a >= b { + return a + } + return b +} + +func min(a, b int32) int32 { + if a <= b { + return a + } + return b +} + +// calculateScaleUpLimitWithScalingRules returns the maximum number of pods that could be added for the given HPAScalingRules +func calculateScaleUpLimitWithScalingRules(currentReplicas int32, scaleUpEvents, scaleDownEvents []timestampedScaleEvent, scalingRules *autoscalingv2.HPAScalingRules) int32 { + var result int32 + var proposed int32 + var selectPolicyFn func(int32, int32) int32 + if *scalingRules.SelectPolicy == autoscalingv2.DisabledPolicySelect { + return currentReplicas // Scaling is disabled + } else if *scalingRules.SelectPolicy == autoscalingv2.MinChangePolicySelect { + result = math.MaxInt32 + selectPolicyFn = min // For scaling up, the lowest change ('min' policy) produces a minimum value + } else { + result = math.MinInt32 + selectPolicyFn = max // Use the default policy otherwise to produce a highest possible change + } + for _, policy := range scalingRules.Policies { + replicasAddedInCurrentPeriod := getReplicasChangePerPeriod(policy.PeriodSeconds, scaleUpEvents) + replicasDeletedInCurrentPeriod := getReplicasChangePerPeriod(policy.PeriodSeconds, scaleDownEvents) + periodStartReplicas := currentReplicas - replicasAddedInCurrentPeriod + replicasDeletedInCurrentPeriod + if policy.Type == autoscalingv2.PodsScalingPolicy { + proposed = periodStartReplicas + policy.Value + } else if policy.Type == autoscalingv2.PercentScalingPolicy { + // the proposal has to be rounded up because the proposed change might not increase the replica count causing the target to never scale up + proposed = int32(math.Ceil(float64(periodStartReplicas) * (1 + float64(policy.Value)/100))) + } + result = selectPolicyFn(result, proposed) + } + return result +} + +// calculateScaleDownLimitWithBehavior returns the maximum number of pods that could be deleted for the given HPAScalingRules +func calculateScaleDownLimitWithBehaviors(currentReplicas int32, scaleUpEvents, scaleDownEvents []timestampedScaleEvent, scalingRules *autoscalingv2.HPAScalingRules) int32 { + var result int32 + var proposed int32 + var selectPolicyFn func(int32, int32) int32 + if *scalingRules.SelectPolicy == autoscalingv2.DisabledPolicySelect { + return currentReplicas // Scaling is disabled + } else if *scalingRules.SelectPolicy == autoscalingv2.MinChangePolicySelect { + result = math.MinInt32 + selectPolicyFn = max // For scaling down, the lowest change ('min' policy) produces a maximum value + } else { + result = math.MaxInt32 + selectPolicyFn = min // Use the default policy otherwise to produce a highest possible change + } + for _, policy := range scalingRules.Policies { + replicasAddedInCurrentPeriod := getReplicasChangePerPeriod(policy.PeriodSeconds, scaleUpEvents) + replicasDeletedInCurrentPeriod := getReplicasChangePerPeriod(policy.PeriodSeconds, scaleDownEvents) + periodStartReplicas := currentReplicas - replicasAddedInCurrentPeriod + replicasDeletedInCurrentPeriod + if policy.Type == autoscalingv2.PodsScalingPolicy { + proposed = periodStartReplicas - policy.Value + } else if policy.Type == autoscalingv2.PercentScalingPolicy { + proposed = int32(float64(periodStartReplicas) * (1 - float64(policy.Value)/100)) + } + result = selectPolicyFn(result, proposed) + } + return result +} + +// getReplicasChangePerPeriod function find all the replica changes per period +func getReplicasChangePerPeriod(periodSeconds int32, scaleEvents []timestampedScaleEvent) int32 { + period := time.Second * time.Duration(periodSeconds) + cutoff := time.Now().Add(-period) + var replicas int32 + for _, rec := range scaleEvents { + if rec.timestamp.After(cutoff) { + replicas += rec.replicaChange + } + } + return replicas + +} + +// storeScaleEvent stores (adds or replaces outdated) scale event. +// outdated events to be replaced were marked as outdated in the `markScaleEventsOutdated` function +func (r *recommender) storeScaleEvent(behavior *autoscalingv2.HorizontalPodAutoscalerBehavior, key model.MpaID, prevReplicas, newReplicas int32) { + if behavior == nil { + return // we should not store any event as they will not be used + } + var oldSampleIndex int + var longestPolicyPeriod int32 + foundOldSample := false + if newReplicas > prevReplicas { + longestPolicyPeriod = getLongestPolicyPeriod(behavior.ScaleUp) + + r.scaleUpEventsLock.Lock() + defer r.scaleUpEventsLock.Unlock() + markScaleEventsOutdated(r.scaleUpEvents[key], longestPolicyPeriod) + replicaChange := newReplicas - prevReplicas + for i, event := range r.scaleUpEvents[key] { + if event.outdated { + foundOldSample = true + oldSampleIndex = i + } + } + newEvent := timestampedScaleEvent{replicaChange, time.Now(), false} + if foundOldSample { + r.scaleUpEvents[key][oldSampleIndex] = newEvent + } else { + r.scaleUpEvents[key] = append(r.scaleUpEvents[key], newEvent) + } + } else { + longestPolicyPeriod = getLongestPolicyPeriod(behavior.ScaleDown) + + r.scaleDownEventsLock.Lock() + defer r.scaleDownEventsLock.Unlock() + markScaleEventsOutdated(r.scaleDownEvents[key], longestPolicyPeriod) + replicaChange := prevReplicas - newReplicas + for i, event := range r.scaleDownEvents[key] { + if event.outdated { + foundOldSample = true + oldSampleIndex = i + } + } + newEvent := timestampedScaleEvent{replicaChange, time.Now(), false} + if foundOldSample { + r.scaleDownEvents[key][oldSampleIndex] = newEvent + } else { + r.scaleDownEvents[key] = append(r.scaleDownEvents[key], newEvent) + } + } +} + +func getLongestPolicyPeriod(scalingRules *autoscalingv2.HPAScalingRules) int32 { + var longestPolicyPeriod int32 + for _, policy := range scalingRules.Policies { + if policy.PeriodSeconds > longestPolicyPeriod { + longestPolicyPeriod = policy.PeriodSeconds + } + } + return longestPolicyPeriod +} + +// markScaleEventsOutdated set 'outdated=true' flag for all scale events that are not used by any MPA object +func markScaleEventsOutdated(scaleEvents []timestampedScaleEvent, longestPolicyPeriod int32) { + period := time.Second * time.Duration(longestPolicyPeriod) + cutoff := time.Now().Add(-period) + for i, event := range scaleEvents { + if event.timestamp.Before(cutoff) { + // outdated scale event are marked for later reuse + scaleEvents[i].outdated = true + } + } +} diff --git a/multidimensional-pod-autoscaler/pkg/recommender/routines/recommendation_post_processor.go b/multidimensional-pod-autoscaler/pkg/recommender/routines/recommendation_post_processor.go new file mode 100644 index 000000000000..352d3410c9bf --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/recommender/routines/recommendation_post_processor.go @@ -0,0 +1,27 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package routines + +import ( + mpa_types "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1alpha1" + vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" +) + +// RecommendationPostProcessor can amend the recommendation according to the defined policies +type RecommendationPostProcessor interface { + Process(mpa *mpa_types.MultidimPodAutoscaler, recommendation *vpa_types.RecommendedPodResources) *vpa_types.RecommendedPodResources +} diff --git a/multidimensional-pod-autoscaler/pkg/recommender/routines/recommender.go b/multidimensional-pod-autoscaler/pkg/recommender/routines/recommender.go new file mode 100644 index 000000000000..ccde426af3b3 --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/recommender/routines/recommender.go @@ -0,0 +1,368 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package routines + +import ( + "context" + "flag" + "sort" + "sync" + "time" + + v1 "k8s.io/api/core/v1" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/client/clientset/versioned/scheme" + mpa_api "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/client/clientset/versioned/typed/autoscaling.k8s.io/v1alpha1" + "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/recommender/checkpoint" + "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/recommender/input" + "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/recommender/model" + "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/target" + metrics_recommender "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/utils/metrics/recommender" + mpa_utils "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/utils/mpa" + vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/logic" + vpa_model "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/model" + controllerfetcher "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/target/controller_fetcher" + vpa_utils "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/vpa" + coreinformers "k8s.io/client-go/informers/core/v1" + v1core "k8s.io/client-go/kubernetes/typed/core/v1" + "k8s.io/client-go/tools/record" + klog "k8s.io/klog/v2" + hpa "k8s.io/kubernetes/pkg/controller/podautoscaler" + metricsclient "k8s.io/kubernetes/pkg/controller/podautoscaler/metrics" +) + +const ( + // AggregateContainerStateGCInterval defines how often expired AggregateContainerStates are garbage collected. + AggregateContainerStateGCInterval = 1 * time.Hour + scaleCacheEntryLifetime time.Duration = time.Hour + scaleCacheEntryFreshnessTime time.Duration = 10 * time.Minute + scaleCacheEntryJitterFactor float64 = 1. + defaultResyncPeriod time.Duration = 10 * time.Minute +) + +var ( + checkpointsWriteTimeout = flag.Duration("checkpoints-timeout", time.Minute, `Timeout for writing checkpoints since the start of the recommender's main loop`) + minCheckpointsPerRun = flag.Int("min-checkpoints", 10, "Minimum number of checkpoints to write per recommender's main loop") +) + +// From HPA +var ( + scaleUpLimitFactor = 2.0 + scaleUpLimitMinimum = 4.0 +) + +type timestampedRecommendation struct { + recommendation int32 + timestamp time.Time +} +type timestampedScaleEvent struct { + replicaChange int32 // absolute value, non-negative + timestamp time.Time + outdated bool +} + +// Recommender recommend resources for certain containers, based on utilization periodically got from metrics api. +type Recommender interface { + // RunOnce performs one iteration of recommender duties followed by update of recommendations in MPA objects. + RunOnce(workers int, vpaOrHpa string) + // GetClusterState returns ClusterState used by Recommender + GetClusterState() *model.ClusterState + // GetClusterStateFeeder returns ClusterStateFeeder used by Recommender + GetClusterStateFeeder() input.ClusterStateFeeder + // UpdateMPAs computes recommendations and sends MPAs status updates to API Server + UpdateMPAs(ctx context.Context, vpaOrHpa string) + // MaintainCheckpoints stores current checkpoints in API Server and garbage collect old ones + // MaintainCheckpoints writes at least minCheckpoints if there are more checkpoints to write. + // Checkpoints are written until ctx permits or all checkpoints are written. + MaintainCheckpoints(ctx context.Context, minCheckpoints int) +} + +type recommender struct { + // Fields inherited from VPA. + clusterState *model.ClusterState + clusterStateFeeder input.ClusterStateFeeder + checkpointWriter checkpoint.CheckpointWriter + checkpointsGCInterval time.Duration + controllerFetcher controllerfetcher.ControllerFetcher + lastCheckpointGC time.Time + mpaClient mpa_api.MultidimPodAutoscalersGetter + selectorFetcher target.MpaTargetSelectorFetcher + podResourceRecommender logic.PodResourceRecommender + useCheckpoints bool + lastAggregateContainerStateGC time.Time + recommendationPostProcessor []RecommendationPostProcessor + + // Fields for HPA. + replicaCalc *hpa.ReplicaCalculator + eventRecorder record.EventRecorder + downscaleStabilisationWindow time.Duration + // Controllers that need to be synced. + // Latest unstabilized recommendations for each autoscaler. + recommendations map[model.MpaID][]timestampedRecommendation + recommendationsLock sync.Mutex + // Latest autoscaler events. + scaleUpEvents map[model.MpaID][]timestampedScaleEvent + scaleUpEventsLock sync.RWMutex + scaleDownEvents map[model.MpaID][]timestampedScaleEvent + scaleDownEventsLock sync.RWMutex +} + +func (r *recommender) GetClusterState() *model.ClusterState { + return r.clusterState +} + +func (r *recommender) GetClusterStateFeeder() input.ClusterStateFeeder { + return r.clusterStateFeeder +} + +// Updates MPA CRD objects' statuses. +// vpaOrHpa can be either 'vpa', 'hpa', or 'both'. +func (r *recommender) UpdateMPAs(ctx context.Context, vpaOrHpa string) { + cnt := metrics_recommender.NewObjectCounter() + defer cnt.Observe() + + for _, observedMpa := range r.clusterState.ObservedMpas { + key := model.MpaID{ + Namespace: observedMpa.Namespace, + MpaName: observedMpa.Name, + } + klog.V(4).Infof("Recommender is checking MPA %v...", key) + + mpa, found := r.clusterState.Mpas[key] + if !found { + klog.V(4).Infof("MPA %v not found in the cluster state map!", key) + continue + } + + // Vertical Pod Autoscaling + if vpaOrHpa != "hpa" { + klog.V(4).Infof("Vertical scaling...") + resources := r.podResourceRecommender.GetRecommendedPodResources(GetContainerNameToAggregateStateMap(mpa)) + had := mpa.HasRecommendation() + + listOfResourceRecommendation := logic.MapToListOfRecommendedContainerResources(resources) + + for _, postProcessor := range r.recommendationPostProcessor { + listOfResourceRecommendation = postProcessor.Process(observedMpa, listOfResourceRecommendation) + } + + mpa.UpdateRecommendation(listOfResourceRecommendation) + klog.V(4).Infof("MPA %v recommendation updated: %v (%v)", key, resources, had) + if mpa.HasRecommendation() && !had { + metrics_recommender.ObserveRecommendationLatency(mpa.Created) + } + hasMatchingPods := mpa.PodCount > 0 + mpa.UpdateConditions(hasMatchingPods) + if err := r.clusterState.RecordRecommendation(mpa, time.Now()); err != nil { + klog.Warningf("%v", err) + if klog.V(4).Enabled() { + klog.Infof("MPA dump") + klog.Infof("%+v", mpa) + klog.Infof("HasMatchingPods: %v", hasMatchingPods) + klog.Infof("PodCount: %v", mpa.PodCount) + pods := r.clusterState.GetMatchingPods(mpa) + klog.Infof("MatchingPods: %+v", pods) + if len(pods) != mpa.PodCount { + klog.Errorf("ClusterState pod count and matching pods disagree for mpa %v/%v", mpa.ID.Namespace, mpa.ID.MpaName) + } + } + } + cnt.Add(mpa) + + _, err := mpa_utils.UpdateMpaStatusIfNeeded( + r.mpaClient.MultidimPodAutoscalers(mpa.ID.Namespace), mpa.ID.MpaName, mpa.AsStatus(), &observedMpa.Status) + if err != nil { + klog.Errorf( + "Cannot update MPA %v object. Reason: %+v", mpa.ID.MpaName, err) + } + } + + // Horizontal Pod Autoscaling + if vpaOrHpa != "vpa" { + observedMpa.Status.Recommendation = mpa.AsStatus().Recommendation + observedMpa.Status.Conditions = mpa.AsStatus().Conditions + klog.V(4).Infof("Horizontal scaling...") + errHPA := r.ReconcileHorizontalAutoscaling(ctx, observedMpa, key) + if errHPA != nil { + klog.Errorf("Error updating MPA status: %v", errHPA.Error()) + } + } + } +} + +// getCappedRecommendation creates a recommendation based on recommended pod +// resources, setting the UncappedTarget to the calculated recommended target +// and if necessary, capping the Target, LowerBound and UpperBound according +// to the ResourcePolicy. +func getCappedRecommendation(mpaID model.MpaID, resources logic.RecommendedPodResources, + policy *vpa_types.PodResourcePolicy) *vpa_types.RecommendedPodResources { + containerResources := make([]vpa_types.RecommendedContainerResources, 0, len(resources)) + // Sort the container names from the map. This is because maps are an + // unordered data structure, and iterating through the map will return + // a different order on every call. + containerNames := make([]string, 0, len(resources)) + for containerName := range resources { + containerNames = append(containerNames, containerName) + } + sort.Strings(containerNames) + // Create the list of recommendations for each container. + for _, name := range containerNames { + containerResources = append(containerResources, vpa_types.RecommendedContainerResources{ + ContainerName: name, + Target: vpa_model.ResourcesAsResourceList(resources[name].Target, logic.GetHumanizeMemory()), + LowerBound: vpa_model.ResourcesAsResourceList(resources[name].LowerBound, logic.GetHumanizeMemory()), + UpperBound: vpa_model.ResourcesAsResourceList(resources[name].UpperBound, logic.GetHumanizeMemory()), + UncappedTarget: vpa_model.ResourcesAsResourceList(resources[name].Target, logic.GetHumanizeMemory()), + }) + } + recommendation := &vpa_types.RecommendedPodResources{ + ContainerRecommendations: containerResources, + } + // Keep the original VPA policy for vertical autoscaling. + cappedRecommendation, err := vpa_utils.ApplyVPAPolicy(recommendation, policy) + if err != nil { + klog.Errorf("Failed to apply policy for MPA %v/%v: %v", mpaID.Namespace, mpaID.MpaName, err) + return recommendation + } + return cappedRecommendation +} + +func (r *recommender) MaintainCheckpoints(ctx context.Context, minCheckpointsPerRun int) { + now := time.Now() + if r.useCheckpoints { + if err := r.checkpointWriter.StoreCheckpoints(ctx, now, minCheckpointsPerRun); err != nil { + klog.Warningf("Failed to store checkpoints. Reason: %+v", err) + } + if time.Since(r.lastCheckpointGC) > r.checkpointsGCInterval { + r.lastCheckpointGC = now + r.clusterStateFeeder.GarbageCollectCheckpoints() + } + } +} + +func (r *recommender) RunOnce(workers int, vpaOrHpa string) { + timer := metrics_recommender.NewExecutionTimer() + defer timer.ObserveTotal() + + ctx := context.Background() + ctx, cancelFunc := context.WithDeadline(ctx, time.Now().Add(*checkpointsWriteTimeout)) + defer cancelFunc() + + // From HPA. + defer utilruntime.HandleCrash() + + klog.V(3).Infof("Recommender Run") + defer klog.V(3).Infof("Shutting down MPA Recommender") + + r.clusterStateFeeder.LoadMPAs(ctx) + timer.ObserveStep("LoadMPAs") + + r.clusterStateFeeder.LoadPods() + timer.ObserveStep("LoadPods") + + r.clusterStateFeeder.LoadRealTimeMetrics() + timer.ObserveStep("LoadMetrics") + klog.V(3).Infof("ClusterState is tracking %v PodStates and %v MPAs", len(r.clusterState.Pods), len(r.clusterState.Mpas)) + + r.UpdateMPAs(ctx, vpaOrHpa) + timer.ObserveStep("UpdateMPAs") + + r.MaintainCheckpoints(ctx, *minCheckpointsPerRun) + timer.ObserveStep("MaintainCheckpoints") + + r.clusterState.RateLimitedGarbageCollectAggregateCollectionStates(ctx, time.Now(), r.controllerFetcher) + timer.ObserveStep("GarbageCollect") + klog.V(3).Infof("ClusterState is tracking %d aggregated container states", r.clusterState.StateMapSize()) +} + +// RecommenderFactory makes instances of Recommender. +type RecommenderFactory struct { + ClusterState *model.ClusterState + + ClusterStateFeeder input.ClusterStateFeeder + ControllerFetcher controllerfetcher.ControllerFetcher + CheckpointWriter checkpoint.CheckpointWriter + PodResourceRecommender logic.PodResourceRecommender + MpaClient mpa_api.MultidimPodAutoscalersGetter + SelectorFetcher target.MpaTargetSelectorFetcher + + RecommendationPostProcessors []RecommendationPostProcessor + + CheckpointsGCInterval time.Duration + UseCheckpoints bool + + // For HPA. + EvtNamespacer v1core.EventsGetter + PodInformer coreinformers.PodInformer + MetricsClient metricsclient.MetricsClient + ResyncPeriod time.Duration + DownscaleStabilisationWindow time.Duration + Tolerance float64 + CpuInitializationPeriod time.Duration + DelayOfInitialReadinessStatus time.Duration +} + +// Make creates a new recommender instance, +// which can be run in order to provide continuous resource recommendations for containers. +func (c RecommenderFactory) Make() Recommender { + // For HPA. + broadcaster := record.NewBroadcaster() + broadcaster.StartStructuredLogging(0) + broadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: c.EvtNamespacer.Events("")}) + recorder := broadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "horizontal-pod-autoscaler"}) + + recommender := &recommender{ + // From VPA. + clusterState: c.ClusterState, + clusterStateFeeder: c.ClusterStateFeeder, + checkpointWriter: c.CheckpointWriter, + checkpointsGCInterval: c.CheckpointsGCInterval, + controllerFetcher: c.ControllerFetcher, + useCheckpoints: c.UseCheckpoints, + mpaClient: c.MpaClient, + selectorFetcher: c.SelectorFetcher, + podResourceRecommender: c.PodResourceRecommender, + recommendationPostProcessor: c.RecommendationPostProcessors, + lastAggregateContainerStateGC: time.Now(), + lastCheckpointGC: time.Now(), + + // From HPA. + downscaleStabilisationWindow: c.DownscaleStabilisationWindow, + // podLister is able to list/get Pods from the shared cache from the informer passed in to + // NewHorizontalController. + eventRecorder: recorder, + recommendations: map[model.MpaID][]timestampedRecommendation{}, + recommendationsLock: sync.Mutex{}, + scaleUpEvents: map[model.MpaID][]timestampedScaleEvent{}, + scaleUpEventsLock: sync.RWMutex{}, + scaleDownEvents: map[model.MpaID][]timestampedScaleEvent{}, + scaleDownEventsLock: sync.RWMutex{}, + } + + replicaCalc := hpa.NewReplicaCalculator( + c.MetricsClient, + recommender.clusterStateFeeder.GetPodLister(), + c.Tolerance, + c.CpuInitializationPeriod, + c.DelayOfInitialReadinessStatus, + ) + recommender.replicaCalc = replicaCalc + + klog.V(3).Infof("New Recommender created!") + return recommender +} diff --git a/multidimensional-pod-autoscaler/pkg/recommender/routines/recommender_test.go b/multidimensional-pod-autoscaler/pkg/recommender/routines/recommender_test.go new file mode 100644 index 000000000000..2d09034ff25f --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/recommender/routines/recommender_test.go @@ -0,0 +1,79 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package routines + +import ( + "testing" + "time" + + labels "k8s.io/apimachinery/pkg/labels" + "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/recommender/model" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/logic" + vpa_model "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/model" + + "github.com/stretchr/testify/assert" +) + +func TestSortedRecommendation(t *testing.T) { + cases := []struct { + name string + resources logic.RecommendedPodResources + expectedLast []string + }{ + { + name: "All recommendations sorted", + resources: logic.RecommendedPodResources{ + "a-container": logic.RecommendedContainerResources{Target: vpa_model.Resources{vpa_model.ResourceCPU: vpa_model.CPUAmountFromCores(1), vpa_model.ResourceMemory: vpa_model.MemoryAmountFromBytes(1000)}}, + "b-container": logic.RecommendedContainerResources{Target: vpa_model.Resources{vpa_model.ResourceCPU: vpa_model.CPUAmountFromCores(1), vpa_model.ResourceMemory: vpa_model.MemoryAmountFromBytes(1000)}}, + "c-container": logic.RecommendedContainerResources{Target: vpa_model.Resources{vpa_model.ResourceCPU: vpa_model.CPUAmountFromCores(1), vpa_model.ResourceMemory: vpa_model.MemoryAmountFromBytes(1000)}}, + "d-container": logic.RecommendedContainerResources{Target: vpa_model.Resources{vpa_model.ResourceCPU: vpa_model.CPUAmountFromCores(1), vpa_model.ResourceMemory: vpa_model.MemoryAmountFromBytes(1000)}}, + }, + expectedLast: []string{ + "a-container", + "b-container", + "c-container", + "d-container", + }, + }, + { + name: "All recommendations unsorted", + resources: logic.RecommendedPodResources{ + "b-container": logic.RecommendedContainerResources{Target: vpa_model.Resources{vpa_model.ResourceCPU: vpa_model.CPUAmountFromCores(1), vpa_model.ResourceMemory: vpa_model.MemoryAmountFromBytes(1000)}}, + "a-container": logic.RecommendedContainerResources{Target: vpa_model.Resources{vpa_model.ResourceCPU: vpa_model.CPUAmountFromCores(1), vpa_model.ResourceMemory: vpa_model.MemoryAmountFromBytes(1000)}}, + "d-container": logic.RecommendedContainerResources{Target: vpa_model.Resources{vpa_model.ResourceCPU: vpa_model.CPUAmountFromCores(1), vpa_model.ResourceMemory: vpa_model.MemoryAmountFromBytes(1000)}}, + "c-container": logic.RecommendedContainerResources{Target: vpa_model.Resources{vpa_model.ResourceCPU: vpa_model.CPUAmountFromCores(1), vpa_model.ResourceMemory: vpa_model.MemoryAmountFromBytes(1000)}}, + }, + expectedLast: []string{ + "a-container", + "b-container", + "c-container", + "d-container", + }, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + namespace := "test-namespace" + mpa := model.NewMpa(model.MpaID{Namespace: namespace, MpaName: "my-mpa"}, labels.Nothing(), time.Unix(0, 0)) + mpa.UpdateRecommendation(getCappedRecommendation(mpa.ID, tc.resources, nil)) + // Check that the slice is in the correct order. + for i := range mpa.Recommendation.ContainerRecommendations { + assert.Equal(t, tc.expectedLast[i], mpa.Recommendation.ContainerRecommendations[i].ContainerName) + } + }) + } +} diff --git a/multidimensional-pod-autoscaler/pkg/recommender/routines/vpa.go b/multidimensional-pod-autoscaler/pkg/recommender/routines/vpa.go new file mode 100644 index 000000000000..89975b00511e --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/recommender/routines/vpa.go @@ -0,0 +1,55 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package routines + +import ( + "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/recommender/model" + vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" + vpa_model "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/model" + api_utils "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/vpa" +) + +// GetContainerNameToAggregateStateMap returns ContainerNameToAggregateStateMap for pods. +// Updated to integrate with the MPA API. +func GetContainerNameToAggregateStateMap(mpa *model.Mpa) vpa_model.ContainerNameToAggregateStateMap { + containerNameToAggregateStateMap := mpa.AggregateStateByContainerName() + filteredContainerNameToAggregateStateMap := make(vpa_model.ContainerNameToAggregateStateMap) + + for containerName, aggregatedContainerState := range containerNameToAggregateStateMap { + containerResourcePolicy := api_utils.GetContainerResourcePolicy(containerName, mpa.ResourcePolicy) + autoscalingDisabled := containerResourcePolicy != nil && containerResourcePolicy.Mode != nil && + *containerResourcePolicy.Mode == vpa_types.ContainerScalingModeOff + if !autoscalingDisabled && aggregatedContainerState.TotalSamplesCount > 0 { + aggregatedContainerState.UpdateFromPolicy(containerResourcePolicy) + vpaAggregatedContainerState := vpa_model.AggregateContainerState{ + AggregateCPUUsage: aggregatedContainerState.AggregateCPUUsage, + AggregateMemoryPeaks: aggregatedContainerState.AggregateMemoryPeaks, + FirstSampleStart: aggregatedContainerState.FirstSampleStart, + LastSampleStart: aggregatedContainerState.LastSampleStart, + TotalSamplesCount: aggregatedContainerState.TotalSamplesCount, + CreationTime: aggregatedContainerState.CreationTime, + LastRecommendation: aggregatedContainerState.LastRecommendation, + IsUnderVPA: aggregatedContainerState.IsUnderVPA, + UpdateMode: aggregatedContainerState.UpdateMode, + ScalingMode: aggregatedContainerState.ScalingMode, + ControlledResources: aggregatedContainerState.ControlledResources, + } + filteredContainerNameToAggregateStateMap[containerName] = &vpaAggregatedContainerState + } + } + return filteredContainerNameToAggregateStateMap +} diff --git a/multidimensional-pod-autoscaler/pkg/target/fetcher.go b/multidimensional-pod-autoscaler/pkg/target/fetcher.go new file mode 100644 index 000000000000..b16f2879171e --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/target/fetcher.go @@ -0,0 +1,214 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package target + +import ( + "context" + "fmt" + "time" + + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/wait" + mpa_types "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1alpha1" + "k8s.io/client-go/discovery" + cacheddiscovery "k8s.io/client-go/discovery/cached" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/informers" + kube_client "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/restmapper" + "k8s.io/client-go/scale" + "k8s.io/client-go/tools/cache" + klog "k8s.io/klog/v2" +) + +const ( + discoveryResetPeriod time.Duration = 5 * time.Minute +) + +// MpaTargetSelectorFetcher gets a labelSelector used to gather Pods controlled by the given MPA. +type MpaTargetSelectorFetcher interface { + // Fetch returns a labelSelector used to gather Pods controlled by the given MPA. + // If error is nil, the returned labelSelector is not nil. + Fetch(ctx context.Context, mpa *mpa_types.MultidimPodAutoscaler) (labels.Selector, error) + + // For updating the Deployments. + GetRESTMappings(groupKind schema.GroupKind) ([]*apimeta.RESTMapping, error) + Scales(namespace string) scale.ScaleInterface +} + +type wellKnownController string + +const ( + daemonSet wellKnownController = "DaemonSet" + deployment wellKnownController = "Deployment" + replicaSet wellKnownController = "ReplicaSet" + statefulSet wellKnownController = "StatefulSet" + replicationController wellKnownController = "ReplicationController" + job wellKnownController = "Job" + cronJob wellKnownController = "CronJob" +) + +// NewMpaTargetSelectorFetcher returns new instance of MpaTargetSelectorFetcher +func NewMpaTargetSelectorFetcher(config *rest.Config, kubeClient kube_client.Interface, factory informers.SharedInformerFactory) MpaTargetSelectorFetcher { + discoveryClient, err := discovery.NewDiscoveryClientForConfig(config) + if err != nil { + klog.Fatalf("Could not create discoveryClient: %v", err) + } + resolver := scale.NewDiscoveryScaleKindResolver(discoveryClient) + restClient := kubeClient.CoreV1().RESTClient() + cachedDiscoveryClient := cacheddiscovery.NewMemCacheClient(discoveryClient) + mapper := restmapper.NewDeferredDiscoveryRESTMapper(cachedDiscoveryClient) + go wait.Until(func() { + mapper.Reset() + }, discoveryResetPeriod, make(chan struct{})) + + informersMap := map[wellKnownController]cache.SharedIndexInformer{ + daemonSet: factory.Apps().V1().DaemonSets().Informer(), + deployment: factory.Apps().V1().Deployments().Informer(), + replicaSet: factory.Apps().V1().ReplicaSets().Informer(), + statefulSet: factory.Apps().V1().StatefulSets().Informer(), + replicationController: factory.Core().V1().ReplicationControllers().Informer(), + job: factory.Batch().V1().Jobs().Informer(), + cronJob: factory.Batch().V1().CronJobs().Informer(), + } + + for kind, informer := range informersMap { + stopCh := make(chan struct{}) + go informer.Run(stopCh) + synced := cache.WaitForCacheSync(stopCh, informer.HasSynced) + if !synced { + klog.Fatalf("Could not sync cache for %s: %v", kind, err) + } else { + klog.Infof("Initial sync of %s completed", kind) + } + } + + scaleNamespacer := scale.New(restClient, mapper, dynamic.LegacyAPIPathResolverFunc, resolver) + return &mpaTargetSelectorFetcher{ + scaleNamespacer: scaleNamespacer, + mapper: mapper, + informersMap: informersMap, + } +} + +// mpaTargetSelectorFetcher implements MpaTargetSelectorFetcher interface +// by querying API server for the controller pointed by MPA's scaleTargetRef +type mpaTargetSelectorFetcher struct { + scaleNamespacer scale.ScalesGetter + mapper apimeta.RESTMapper + informersMap map[wellKnownController]cache.SharedIndexInformer +} + +func (f *mpaTargetSelectorFetcher) Fetch(ctx context.Context, mpa *mpa_types.MultidimPodAutoscaler) (labels.Selector, error) { + if mpa.Spec.ScaleTargetRef == nil { + return nil, fmt.Errorf("scaleTargetRef not defined.") + } + kind := wellKnownController(mpa.Spec.ScaleTargetRef.Kind) + informer, exists := f.informersMap[kind] + if exists { + return getLabelSelector(informer, mpa.Spec.ScaleTargetRef.Kind, mpa.Namespace, mpa.Spec.ScaleTargetRef.Name) + } + + // not on a list of known controllers, use scale sub-resource + // TODO: cache response + groupVersion, err := schema.ParseGroupVersion(mpa.Spec.ScaleTargetRef.APIVersion) + if err != nil { + return nil, err + } + groupKind := schema.GroupKind{ + Group: groupVersion.Group, + Kind: mpa.Spec.ScaleTargetRef.Kind, + } + + selector, err := f.getLabelSelectorFromResource(ctx, groupKind, mpa.Namespace, mpa.Spec.ScaleTargetRef.Name) + if err != nil { + return nil, fmt.Errorf("Unhandled ScaleTargetRef %s / %s / %s, last error %v", + mpa.Spec.ScaleTargetRef.APIVersion, mpa.Spec.ScaleTargetRef.Kind, mpa.Spec.ScaleTargetRef.Name, err) + } + return selector, nil +} + +func getLabelSelector(informer cache.SharedIndexInformer, kind, namespace, name string) (labels.Selector, error) { + obj, exists, err := informer.GetStore().GetByKey(namespace + "/" + name) + if err != nil { + return nil, err + } + if !exists { + return nil, fmt.Errorf("%s %s/%s does not exist", kind, namespace, name) + } + switch apiObj := obj.(type) { + case (*appsv1.DaemonSet): + return metav1.LabelSelectorAsSelector(apiObj.Spec.Selector) + case (*appsv1.Deployment): + return metav1.LabelSelectorAsSelector(apiObj.Spec.Selector) + case (*appsv1.StatefulSet): + return metav1.LabelSelectorAsSelector(apiObj.Spec.Selector) + case (*appsv1.ReplicaSet): + return metav1.LabelSelectorAsSelector(apiObj.Spec.Selector) + case (*batchv1.Job): + return metav1.LabelSelectorAsSelector(apiObj.Spec.Selector) + case (*batchv1.CronJob): + return metav1.LabelSelectorAsSelector(metav1.SetAsLabelSelector(apiObj.Spec.JobTemplate.Spec.Template.Labels)) + case (*corev1.ReplicationController): + return metav1.LabelSelectorAsSelector(metav1.SetAsLabelSelector(apiObj.Spec.Selector)) + } + return nil, fmt.Errorf("don't know how to read label seletor") +} + +func (f *mpaTargetSelectorFetcher) getLabelSelectorFromResource( + ctx context.Context, groupKind schema.GroupKind, namespace, name string, +) (labels.Selector, error) { + mappings, err := f.mapper.RESTMappings(groupKind) + if err != nil { + return nil, err + } + + var lastError error + for _, mapping := range mappings { + groupResource := mapping.Resource.GroupResource() + scale, err := f.scaleNamespacer.Scales(namespace).Get(ctx, groupResource, name, metav1.GetOptions{}) + if err == nil { + if scale.Status.Selector == "" { + return nil, fmt.Errorf("Resource %s/%s has an empty selector for scale sub-resource", namespace, name) + } + selector, err := labels.Parse(scale.Status.Selector) + if err != nil { + return nil, err + } + return selector, nil + } + lastError = err + } + + // nothing found, apparently the resource does support scale (or we lack RBAC) + return nil, lastError +} + +func (f *mpaTargetSelectorFetcher) GetRESTMappings(groupKind schema.GroupKind) ([]*apimeta.RESTMapping, error) { + return f.mapper.RESTMappings(groupKind) +} + +func (f *mpaTargetSelectorFetcher) Scales(namespace string) scale.ScaleInterface { + return f.scaleNamespacer.Scales(namespace) +} diff --git a/multidimensional-pod-autoscaler/pkg/target/mock/fetcher_mock.go b/multidimensional-pod-autoscaler/pkg/target/mock/fetcher_mock.go new file mode 100644 index 000000000000..6821f9ae3b82 --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/target/mock/fetcher_mock.go @@ -0,0 +1,76 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mocktarget + +import ( + "context" + + gomock "github.com/golang/mock/gomock" + apimeta "k8s.io/apimachinery/pkg/api/meta" + labels "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime/schema" + mpa_types "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1alpha1" + "k8s.io/client-go/restmapper" + "k8s.io/client-go/scale" +) + +// MockMpaTargetSelectorFetcher is a mock of MpaTargetSelectorFetcher interface +type MockMpaTargetSelectorFetcher struct { + ctrl *gomock.Controller + recorder *_MockMpaTargetSelectorFetcherRecorder + mapper restmapper.DeferredDiscoveryRESTMapper +} + +// Recorder for MockMpaTargetSelectorFetcher (not exported) +type _MockMpaTargetSelectorFetcherRecorder struct { + mock *MockMpaTargetSelectorFetcher +} + +// NewMockMpaTargetSelectorFetcher returns mock instance of a mock of MpaTargetSelectorFetcher +func NewMockMpaTargetSelectorFetcher(ctrl *gomock.Controller) *MockMpaTargetSelectorFetcher { + mock := &MockMpaTargetSelectorFetcher{ctrl: ctrl} + mock.recorder = &_MockMpaTargetSelectorFetcherRecorder{mock} + return mock +} + +// EXPECT enables configuring expectaions +func (_m *MockMpaTargetSelectorFetcher) EXPECT() *_MockMpaTargetSelectorFetcherRecorder { + return _m.recorder +} + +// Fetch enables configuring expectations on Fetch method +func (_m *MockMpaTargetSelectorFetcher) Fetch(_ context.Context, mpa *mpa_types.MultidimPodAutoscaler) (labels.Selector, error) { + ret := _m.ctrl.Call(_m, "Fetch", mpa) + ret0, _ := ret[0].(labels.Selector) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Fetch enables configuring expectations on Fetch method +func (_mr *_MockMpaTargetSelectorFetcherRecorder) Fetch(arg0 interface{}) *gomock.Call { + return _mr.mock.ctrl.RecordCall(_mr.mock, "Fetch", arg0) +} + +// GetRESTMappings enables configuring expectations on GetRESTMappings method +func (_m *MockMpaTargetSelectorFetcher) GetRESTMappings(groupKind schema.GroupKind) ([]*apimeta.RESTMapping, error) { + return _m.mapper.RESTMappings(groupKind) +} + +// Scales enables configuring expectations on Scales method +func (_m *MockMpaTargetSelectorFetcher) Scales(namespace string) scale.ScaleInterface { + return nil +} diff --git a/multidimensional-pod-autoscaler/pkg/updater/.gitignore b/multidimensional-pod-autoscaler/pkg/updater/.gitignore new file mode 100644 index 000000000000..3834e7718699 --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/updater/.gitignore @@ -0,0 +1,7 @@ +# Updater binary +updater +updater-amd64 +updater-arm64 +updater-arm +updater-ppc64le +updater-s390x diff --git a/multidimensional-pod-autoscaler/pkg/updater/Dockerfile b/multidimensional-pod-autoscaler/pkg/updater/Dockerfile new file mode 100644 index 000000000000..4ec77af365a7 --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/updater/Dockerfile @@ -0,0 +1,40 @@ +# Copyright 2016 The Kubernetes Authors. All rights reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM --platform=$BUILDPLATFORM golang:1.23.4 AS builder + +WORKDIR /workspace + +# Copy the Go Modules manifests +COPY multidimensional-pod-autoscaler/go.mod go.mod +COPY multidimensional-pod-autoscaler/go.sum go.sum +# TODO: This is temporary until the VPA has cut a new release +COPY vertical-pod-autoscaler /vertical-pod-autoscaler + +RUN go mod download + +COPY multidimensional-pod-autoscaler/common common +COPY multidimensional-pod-autoscaler/pkg pkg + +ARG TARGETOS TARGETARCH + +RUN CGO_ENABLED=0 LD_FLAGS=-s GOARCH=$TARGETARCH GOOS=$TARGETOS go build -C pkg/updater -o updater-$TARGETARCH + +FROM gcr.io/distroless/static:nonroot + +ARG TARGETARCH + +COPY --from=builder /workspace/pkg/updater/updater-$TARGETARCH /updater + +ENTRYPOINT ["/updater"] diff --git a/multidimensional-pod-autoscaler/pkg/updater/Makefile b/multidimensional-pod-autoscaler/pkg/updater/Makefile new file mode 100644 index 000000000000..a71fdb54bbd9 --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/updater/Makefile @@ -0,0 +1,90 @@ +all: build + +TAG?=dev +REGISTRY?=staging-k8s.gcr.io +FLAGS= +TEST_ENVVAR=LD_FLAGS=-s GO111MODULE=on +ENVVAR=CGO_ENABLED=0 $(TEST_ENVVAR) +GOOS?=linux +COMPONENT=updater +FULL_COMPONENT=mpa-${COMPONENT} + +ALL_ARCHITECTURES?=amd64 arm arm64 ppc64le s390x +export DOCKER_CLI_EXPERIMENTAL=enabled + +build: clean + $(ENVVAR) GOOS=$(GOOS) go build ./... + $(ENVVAR) GOOS=$(GOOS) go build -o ${COMPONENT} + +build-binary: clean + $(ENVVAR) GOOS=$(GOOS) go build -o ${COMPONENT} + +test-unit: clean build + $(TEST_ENVVAR) go test --test.short -race ./... $(FLAGS) + +.PHONY: docker-build +docker-build: $(addprefix docker-build-,$(ALL_ARCHITECTURES)) + +.PHONY: docker-build-* +docker-build-%: +ifndef REGISTRY + ERR = $(error REGISTRY is undefined) + $(ERR) +endif +ifndef TAG + ERR = $(error TAG is undefined) + $(ERR) +endif + docker buildx build --pull --load --platform linux/$* -t ${REGISTRY}/${FULL_COMPONENT}-$*:${TAG} -f ./Dockerfile ../../../ + +.PHONY: docker-push +docker-push: $(addprefix do-push-,$(ALL_ARCHITECTURES)) push-multi-arch; + +.PHONY: do-push-* +do-push-%: +ifndef REGISTRY + ERR = $(error REGISTRY is undefined) + $(ERR) +endif +ifndef TAG + ERR = $(error TAG is undefined) + $(ERR) +endif + docker push ${REGISTRY}/${FULL_COMPONENT}-$*:${TAG} + +.PHONY: push-multi-arch +push-multi-arch: + docker manifest create --amend $(REGISTRY)/${FULL_COMPONENT}:$(TAG) $(shell echo $(ALL_ARCHITECTURES) | sed -e "s~[^ ]*~$(REGISTRY)/${FULL_COMPONENT}\-&:$(TAG)~g") + @for arch in $(ALL_ARCHITECTURES); do docker manifest annotate --arch $${arch} $(REGISTRY)/${FULL_COMPONENT}:$(TAG) $(REGISTRY)/${FULL_COMPONENT}-$${arch}:${TAG}; done + docker manifest push --purge $(REGISTRY)/${FULL_COMPONENT}:$(TAG) + +.PHONY: show-git-info +show-git-info: + echo '=============== local git status ===============' + git status + echo '=============== last commit ===============' + git log -1 + echo '=============== bulding from the above ===============' + +.PHONY: create-buildx-builder +create-buildx-builder: + BUILDER=$(shell docker buildx create --driver=docker-container --use) + +.PHONY: remove-buildx-builder +remove-buildx-builder: + docker buildx rm ${BUILDER} + +.PHONY: release +release: show-git-info create-buildx-builder docker-build remove-buildx-builder docker-push + @echo "Full in-docker release ${FULL_COMPONENT}:${TAG} completed" + +clean: $(addprefix clean-,$(ALL_ARCHITECTURES)) + +clean-%: + rm -f ${COMPONENT}-$* + +format: + test -z "$$(find . -path ./vendor -prune -type f -o -name '*.go' -exec gofmt -s -d {} + | tee /dev/stderr)" || \ + test -z "$$(find . -path ./vendor -prune -type f -o -name '*.go' -exec gofmt -s -w {} + | tee /dev/stderr)" + +.PHONY: all build test-unit clean format release diff --git a/multidimensional-pod-autoscaler/pkg/updater/README.md b/multidimensional-pod-autoscaler/pkg/updater/README.md new file mode 100644 index 000000000000..91965e3f126b --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/updater/README.md @@ -0,0 +1,44 @@ +# MPA Updater + +- [Introduction](#introduction) +- [Current implementation](current-implementation) +- [Missing parts](#missing-parts) + +## Introduction +Updater component for Multidimensional Pod Autoscaler described in https://github.com/kubernetes/community/pull/338 (To be updated) + +Updater runs in Kubernetes cluster and decides which pods should be restarted +based on resources allocation recommendation calculated by Recommender. +If a pod should be updated, Updater will try to evict the pod. +It respects the pod disruption budget, by using Eviction API to evict pods. +Updater does not perform the actual resources update, but relies on Multidimensional Pod Autoscaler admission plugin +to update pod resources when the pod is recreated after eviction. + +## Running the Updater + +* Create RBAC configuration from `../../deploy/mpa-rbac.yaml` if not yet done so. +* Create a deployment with the updater pod from `../../deploy/updater-deployment.yaml`. +* The updater will start running and watch MPA object statuses for pod eviction and replica updates. + +## Current implementation +Runs in a loop. On one iteration performs: +* Fetching Multidimensional Pod Autoscaler configuration using a lister implementation. +* Fetching live pods information with their current resource allocation. +* For each replicated pods group calculating if pod update is required and how many replicas can be evicted. +Updater will always allow eviction of at least one pod in replica set. Maximum ratio of evicted replicas is specified by flag. +* Evicting pods if recommended resources significantly vary from the actual resources allocation. +Threshold for evicting pods is specified by recommended min/max values from VPA resource. +Priority of evictions within a set of replicated pods is proportional to sum of percentages of changes in resources +(i.e. pod with 15% memory increase 15% cpu decrease recommended will be evicted +before pod with 20% memory increase and no change in cpu). +* Updating the Deployment with the desired number of replicas. + +## Missing parts +* Recommendation API for fetching data from Multidimensional Pod Autoscaler Recommender. + +## Building the Docker Image + +``` +make build-binary-with-vendor-amd64 +make docker-build-amd64 +``` diff --git a/multidimensional-pod-autoscaler/pkg/updater/eviction/pods_eviction_restriction.go b/multidimensional-pod-autoscaler/pkg/updater/eviction/pods_eviction_restriction.go new file mode 100644 index 000000000000..8b063f055cd2 --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/updater/eviction/pods_eviction_restriction.go @@ -0,0 +1,396 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package eviction + +import ( + "context" + "fmt" + "time" + + appsv1 "k8s.io/api/apps/v1" + apiv1 "k8s.io/api/core/v1" + policyv1 "k8s.io/api/policy/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + mpa_types "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1alpha1" + appsinformer "k8s.io/client-go/informers/apps/v1" + coreinformer "k8s.io/client-go/informers/core/v1" + kube_client "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/tools/record" + klog "k8s.io/klog/v2" +) + +const ( + resyncPeriod time.Duration = 1 * time.Minute +) + +// PodsEvictionRestriction controls pods evictions. It ensures that we will not evict too +// many pods from one replica set. For replica set will allow to evict one pod or more if +// evictionToleranceFraction is configured. +type PodsEvictionRestriction interface { + // Evict sends eviction instruction to the api client. + // Returns error if pod cannot be evicted or if client returned error. + Evict(pod *apiv1.Pod, mpa *mpa_types.MultidimPodAutoscaler, eventRecorder record.EventRecorder) error + // CanEvict checks if pod can be safely evicted + CanEvict(pod *apiv1.Pod) bool +} + +type podsEvictionRestrictionImpl struct { + client kube_client.Interface + podToReplicaCreatorMap map[string]podReplicaCreator + creatorToSingleGroupStatsMap map[podReplicaCreator]singleGroupStats +} + +type singleGroupStats struct { + configured int + pending int + running int + evictionTolerance int + evicted int +} + +// PodsEvictionRestrictionFactory creates PodsEvictionRestriction +type PodsEvictionRestrictionFactory interface { + // NewPodsEvictionRestriction creates PodsEvictionRestriction for given set of pods, + // controlled by a single MPA object. + NewPodsEvictionRestriction(pods []*apiv1.Pod, mpa *mpa_types.MultidimPodAutoscaler) PodsEvictionRestriction +} + +type podsEvictionRestrictionFactoryImpl struct { + client kube_client.Interface + rcInformer cache.SharedIndexInformer // informer for Replication Controllers + ssInformer cache.SharedIndexInformer // informer for Stateful Sets + rsInformer cache.SharedIndexInformer // informer for Replica Sets + dsInformer cache.SharedIndexInformer // informer for Daemon Sets + minReplicas int + evictionToleranceFraction float64 +} + +type controllerKind string + +const ( + replicationController controllerKind = "ReplicationController" + statefulSet controllerKind = "StatefulSet" + replicaSet controllerKind = "ReplicaSet" + job controllerKind = "Job" + daemonSet controllerKind = "DaemonSet" +) + +type podReplicaCreator struct { + Namespace string + Name string + Kind controllerKind +} + +// CanEvict checks if pod can be safely evicted +func (e *podsEvictionRestrictionImpl) CanEvict(pod *apiv1.Pod) bool { + cr, present := e.podToReplicaCreatorMap[getPodID(pod)] + if present { + singleGroupStats, present := e.creatorToSingleGroupStatsMap[cr] + if pod.Status.Phase == apiv1.PodPending { + return true + } + if present { + shouldBeAlive := singleGroupStats.configured - singleGroupStats.evictionTolerance + if singleGroupStats.running-singleGroupStats.evicted > shouldBeAlive { + return true + } + // If all pods are running and eviction tollerance is small evict 1 pod. + if singleGroupStats.running == singleGroupStats.configured && + singleGroupStats.evictionTolerance == 0 && + singleGroupStats.evicted == 0 { + return true + } + } + } + return false +} + +// Evict sends eviction instruction to api client. Returns error if pod cannot be evicted or if client returned error +// Does not check if pod was actually evicted after eviction grace period. +func (e *podsEvictionRestrictionImpl) Evict(podToEvict *apiv1.Pod, mpa *mpa_types.MultidimPodAutoscaler, eventRecorder record.EventRecorder) error { + cr, present := e.podToReplicaCreatorMap[getPodID(podToEvict)] + if !present { + return fmt.Errorf("pod not suitable for eviction %v : not in replicated pods map", podToEvict.Name) + } + + if !e.CanEvict(podToEvict) { + return fmt.Errorf("cannot evict pod %v : eviction budget exceeded", podToEvict.Name) + } + + eviction := &policyv1.Eviction{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: podToEvict.Namespace, + Name: podToEvict.Name, + }, + } + err := e.client.CoreV1().Pods(podToEvict.Namespace).EvictV1(context.TODO(), eviction) + if err != nil { + klog.Errorf("failed to evict pod %s/%s, error: %v", podToEvict.Namespace, podToEvict.Name, err) + return err + } + eventRecorder.Event(podToEvict, apiv1.EventTypeNormal, "EvictedByMPA", + "Pod was evicted by MPA Updater to apply resource recommendation.") + + eventRecorder.Event(mpa, apiv1.EventTypeNormal, "EvictedPod", + "MPA Updater evicted Pod "+podToEvict.Name+" to apply resource recommendation.") + + if podToEvict.Status.Phase != apiv1.PodPending { + singleGroupStats, present := e.creatorToSingleGroupStatsMap[cr] + if !present { + return fmt.Errorf("Internal error - cannot find stats for replication group %v", cr) + } + singleGroupStats.evicted = singleGroupStats.evicted + 1 + e.creatorToSingleGroupStatsMap[cr] = singleGroupStats + } + + return nil +} + +// NewPodsEvictionRestrictionFactory creates PodsEvictionRestrictionFactory +func NewPodsEvictionRestrictionFactory(client kube_client.Interface, minReplicas int, + evictionToleranceFraction float64) (PodsEvictionRestrictionFactory, error) { + rcInformer, err := setUpInformer(client, replicationController) + if err != nil { + return nil, fmt.Errorf("Failed to create rcInformer: %v", err) + } + ssInformer, err := setUpInformer(client, statefulSet) + if err != nil { + return nil, fmt.Errorf("Failed to create ssInformer: %v", err) + } + rsInformer, err := setUpInformer(client, replicaSet) + if err != nil { + return nil, fmt.Errorf("Failed to create rsInformer: %v", err) + } + dsInformer, err := setUpInformer(client, daemonSet) + if err != nil { + return nil, fmt.Errorf("Failed to create dsInformer: %v", err) + } + return &podsEvictionRestrictionFactoryImpl{ + client: client, + rcInformer: rcInformer, // informer for Replication Controllers + ssInformer: ssInformer, // informer for Replica Sets + rsInformer: rsInformer, // informer for Stateful Sets + dsInformer: dsInformer, // informer for Daemon Sets + minReplicas: minReplicas, + evictionToleranceFraction: evictionToleranceFraction}, nil +} + +// NewPodsEvictionRestriction creates PodsEvictionRestriction for a given set of pods, +// controlled by a single MPA object. +func (f *podsEvictionRestrictionFactoryImpl) NewPodsEvictionRestriction(pods []*apiv1.Pod, mpa *mpa_types.MultidimPodAutoscaler) PodsEvictionRestriction { + // We can evict pod only if it is a part of replica set + // For each replica set we can evict only a fraction of pods. + // Evictions may be later limited by pod disruption budget if configured. + + livePods := make(map[podReplicaCreator][]*apiv1.Pod) + + for _, pod := range pods { + creator, err := getPodReplicaCreator(pod) + if err != nil { + klog.Errorf("failed to obtain replication info for pod %s: %v", pod.Name, err) + continue + } + if creator == nil { + klog.Warningf("pod %s not replicated", pod.Name) + continue + } + livePods[*creator] = append(livePods[*creator], pod) + } + + podToReplicaCreatorMap := make(map[string]podReplicaCreator) + creatorToSingleGroupStatsMap := make(map[podReplicaCreator]singleGroupStats) + + // Use per-MPA minReplicas if present, fall back to the global setting. + required := f.minReplicas + if mpa.Spec.Constraints != nil && mpa.Spec.Constraints.Global.MinReplicas != nil { + required = int(*mpa.Spec.Constraints.Global.MinReplicas) + klog.V(3).Infof("overriding minReplicas from global %v to per-MPA %v for MPA %v/%v", + f.minReplicas, required, mpa.Namespace, mpa.Name) + } + + for creator, replicas := range livePods { + actual := len(replicas) + if actual < required { + klog.V(2).Infof("too few replicas for %v %v/%v. Found %v live pods, needs %v (global %v)", + creator.Kind, creator.Namespace, creator.Name, actual, required, f.minReplicas) + continue + } + + var configured int + if creator.Kind == job { + // Job has no replicas configuration, so we will use actual number of live pods as replicas count. + configured = actual + } else { + var err error + configured, err = f.getReplicaCount(creator) + if err != nil { + klog.Errorf("failed to obtain replication info for %v %v/%v. %v", + creator.Kind, creator.Namespace, creator.Name, err) + continue + } + } + + singleGroup := singleGroupStats{} + singleGroup.configured = configured + singleGroup.evictionTolerance = int(float64(configured) * f.evictionToleranceFraction) + for _, pod := range replicas { + podToReplicaCreatorMap[getPodID(pod)] = creator + if pod.Status.Phase == apiv1.PodPending { + singleGroup.pending = singleGroup.pending + 1 + } + } + singleGroup.running = len(replicas) - singleGroup.pending + creatorToSingleGroupStatsMap[creator] = singleGroup + } + return &podsEvictionRestrictionImpl{ + client: f.client, + podToReplicaCreatorMap: podToReplicaCreatorMap, + creatorToSingleGroupStatsMap: creatorToSingleGroupStatsMap} +} + +func getPodReplicaCreator(pod *apiv1.Pod) (*podReplicaCreator, error) { + creator := managingControllerRef(pod) + if creator == nil { + return nil, nil + } + podReplicaCreator := &podReplicaCreator{ + Namespace: pod.Namespace, + Name: creator.Name, + Kind: controllerKind(creator.Kind), + } + return podReplicaCreator, nil +} + +func getPodID(pod *apiv1.Pod) string { + if pod == nil { + return "" + } + return pod.Namespace + "/" + pod.Name +} + +func (f *podsEvictionRestrictionFactoryImpl) getReplicaCount(creator podReplicaCreator) (int, error) { + switch creator.Kind { + case replicationController: + rcObj, exists, err := f.rcInformer.GetStore().GetByKey(creator.Namespace + "/" + creator.Name) + if err != nil { + return 0, fmt.Errorf("replication controller %s/%s is not available, err: %v", creator.Namespace, creator.Name, err) + } + if !exists { + return 0, fmt.Errorf("replication controller %s/%s does not exist", creator.Namespace, creator.Name) + } + rc, ok := rcObj.(*apiv1.ReplicationController) + if !ok { + return 0, fmt.Errorf("Failed to parse Replication Controller") + } + if rc.Spec.Replicas == nil || *rc.Spec.Replicas == 0 { + return 0, fmt.Errorf("replication controller %s/%s has no replicas config", creator.Namespace, creator.Name) + } + return int(*rc.Spec.Replicas), nil + + case replicaSet: + rsObj, exists, err := f.rsInformer.GetStore().GetByKey(creator.Namespace + "/" + creator.Name) + if err != nil { + return 0, fmt.Errorf("replica set %s/%s is not available, err: %v", creator.Namespace, creator.Name, err) + } + if !exists { + return 0, fmt.Errorf("replica set %s/%s does not exist", creator.Namespace, creator.Name) + } + rs, ok := rsObj.(*appsv1.ReplicaSet) + if !ok { + return 0, fmt.Errorf("Failed to parse Replicaset") + } + if rs.Spec.Replicas == nil || *rs.Spec.Replicas == 0 { + return 0, fmt.Errorf("replica set %s/%s has no replicas config", creator.Namespace, creator.Name) + } + return int(*rs.Spec.Replicas), nil + + case statefulSet: + ssObj, exists, err := f.ssInformer.GetStore().GetByKey(creator.Namespace + "/" + creator.Name) + if err != nil { + return 0, fmt.Errorf("stateful set %s/%s is not available, err: %v", creator.Namespace, creator.Name, err) + } + if !exists { + return 0, fmt.Errorf("stateful set %s/%s does not exist", creator.Namespace, creator.Name) + } + ss, ok := ssObj.(*appsv1.StatefulSet) + if !ok { + return 0, fmt.Errorf("Failed to parse StatefulSet") + } + if ss.Spec.Replicas == nil || *ss.Spec.Replicas == 0 { + return 0, fmt.Errorf("stateful set %s/%s has no replicas config", creator.Namespace, creator.Name) + } + return int(*ss.Spec.Replicas), nil + + case daemonSet: + dsObj, exists, err := f.dsInformer.GetStore().GetByKey(creator.Namespace + "/" + creator.Name) + if err != nil { + return 0, fmt.Errorf("daemon set %s/%s is not available, err: %v", creator.Namespace, creator.Name, err) + } + if !exists { + return 0, fmt.Errorf("daemon set %s/%s does not exist", creator.Namespace, creator.Name) + } + ds, ok := dsObj.(*appsv1.DaemonSet) + if !ok { + return 0, fmt.Errorf("Failed to parse DaemonSet") + } + if ds.Status.NumberReady == 0 { + return 0, fmt.Errorf("daemon set %s/%s has no number ready pods", creator.Namespace, creator.Name) + } + return int(ds.Status.NumberReady), nil + } + + return 0, nil +} + +func managingControllerRef(pod *apiv1.Pod) *metav1.OwnerReference { + var managingController metav1.OwnerReference + for _, ownerReference := range pod.ObjectMeta.GetOwnerReferences() { + if *ownerReference.Controller { + managingController = ownerReference + break + } + } + return &managingController +} + +func setUpInformer(kubeClient kube_client.Interface, kind controllerKind) (cache.SharedIndexInformer, error) { + var informer cache.SharedIndexInformer + switch kind { + case replicationController: + informer = coreinformer.NewReplicationControllerInformer(kubeClient, apiv1.NamespaceAll, + resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + case replicaSet: + informer = appsinformer.NewReplicaSetInformer(kubeClient, apiv1.NamespaceAll, + resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + case statefulSet: + informer = appsinformer.NewStatefulSetInformer(kubeClient, apiv1.NamespaceAll, + resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + case daemonSet: + informer = appsinformer.NewDaemonSetInformer(kubeClient, apiv1.NamespaceAll, + resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + default: + return nil, fmt.Errorf("Unknown controller kind: %v", kind) + } + stopCh := make(chan struct{}) + go informer.Run(stopCh) + synced := cache.WaitForCacheSync(stopCh, informer.HasSynced) + if !synced { + return nil, fmt.Errorf("Failed to sync %v cache.", kind) + } + return informer, nil +} diff --git a/multidimensional-pod-autoscaler/pkg/updater/eviction/pods_eviction_restriction_test.go b/multidimensional-pod-autoscaler/pkg/updater/eviction/pods_eviction_restriction_test.go new file mode 100644 index 000000000000..56da4002e508 --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/updater/eviction/pods_eviction_restriction_test.go @@ -0,0 +1,612 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package eviction + +import ( + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/assert" + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + apiv1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + mpa_types "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1alpha1" + mpa_test "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/utils/test" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/test" + appsinformer "k8s.io/client-go/informers/apps/v1" + coreinformer "k8s.io/client-go/informers/core/v1" + "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/tools/cache" +) + +type podWithExpectations struct { + pod *apiv1.Pod + canEvict bool + evictionSuccess bool +} + +func getBasicMpa() *mpa_types.MultidimPodAutoscaler { + return mpa_test.MultidimPodAutoscaler().WithContainer("any").Get() +} + +func TestEvictReplicatedByController(t *testing.T) { + + rc := apiv1.ReplicationController{ + ObjectMeta: metav1.ObjectMeta{ + Name: "rc", + Namespace: "default", + }, + TypeMeta: metav1.TypeMeta{ + Kind: "ReplicationController", + }, + } + + mpaSingleReplica := getBasicMpa() + minReplicas := int32(1) + mpaSingleReplica.Spec.Constraints = &mpa_types.ScalingConstraints{ + Global: &mpa_types.HorizontalScalingConstraints{MinReplicas: &minReplicas}, + } + + index := 0 + generatePod := func() test.PodBuilder { + index++ + return test.Pod().WithName(fmt.Sprintf("test-%v", index)).WithCreator(&rc.ObjectMeta, &rc.TypeMeta) + } + + testCases := []struct { + name string + replicas int32 + evictionTollerance float64 + mpa *mpa_types.MultidimPodAutoscaler + pods []podWithExpectations + }{ + { + name: "Evict only first pod (half of 3).", + replicas: 3, + evictionTollerance: 0.5, + mpa: getBasicMpa(), + pods: []podWithExpectations{ + { + pod: generatePod().Get(), + canEvict: true, + evictionSuccess: true, + }, + { + pod: generatePod().Get(), + canEvict: true, + evictionSuccess: false, + }, + { + pod: generatePod().Get(), + canEvict: true, + evictionSuccess: false, + }, + }, + }, + { + name: "Evict two pods (half of 4).", + replicas: 4, + evictionTollerance: 0.5, + mpa: getBasicMpa(), + pods: []podWithExpectations{ + { + + pod: generatePod().Get(), + canEvict: true, + evictionSuccess: true, + }, + { + pod: generatePod().Get(), + canEvict: true, + evictionSuccess: true, + }, + { + pod: generatePod().Get(), + canEvict: true, + evictionSuccess: false, + }, + { + pod: generatePod().Get(), + canEvict: true, + evictionSuccess: false, + }, + }, + }, + { + name: "Half of the population can be evicted. One pod is missing already.", + replicas: 4, + evictionTollerance: 0.5, + mpa: getBasicMpa(), + pods: []podWithExpectations{ + { + pod: generatePod().Get(), + canEvict: true, + evictionSuccess: true, + }, + { + pod: generatePod().Get(), + canEvict: true, + evictionSuccess: false, + }, + { + pod: generatePod().Get(), + canEvict: true, + evictionSuccess: false, + }, + }, + }, + { + name: "For small eviction tollerance at least one pod is evicted.", + replicas: 3, + evictionTollerance: 0.1, + mpa: getBasicMpa(), + pods: []podWithExpectations{ + { + pod: generatePod().Get(), + canEvict: true, + evictionSuccess: true, + }, + { + pod: generatePod().Get(), + canEvict: true, + evictionSuccess: false, + }, + { + pod: generatePod().Get(), + canEvict: true, + evictionSuccess: false, + }, + }, + }, + { + name: "Only 2 pods in replica of 3 and tollerance is 0. None of pods can be evicted.", + replicas: 3, + evictionTollerance: 0.1, + mpa: getBasicMpa(), + pods: []podWithExpectations{ + { + pod: generatePod().Get(), + canEvict: false, + evictionSuccess: false, + }, + { + pod: generatePod().Get(), + canEvict: false, + evictionSuccess: false, + }, + }, + }, + { + name: "Only pending pod can be evicted without violation of tollerance.", + replicas: 3, + evictionTollerance: 0.5, + mpa: getBasicMpa(), + pods: []podWithExpectations{ + { + pod: generatePod().Get(), + canEvict: false, + evictionSuccess: false, + }, + { + pod: generatePod().WithPhase(apiv1.PodPending).Get(), + canEvict: true, + evictionSuccess: true, + }, + { + pod: generatePod().Get(), + canEvict: false, + evictionSuccess: false, + }, + }, + }, + { + name: "Pending pods are always evictable.", + replicas: 4, + evictionTollerance: 0.5, + mpa: getBasicMpa(), + pods: []podWithExpectations{ + { + pod: generatePod().Get(), + canEvict: false, + evictionSuccess: false, + }, + { + pod: generatePod().WithPhase(apiv1.PodPending).Get(), + canEvict: true, + evictionSuccess: true, + }, + { + pod: generatePod().WithPhase(apiv1.PodPending).Get(), + canEvict: true, + evictionSuccess: true, + }, + { + pod: generatePod().WithPhase(apiv1.PodPending).Get(), + canEvict: true, + evictionSuccess: true, + }, + }, + }, + { + name: "Cannot evict a single Pod under default settings.", + replicas: 1, + evictionTollerance: 0.5, + mpa: getBasicMpa(), + pods: []podWithExpectations{ + { + pod: generatePod().Get(), + canEvict: false, + evictionSuccess: false, + }, + }, + }, + { + name: "Can evict even a single Pod using HorizontalScalingConstraints.MinReplicas.", + replicas: 1, + evictionTollerance: 0.5, + mpa: mpaSingleReplica, + pods: []podWithExpectations{ + { + pod: generatePod().Get(), + canEvict: true, + evictionSuccess: true, + }, + }, + }, + } + + for _, testCase := range testCases { + rc.Spec = apiv1.ReplicationControllerSpec{ + Replicas: &testCase.replicas, + } + pods := make([]*apiv1.Pod, 0, len(testCase.pods)) + for _, p := range testCase.pods { + pods = append(pods, p.pod) + } + factory, _ := getEvictionRestrictionFactory(&rc, nil, nil, nil, 2, testCase.evictionTollerance) + eviction := factory.NewPodsEvictionRestriction(pods, testCase.mpa) + for i, p := range testCase.pods { + assert.Equalf(t, p.canEvict, eviction.CanEvict(p.pod), "TC %v - unexpected CanEvict result for pod-%v %#v", testCase.name, i, p.pod) + } + for i, p := range testCase.pods { + err := eviction.Evict(p.pod, testCase.mpa, test.FakeEventRecorder()) + if p.evictionSuccess { + assert.NoErrorf(t, err, "TC %v - unexpected Evict result for pod-%v %#v", testCase.name, i, p.pod) + } else { + assert.Errorf(t, err, "TC %v - unexpected Evict result for pod-%v %#v", testCase.name, i, p.pod) + } + } + } + +} + +func TestEvictReplicatedByReplicaSet(t *testing.T) { + replicas := int32(5) + livePods := 5 + + rs := appsv1.ReplicaSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "rs", + Namespace: "default", + }, + TypeMeta: metav1.TypeMeta{ + Kind: "ReplicaSet", + }, + Spec: appsv1.ReplicaSetSpec{ + Replicas: &replicas, + }, + } + + pods := make([]*apiv1.Pod, livePods) + for i := range pods { + pods[i] = test.Pod().WithName(getTestPodName(i)).WithCreator(&rs.ObjectMeta, &rs.TypeMeta).Get() + } + + factory, _ := getEvictionRestrictionFactory(nil, &rs, nil, nil, 2, 0.5) + eviction := factory.NewPodsEvictionRestriction(pods, getBasicMpa()) + + for _, pod := range pods { + assert.True(t, eviction.CanEvict(pod)) + } + + for _, pod := range pods[:2] { + err := eviction.Evict(pod, getBasicMpa(), test.FakeEventRecorder()) + assert.Nil(t, err, "Should evict with no error") + } + for _, pod := range pods[2:] { + err := eviction.Evict(pod, getBasicMpa(), test.FakeEventRecorder()) + assert.Error(t, err, "Error expected") + } +} + +func TestEvictReplicatedByStatefulSet(t *testing.T) { + replicas := int32(5) + livePods := 5 + + ss := appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ss", + Namespace: "default", + }, + TypeMeta: metav1.TypeMeta{ + Kind: "StatefulSet", + }, + Spec: appsv1.StatefulSetSpec{ + Replicas: &replicas, + }, + } + + pods := make([]*apiv1.Pod, livePods) + for i := range pods { + pods[i] = test.Pod().WithName(getTestPodName(i)).WithCreator(&ss.ObjectMeta, &ss.TypeMeta).Get() + } + + factory, _ := getEvictionRestrictionFactory(nil, nil, &ss, nil, 2, 0.5) + eviction := factory.NewPodsEvictionRestriction(pods, getBasicMpa()) + + for _, pod := range pods { + assert.True(t, eviction.CanEvict(pod)) + } + + for _, pod := range pods[:2] { + err := eviction.Evict(pod, getBasicMpa(), test.FakeEventRecorder()) + assert.Nil(t, err, "Should evict with no error") + } + for _, pod := range pods[2:] { + err := eviction.Evict(pod, getBasicMpa(), test.FakeEventRecorder()) + assert.Error(t, err, "Error expected") + } +} + +func TestEvictReplicatedByDaemonSet(t *testing.T) { + livePods := int32(5) + + ds := appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ds", + Namespace: "default", + }, + TypeMeta: metav1.TypeMeta{ + Kind: "DaemonSet", + }, + Status: appsv1.DaemonSetStatus{ + NumberReady: livePods, + }, + } + + pods := make([]*apiv1.Pod, livePods) + for i := range pods { + pods[i] = test.Pod().WithName(getTestPodName(i)).WithCreator(&ds.ObjectMeta, &ds.TypeMeta).Get() + } + factory, _ := getEvictionRestrictionFactory(nil, nil, nil, &ds, 2, 0.5) + eviction := factory.NewPodsEvictionRestriction(pods, getBasicMpa()) + + for _, pod := range pods { + assert.True(t, eviction.CanEvict(pod)) + } + for _, pod := range pods[:2] { + err := eviction.Evict(pod, getBasicMpa(), test.FakeEventRecorder()) + assert.Nil(t, err, "Should evict with no error") + } + for _, pod := range pods[2:] { + err := eviction.Evict(pod, getBasicMpa(), test.FakeEventRecorder()) + assert.Error(t, err, "Error expected") + } +} + +func TestEvictReplicatedByJob(t *testing.T) { + job := batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: "job", + Namespace: "default", + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Job", + }, + } + + livePods := 5 + + pods := make([]*apiv1.Pod, livePods) + for i := range pods { + pods[i] = test.Pod().WithName(getTestPodName(i)).WithCreator(&job.ObjectMeta, &job.TypeMeta).Get() + } + + factory, _ := getEvictionRestrictionFactory(nil, nil, nil, nil, 2, 0.5) + eviction := factory.NewPodsEvictionRestriction(pods, getBasicMpa()) + + for _, pod := range pods { + assert.True(t, eviction.CanEvict(pod)) + } + + for _, pod := range pods[:2] { + err := eviction.Evict(pod, getBasicMpa(), test.FakeEventRecorder()) + assert.Nil(t, err, "Should evict with no error") + } + for _, pod := range pods[2:] { + err := eviction.Evict(pod, getBasicMpa(), test.FakeEventRecorder()) + assert.Error(t, err, "Error expected") + } +} + +func TestEvictTooFewReplicas(t *testing.T) { + replicas := int32(5) + livePods := 5 + + rc := apiv1.ReplicationController{ + ObjectMeta: metav1.ObjectMeta{ + Name: "rc", + Namespace: "default", + }, + TypeMeta: metav1.TypeMeta{ + Kind: "ReplicationController", + }, + Spec: apiv1.ReplicationControllerSpec{ + Replicas: &replicas, + }, + } + + pods := make([]*apiv1.Pod, livePods) + for i := range pods { + pods[i] = test.Pod().WithName(getTestPodName(i)).WithCreator(&rc.ObjectMeta, &rc.TypeMeta).Get() + } + + factory, _ := getEvictionRestrictionFactory(&rc, nil, nil, nil, 10, 0.5) + eviction := factory.NewPodsEvictionRestriction(pods, getBasicMpa()) + + for _, pod := range pods { + assert.False(t, eviction.CanEvict(pod)) + } + + for _, pod := range pods { + err := eviction.Evict(pod, getBasicMpa(), test.FakeEventRecorder()) + assert.Error(t, err, "Error expected") + } +} + +func TestEvictionTolerance(t *testing.T) { + replicas := int32(5) + livePods := 5 + tolerance := 0.8 + + rc := apiv1.ReplicationController{ + ObjectMeta: metav1.ObjectMeta{ + Name: "rc", + Namespace: "default", + }, + TypeMeta: metav1.TypeMeta{ + Kind: "ReplicationController", + }, + Spec: apiv1.ReplicationControllerSpec{ + Replicas: &replicas, + }, + } + + pods := make([]*apiv1.Pod, livePods) + for i := range pods { + pods[i] = test.Pod().WithName(getTestPodName(i)).WithCreator(&rc.ObjectMeta, &rc.TypeMeta).Get() + } + + factory, _ := getEvictionRestrictionFactory(&rc, nil, nil, nil, 2 /*minReplicas*/, tolerance) + eviction := factory.NewPodsEvictionRestriction(pods, getBasicMpa()) + + for _, pod := range pods { + assert.True(t, eviction.CanEvict(pod)) + } + + for _, pod := range pods[:4] { + err := eviction.Evict(pod, getBasicMpa(), test.FakeEventRecorder()) + assert.Nil(t, err, "Should evict with no error") + } + for _, pod := range pods[4:] { + err := eviction.Evict(pod, getBasicMpa(), test.FakeEventRecorder()) + assert.Error(t, err, "Error expected") + } +} + +func TestEvictAtLeastOne(t *testing.T) { + replicas := int32(5) + livePods := 5 + tolerance := 0.1 + + rc := apiv1.ReplicationController{ + ObjectMeta: metav1.ObjectMeta{ + Name: "rc", + Namespace: "default", + }, + TypeMeta: metav1.TypeMeta{ + Kind: "ReplicationController", + }, + Spec: apiv1.ReplicationControllerSpec{ + Replicas: &replicas, + }, + } + + pods := make([]*apiv1.Pod, livePods) + for i := range pods { + pods[i] = test.Pod().WithName(getTestPodName(i)).WithCreator(&rc.ObjectMeta, &rc.TypeMeta).Get() + } + + factory, _ := getEvictionRestrictionFactory(&rc, nil, nil, nil, 2, tolerance) + eviction := factory.NewPodsEvictionRestriction(pods, getBasicMpa()) + + for _, pod := range pods { + assert.True(t, eviction.CanEvict(pod)) + } + + for _, pod := range pods[:1] { + err := eviction.Evict(pod, getBasicMpa(), test.FakeEventRecorder()) + assert.Nil(t, err, "Should evict with no error") + } + for _, pod := range pods[1:] { + err := eviction.Evict(pod, getBasicMpa(), test.FakeEventRecorder()) + assert.Error(t, err, "Error expected") + } +} + +func getEvictionRestrictionFactory(rc *apiv1.ReplicationController, rs *appsv1.ReplicaSet, + ss *appsv1.StatefulSet, ds *appsv1.DaemonSet, minReplicas int, + evictionToleranceFraction float64) (PodsEvictionRestrictionFactory, error) { + kubeClient := &fake.Clientset{} + rcInformer := coreinformer.NewReplicationControllerInformer(kubeClient, apiv1.NamespaceAll, + 0*time.Second, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + rsInformer := appsinformer.NewReplicaSetInformer(kubeClient, apiv1.NamespaceAll, + 0*time.Second, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + ssInformer := appsinformer.NewStatefulSetInformer(kubeClient, apiv1.NamespaceAll, + 0*time.Second, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + dsInformer := appsinformer.NewDaemonSetInformer(kubeClient, apiv1.NamespaceAll, + 0*time.Second, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + if rc != nil { + err := rcInformer.GetIndexer().Add(rc) + if err != nil { + return nil, fmt.Errorf("Error adding object to cache: %v", err) + } + } + if rs != nil { + err := rsInformer.GetIndexer().Add(rs) + if err != nil { + return nil, fmt.Errorf("Error adding object to cache: %v", err) + } + } + if ss != nil { + err := ssInformer.GetIndexer().Add(ss) + if err != nil { + return nil, fmt.Errorf("Error adding object to cache: %v", err) + } + } + if ds != nil { + err := dsInformer.GetIndexer().Add(ds) + if err != nil { + return nil, fmt.Errorf("Error adding object to cache: %v", err) + } + } + return &podsEvictionRestrictionFactoryImpl{ + client: kubeClient, + rsInformer: rsInformer, + rcInformer: rcInformer, + ssInformer: ssInformer, + dsInformer: dsInformer, + minReplicas: minReplicas, + evictionToleranceFraction: evictionToleranceFraction, + }, nil +} + +func getTestPodName(index int) string { + return fmt.Sprintf("test-%v", index) +} diff --git a/multidimensional-pod-autoscaler/pkg/updater/logic/updater.go b/multidimensional-pod-autoscaler/pkg/updater/logic/updater.go new file mode 100644 index 000000000000..1219f0871f42 --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/updater/logic/updater.go @@ -0,0 +1,528 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package logic + +import ( + "context" + "fmt" + "time" + + "golang.org/x/time/rate" + + autoscalingv1 "k8s.io/api/autoscaling/v1" + apiv1 "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime/schema" + mpa_types "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1alpha1" + mpa_clientset "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/client/clientset/versioned" + mpa_lister "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/client/listers/autoscaling.k8s.io/v1alpha1" + "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/target" + "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/updater/eviction" + "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/updater/priority" + mpa_api_util "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/utils/mpa" + vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" + controllerfetcher "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/target/controller_fetcher" + metrics_updater "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/metrics/updater" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/status" + kube_client "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/kubernetes/scheme" + clientv1 "k8s.io/client-go/kubernetes/typed/core/v1" + v1lister "k8s.io/client-go/listers/core/v1" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/tools/record" + "k8s.io/klog/v2" +) + +// Updater performs updates on pods if recommended by Multidimensional Pod Autoscaler +type Updater interface { + // RunOnce represents single iteration in the main-loop of Updater. + RunOnce(context.Context) + + // RunOnceUpdatingDeployment represents single iteration in the main-loop of Updater + // which does pod eviction and deployment updates. + RunOnceUpdatingDeployment(context.Context) +} + +type updater struct { + mpaLister mpa_lister.MultidimPodAutoscalerLister + podLister v1lister.PodLister + eventRecorder record.EventRecorder + evictionFactory eviction.PodsEvictionRestrictionFactory + recommendationProcessor mpa_api_util.RecommendationProcessor + evictionAdmission priority.PodEvictionAdmission + priorityProcessor priority.PriorityProcessor + evictionRateLimiter *rate.Limiter + selectorFetcher target.MpaTargetSelectorFetcher + useAdmissionControllerStatus bool + statusValidator status.Validator + controllerFetcher controllerfetcher.ControllerFetcher + ignoredNamespaces []string +} + +// NewUpdater creates Updater with given configuration +func NewUpdater( + kubeClient kube_client.Interface, + mpaClient *mpa_clientset.Clientset, + minReplicasForEvicition int, + evictionRateLimit float64, + evictionRateBurst int, + evictionToleranceFraction float64, + useAdmissionControllerStatus bool, + statusNamespace string, + recommendationProcessor mpa_api_util.RecommendationProcessor, + evictionAdmission priority.PodEvictionAdmission, + selectorFetcher target.MpaTargetSelectorFetcher, + controllerFetcher controllerfetcher.ControllerFetcher, + priorityProcessor priority.PriorityProcessor, + namespace string, + ignoredNamespaces []string, +) (Updater, error) { + evictionRateLimiter := getRateLimiter(evictionRateLimit, evictionRateBurst) + factory, err := eviction.NewPodsEvictionRestrictionFactory(kubeClient, minReplicasForEvicition, evictionToleranceFraction) + if err != nil { + return nil, fmt.Errorf("Failed to create eviction restriction factory: %v", err) + } + return &updater{ + mpaLister: mpa_api_util.NewMpasLister(mpaClient, make(chan struct{}), namespace), + podLister: newPodLister(kubeClient, namespace), + eventRecorder: newEventRecorder(kubeClient), + evictionFactory: factory, + recommendationProcessor: recommendationProcessor, + evictionRateLimiter: evictionRateLimiter, + evictionAdmission: evictionAdmission, + priorityProcessor: priorityProcessor, + selectorFetcher: selectorFetcher, + controllerFetcher: controllerFetcher, + useAdmissionControllerStatus: useAdmissionControllerStatus, + statusValidator: status.NewValidator( + kubeClient, + status.AdmissionControllerStatusName, + statusNamespace, + ), + ignoredNamespaces: ignoredNamespaces, + }, nil +} + +// RunOnce represents single iteration in the main-loop of Updater +func (u *updater) RunOnce(ctx context.Context) { + timer := metrics_updater.NewExecutionTimer() + defer timer.ObserveTotal() + + if u.useAdmissionControllerStatus { + isValid, err := u.statusValidator.IsStatusValid(ctx, status.AdmissionControllerStatusTimeout) + if err != nil { + klog.Errorf("Error getting Admission Controller status: %v. Skipping eviction loop", err) + return + } + if !isValid { + klog.Warningf("Admission Controller status has been refreshed more than %v ago. Skipping eviction loop", + status.AdmissionControllerStatusTimeout) + return + } + } + + mpaList, err := u.mpaLister.List(labels.Everything()) + if err != nil { + klog.Fatalf("failed get MPA list: %v", err) + } + timer.ObserveStep("ListMPAs") + klog.V(4).Infof("Retrieved all MPA objects.") + + mpas := make([]*mpa_api_util.MpaWithSelector, 0) + + for _, mpa := range mpaList { + if mpa_api_util.GetUpdateMode(mpa) != vpa_types.UpdateModeRecreate && + mpa_api_util.GetUpdateMode(mpa) != vpa_types.UpdateModeAuto { + klog.V(3).Infof("skipping MPA object %v because its mode is not \"Recreate\" or \"Auto\"", mpa.Name) + continue + } + selector, err := u.selectorFetcher.Fetch(ctx, mpa) + if err != nil { + klog.V(3).Infof("skipping MPA object %v because we cannot fetch selector", mpa.Name) + continue + } + + mpas = append(mpas, &mpa_api_util.MpaWithSelector{ + Mpa: mpa, + Selector: selector, + }) + } + + if len(mpas) == 0 { + klog.Warningf("no MPA objects to process") + if u.evictionAdmission != nil { + u.evictionAdmission.CleanUp() + } + return + } + + podsList, err := u.podLister.List(labels.Everything()) + if err != nil { + klog.Errorf("failed to get pods list: %v", err) + return + } + timer.ObserveStep("ListPods") + allLivePods := filterDeletedPods(podsList) + klog.V(4).Infof("Retrieved all live pods.") + + controlledPods := make(map[*mpa_types.MultidimPodAutoscaler][]*apiv1.Pod) + for _, pod := range allLivePods { + controllingMPA := mpa_api_util.GetControllingMPAForPod(ctx, pod, mpas, u.controllerFetcher) + if controllingMPA != nil { + controlledPods[controllingMPA.Mpa] = append(controlledPods[controllingMPA.Mpa], pod) + } + } + timer.ObserveStep("FilterPods") + klog.V(4).Infof("Matched the MPA object for all pods.") + + if u.evictionAdmission != nil { + u.evictionAdmission.LoopInit(allLivePods, controlledPods) + } + timer.ObserveStep("AdmissionInit") + + // wrappers for metrics which are computed every loop run + controlledPodsCounter := metrics_updater.NewControlledPodsCounter() + evictablePodsCounter := metrics_updater.NewEvictablePodsCounter() + mpasWithEvictablePodsCounter := metrics_updater.NewVpasWithEvictablePodsCounter() + mpasWithEvictedPodsCounter := metrics_updater.NewVpasWithEvictedPodsCounter() + + // using defer to protect against 'return' after evictionRateLimiter.Wait + defer controlledPodsCounter.Observe() + defer evictablePodsCounter.Observe() + defer mpasWithEvictablePodsCounter.Observe() + defer mpasWithEvictedPodsCounter.Observe() + + // NOTE: this loop assumes that controlledPods are filtered + // to contain only Pods controlled by a MPA in auto or recreate mode + for mpa, livePods := range controlledPods { + mpaSize := len(livePods) + controlledPodsCounter.Add(mpaSize, mpaSize) + evictionLimiter := u.evictionFactory.NewPodsEvictionRestriction(livePods, mpa) + podsForUpdate := u.getPodsUpdateOrder(filterNonEvictablePods(livePods, evictionLimiter), mpa) + evictablePodsCounter.Add(mpaSize, len(podsForUpdate)) + + withEvictable := false + withEvicted := false + for _, pod := range podsForUpdate { + withEvictable = true + if !evictionLimiter.CanEvict(pod) { + continue + } + err := u.evictionRateLimiter.Wait(ctx) + if err != nil { + klog.Warningf("evicting pod %v failed: %v", pod.Name, err) + return + } + klog.V(2).Infof("evicting pod %v", pod.Name) + evictErr := evictionLimiter.Evict(pod, mpa, u.eventRecorder) + if evictErr != nil { + klog.Warningf("evicting pod %v failed: %v", pod.Name, evictErr) + } else { + withEvicted = true + metrics_updater.AddEvictedPod(mpaSize) + } + } + + if withEvictable { + mpasWithEvictablePodsCounter.Add(mpaSize, 1) + } + if withEvicted { + mpasWithEvictedPodsCounter.Add(mpaSize, 1) + } + } + timer.ObserveStep("EvictPods") + klog.V(4).Infof("Evicted all eligible pods.") +} + +// RunOnceUpdatingDeployment represents single iteration in the main-loop of Updater which evicts +// pods for MPA and updates the Deployment for HPA. +func (u *updater) RunOnceUpdatingDeployment(ctx context.Context) { + timer := metrics_updater.NewExecutionTimer() + defer timer.ObserveTotal() + + if u.useAdmissionControllerStatus { + isValid, err := u.statusValidator.IsStatusValid(ctx, status.AdmissionControllerStatusTimeout) + if err != nil { + klog.Errorf("Error getting Admission Controller status: %v. Skipping eviction loop", err) + return + } + if !isValid { + klog.Warningf("Admission Controller status has been refreshed more than %v ago. Skipping eviction loop", + status.AdmissionControllerStatusTimeout) + return + } + } + + mpaList, err := u.mpaLister.List(labels.Everything()) + if err != nil { + klog.Fatalf("failed get MPA list: %v", err) + } + timer.ObserveStep("ListMPAs") + klog.V(4).Infof("Retrieved all MPA objects.") + + mpas := make([]*mpa_api_util.MpaWithSelector, 0) + + for _, mpa := range mpaList { + if mpa_api_util.GetUpdateMode(mpa) != vpa_types.UpdateModeRecreate && + mpa_api_util.GetUpdateMode(mpa) != vpa_types.UpdateModeAuto { + klog.V(3).Infof("skipping MPA object %v because its mode is not \"Recreate\" or \"Auto\"", mpa.Name) + continue + } + selector, err := u.selectorFetcher.Fetch(ctx, mpa) + if err != nil { + klog.V(3).Infof("skipping MPA object %v because we cannot fetch selector", mpa.Name) + continue + } + + mpas = append(mpas, &mpa_api_util.MpaWithSelector{ + Mpa: mpa, + Selector: selector, + }) + + // Update the number of replicas. + targetGV, err := schema.ParseGroupVersion(mpa.Spec.ScaleTargetRef.APIVersion) + if err != nil { + klog.Errorf("%s: FailedGetScale - error: %v", v1.EventTypeWarning, err.Error()) + u.eventRecorder.Event(mpa, v1.EventTypeWarning, "FailedGetScale", err.Error()) + return + } + targetGK := schema.GroupKind{ + Group: targetGV.Group, + Kind: mpa.Spec.ScaleTargetRef.Kind, + } + mappings, err := u.selectorFetcher.GetRESTMappings(targetGK) + if err != nil { + klog.Errorf("%s: FailedGetScale - error: %v", v1.EventTypeWarning, err.Error()) + u.eventRecorder.Event(mpa, v1.EventTypeWarning, "FailedGetScale", err.Error()) + return + } + scale, targetGR, err := u.scaleForResourceMappings(ctx, mpa.Namespace, mpa.Spec.ScaleTargetRef.Name, mappings) + if err != nil { + klog.Errorf("%s: FailedGetScale - error: %v", v1.EventTypeWarning, err.Error()) + u.eventRecorder.Event(mpa, v1.EventTypeWarning, "FailedGetScale", err.Error()) + return + } + desiredReplicas := mpa.Status.DesiredReplicas + if desiredReplicas == scale.Spec.Replicas { + // No need to update the number of replicas. + klog.V(4).Infof("No need to change the number of replicas for MPA %v", mpa.Name) + continue + } else if desiredReplicas > *mpa.Spec.Constraints.Global.MaxReplicas || desiredReplicas < *mpa.Spec.Constraints.Global.MinReplicas { + // Constraints not satisfied. Should not be out of bound because it should have been + // checked in the recommender. + continue + } else { + klog.V(4).Infof("Updating the number of replicas from %d to %d for MPA %v", scale.Spec.Replicas, desiredReplicas, mpa.Name) + scale.Spec.Replicas = desiredReplicas + _, err = u.selectorFetcher.Scales(mpa.Namespace).Update(ctx, targetGR, scale, metav1.UpdateOptions{}) + if err != nil { + u.eventRecorder.Eventf(mpa, v1.EventTypeWarning, "FailedRescale", "New size: %d; error: %v", desiredReplicas, err.Error()) + klog.Errorf("%s: FailedRescale - New size: %d; error: %v", v1.EventTypeWarning, desiredReplicas, err.Error()) + return + } + klog.V(4).Infof("%s: Successfully rescaled the number of replicas to %d based on MPA %v", v1.EventTypeNormal, desiredReplicas, mpa.Name) + } + } + + if len(mpas) == 0 { + klog.Warningf("no MPA objects to process") + if u.evictionAdmission != nil { + u.evictionAdmission.CleanUp() + } + return + } + + podsList, err := u.podLister.List(labels.Everything()) + if err != nil { + klog.Errorf("failed to get pods list: %v", err) + return + } + timer.ObserveStep("ListPods") + allLivePods := filterDeletedPods(podsList) + klog.V(4).Infof("Retrieved all live pods.") + + controlledPods := make(map[*mpa_types.MultidimPodAutoscaler][]*apiv1.Pod) + for _, pod := range allLivePods { + controllingMPA := mpa_api_util.GetControllingMPAForPod(ctx, pod, mpas, u.controllerFetcher) + if controllingMPA != nil { + controlledPods[controllingMPA.Mpa] = append(controlledPods[controllingMPA.Mpa], pod) + } + } + timer.ObserveStep("FilterPods") + klog.V(4).Infof("Matched the MPA object for all pods.") + + if u.evictionAdmission != nil { + u.evictionAdmission.LoopInit(allLivePods, controlledPods) + } + timer.ObserveStep("AdmissionInit") + + // wrappers for metrics which are computed every loop run + controlledPodsCounter := metrics_updater.NewControlledPodsCounter() + evictablePodsCounter := metrics_updater.NewEvictablePodsCounter() + mpasWithEvictablePodsCounter := metrics_updater.NewVpasWithEvictablePodsCounter() + mpasWithEvictedPodsCounter := metrics_updater.NewVpasWithEvictedPodsCounter() + + // using defer to protect against 'return' after evictionRateLimiter.Wait + defer controlledPodsCounter.Observe() + defer evictablePodsCounter.Observe() + defer mpasWithEvictablePodsCounter.Observe() + defer mpasWithEvictedPodsCounter.Observe() + + // NOTE: this loop assumes that controlledPods are filtered + // to contain only Pods controlled by a MPA in auto or recreate mode + for mpa, livePods := range controlledPods { + mpaSize := len(livePods) + controlledPodsCounter.Add(mpaSize, mpaSize) + evictionLimiter := u.evictionFactory.NewPodsEvictionRestriction(livePods, mpa) + podsForUpdate := u.getPodsUpdateOrder(filterNonEvictablePods(livePods, evictionLimiter), mpa) + evictablePodsCounter.Add(mpaSize, len(podsForUpdate)) + + withEvictable := false + withEvicted := false + for _, pod := range podsForUpdate { + withEvictable = true + if !evictionLimiter.CanEvict(pod) { + continue + } + err := u.evictionRateLimiter.Wait(ctx) + if err != nil { + klog.Warningf("evicting pod %v failed: %v", pod.Name, err) + return + } + klog.V(2).Infof("evicting pod %v", pod.Name) + evictErr := evictionLimiter.Evict(pod, mpa, u.eventRecorder) + if evictErr != nil { + klog.Warningf("evicting pod %v failed: %v", pod.Name, evictErr) + } else { + withEvicted = true + metrics_updater.AddEvictedPod(mpaSize) + } + } + + if withEvictable { + mpasWithEvictablePodsCounter.Add(mpaSize, 1) + } + if withEvicted { + mpasWithEvictedPodsCounter.Add(mpaSize, 1) + } + } + timer.ObserveStep("EvictPods") + klog.V(4).Infof("Evicted all eligible pods.") +} + +// scaleForResourceMappings attempts to fetch the scale for the resource with the given name and +// namespace, trying each RESTMapping in turn until a working one is found. If none work, the first +// error is returned. It returns both the scale, as well as the group-resource from the working +// mapping. +func (u *updater) scaleForResourceMappings(ctx context.Context, namespace, name string, mappings []*apimeta.RESTMapping) (*autoscalingv1.Scale, schema.GroupResource, error) { + var firstErr error + for i, mapping := range mappings { + targetGR := mapping.Resource.GroupResource() + scale, err := u.selectorFetcher.Scales(namespace).Get(ctx, targetGR, name, metav1.GetOptions{}) + if err == nil { + return scale, targetGR, nil + } + + // if this is the first error, remember it, + // then go on and try other mappings until we find a good one + if i == 0 { + firstErr = err + } + } + + // make sure we handle an empty set of mappings + if firstErr == nil { + firstErr = fmt.Errorf("unrecognized resource") + } + + return nil, schema.GroupResource{}, firstErr +} + +func getRateLimiter(evictionRateLimit float64, evictionRateLimitBurst int) *rate.Limiter { + var evictionRateLimiter *rate.Limiter + if evictionRateLimit <= 0 { + // As a special case if the rate is set to rate.Inf, the burst rate is ignored + // see https://github.com/golang/time/blob/master/rate/rate.go#L37 + evictionRateLimiter = rate.NewLimiter(rate.Inf, 0) + klog.V(1).Info("Rate limit disabled") + } else { + evictionRateLimiter = rate.NewLimiter(rate.Limit(evictionRateLimit), evictionRateLimitBurst) + } + return evictionRateLimiter +} + +// getPodsUpdateOrder returns list of pods that should be updated ordered by update priority +func (u *updater) getPodsUpdateOrder(pods []*apiv1.Pod, mpa *mpa_types.MultidimPodAutoscaler) []*apiv1.Pod { + priorityCalculator := priority.NewUpdatePriorityCalculator( + mpa, + nil, + u.recommendationProcessor, + u.priorityProcessor) + + for _, pod := range pods { + priorityCalculator.AddPod(pod, time.Now()) + } + + return priorityCalculator.GetSortedPods(u.evictionAdmission) +} + +func filterNonEvictablePods(pods []*apiv1.Pod, evictionRestriciton eviction.PodsEvictionRestriction) []*apiv1.Pod { + result := make([]*apiv1.Pod, 0) + for _, pod := range pods { + if evictionRestriciton.CanEvict(pod) { + result = append(result, pod) + } + } + return result +} + +func filterDeletedPods(pods []*apiv1.Pod) []*apiv1.Pod { + result := make([]*apiv1.Pod, 0) + for _, pod := range pods { + if pod.DeletionTimestamp == nil { + result = append(result, pod) + } + } + return result +} + +func newPodLister(kubeClient kube_client.Interface, namespace string) v1lister.PodLister { + selector := fields.ParseSelectorOrDie("spec.nodeName!=" + "" + ",status.phase!=" + + string(apiv1.PodSucceeded) + ",status.phase!=" + string(apiv1.PodFailed)) + podListWatch := cache.NewListWatchFromClient(kubeClient.CoreV1().RESTClient(), "pods", namespace, selector) + store := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + podLister := v1lister.NewPodLister(store) + podReflector := cache.NewReflector(podListWatch, &apiv1.Pod{}, store, time.Hour) + stopCh := make(chan struct{}) + go podReflector.Run(stopCh) + + return podLister +} + +func newEventRecorder(kubeClient kube_client.Interface) record.EventRecorder { + eventBroadcaster := record.NewBroadcaster() + eventBroadcaster.StartLogging(klog.V(4).Infof) + if _, isFake := kubeClient.(*fake.Clientset); !isFake { + eventBroadcaster.StartRecordingToSink(&clientv1.EventSinkImpl{Interface: clientv1.New(kubeClient.CoreV1().RESTClient()).Events("")}) + } + return eventBroadcaster.NewRecorder(scheme.Scheme, apiv1.EventSource{Component: "mpa-updater"}) +} diff --git a/multidimensional-pod-autoscaler/pkg/updater/logic/updater_test.go b/multidimensional-pod-autoscaler/pkg/updater/logic/updater_test.go new file mode 100644 index 000000000000..71249be3dfe7 --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/updater/logic/updater_test.go @@ -0,0 +1,252 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package logic + +import ( + "context" + "strconv" + "testing" + "time" + + "golang.org/x/time/rate" + + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/assert" + apiv1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + mpa_types "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1alpha1" + target_mock "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/target/mock" + "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/updater/eviction" + "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/updater/priority" + mpa_test "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/utils/test" + vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/status" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/test" +) + +func parseLabelSelector(selector string) labels.Selector { + labelSelector, _ := metav1.ParseToLabelSelector(selector) + parsedSelector, _ := metav1.LabelSelectorAsSelector(labelSelector) + return parsedSelector +} + +func TestRunOnce_Mode(t *testing.T) { + tests := []struct { + name string + updateMode vpa_types.UpdateMode + expectFetchCalls bool + expectedEvictionCount int + }{ + { + name: "with Auto mode", + updateMode: vpa_types.UpdateModeAuto, + expectFetchCalls: true, + expectedEvictionCount: 5, + }, + { + name: "with Initial mode", + updateMode: vpa_types.UpdateModeInitial, + expectFetchCalls: false, + expectedEvictionCount: 0, + }, + { + name: "with Off mode", + updateMode: vpa_types.UpdateModeOff, + expectFetchCalls: false, + expectedEvictionCount: 0, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + testRunOnceBase( + t, + tc.updateMode, + newFakeValidator(true), + tc.expectFetchCalls, + tc.expectedEvictionCount, + ) + }) + } +} + +func TestRunOnce_Status(t *testing.T) { + tests := []struct { + name string + statusValidator status.Validator + expectFetchCalls bool + expectedEvictionCount int + }{ + { + name: "with valid status", + statusValidator: newFakeValidator(true), + expectFetchCalls: true, + expectedEvictionCount: 5, + }, + { + name: "with invalid status", + statusValidator: newFakeValidator(false), + expectFetchCalls: false, + expectedEvictionCount: 0, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + testRunOnceBase( + t, + vpa_types.UpdateModeAuto, + tc.statusValidator, + tc.expectFetchCalls, + tc.expectedEvictionCount, + ) + }) + } +} + +func testRunOnceBase( + t *testing.T, + updateMode vpa_types.UpdateMode, + statusValidator status.Validator, + expectFetchCalls bool, + expectedEvictionCount int, +) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + replicas := int32(5) + livePods := 5 + labels := map[string]string{"app": "testingApp"} + selector := parseLabelSelector("app = testingApp") + containerName := "container1" + rc := apiv1.ReplicationController{ + ObjectMeta: metav1.ObjectMeta{ + Name: "rc", + Namespace: "default", + }, + Spec: apiv1.ReplicationControllerSpec{ + Replicas: &replicas, + }, + } + pods := make([]*apiv1.Pod, livePods) + eviction := &mpa_test.PodsEvictionRestrictionMock{} + + for i := range pods { + pods[i] = test.Pod().WithName("test_"+strconv.Itoa(i)). + AddContainer(test.Container().WithName(containerName). + WithCPURequest(resource.MustParse("100m")). + WithMemRequest(resource.MustParse("100M")). + Get()). + WithCreator(&rc.ObjectMeta, &rc.TypeMeta). + Get() + + pods[i].Labels = labels + eviction.On("CanEvict", pods[i]).Return(true) + eviction.On("Evict", pods[i], nil).Return(nil) + } + + factory := &fakeEvictFactory{eviction} + mpaLister := &mpa_test.MultidimPodAutoscalerListerMock{} + + podLister := &test.PodListerMock{} + podLister.On("List").Return(pods, nil) + + mpaObj := mpa_test.MultidimPodAutoscaler(). + WithContainer(containerName). + WithTarget("2", "200M"). + WithMinAllowed(containerName, "1", "100M"). + WithMaxAllowed(containerName, "3", "1G"). + Get() + mpaObj.Spec.Policy = &mpa_types.PodUpdatePolicy{UpdateMode: &updateMode} + mpaLister.On("List").Return([]*mpa_types.MultidimPodAutoscaler{mpaObj}, nil).Once() + + mockSelectorFetcher := target_mock.NewMockMpaTargetSelectorFetcher(ctrl) + + updater := &updater{ + mpaLister: mpaLister, + podLister: podLister, + evictionFactory: factory, + evictionRateLimiter: rate.NewLimiter(rate.Inf, 0), + recommendationProcessor: &mpa_test.FakeRecommendationProcessor{}, + selectorFetcher: mockSelectorFetcher, + useAdmissionControllerStatus: true, + statusValidator: statusValidator, + priorityProcessor: priority.NewProcessor(), + } + + if expectFetchCalls { + mockSelectorFetcher.EXPECT().Fetch(gomock.Eq(mpaObj)).Return(selector, nil) + } + updater.RunOnce(context.Background()) + eviction.AssertNumberOfCalls(t, "Evict", expectedEvictionCount) +} + +func TestRunOnceNotingToProcess(t *testing.T) { + eviction := &mpa_test.PodsEvictionRestrictionMock{} + factory := &fakeEvictFactory{eviction} + mpaLister := &mpa_test.MultidimPodAutoscalerListerMock{} + podLister := &test.PodListerMock{} + mpaLister.On("List").Return(nil, nil).Once() + + updater := &updater{ + mpaLister: mpaLister, + podLister: podLister, + evictionFactory: factory, + evictionRateLimiter: rate.NewLimiter(rate.Inf, 0), + recommendationProcessor: &mpa_test.FakeRecommendationProcessor{}, + useAdmissionControllerStatus: true, + statusValidator: newFakeValidator(true), + } + updater.RunOnce(context.Background()) +} + +func TestGetRateLimiter(t *testing.T) { + cases := []struct { + rateLimit float64 + rateLimitBurst int + expectedLimiter *rate.Limiter + }{ + {0.0, 1, rate.NewLimiter(rate.Inf, 0)}, + {-1.0, 2, rate.NewLimiter(rate.Inf, 0)}, + {10.0, 3, rate.NewLimiter(rate.Limit(10), 3)}, + } + for _, tc := range cases { + limiter := getRateLimiter(tc.rateLimit, tc.rateLimitBurst) + assert.Equal(t, tc.expectedLimiter.Burst(), limiter.Burst()) + assert.InDelta(t, float64(tc.expectedLimiter.Limit()), float64(limiter.Limit()), 1e-6) + } +} + +type fakeEvictFactory struct { + evict eviction.PodsEvictionRestriction +} + +func (f fakeEvictFactory) NewPodsEvictionRestriction(pods []*apiv1.Pod, mpa *mpa_types.MultidimPodAutoscaler) eviction.PodsEvictionRestriction { + return f.evict +} + +type fakeValidator struct { + isValid bool +} + +func newFakeValidator(isValid bool) status.Validator { + return &fakeValidator{isValid} +} + +func (f *fakeValidator) IsStatusValid(ctx context.Context, statusTimeout time.Duration) (bool, error) { + return f.isValid, nil +} diff --git a/multidimensional-pod-autoscaler/pkg/updater/main.go b/multidimensional-pod-autoscaler/pkg/updater/main.go new file mode 100644 index 000000000000..b34e54b7351c --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/updater/main.go @@ -0,0 +1,222 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + "flag" + "os" + "strings" + "time" + + "github.com/spf13/pflag" + apiv1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/uuid" + "k8s.io/autoscaler/multidimensional-pod-autoscaler/common" + mpa_clientset "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/client/clientset/versioned" + "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/target" + updater "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/updater/logic" + "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/updater/priority" + mpa_api_util "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/utils/mpa" + vpa_common "k8s.io/autoscaler/vertical-pod-autoscaler/common" + controllerfetcher "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/target/controller_fetcher" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/limitrange" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/metrics" + metrics_updater "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/metrics/updater" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/server" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/status" + "k8s.io/client-go/informers" + kube_client "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/leaderelection" + "k8s.io/client-go/tools/leaderelection/resourcelock" + kube_flag "k8s.io/component-base/cli/flag" + componentbaseconfig "k8s.io/component-base/config" + componentbaseoptions "k8s.io/component-base/config/options" + "k8s.io/klog/v2" +) + +var ( + updaterInterval = flag.Duration("updater-interval", 10*time.Second, + `How often updater should run (default: 10s)`) + + minReplicas = flag.Int("min-replicas", 2, + `Minimum number of replicas to perform update (global setting) which can be overridden by the per-MPA setting.`) + + evictionToleranceFraction = flag.Float64("eviction-tolerance", 0.5, + `Fraction of replica count that can be evicted for update, if more than one pod can be evicted.`) + + evictionRateLimit = flag.Float64("eviction-rate-limit", -1, + `Number of pods that can be evicted per seconds. A rate limit set to 0 or -1 will disable + the rate limiter.`) + + evictionRateBurst = flag.Int("eviction-rate-burst", 1, `Burst of pods that can be evicted.`) + + address = flag.String("address", ":8943", "The address to expose Prometheus metrics.") + + useAdmissionControllerStatus = flag.Bool("use-admission-controller-status", true, + "If true, updater will only evict pods when admission controller status is valid.") + + namespace = os.Getenv("NAMESPACE") + mpaObjectNamespace = flag.String("mpa-object-namespace", apiv1.NamespaceAll, "Namespace to search for MPA objects. Empty means all namespaces will be used.") + ignoredMpaObjectNamespaces = flag.String("ignored-mpa-object-namespaces", "", "Comma separated list of namespaces to ignore when searching for MPA objects. Empty means no namespaces will be ignored.") +) + +const ( + defaultResyncPeriod time.Duration = 10 * time.Minute + scaleCacheEntryLifetime time.Duration = time.Hour + scaleCacheEntryFreshnessTime time.Duration = 10 * time.Minute + scaleCacheEntryJitterFactor float64 = 1. +) + +func main() { + commonFlags := vpa_common.InitCommonFlags() + klog.InitFlags(nil) + vpa_common.InitLoggingFlags() + + leaderElection := defaultLeaderElectionConfiguration() + componentbaseoptions.BindLeaderElectionFlags(&leaderElection, pflag.CommandLine) + + kube_flag.InitFlags() + klog.V(1).Infof("Multidimensional Pod Autoscaler %s Updater", common.MultidimPodAutoscalerVersion) + + if len(*mpaObjectNamespace) > 0 && len(*ignoredMpaObjectNamespaces) > 0 { + klog.Fatalf("--mpa-object-namespace and --ignored-mpa-object-namespaces are mutually exclusive and can't be set together.") + } + + healthCheck := metrics.NewHealthCheck(*updaterInterval * 5) + server.Initialize(&commonFlags.EnableProfiling, healthCheck, address) + + metrics_updater.Register() + + if !leaderElection.LeaderElect { + run(healthCheck, commonFlags) + } else { + id, err := os.Hostname() + if err != nil { + klog.Fatalf("Unable to get hostname: %v", err) + } + id = id + "_" + string(uuid.NewUUID()) + + config := common.CreateKubeConfigOrDie(commonFlags.KubeConfig, float32(commonFlags.KubeApiQps), int(commonFlags.KubeApiBurst)) + kubeClient := kube_client.NewForConfigOrDie(config) + + lock, err := resourcelock.New( + leaderElection.ResourceLock, + leaderElection.ResourceNamespace, + leaderElection.ResourceName, + kubeClient.CoreV1(), + kubeClient.CoordinationV1(), + resourcelock.ResourceLockConfig{ + Identity: id, + }, + ) + if err != nil { + klog.Fatalf("Unable to create leader election lock: %v", err) + } + + leaderelection.RunOrDie(context.TODO(), leaderelection.LeaderElectionConfig{ + Lock: lock, + LeaseDuration: leaderElection.LeaseDuration.Duration, + RenewDeadline: leaderElection.RenewDeadline.Duration, + RetryPeriod: leaderElection.RetryPeriod.Duration, + ReleaseOnCancel: true, + Callbacks: leaderelection.LeaderCallbacks{ + OnStartedLeading: func(_ context.Context) { + run(healthCheck, commonFlags) + }, + OnStoppedLeading: func() { + klog.Fatal("lost master") + }, + }, + }) + } +} + +const ( + defaultLeaseDuration = 15 * time.Second + defaultRenewDeadline = 10 * time.Second + defaultRetryPeriod = 2 * time.Second +) + +func defaultLeaderElectionConfiguration() componentbaseconfig.LeaderElectionConfiguration { + return componentbaseconfig.LeaderElectionConfiguration{ + LeaderElect: false, + LeaseDuration: metav1.Duration{Duration: defaultLeaseDuration}, + RenewDeadline: metav1.Duration{Duration: defaultRenewDeadline}, + RetryPeriod: metav1.Duration{Duration: defaultRetryPeriod}, + ResourceLock: resourcelock.LeasesResourceLock, + ResourceName: "mpa-updater", + ResourceNamespace: metav1.NamespaceSystem, + } +} + +func run(healthCheck *metrics.HealthCheck, commonFlag *vpa_common.CommonFlags) { + config := common.CreateKubeConfigOrDie(commonFlag.KubeConfig, float32(commonFlag.KubeApiQps), int(commonFlag.KubeApiBurst)) + kubeClient := kube_client.NewForConfigOrDie(config) + mpaClient := mpa_clientset.NewForConfigOrDie(config) + factory := informers.NewSharedInformerFactory(kubeClient, defaultResyncPeriod) + targetSelectorFetcher := target.NewMpaTargetSelectorFetcher(config, kubeClient, factory) + controllerFetcher := controllerfetcher.NewControllerFetcher(config, kubeClient, factory, scaleCacheEntryFreshnessTime, scaleCacheEntryLifetime, scaleCacheEntryJitterFactor) + var limitRangeCalculator limitrange.LimitRangeCalculator + limitRangeCalculator, err := limitrange.NewLimitsRangeCalculator(factory) + if err != nil { + klog.ErrorS(err, "Failed to create limitRangeCalculator, falling back to not checking limits") + limitRangeCalculator = limitrange.NewNoopLimitsCalculator() + } + admissionControllerStatusNamespace := status.AdmissionControllerStatusNamespace + if namespace != "" { + admissionControllerStatusNamespace = namespace + } + + ignoredNamespaces := strings.Split(*ignoredMpaObjectNamespaces, ",") + + // TODO: use SharedInformerFactory in updater + updater, err := updater.NewUpdater( + kubeClient, + mpaClient, + *minReplicas, + *evictionRateLimit, + *evictionRateBurst, + *evictionToleranceFraction, + *useAdmissionControllerStatus, + admissionControllerStatusNamespace, + mpa_api_util.NewCappingRecommendationProcessor(limitRangeCalculator), + nil, + // priority.NewScalingDirectionPodEvictionAdmission(), + targetSelectorFetcher, + controllerFetcher, + priority.NewProcessor(), + *mpaObjectNamespace, + ignoredNamespaces, + ) + if err != nil { + klog.Fatalf("Failed to create updater: %v", err) + } + + // Start updating health check endpoint. + healthCheck.StartMonitoring() + + ticker := time.Tick(*updaterInterval) + for range ticker { + ctx, cancel := context.WithTimeout(context.Background(), *updaterInterval) + // updater.RunOnce(ctx) + updater.RunOnceUpdatingDeployment(ctx) + healthCheck.UpdateLastActivity() + cancel() + } +} diff --git a/multidimensional-pod-autoscaler/pkg/updater/priority/pod_eviction_admission.go b/multidimensional-pod-autoscaler/pkg/updater/priority/pod_eviction_admission.go new file mode 100644 index 000000000000..8a6d993baf94 --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/updater/priority/pod_eviction_admission.go @@ -0,0 +1,81 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package priority + +import ( + apiv1 "k8s.io/api/core/v1" + mpa_types "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1alpha1" + vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" +) + +// PodEvictionAdmission controls evictions of pods. +type PodEvictionAdmission interface { + // LoopInit initializes PodEvictionAdmission for next Updater loop with the live pods and + // pods currently controlled by MPA in this cluster. + LoopInit(allLivePods []*apiv1.Pod, mpaControlledPods map[*mpa_types.MultidimPodAutoscaler][]*apiv1.Pod) + // Admit returns true if PodEvictionAdmission decides that pod can be evicted with given recommendation. + Admit(pod *apiv1.Pod, recommendation *vpa_types.RecommendedPodResources) bool + // CleanUp cleans up any state that PodEvictionAdmission may keep. Called + // when no MPA objects are present in the cluster. + CleanUp() +} + +// NewDefaultPodEvictionAdmission constructs new PodEvictionAdmission that admits all pods. +func NewDefaultPodEvictionAdmission() PodEvictionAdmission { + return &noopPodEvictionAdmission{} +} + +// NewSequentialPodEvictionAdmission constructs PodEvictionAdmission that will chain provided PodEvictionAdmission objects +func NewSequentialPodEvictionAdmission(admissions []PodEvictionAdmission) PodEvictionAdmission { + return &sequentialPodEvictionAdmission{admissions: admissions} +} + +type sequentialPodEvictionAdmission struct { + admissions []PodEvictionAdmission +} + +func (a *sequentialPodEvictionAdmission) LoopInit(allLivePods []*apiv1.Pod, mpaControlledPods map[*mpa_types.MultidimPodAutoscaler][]*apiv1.Pod) { + for _, admission := range a.admissions { + admission.LoopInit(allLivePods, mpaControlledPods) + } +} + +func (a *sequentialPodEvictionAdmission) Admit(pod *apiv1.Pod, recommendation *vpa_types.RecommendedPodResources) bool { + for _, admission := range a.admissions { + admit := admission.Admit(pod, recommendation) + if !admit { + return false + } + } + return true +} + +func (a *sequentialPodEvictionAdmission) CleanUp() { + for _, admission := range a.admissions { + admission.CleanUp() + } +} + +type noopPodEvictionAdmission struct{} + +func (n *noopPodEvictionAdmission) LoopInit(allLivePods []*apiv1.Pod, mpaControlledPods map[*mpa_types.MultidimPodAutoscaler][]*apiv1.Pod) { +} +func (n *noopPodEvictionAdmission) Admit(pod *apiv1.Pod, recommendation *vpa_types.RecommendedPodResources) bool { + return true +} +func (n *noopPodEvictionAdmission) CleanUp() { +} diff --git a/multidimensional-pod-autoscaler/pkg/updater/priority/priority_processor.go b/multidimensional-pod-autoscaler/pkg/updater/priority/priority_processor.go new file mode 100644 index 000000000000..76825c220d22 --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/updater/priority/priority_processor.go @@ -0,0 +1,98 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package priority + +import ( + "math" + + apiv1 "k8s.io/api/core/v1" + mpa_types "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1alpha1" + vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/annotations" + vpa_api_util "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/vpa" + "k8s.io/klog/v2" +) + +// PriorityProcessor calculates priority for pod updates. +type PriorityProcessor interface { + GetUpdatePriority(pod *apiv1.Pod, mpa *mpa_types.MultidimPodAutoscaler, + recommendation *vpa_types.RecommendedPodResources) PodPriority +} + +// NewProcessor creates a new default PriorityProcessor. +func NewProcessor() PriorityProcessor { + return &defaultPriorityProcessor{} +} + +type defaultPriorityProcessor struct { +} + +func (*defaultPriorityProcessor) GetUpdatePriority(pod *apiv1.Pod, _ *mpa_types.MultidimPodAutoscaler, + recommendation *vpa_types.RecommendedPodResources) PodPriority { + outsideRecommendedRange := false + scaleUp := false + // Sum of requests over all containers, per resource type. + totalRequestPerResource := make(map[apiv1.ResourceName]int64) + // Sum of recommendations over all containers, per resource type. + totalRecommendedPerResource := make(map[apiv1.ResourceName]int64) + + hasObservedContainers, vpaContainerSet := parseVpaObservedContainers(pod) + + for _, podContainer := range pod.Spec.Containers { + if hasObservedContainers && !vpaContainerSet.Has(podContainer.Name) { + klog.V(4).Infof("Not listed in %s:%s. Skipping container %s priority calculations", + annotations.VpaObservedContainersLabel, pod.GetAnnotations()[annotations.VpaObservedContainersLabel], podContainer.Name) + continue + } + recommendedRequest := vpa_api_util.GetRecommendationForContainer(podContainer.Name, recommendation) + if recommendedRequest == nil { + continue + } + for resourceName, recommended := range recommendedRequest.Target { + totalRecommendedPerResource[resourceName] += recommended.MilliValue() + lowerBound, hasLowerBound := recommendedRequest.LowerBound[resourceName] + upperBound, hasUpperBound := recommendedRequest.UpperBound[resourceName] + if request, hasRequest := podContainer.Resources.Requests[resourceName]; hasRequest { + totalRequestPerResource[resourceName] += request.MilliValue() + if recommended.MilliValue() > request.MilliValue() { + scaleUp = true + } + if (hasLowerBound && request.Cmp(lowerBound) < 0) || + (hasUpperBound && request.Cmp(upperBound) > 0) { + outsideRecommendedRange = true + } + } else { + // Note: if the request is not specified, the container will use the + // namespace default request. Currently we ignore it and treat such + // containers as if they had 0 request. A more correct approach would + // be to always calculate the 'effective' request. + scaleUp = true + outsideRecommendedRange = true + } + } + } + resourceDiff := 0.0 + for resource, totalRecommended := range totalRecommendedPerResource { + totalRequest := math.Max(float64(totalRequestPerResource[resource]), 1.0) + resourceDiff += math.Abs(totalRequest-float64(totalRecommended)) / totalRequest + } + return PodPriority{ + OutsideRecommendedRange: outsideRecommendedRange, + ScaleUp: scaleUp, + ResourceDiff: resourceDiff, + } +} diff --git a/multidimensional-pod-autoscaler/pkg/updater/priority/priority_processor_fake.go b/multidimensional-pod-autoscaler/pkg/updater/priority/priority_processor_fake.go new file mode 100644 index 000000000000..fc684aea32d2 --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/updater/priority/priority_processor_fake.go @@ -0,0 +1,50 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package priority + +import ( + "fmt" + + corev1 "k8s.io/api/core/v1" + mpa_types "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1alpha1" + vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" +) + +type fakePriorityProcessor struct { + priorities map[string]PodPriority +} + +// NewFakeProcessor returns a fake processor for testing that can be initialized +// with a map from pod name to priority expected to be returned. +func NewFakeProcessor(priorities map[string]PodPriority) PriorityProcessor { + return &fakePriorityProcessor{ + priorities: priorities, + } +} + +func (f *fakePriorityProcessor) GetUpdatePriority(pod *corev1.Pod, mpa *mpa_types.MultidimPodAutoscaler, + recommendation *vpa_types.RecommendedPodResources) PodPriority { + prio, ok := f.priorities[pod.Name] + if !ok { + panic(fmt.Sprintf("Unexpected pod name: %v", pod.Name)) + } + return PodPriority{ + ScaleUp: prio.ScaleUp, + ResourceDiff: prio.ResourceDiff, + OutsideRecommendedRange: prio.OutsideRecommendedRange, + } +} diff --git a/multidimensional-pod-autoscaler/pkg/updater/priority/priority_processor_test.go b/multidimensional-pod-autoscaler/pkg/updater/priority/priority_processor_test.go new file mode 100644 index 000000000000..e62708d7093c --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/updater/priority/priority_processor_test.go @@ -0,0 +1,288 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package priority + +import ( + "testing" + + corev1 "k8s.io/api/core/v1" + mpa_types "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1alpha1" + "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/utils/test" + vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/annotations" + vpa_test "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/test" + + "github.com/stretchr/testify/assert" +) + +func TestGetUpdatePriority(t *testing.T) { + containerName := "test-container" + testCases := []struct { + name string + pod *corev1.Pod + mpa *mpa_types.MultidimPodAutoscaler + expectedPrio PodPriority + }{ + { + name: "simple scale up", + pod: vpa_test.Pod().WithName("POD1").AddContainer(test.BuildTestContainer(containerName, "2", "")).Get(), + mpa: test.MultidimPodAutoscaler().WithContainer(containerName).WithTarget("10", "").Get(), + expectedPrio: PodPriority{ + OutsideRecommendedRange: false, + ResourceDiff: 4.0, + ScaleUp: true, + }, + }, { + name: "simple scale down", + pod: vpa_test.Pod().WithName("POD1").AddContainer(test.BuildTestContainer(containerName, "4", "")).Get(), + mpa: test.MultidimPodAutoscaler().WithContainer(containerName).WithTarget("2", "").Get(), + expectedPrio: PodPriority{ + OutsideRecommendedRange: false, + ResourceDiff: 0.5, + ScaleUp: false, + }, + }, { + name: "no resource diff", + pod: vpa_test.Pod().WithName("POD1").AddContainer(test.BuildTestContainer(containerName, "2", "")).Get(), + mpa: test.MultidimPodAutoscaler().WithContainer(containerName).WithTarget("2", "").Get(), + expectedPrio: PodPriority{ + OutsideRecommendedRange: false, + ResourceDiff: 0.0, + ScaleUp: false, + }, + }, { + name: "scale up on milliquanitites", + pod: vpa_test.Pod().WithName("POD1").AddContainer(test.BuildTestContainer(containerName, "10m", "")).Get(), + mpa: test.MultidimPodAutoscaler().WithContainer(containerName).WithTarget("900m", "").Get(), + expectedPrio: PodPriority{ + OutsideRecommendedRange: false, + ResourceDiff: 89.0, + ScaleUp: true, + }, + }, { + name: "scale up outside recommended range", + pod: vpa_test.Pod().WithName("POD1").AddContainer(test.BuildTestContainer(containerName, "4", "")).Get(), + mpa: test.MultidimPodAutoscaler().WithContainer(containerName). + WithTarget("10", ""). + WithLowerBound("6", ""). + WithUpperBound("14", "").Get(), + expectedPrio: PodPriority{ + OutsideRecommendedRange: true, + ResourceDiff: 1.5, + ScaleUp: true, + }, + }, { + name: "scale down outside recommended range", + pod: vpa_test.Pod().WithName("POD1").AddContainer(test.BuildTestContainer(containerName, "8", "")).Get(), + mpa: test.MultidimPodAutoscaler().WithContainer(containerName). + WithTarget("2", ""). + WithLowerBound("1", ""). + WithUpperBound("3", "").Get(), + expectedPrio: PodPriority{ + OutsideRecommendedRange: true, + ResourceDiff: 0.75, + ScaleUp: false, + }, + }, { + name: "scale up with multiple quantities", + pod: vpa_test.Pod().WithName("POD1").AddContainer(test.BuildTestContainer(containerName, "2", "")).Get(), + mpa: test.MultidimPodAutoscaler().WithContainer(containerName).WithTarget("10", "").Get(), + expectedPrio: PodPriority{ + OutsideRecommendedRange: false, + ResourceDiff: 4.0, + ScaleUp: true, + }, + }, { + name: "multiple resources, both scale up", + pod: vpa_test.Pod().WithName("POD1").AddContainer(test.BuildTestContainer(containerName, "3", "10M")).Get(), + mpa: test.MultidimPodAutoscaler().WithContainer(containerName).WithTarget("6", "20M").Get(), + expectedPrio: PodPriority{ + OutsideRecommendedRange: false, + ResourceDiff: 1.0 + 1.0, // summed relative diffs for resources + ScaleUp: true, + }, + }, { + name: "multiple resources, only one scale up", + pod: vpa_test.Pod().WithName("POD1").AddContainer(test.BuildTestContainer(containerName, "4", "10M")).Get(), + mpa: test.MultidimPodAutoscaler().WithContainer(containerName).WithTarget("2", "20M").Get(), + expectedPrio: PodPriority{ + OutsideRecommendedRange: false, + ResourceDiff: 1.5 + 0.0, // summed relative diffs for resources + ScaleUp: true, + }, + }, { + name: "multiple resources, both scale down", + pod: vpa_test.Pod().WithName("POD1").AddContainer(test.BuildTestContainer(containerName, "4", "20M")).Get(), + mpa: test.MultidimPodAutoscaler().WithContainer(containerName).WithTarget("2", "10M").Get(), + expectedPrio: PodPriority{ + OutsideRecommendedRange: false, + ResourceDiff: 0.5 + 0.5, // summed relative diffs for resources + ScaleUp: false, + }, + }, { + name: "multiple resources, one outside recommended range", + pod: vpa_test.Pod().WithName("POD1").AddContainer(test.BuildTestContainer(containerName, "4", "20M")).Get(), + mpa: test.MultidimPodAutoscaler().WithContainer(containerName). + WithTarget("2", "10M"). + WithLowerBound("1", "5M"). + WithUpperBound("3", "30M").Get(), + expectedPrio: PodPriority{ + OutsideRecommendedRange: true, + ResourceDiff: 0.5 + 0.5, // summed relative diffs for resources + ScaleUp: false, + }, + }, { + name: "multiple containers, both scale up", + pod: vpa_test.Pod().WithName("POD1").AddContainer(test.BuildTestContainer(containerName, "1", "")). + AddContainer(test.BuildTestContainer("test-container-2", "2", "")).Get(), + mpa: test.MultidimPodAutoscaler().WithContainer(containerName). + WithTarget("4", "").AppendRecommendation( + test.Recommendation(). + WithContainer("test-container-2"). + WithTarget("8", "").GetContainerResources()).Get(), + expectedPrio: PodPriority{ + OutsideRecommendedRange: false, + ResourceDiff: 3.0, // relative diff between summed requests and summed recommendations + ScaleUp: true, + }, + }, { + name: "multiple containers, both scale down", + pod: vpa_test.Pod().WithName("POD1").AddContainer(test.BuildTestContainer(containerName, "3", "")). + AddContainer(test.BuildTestContainer("test-container-2", "7", "")).Get(), + mpa: test.MultidimPodAutoscaler().WithContainer(containerName). + WithTarget("1", "").AppendRecommendation( + test.Recommendation(). + WithContainer("test-container-2"). + WithTarget("2", "").GetContainerResources()).Get(), + expectedPrio: PodPriority{ + OutsideRecommendedRange: false, + ResourceDiff: 0.7, // relative diff between summed requests and summed recommendations + ScaleUp: false, + }, + }, { + name: "multiple containers, both scale up, one outside range", + pod: vpa_test.Pod().WithName("POD1").AddContainer(test.BuildTestContainer(containerName, "1", "")). + AddContainer(test.BuildTestContainer("test-container-2", "2", "")).Get(), + mpa: test.MultidimPodAutoscaler().WithContainer(containerName). + WithTarget("4", ""). + WithLowerBound("1", "").AppendRecommendation( + test.Recommendation(). + WithContainer("test-container-2"). + WithTarget("8", ""). + WithLowerBound("3", ""). + WithUpperBound("10", "").GetContainerResources()).Get(), + expectedPrio: PodPriority{ + OutsideRecommendedRange: true, + ResourceDiff: 3.0, // relative diff between summed requests and summed recommendations + ScaleUp: true, + }, + }, { + name: "multiple containers, multiple resources", + // container1: request={6 CPU, 10 MB}, recommended={8 CPU, 20 MB} + // container2: request={4 CPU, 30 MB}, recommended={7 CPU, 30 MB} + // total: request={10 CPU, 40 MB}, recommended={15 CPU, 50 MB} + pod: vpa_test.Pod().WithName("POD1").AddContainer(test.BuildTestContainer(containerName, "6", "10M")). + AddContainer(test.BuildTestContainer("test-container-2", "4", "30M")).Get(), + mpa: test.MultidimPodAutoscaler().WithContainer(containerName). + WithTarget("8", "20M").AppendRecommendation( + test.Recommendation(). + WithContainer("test-container-2"). + WithTarget("7", "30M").GetContainerResources()).Get(), + expectedPrio: PodPriority{ + OutsideRecommendedRange: false, + // relative diff between summed requests and summed recommendations, summed over resources + ResourceDiff: 0.5 + 0.25, + ScaleUp: true, + }, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + processor := NewProcessor() + prio := processor.GetUpdatePriority(tc.pod, tc.mpa, tc.mpa.Status.Recommendation) + assert.Equal(t, tc.expectedPrio, prio) + }) + } +} + +// Verify GetUpdatePriorty does not encounter a NPE when there is no +// recommendation for a container. +func TestGetUpdatePriority_NoRecommendationForContainer(t *testing.T) { + p := NewProcessor() + pod := vpa_test.Pod().WithName("POD1").AddContainer(test.BuildTestContainer("test-container", "5", "10")).Get() + mpa := test.MultidimPodAutoscaler().WithName("test-mpa").WithContainer("test-container").Get() + result := p.GetUpdatePriority(pod, mpa, nil) + assert.NotNil(t, result) +} + +func TestGetUpdatePriority_VpaObservedContainers(t *testing.T) { + const ( + // There is no VpaObservedContainers annotation + // or the container is listed in the annotation. + optedInContainerDiff = 9 + // There is VpaObservedContainers annotation + // and the container is not listed in. + optedOutContainerDiff = 0 + ) + testVpa := test.MultidimPodAutoscaler().WithName("test-mpa").WithContainer(containerName).Get() + tests := []struct { + name string + pod *corev1.Pod + recommendation *vpa_types.RecommendedPodResources + want float64 + }{ + { + name: "with no VpaObservedContainers annotation", + pod: vpa_test.Pod().WithName("POD1").AddContainer(test.BuildTestContainer(containerName, "1", "")).Get(), + recommendation: test.Recommendation().WithContainer(containerName).WithTarget("10", "").Get(), + want: optedInContainerDiff, + }, + { + name: "with container listed in VpaObservedContainers annotation", + pod: vpa_test.Pod().WithAnnotations(map[string]string{annotations.VpaObservedContainersLabel: containerName}). + WithName("POD1").AddContainer(test.BuildTestContainer(containerName, "1", "")).Get(), + recommendation: test.Recommendation().WithContainer(containerName).WithTarget("10", "").Get(), + want: optedInContainerDiff, + }, + { + name: "with container not listed in VpaObservedContainers annotation", + pod: vpa_test.Pod().WithAnnotations(map[string]string{annotations.VpaObservedContainersLabel: ""}). + WithName("POD1").AddContainer(test.BuildTestContainer(containerName, "1", "")).Get(), + recommendation: test.Recommendation().WithContainer(containerName).WithTarget("10", "").Get(), + want: optedOutContainerDiff, + }, + { + name: "with incorrect VpaObservedContainers annotation", + pod: vpa_test.Pod().WithAnnotations(map[string]string{annotations.VpaObservedContainersLabel: "abcd;';"}). + WithName("POD1").AddContainer(test.BuildTestContainer(containerName, "1", "")).Get(), + recommendation: test.Recommendation().WithContainer(containerName).WithTarget("10", "").Get(), + want: optedInContainerDiff, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + p := NewProcessor() + result := p.GetUpdatePriority(tc.pod, testVpa, tc.recommendation) + assert.NotNil(t, result) + // The resourceDiff should be a difference between container resources + // and container resource recommendations. Containers not listed + // in an existing mpaObservedContainers annotations shouldn't be taken + // into account during calculations. + assert.InDelta(t, result.ResourceDiff, tc.want, 0.0001) + }) + } +} diff --git a/multidimensional-pod-autoscaler/pkg/updater/priority/update_priority_calculator.go b/multidimensional-pod-autoscaler/pkg/updater/priority/update_priority_calculator.go new file mode 100644 index 000000000000..ad87f637cf52 --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/updater/priority/update_priority_calculator.go @@ -0,0 +1,225 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package priority + +import ( + "flag" + "sort" + "time" + + apiv1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/sets" + mpa_types "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1alpha1" + mpa_api_util "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/utils/mpa" + vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/annotations" + vpa_api_util "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/vpa" + "k8s.io/klog/v2" +) + +var ( + defaultUpdateThreshold = flag.Float64("pod-update-threshold", 0.1, "Ignore updates that have priority lower than the value of this flag") + + podLifetimeUpdateThreshold = flag.Duration("in-recommendation-bounds-eviction-lifetime-threshold", time.Hour*12, "Pods that live for at least that long can be evicted even if their request is within the [MinRecommended...MaxRecommended] range") + + evictAfterOOMThreshold = flag.Duration("evict-after-oom-threshold", 10*time.Minute, + `Evict pod that has only one container and it OOMed in less than + evict-after-oom-threshold since start.`) +) + +// UpdatePriorityCalculator is responsible for prioritizing updates on pods. +// It can returns a sorted list of pods in order of update priority. +// Update priority is proportional to fraction by which resources should be increased / decreased. +// i.e. pod with 10M current memory and recommendation 20M will have higher update priority +// than pod with 100M current memory and 150M recommendation (100% increase vs 50% increase) +type UpdatePriorityCalculator struct { + mpa *mpa_types.MultidimPodAutoscaler + pods []prioritizedPod + config *UpdateConfig + recommendationProcessor mpa_api_util.RecommendationProcessor + priorityProcessor PriorityProcessor +} + +// UpdateConfig holds configuration for UpdatePriorityCalculator +type UpdateConfig struct { + // MinChangePriority is the minimum change priority that will trigger a update. + // TODO: should have separate for Mem and CPU? + MinChangePriority float64 +} + +// NewUpdatePriorityCalculator creates new UpdatePriorityCalculator for the given MPA object +// an update config. +// If the MPA resource policy is nil, there will be no policy restriction on update. +// If the given update config is nil, default values are used. +func NewUpdatePriorityCalculator(mpa *mpa_types.MultidimPodAutoscaler, + config *UpdateConfig, + recommendationProcessor mpa_api_util.RecommendationProcessor, + priorityProcessor PriorityProcessor) UpdatePriorityCalculator { + if config == nil { + config = &UpdateConfig{MinChangePriority: *defaultUpdateThreshold} + } + return UpdatePriorityCalculator{ + mpa: mpa, + config: config, + recommendationProcessor: recommendationProcessor, + priorityProcessor: priorityProcessor} +} + +// AddPod adds pod to the UpdatePriorityCalculator. +func (calc *UpdatePriorityCalculator) AddPod(pod *apiv1.Pod, now time.Time) { + processedRecommendation, _, err := calc.recommendationProcessor.Apply(calc.mpa.Status.Recommendation, calc.mpa.Spec.ResourcePolicy, calc.mpa.Status.Conditions, pod) + if err != nil { + klog.V(2).Infof("cannot process recommendation for pod %s/%s: %v", pod.Namespace, pod.Name, err) + return + } + + hasObservedContainers, vpaContainerSet := parseVpaObservedContainers(pod) + + updatePriority := calc.priorityProcessor.GetUpdatePriority(pod, calc.mpa, processedRecommendation) + + quickOOM := false + for i := range pod.Status.ContainerStatuses { + cs := &pod.Status.ContainerStatuses[i] + if hasObservedContainers && !vpaContainerSet.Has(cs.Name) { + // Containers not observed by Admission Controller are not supported + // by the quick OOM logic. + klog.V(4).Infof("Not listed in %s:%s. Skipping container %s quick OOM calculations", + annotations.VpaObservedContainersLabel, pod.GetAnnotations()[annotations.VpaObservedContainersLabel], cs.Name) + continue + } + crp := vpa_api_util.GetContainerResourcePolicy(cs.Name, calc.mpa.Spec.ResourcePolicy) + if crp != nil && crp.Mode != nil && *crp.Mode == vpa_types.ContainerScalingModeOff { + // Containers with ContainerScalingModeOff are not considered + // during the quick OOM calculation. + klog.V(4).Infof("Container with ContainerScalingModeOff. Skipping container %s quick OOM calculations", cs.Name) + continue + } + terminationState := &cs.LastTerminationState + if terminationState.Terminated != nil && + terminationState.Terminated.Reason == "OOMKilled" && + terminationState.Terminated.FinishedAt.Time.Sub(terminationState.Terminated.StartedAt.Time) < *evictAfterOOMThreshold { + quickOOM = true + klog.V(2).Infof("quick OOM detected in pod %v/%v, container %v", pod.Namespace, pod.Name, cs.Name) + } + } + + // The update is allowed in following cases: + // - the request is outside the recommended range for some container. + // - the pod lives for at least 24h and the resource diff is >= MinChangePriority. + // - a MPA scaled container OOMed in less than evictAfterOOMThreshold. + if !updatePriority.OutsideRecommendedRange && !quickOOM { + if pod.Status.StartTime == nil { + // TODO: Set proper condition on the MPA. + klog.V(4).Infof("not updating pod %v/%v, missing field pod.Status.StartTime", pod.Namespace, pod.Name) + return + } + if now.Before(pod.Status.StartTime.Add(*podLifetimeUpdateThreshold)) { + klog.V(4).Infof("not updating a short-lived pod %v/%v, request within recommended range", pod.Namespace, pod.Name) + return + } + if updatePriority.ResourceDiff < calc.config.MinChangePriority { + klog.V(4).Infof("not updating pod %v/%v, resource diff too low: %v", pod.Namespace, pod.Name, updatePriority) + return + } + } + + // If the pod has quick OOMed then evict only if the resources will change + if quickOOM && updatePriority.ResourceDiff == 0 { + klog.V(4).Infof("not updating pod %v/%v because resource would not change", pod.Namespace, pod.Name) + return + } + klog.V(2).Infof("pod accepted for update %v/%v with priority %v", pod.Namespace, pod.Name, updatePriority.ResourceDiff) + calc.pods = append(calc.pods, prioritizedPod{ + pod: pod, + priority: updatePriority, + recommendation: processedRecommendation}) +} + +// GetSortedPods returns a list of pods ordered by update priority (highest update priority first) +func (calc *UpdatePriorityCalculator) GetSortedPods(admission PodEvictionAdmission) []*apiv1.Pod { + sort.Sort(byPriorityDesc(calc.pods)) + + result := []*apiv1.Pod{} + for _, podPrio := range calc.pods { + if admission == nil || admission.Admit(podPrio.pod, podPrio.recommendation) { + result = append(result, podPrio.pod) + } else { + klog.V(2).Infof("pod removed from update queue by PodEvictionAdmission: %v", podPrio.pod.Name) + } + } + + return result +} + +func parseVpaObservedContainers(pod *apiv1.Pod) (bool, sets.String) { + observedContainers, hasObservedContainers := pod.GetAnnotations()[annotations.VpaObservedContainersLabel] + vpaContainerSet := sets.NewString() + if hasObservedContainers { + if containers, err := annotations.ParseVpaObservedContainersValue(observedContainers); err != nil { + klog.Errorf("MPA annotation %s failed to parse: %v", observedContainers, err) + hasObservedContainers = false + } else { + vpaContainerSet.Insert(containers...) + } + } + return hasObservedContainers, vpaContainerSet +} + +type prioritizedPod struct { + pod *apiv1.Pod + priority PodPriority + recommendation *vpa_types.RecommendedPodResources +} + +// PodPriority contains data for a pod update that can be used to prioritize between updates. +type PodPriority struct { + // Is any container outside of the recommended range. + OutsideRecommendedRange bool + // Does any container want to grow. + ScaleUp bool + // Relative difference between the total requested and total recommended resources. + ResourceDiff float64 +} + +type byPriorityDesc []prioritizedPod + +func (list byPriorityDesc) Len() int { + return len(list) +} +func (list byPriorityDesc) Swap(i, j int) { + list[i], list[j] = list[j], list[i] +} + +// Less implements reverse ordering by priority (highest priority first). +// This means we return true if priority at index j is lower than at index i. +func (list byPriorityDesc) Less(i, j int) bool { + return list[j].priority.Less(list[i].priority) +} + +// Less returns true if p is lower than other. +func (p PodPriority) Less(other PodPriority) bool { + // 1. If any container wants to grow, the pod takes precedence. + // TODO: A better policy would be to prioritize scaling down when + // (a) the pod is pending + // (b) there is general resource shortage + // and prioritize scaling up otherwise. + if p.ScaleUp != other.ScaleUp { + return other.ScaleUp + } + // 2. A pod with larger value of resourceDiff takes precedence. + return p.ResourceDiff < other.ResourceDiff +} diff --git a/multidimensional-pod-autoscaler/pkg/updater/priority/update_priority_calculator_test.go b/multidimensional-pod-autoscaler/pkg/updater/priority/update_priority_calculator_test.go new file mode 100644 index 000000000000..5e918b74d9ef --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/updater/priority/update_priority_calculator_test.go @@ -0,0 +1,559 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package priority + +import ( + "fmt" + "testing" + "time" + + mpa_types "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1alpha1" + mpa_test "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/utils/test" + vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/annotations" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/test" + + apiv1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/stretchr/testify/assert" +) + +const ( + containerName = "container1" +) + +// TODO(bskiba): Refactor the SortPriority tests as a testcase list test. +func TestSortPriority(t *testing.T) { + pod1 := test.Pod().WithName("POD1").AddContainer(test.Container().WithName(containerName).WithCPURequest(resource.MustParse("2")).Get()).Get() + pod2 := test.Pod().WithName("POD2").AddContainer(test.Container().WithName(containerName).WithCPURequest(resource.MustParse("4")).Get()).Get() + pod3 := test.Pod().WithName("POD3").AddContainer(test.Container().WithName(containerName).WithCPURequest(resource.MustParse("1")).Get()).Get() + pod4 := test.Pod().WithName("POD4").AddContainer(test.Container().WithName(containerName).WithCPURequest(resource.MustParse("3")).Get()).Get() + + mpa := mpa_test.MultidimPodAutoscaler().WithContainer(containerName).WithTarget("10", "").Get() + + priorityProcessor := NewFakeProcessor(map[string]PodPriority{ + "POD1": {ResourceDiff: 4.0}, + "POD2": {ResourceDiff: 1.5}, + "POD3": {ResourceDiff: 9.0}, + "POD4": {ResourceDiff: 2.33}, + }) + calculator := NewUpdatePriorityCalculator(mpa, nil, &mpa_test.FakeRecommendationProcessor{}, priorityProcessor) + + timestampNow := pod1.Status.StartTime.Time.Add(time.Hour * 24) + calculator.AddPod(pod1, timestampNow) + calculator.AddPod(pod2, timestampNow) + calculator.AddPod(pod3, timestampNow) + calculator.AddPod(pod4, timestampNow) + + result := calculator.GetSortedPods(NewDefaultPodEvictionAdmission()) + assert.Exactly(t, []*apiv1.Pod{pod3, pod1, pod4, pod2}, result, "Wrong priority order") +} + +func TestSortPriorityResourcesDecrease(t *testing.T) { + pod1 := test.Pod().WithName("POD1").AddContainer(test.Container().WithName(containerName).WithCPURequest(resource.MustParse("4")).Get()).Get() + pod2 := test.Pod().WithName("POD2").AddContainer(test.Container().WithName(containerName).WithCPURequest(resource.MustParse("8")).Get()).Get() + pod3 := test.Pod().WithName("POD3").AddContainer(test.Container().WithName(containerName).WithCPURequest(resource.MustParse("10")).Get()).Get() + + mpa := mpa_test.MultidimPodAutoscaler().WithContainer(containerName).WithTarget("5", "").Get() + + priorityProcessor := NewFakeProcessor(map[string]PodPriority{ + "POD1": {ScaleUp: true, ResourceDiff: 0.25}, + "POD2": {ScaleUp: false, ResourceDiff: 0.25}, + "POD3": {ScaleUp: false, ResourceDiff: 0.5}, + }) + calculator := NewUpdatePriorityCalculator(mpa, nil, &mpa_test.FakeRecommendationProcessor{}, priorityProcessor) + + timestampNow := pod1.Status.StartTime.Time.Add(time.Hour * 24) + calculator.AddPod(pod1, timestampNow) + calculator.AddPod(pod2, timestampNow) + calculator.AddPod(pod3, timestampNow) + + // Expect the following order: + // 1. pod1 - wants to grow by 1 unit. + // 2. pod3 - can reclaim 5 units. + // 3. pod2 - can reclaim 3 units. + result := calculator.GetSortedPods(NewDefaultPodEvictionAdmission()) + assert.Exactly(t, []*apiv1.Pod{pod1, pod3, pod2}, result, "Wrong priority order") +} + +func TestUpdateNotRequired(t *testing.T) { + pod1 := test.Pod().WithName("POD1").AddContainer(test.Container().WithName(containerName).WithCPURequest(resource.MustParse("4")).Get()).Get() + mpa := mpa_test.MultidimPodAutoscaler().WithContainer(containerName).WithTarget("4", "").Get() + + priorityProcessor := NewFakeProcessor(map[string]PodPriority{"POD1": { + ResourceDiff: 0.0, + }}) + calculator := NewUpdatePriorityCalculator(mpa, nil, &mpa_test.FakeRecommendationProcessor{}, + priorityProcessor) + + timestampNow := pod1.Status.StartTime.Time.Add(time.Hour * 24) + calculator.AddPod(pod1, timestampNow) + + result := calculator.GetSortedPods(NewDefaultPodEvictionAdmission()) + assert.Exactly(t, []*apiv1.Pod{}, result, "Pod should not be updated") +} + +// TODO: add expects to fake processor +func TestUseProcessor(t *testing.T) { + processedRecommendation := test.Recommendation().WithContainer(containerName).WithTarget("4", "10M").Get() + recommendationProcessor := &mpa_test.RecommendationProcessorMock{} + recommendationProcessor.On("Apply").Return(processedRecommendation, nil) + + mpa := mpa_test.MultidimPodAutoscaler().WithContainer(containerName).WithTarget("5", "5M").Get() + pod1 := test.Pod().WithName("POD1").AddContainer(test.Container().WithName(containerName).WithCPURequest(resource.MustParse("4")).WithMemRequest(resource.MustParse("10M")).Get()).Get() + + priorityProcessor := NewFakeProcessor(map[string]PodPriority{ + "POD1": {ResourceDiff: 0.0}, + }) + calculator := NewUpdatePriorityCalculator( + mpa, nil, recommendationProcessor, priorityProcessor) + + timestampNow := pod1.Status.StartTime.Time.Add(time.Hour * 24) + calculator.AddPod(pod1, timestampNow) + + result := calculator.GetSortedPods(NewDefaultPodEvictionAdmission()) + assert.Exactly(t, []*apiv1.Pod{}, result, "Pod should not be updated") +} + +// Verify that a pod that lives for more than podLifetimeUpdateThreshold is +// updated if it has at least one container with the request: +// 1. outside the [MinRecommended...MaxRecommended] range or +// 2. diverging from the target by more than MinChangePriority. +func TestUpdateLonglivedPods(t *testing.T) { + pods := []*apiv1.Pod{ + test.Pod().WithName("POD1").AddContainer(test.Container().WithName(containerName).WithCPURequest(resource.MustParse("4")).Get()).Get(), + test.Pod().WithName("POD2").AddContainer(test.Container().WithName(containerName).WithCPURequest(resource.MustParse("1")).Get()).Get(), + test.Pod().WithName("POD3").AddContainer(test.Container().WithName(containerName).WithCPURequest(resource.MustParse("8")).Get()).Get(), + } + + // Both pods are within the recommended range. + mpa := mpa_test.MultidimPodAutoscaler().WithContainer(containerName). + WithTarget("5", ""). + WithLowerBound("1", ""). + WithUpperBound("6", "").Get() + + priorityProcessor := NewFakeProcessor(map[string]PodPriority{ + "POD1": {OutsideRecommendedRange: false, ScaleUp: true, ResourceDiff: 0.25}, + "POD2": {OutsideRecommendedRange: false, ScaleUp: true, ResourceDiff: 4.0}, + "POD3": {OutsideRecommendedRange: true, ScaleUp: false, ResourceDiff: 0.25}, + }) + + calculator := NewUpdatePriorityCalculator( + mpa, &UpdateConfig{MinChangePriority: 0.5}, &mpa_test.FakeRecommendationProcessor{}, priorityProcessor) + + // Pretend that the test pods started 13 hours ago. + timestampNow := pods[0].Status.StartTime.Time.Add(time.Hour * 13) + for i := 0; i < 3; i++ { + calculator.AddPod(pods[i], timestampNow) + } + result := calculator.GetSortedPods(NewDefaultPodEvictionAdmission()) + assert.Exactly(t, []*apiv1.Pod{pods[1], pods[2]}, result, "Exactly POD2 and POD3 should be updated") +} + +// Verify that a pod that lives for less than podLifetimeUpdateThreshold is +// updated only if the request is outside the [MinRecommended...MaxRecommended] +// range for at least one container. +func TestUpdateShortlivedPods(t *testing.T) { + pods := []*apiv1.Pod{ + test.Pod().WithName("POD1").AddContainer(test.Container().WithName(containerName).WithCPURequest(resource.MustParse("4")).Get()).Get(), + test.Pod().WithName("POD2").AddContainer(test.Container().WithName(containerName).WithCPURequest(resource.MustParse("1")).Get()).Get(), + test.Pod().WithName("POD3").AddContainer(test.Container().WithName(containerName).WithCPURequest(resource.MustParse("10")).Get()).Get(), + } + + // Pods 1 and 2 are within the recommended range. + mpa := mpa_test.MultidimPodAutoscaler().WithContainer(containerName). + WithTarget("5", ""). + WithLowerBound("1", ""). + WithUpperBound("6", "").Get() + + priorityProcessor := NewFakeProcessor(map[string]PodPriority{ + "POD1": {OutsideRecommendedRange: false, ScaleUp: true, ResourceDiff: 0.25}, + "POD2": {OutsideRecommendedRange: false, ScaleUp: true, ResourceDiff: 0.0}, + "POD3": {OutsideRecommendedRange: true, ScaleUp: false, ResourceDiff: 0.9}, + }) + + calculator := NewUpdatePriorityCalculator( + mpa, &UpdateConfig{MinChangePriority: 0.5}, &mpa_test.FakeRecommendationProcessor{}, priorityProcessor) + + // Pretend that the test pods started 11 hours ago. + timestampNow := pods[0].Status.StartTime.Time.Add(time.Hour * 11) + for i := 0; i < 3; i++ { + calculator.AddPod(pods[i], timestampNow) + } + result := calculator.GetSortedPods(NewDefaultPodEvictionAdmission()) + assert.Exactly(t, []*apiv1.Pod{pods[2]}, result, "Only POD3 should be updated") +} + +func TestUpdatePodWithQuickOOM(t *testing.T) { + pod := test.Pod().WithName("POD1").AddContainer(test.Container().WithName(containerName).WithCPURequest(resource.MustParse("4")).Get()).Get() + + // Pretend that the test pod started 11 hours ago. + timestampNow := pod.Status.StartTime.Time.Add(time.Hour * 11) + + pod.Status.ContainerStatuses = []apiv1.ContainerStatus{ + { + LastTerminationState: apiv1.ContainerState{ + Terminated: &apiv1.ContainerStateTerminated{ + Reason: "OOMKilled", + FinishedAt: metav1.NewTime(timestampNow.Add(-1 * 3 * time.Minute)), + StartedAt: metav1.NewTime(timestampNow.Add(-1 * 5 * time.Minute)), + }, + }, + }, + } + + // Pod is within the recommended range. + mpa := mpa_test.MultidimPodAutoscaler().WithContainer(containerName). + WithTarget("5", ""). + WithLowerBound("1", ""). + WithUpperBound("6", "").Get() + + priorityProcessor := NewFakeProcessor(map[string]PodPriority{ + "POD1": {ScaleUp: true, ResourceDiff: 0.25}, + }) + + calculator := NewUpdatePriorityCalculator( + mpa, &UpdateConfig{MinChangePriority: 0.5}, &mpa_test.FakeRecommendationProcessor{}, priorityProcessor) + + calculator.AddPod(pod, timestampNow) + result := calculator.GetSortedPods(NewDefaultPodEvictionAdmission()) + assert.Exactly(t, []*apiv1.Pod{pod}, result, "Pod should be updated") +} + +func TestDontUpdatePodWithQuickOOMNoResourceChange(t *testing.T) { + pod := test.Pod().WithName("POD1").AddContainer(test.Container().WithName(containerName).WithCPURequest(resource.MustParse("2")).WithMemRequest(resource.MustParse("8Gi")).Get()).Get() + + // Pretend that the test pod started 11 hours ago. + timestampNow := pod.Status.StartTime.Time.Add(time.Hour * 11) + + pod.Status.ContainerStatuses = []apiv1.ContainerStatus{ + { + LastTerminationState: apiv1.ContainerState{ + Terminated: &apiv1.ContainerStateTerminated{ + Reason: "OOMKilled", + FinishedAt: metav1.NewTime(timestampNow.Add(-1 * 3 * time.Minute)), + StartedAt: metav1.NewTime(timestampNow.Add(-1 * 5 * time.Minute)), + }, + }, + }, + } + + // Pod is within the recommended range. + mpa := mpa_test.MultidimPodAutoscaler().WithContainer(containerName). + WithTarget("4", "8Gi"). + WithLowerBound("2", "5Gi"). + WithUpperBound("5", "10Gi").Get() + + priorityProcessor := NewFakeProcessor(map[string]PodPriority{ + "POD1": {ScaleUp: true, ResourceDiff: 0.0}, + }) + + calculator := NewUpdatePriorityCalculator( + mpa, &UpdateConfig{MinChangePriority: 0.1}, &mpa_test.FakeRecommendationProcessor{}, priorityProcessor) + + calculator.AddPod(pod, timestampNow) + result := calculator.GetSortedPods(NewDefaultPodEvictionAdmission()) + assert.Exactly(t, []*apiv1.Pod{}, result, "Pod should not be updated") +} + +func TestDontUpdatePodWithOOMAfterLongRun(t *testing.T) { + pod := test.Pod().WithName("POD1").AddContainer(test.Container().WithName(containerName).WithCPURequest(resource.MustParse("4")).Get()).Get() + + // Pretend that the test pod started 11 hours ago. + timestampNow := pod.Status.StartTime.Time.Add(time.Hour * 11) + + pod.Status.ContainerStatuses = []apiv1.ContainerStatus{ + { + LastTerminationState: apiv1.ContainerState{ + Terminated: &apiv1.ContainerStateTerminated{ + Reason: "OOMKilled", + FinishedAt: metav1.NewTime(timestampNow.Add(-1 * 3 * time.Minute)), + StartedAt: metav1.NewTime(timestampNow.Add(-1 * 60 * time.Minute)), + }, + }, + }, + } + + // Pod is within the recommended range. + mpa := mpa_test.MultidimPodAutoscaler().WithContainer(containerName). + WithTarget("5", ""). + WithLowerBound("1", ""). + WithUpperBound("6", "").Get() + + priorityProcessor := NewFakeProcessor(map[string]PodPriority{ + "POD1": {ScaleUp: true, ResourceDiff: 0.0}, + }) + calculator := NewUpdatePriorityCalculator( + mpa, &UpdateConfig{MinChangePriority: 0.5}, &mpa_test.FakeRecommendationProcessor{}, priorityProcessor) + + calculator.AddPod(pod, timestampNow) + result := calculator.GetSortedPods(NewDefaultPodEvictionAdmission()) + assert.Exactly(t, []*apiv1.Pod{}, result, "Pod shouldn't be updated") +} + +func TestQuickOOM_VpaOvservedContainers(t *testing.T) { + tests := []struct { + name string + annotation map[string]string + want bool + }{ + { + name: "no VpaOvservedContainers annotation", + annotation: map[string]string{}, + want: true, + }, + { + name: "container listed in VpaOvservedContainers annotation", + annotation: map[string]string{annotations.VpaObservedContainersLabel: containerName}, + want: true, + }, + { + // Containers not listed in VpaOvservedContainers annotation + // shouldn't trigger the quick OOM. + name: "container not listed in VpaOvservedContainers annotation", + annotation: map[string]string{annotations.VpaObservedContainersLabel: ""}, + want: false, + }, + } + for _, tc := range tests { + t.Run(fmt.Sprintf("test case: %s", tc.name), func(t *testing.T) { + pod := test.Pod().WithAnnotations(tc.annotation). + WithName("POD1").AddContainer(test.Container().WithName(containerName).WithCPURequest(resource.MustParse("4")).Get()).Get() + + // Pretend that the test pod started 11 hours ago. + timestampNow := pod.Status.StartTime.Time.Add(time.Hour * 11) + + pod.Status.ContainerStatuses = []apiv1.ContainerStatus{ + { + Name: containerName, + LastTerminationState: apiv1.ContainerState{ + Terminated: &apiv1.ContainerStateTerminated{ + Reason: "OOMKilled", + FinishedAt: metav1.NewTime(timestampNow.Add(-1 * 3 * time.Minute)), + StartedAt: metav1.NewTime(timestampNow.Add(-1 * 5 * time.Minute)), + }, + }, + }, + } + + // Pod is within the recommended range. + mpa := mpa_test.MultidimPodAutoscaler().WithContainer(containerName). + WithTarget("5", ""). + WithLowerBound("1", ""). + WithUpperBound("6", "").Get() + + priorityProcessor := NewFakeProcessor(map[string]PodPriority{ + "POD1": {ScaleUp: true, ResourceDiff: 0.25}}) + calculator := NewUpdatePriorityCalculator( + mpa, &UpdateConfig{MinChangePriority: 0.5}, &mpa_test.FakeRecommendationProcessor{}, priorityProcessor) + + calculator.AddPod(pod, timestampNow) + result := calculator.GetSortedPods(NewDefaultPodEvictionAdmission()) + isUpdate := len(result) != 0 + assert.Equal(t, tc.want, isUpdate) + }) + } +} + +func TestQuickOOM_ContainerResourcePolicy(t *testing.T) { + scalingModeAuto := vpa_types.ContainerScalingModeAuto + scalingModeOff := vpa_types.ContainerScalingModeOff + tests := []struct { + name string + resourcePolicy vpa_types.ContainerResourcePolicy + want bool + }{ + { + name: "ContainerScalingModeAuto", + resourcePolicy: vpa_types.ContainerResourcePolicy{ + ContainerName: containerName, + Mode: &scalingModeAuto, + }, + want: true, + }, + { + // Containers with ContainerScalingModeOff + // shouldn't trigger the quick OOM. + name: "ContainerScalingModeOff", + resourcePolicy: vpa_types.ContainerResourcePolicy{ + ContainerName: containerName, + Mode: &scalingModeOff, + }, + want: false, + }, + { + name: "ContainerScalingModeAuto as default", + resourcePolicy: vpa_types.ContainerResourcePolicy{ + ContainerName: vpa_types.DefaultContainerResourcePolicy, + Mode: &scalingModeAuto, + }, + want: true, + }, + { + // When ContainerScalingModeOff is default + // container shouldn't trigger the quick OOM. + name: "ContainerScalingModeOff as default", + resourcePolicy: vpa_types.ContainerResourcePolicy{ + ContainerName: vpa_types.DefaultContainerResourcePolicy, + Mode: &scalingModeOff, + }, + want: false, + }, + } + for _, tc := range tests { + t.Run(fmt.Sprintf("test case: %s", tc.name), func(t *testing.T) { + pod := test.Pod().WithAnnotations(map[string]string{annotations.VpaObservedContainersLabel: containerName}). + WithName("POD1").AddContainer(test.Container().WithName(containerName).WithCPURequest(resource.MustParse("4")).Get()).Get() + + // Pretend that the test pod started 11 hours ago. + timestampNow := pod.Status.StartTime.Time.Add(time.Hour * 11) + + pod.Status.ContainerStatuses = []apiv1.ContainerStatus{ + { + Name: containerName, + LastTerminationState: apiv1.ContainerState{ + Terminated: &apiv1.ContainerStateTerminated{ + Reason: "OOMKilled", + FinishedAt: metav1.NewTime(timestampNow.Add(-1 * 3 * time.Minute)), + StartedAt: metav1.NewTime(timestampNow.Add(-1 * 5 * time.Minute)), + }, + }, + }, + } + + // Pod is within the recommended range. + mpa := mpa_test.MultidimPodAutoscaler().WithContainer(containerName). + WithTarget("5", ""). + WithLowerBound("1", ""). + WithUpperBound("6", "").Get() + + mpa.Spec.ResourcePolicy = &vpa_types.PodResourcePolicy{ + ContainerPolicies: []vpa_types.ContainerResourcePolicy{ + tc.resourcePolicy, + }, + } + priorityProcessor := NewFakeProcessor(map[string]PodPriority{ + "POD1": {ScaleUp: true, ResourceDiff: 0.25}}) + calculator := NewUpdatePriorityCalculator( + mpa, &UpdateConfig{MinChangePriority: 0.5}, &mpa_test.FakeRecommendationProcessor{}, priorityProcessor) + + calculator.AddPod(pod, timestampNow) + result := calculator.GetSortedPods(NewDefaultPodEvictionAdmission()) + isUpdate := len(result) != 0 + assert.Equal(t, tc.want, isUpdate) + }) + } +} + +func TestNoPods(t *testing.T) { + calculator := NewUpdatePriorityCalculator(nil, nil, &mpa_test.FakeRecommendationProcessor{}, + NewFakeProcessor(map[string]PodPriority{})) + result := calculator.GetSortedPods(NewDefaultPodEvictionAdmission()) + assert.Exactly(t, []*apiv1.Pod{}, result) +} + +type pod1Admission struct{} + +func (p *pod1Admission) LoopInit([]*apiv1.Pod, map[*mpa_types.MultidimPodAutoscaler][]*apiv1.Pod) {} +func (p *pod1Admission) Admit(pod *apiv1.Pod, recommendation *vpa_types.RecommendedPodResources) bool { + return pod.Name == "POD1" +} +func (p *pod1Admission) CleanUp() {} + +func TestAdmission(t *testing.T) { + + pod1 := test.Pod().WithName("POD1").AddContainer(test.Container().WithName(containerName).WithCPURequest(resource.MustParse("2")).Get()).Get() + pod2 := test.Pod().WithName("POD2").AddContainer(test.Container().WithName(containerName).WithCPURequest(resource.MustParse("4")).Get()).Get() + pod3 := test.Pod().WithName("POD3").AddContainer(test.Container().WithName(containerName).WithCPURequest(resource.MustParse("1")).Get()).Get() + pod4 := test.Pod().WithName("POD4").AddContainer(test.Container().WithName(containerName).WithCPURequest(resource.MustParse("3")).Get()).Get() + + mpa := mpa_test.MultidimPodAutoscaler().WithContainer(containerName).WithTarget("10", "").Get() + + priorityProcessor := NewFakeProcessor(map[string]PodPriority{ + "POD1": {ScaleUp: true, ResourceDiff: 4.0}, + "POD2": {ScaleUp: true, ResourceDiff: 1.5}, + "POD3": {ScaleUp: true, ResourceDiff: 9.0}, + "POD4": {ScaleUp: true, ResourceDiff: 2.33}}) + calculator := NewUpdatePriorityCalculator(mpa, nil, + &mpa_test.FakeRecommendationProcessor{}, priorityProcessor) + + timestampNow := pod1.Status.StartTime.Time.Add(time.Hour * 24) + calculator.AddPod(pod1, timestampNow) + calculator.AddPod(pod2, timestampNow) + calculator.AddPod(pod3, timestampNow) + calculator.AddPod(pod4, timestampNow) + + result := calculator.GetSortedPods(&pod1Admission{}) + assert.Exactly(t, []*apiv1.Pod{pod1}, result, "Wrong priority order") +} + +func TestLessPodPriority(t *testing.T) { + testCases := []struct { + name string + prio, other PodPriority + isLess bool + }{ + { + name: "scale down more than empty", + prio: PodPriority{ + ScaleUp: false, + ResourceDiff: 0.1, + }, + other: PodPriority{}, + isLess: false, + }, { + name: "scale up more than empty", + prio: PodPriority{ + ScaleUp: true, + ResourceDiff: 0.1, + }, + other: PodPriority{}, + isLess: false, + }, { + name: "two scale ups", + prio: PodPriority{ + ScaleUp: true, + ResourceDiff: 0.1, + }, + other: PodPriority{ + ScaleUp: true, + ResourceDiff: 1.0, + }, + isLess: true, + }, { + name: "two scale downs", + prio: PodPriority{ + ScaleUp: false, + ResourceDiff: 0.9, + }, + other: PodPriority{ + ScaleUp: false, + ResourceDiff: 0.1, + }, + isLess: false, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.isLess, tc.prio.Less(tc.other)) + assert.Equal(t, !tc.isLess, tc.other.Less(tc.prio)) + }) + } + +} diff --git a/multidimensional-pod-autoscaler/pkg/utils/annotations/vpa_observed_containers.go b/multidimensional-pod-autoscaler/pkg/utils/annotations/vpa_observed_containers.go new file mode 100644 index 000000000000..c65899565df2 --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/utils/annotations/vpa_observed_containers.go @@ -0,0 +1,55 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package annotations + +import ( + "fmt" + "strings" + + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/validation" +) + +const ( + // MpaObservedContainersLabel is a label used by the vpa observed containers annotation. + MpaObservedContainersLabel = "vpaObservedContainers" + listSeparator = ", " +) + +// GetMpaObservedContainersValue creates an annotation value for a given pod. +func GetMpaObservedContainersValue(pod *v1.Pod) string { + containerNames := make([]string, len(pod.Spec.Containers)) + for i := range pod.Spec.Containers { + containerNames[i] = pod.Spec.Containers[i].Name + } + return strings.Join(containerNames, listSeparator) +} + +// ParseMpaObservedContainersValue returns list of containers +// based on a given mpa observed containers annotation value. +func ParseMpaObservedContainersValue(value string) ([]string, error) { + if value == "" { + return []string{}, nil + } + containerNames := strings.Split(value, listSeparator) + for i := range containerNames { + if errs := validation.IsDNS1123Label(containerNames[i]); len(errs) != 0 { + return nil, fmt.Errorf("incorrect format: %s is not a valid container name: %v", containerNames[i], errs) + } + } + return containerNames, nil +} diff --git a/multidimensional-pod-autoscaler/pkg/utils/annotations/vpa_observed_containers_test.go b/multidimensional-pod-autoscaler/pkg/utils/annotations/vpa_observed_containers_test.go new file mode 100644 index 000000000000..7cc146cf7643 --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/utils/annotations/vpa_observed_containers_test.go @@ -0,0 +1,87 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package annotations + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + v1 "k8s.io/api/core/v1" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/test" +) + +func TestGetMpaObservedContainersValue(t *testing.T) { + tests := []struct { + name string + pod *v1.Pod + want string + }{ + { + name: "creating mpa observed containers annotation", + pod: test.Pod(). + AddContainer(test.Container().WithName("test1").Get()). + AddContainer(test.Container().WithName("test2").Get()). + AddContainer(test.Container().WithName("test3").Get()). + Get(), + want: "test1, test2, test3", + }, + } + for _, tc := range tests { + t.Run(fmt.Sprintf("test case: %s", tc.name), func(t *testing.T) { + got := GetMpaObservedContainersValue(tc.pod) + assert.Equal(t, got, tc.want) + }) + } +} + +func TestParseMpaObservedContainersValue(t *testing.T) { + tests := []struct { + name string + annotation string + want []string + wantErr bool + }{ + { + name: "parsing correct mpa observed containers annotation", + annotation: "test1, test2, test3", + want: []string{"test1", "test2", "test3"}, + wantErr: false, + }, + { + name: "parsing mpa observed containers annotation with incorrect container name", + annotation: "test1, test2, test3_;';s", + want: []string(nil), + wantErr: true, + }, + { + name: "parsing empty mpa observed containers annotation", + annotation: "", + want: []string{}, + wantErr: false, + }, + } + for _, tc := range tests { + t.Run(fmt.Sprintf("test case: %s", tc.name), func(t *testing.T) { + got, gotErr := ParseMpaObservedContainersValue(tc.annotation) + if (gotErr != nil) != tc.wantErr { + t.Errorf("gotErr %v, wantErr %v", (gotErr != nil), tc.wantErr) + } + assert.Equal(t, got, tc.want) + }) + } +} diff --git a/multidimensional-pod-autoscaler/pkg/utils/metrics/recommender/recommender.go b/multidimensional-pod-autoscaler/pkg/utils/metrics/recommender/recommender.go new file mode 100644 index 000000000000..0c502957ba5c --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/utils/metrics/recommender/recommender.go @@ -0,0 +1,186 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package recommender (aka metrics_recommender) - code for metrics of VPA/MPA Recommender +package recommender + +import ( + "fmt" + "strconv" + "time" + + "github.com/prometheus/client_golang/prometheus" + mpa_types "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1alpha1" + "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/recommender/model" + vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/metrics" +) + +const ( + metricsNamespace = metrics.TopMetricsNamespace + "recommender" +) + +var ( + modes = []string{string(vpa_types.UpdateModeOff), string(vpa_types.UpdateModeInitial), string(vpa_types.UpdateModeRecreate), string(vpa_types.UpdateModeAuto)} +) + +type apiVersion string + +const ( + v1beta1 apiVersion = "v1beta1" + v1beta2 apiVersion = "v1beta2" + v1 apiVersion = "v1" + v1alpha1 apiVersion = "v1alpha1" +) + +var ( + mpaObjectCount = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: metricsNamespace, + Name: "mpa_objects_count", + Help: "Number of MPA objects present in the cluster.", + }, []string{"update_mode", "has_recommendation", "api", "matches_pods", "unsupported_config"}, + ) + + recommendationLatency = prometheus.NewHistogram( + prometheus.HistogramOpts{ + Namespace: metricsNamespace, + Name: "recommendation_latency_seconds", + Help: "Time elapsed from creating a valid MPA configuration to the first recommendation.", + Buckets: []float64{1.0, 2.0, 5.0, 7.5, 10.0, 20.0, 30.0, 40.00, 50.0, 60.0, 90.0, 120.0, 150.0, 180.0, 240.0, 300.0, 600.0, 900.0, 1800.0}, + }, + ) + + functionLatency = metrics.CreateExecutionTimeMetric(metricsNamespace, + "Time spent in various parts of MPA Recommender main loop.") + + aggregateContainerStatesCount = prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: metricsNamespace, + Name: "aggregate_container_states_count", + Help: "Number of aggregate container states being tracked by the recommender", + }, + ) + + metricServerResponses = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: metricsNamespace, + Name: "metric_server_responses", + Help: "Count of responses to queries to metrics server", + }, []string{"is_error", "client_name"}, + ) +) + +type objectCounterKey struct { + mode string + has bool + matchesPods bool + apiVersion apiVersion + unsupportedConfig bool +} + +// ObjectCounter helps split all MPA objects into buckets +type ObjectCounter struct { + cnt map[objectCounterKey]int +} + +// Register initializes all metrics for MPA Recommender +func Register() { + prometheus.MustRegister(mpaObjectCount, recommendationLatency, functionLatency, aggregateContainerStatesCount, metricServerResponses) +} + +// NewExecutionTimer provides a timer for Recommender's RunOnce execution +func NewExecutionTimer() *metrics.ExecutionTimer { + return metrics.NewExecutionTimer(functionLatency) +} + +// ObserveRecommendationLatency observes the time it took for the first recommendation to appear +func ObserveRecommendationLatency(created time.Time) { + recommendationLatency.Observe(time.Since(created).Seconds()) +} + +// RecordAggregateContainerStatesCount records the number of containers being tracked by the recommender +func RecordAggregateContainerStatesCount(statesCount int) { + aggregateContainerStatesCount.Set(float64(statesCount)) +} + +// RecordMetricsServerResponse records result of a query to metrics server +func RecordMetricsServerResponse(err error, clientName string) { + metricServerResponses.WithLabelValues(strconv.FormatBool(err != nil), clientName).Inc() +} + +// NewObjectCounter creates a new helper to split MPA objects into buckets +func NewObjectCounter() *ObjectCounter { + obj := ObjectCounter{ + cnt: make(map[objectCounterKey]int), + } + + // initialize with empty data so we can clean stale gauge values in Observe + for _, m := range modes { + for _, h := range []bool{false, true} { + for _, api := range []apiVersion{v1beta1, v1beta2, v1, v1alpha1} { + for _, mp := range []bool{false, true} { + for _, uc := range []bool{false, true} { + obj.cnt[objectCounterKey{ + mode: m, + has: h, + apiVersion: api, + matchesPods: mp, + unsupportedConfig: uc, + }] = 0 + } + } + } + } + } + + return &obj +} + +// Add updates the helper state to include the given MPA object +func (oc *ObjectCounter) Add(mpa *model.Mpa) { + mode := string(vpa_types.UpdateModeAuto) + if mpa.UpdateMode != nil && string(*mpa.UpdateMode) != "" { + mode = string(*mpa.UpdateMode) + } + // TODO: Maybe report v1 version as well. + api := v1beta2 + if mpa.IsV1Beta1API { + api = v1beta1 + } + + key := objectCounterKey{ + mode: mode, + has: mpa.HasRecommendation(), + apiVersion: api, + matchesPods: mpa.HasMatchedPods(), + unsupportedConfig: mpa.Conditions.ConditionActive(mpa_types.ConfigUnsupported), + } + oc.cnt[key]++ +} + +// Observe passes all the computed bucket values to metrics +func (oc *ObjectCounter) Observe() { + for k, v := range oc.cnt { + mpaObjectCount.WithLabelValues( + k.mode, + fmt.Sprintf("%v", k.has), + string(k.apiVersion), + fmt.Sprintf("%v", k.matchesPods), + fmt.Sprintf("%v", k.unsupportedConfig), + ).Set(float64(v)) + } +} diff --git a/multidimensional-pod-autoscaler/pkg/utils/mpa/api.go b/multidimensional-pod-autoscaler/pkg/utils/mpa/api.go new file mode 100644 index 000000000000..2b8f99a9986e --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/utils/mpa/api.go @@ -0,0 +1,218 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "time" + + core "k8s.io/api/core/v1" + meta "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" + + apiequality "k8s.io/apimachinery/pkg/api/equality" + mpa_types "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1alpha1" + mpa_clientset "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/client/clientset/versioned" + mpa_api "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/client/clientset/versioned/typed/autoscaling.k8s.io/v1alpha1" + mpa_lister "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/client/listers/autoscaling.k8s.io/v1alpha1" + vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" + controllerfetcher "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/target/controller_fetcher" + "k8s.io/client-go/tools/cache" + "k8s.io/klog/v2" +) + +// MpaWithSelector is a pair of MPA and its selector. +type MpaWithSelector struct { + Mpa *mpa_types.MultidimPodAutoscaler + Selector labels.Selector +} + +type patchRecord struct { + Op string `json:"op,inline"` + Path string `json:"path,inline"` + Value interface{} `json:"value"` +} + +func patchMpaStatus(mpaClient mpa_api.MultidimPodAutoscalerInterface, mpaName string, patches []patchRecord) (result *mpa_types.MultidimPodAutoscaler, err error) { + bytes, err := json.Marshal(patches) + if err != nil { + klog.Errorf("Cannot marshal MPA status patches %+v. Reason: %+v", patches, err) + return + } + + return mpaClient.Patch(context.TODO(), mpaName, types.JSONPatchType, bytes, meta.PatchOptions{}, "status") +} + +// UpdateMpaStatusIfNeeded updates the status field of the MPA API object. +func UpdateMpaStatusIfNeeded(mpaClient mpa_api.MultidimPodAutoscalerInterface, mpaName string, newStatus, + oldStatus *mpa_types.MultidimPodAutoscalerStatus) (result *mpa_types.MultidimPodAutoscaler, err error) { + patches := []patchRecord{{ + Op: "add", + Path: "/status", + Value: *newStatus, + }} + + if !apiequality.Semantic.DeepEqual(*oldStatus, *newStatus) { + return patchMpaStatus(mpaClient, mpaName, patches) + } + return nil, nil +} + +// NewMpasLister returns MultidimPodAutoscalerLister configured to fetch all MPA objects from +// namespace, set namespace to k8sapiv1.NamespaceAll to select all namespaces. +// The method blocks until mpaLister is initially populated. +func NewMpasLister(mpaClient *mpa_clientset.Clientset, stopChannel <-chan struct{}, namespace string) mpa_lister.MultidimPodAutoscalerLister { + mpaListWatch := cache.NewListWatchFromClient(mpaClient.AutoscalingV1alpha1().RESTClient(), "multidimpodautoscalers", namespace, fields.Everything()) + indexer, controller := cache.NewIndexerInformer(mpaListWatch, + &mpa_types.MultidimPodAutoscaler{}, + 1*time.Hour, + &cache.ResourceEventHandlerFuncs{}, + cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + mpaLister := mpa_lister.NewMultidimPodAutoscalerLister(indexer) + go controller.Run(stopChannel) + if !cache.WaitForCacheSync(make(chan struct{}), controller.HasSynced) { + klog.Fatalf("Failed to sync MPA cache during initialization") + } else { + klog.Info("Initial MPA synced successfully") + } + return mpaLister +} + +// PodMatchesMPA returns true iff the mpaWithSelector matches the Pod. +func PodMatchesMPA(pod *core.Pod, mpaWithSelector *MpaWithSelector) bool { + return PodLabelsMatchMPA(pod.Namespace, labels.Set(pod.GetLabels()), mpaWithSelector.Mpa.Namespace, mpaWithSelector.Selector) +} + +// PodLabelsMatchMPA returns true iff the mpaWithSelector matches the pod labels. +func PodLabelsMatchMPA(podNamespace string, labels labels.Set, mpaNamespace string, mpaSelector labels.Selector) bool { + if podNamespace != mpaNamespace { + return false + } + return mpaSelector.Matches(labels) +} + +// Stronger returns true iff a is before b in the order to control a Pod (that matches both MPAs). +func Stronger(a, b *mpa_types.MultidimPodAutoscaler) bool { + // Assume a is not nil and each valid object is before nil object. + if b == nil { + return true + } + // Compare creation timestamps of the MPA objects. This is the clue of the stronger logic. + var aTime, bTime meta.Time + aTime = a.GetCreationTimestamp() + bTime = b.GetCreationTimestamp() + if !aTime.Equal(&bTime) { + return aTime.Before(&bTime) + } + // If the timestamps are the same (unlikely, but possible e.g. in test environments): compare by name to have a complete deterministic order. + return a.GetName() < b.GetName() +} + +// GetControllingMPAForPod chooses the earliest created MPA from the input list that matches the given Pod. +func GetControllingMPAForPod(ctx context.Context, pod *core.Pod, mpas []*MpaWithSelector, ctrlFetcher controllerfetcher.ControllerFetcher) *MpaWithSelector { + + parentController, err := FindParentControllerForPod(ctx, pod, ctrlFetcher) + if err != nil { + klog.ErrorS(err, "Failed to get parent controller for pod", "pod", klog.KObj(pod)) + return nil + } + if parentController == nil { + return nil + } + + var controlling *MpaWithSelector + var controllingMpa *mpa_types.MultidimPodAutoscaler + // Choose the strongest MPA from the ones that match this Pod. + for _, mpaWithSelector := range mpas { + if mpaWithSelector.Mpa.Spec.ScaleTargetRef == nil { + klog.V(5).InfoS("Skipping MPA object because scaleTargetRef is not defined.", "mpa", klog.KObj(mpaWithSelector.Mpa)) + continue + } + if mpaWithSelector.Mpa.Spec.ScaleTargetRef.Kind != parentController.Kind || + mpaWithSelector.Mpa.Namespace != parentController.Namespace || + mpaWithSelector.Mpa.Spec.ScaleTargetRef.Name != parentController.Name { + continue // This pod is not associated to the right controller + } + if PodMatchesMPA(pod, mpaWithSelector) && Stronger(mpaWithSelector.Mpa, controllingMpa) { + controlling = mpaWithSelector + controllingMpa = controlling.Mpa + } + } + return controlling +} + +// FindParentControllerForPod returns the parent controller (topmost well-known or scalable controller) for the given Pod. +func FindParentControllerForPod(ctx context.Context, pod *core.Pod, ctrlFetcher controllerfetcher.ControllerFetcher) (*controllerfetcher.ControllerKeyWithAPIVersion, error) { + var ownerRefrence *meta.OwnerReference + for i := range pod.OwnerReferences { + r := pod.OwnerReferences[i] + if r.Controller != nil && *r.Controller { + ownerRefrence = &r + } + } + if ownerRefrence == nil { + // If the pod has no ownerReference, it cannot be under a VPA. + return nil, nil + } + k := &controllerfetcher.ControllerKeyWithAPIVersion{ + ControllerKey: controllerfetcher.ControllerKey{ + Namespace: pod.Namespace, + Kind: ownerRefrence.Kind, + Name: ownerRefrence.Name, + }, + ApiVersion: ownerRefrence.APIVersion, + } + return ctrlFetcher.FindTopMostWellKnownOrScalable(ctx, k) +} + +// GetUpdateMode returns the updatePolicy.updateMode for a given MPA. +// If the mode is not specified it returns the default (UpdateModeAuto). +func GetUpdateMode(mpa *mpa_types.MultidimPodAutoscaler) vpa_types.UpdateMode { + if mpa.Spec.Policy == nil || mpa.Spec.Policy.UpdateMode == nil || *mpa.Spec.Policy.UpdateMode == "" { + return vpa_types.UpdateModeAuto + } + return *mpa.Spec.Policy.UpdateMode +} + +// CreateOrUpdateMpaCheckpoint updates the status field of the MPA Checkpoint API object. +// If object doesn't exits it is created. +func CreateOrUpdateMpaCheckpoint(mpaCheckpointClient mpa_api.MultidimPodAutoscalerCheckpointInterface, + mpaCheckpoint *mpa_types.MultidimPodAutoscalerCheckpoint) error { + patches := make([]patchRecord, 0) + patches = append(patches, patchRecord{ + Op: "replace", + Path: "/status", + Value: mpaCheckpoint.Status, + }) + bytes, err := json.Marshal(patches) + if err != nil { + return fmt.Errorf("Cannot marshal MPA checkpoint status patches %+v. Reason: %+v", patches, err) + } + _, err = mpaCheckpointClient.Patch(context.TODO(), mpaCheckpoint.ObjectMeta.Name, types.JSONPatchType, bytes, meta.PatchOptions{}) + if err != nil && strings.Contains(err.Error(), fmt.Sprintf("\"%s\" not found", mpaCheckpoint.ObjectMeta.Name)) { + _, err = mpaCheckpointClient.Create(context.TODO(), mpaCheckpoint, meta.CreateOptions{}) + } + if err != nil { + return fmt.Errorf("Cannot save checkpoint for mpa %v container %v. Reason: %+v", mpaCheckpoint.ObjectMeta.Name, mpaCheckpoint.Spec.ContainerName, err) + } + return nil +} diff --git a/multidimensional-pod-autoscaler/pkg/utils/mpa/capping.go b/multidimensional-pod-autoscaler/pkg/utils/mpa/capping.go new file mode 100644 index 000000000000..e9d5983076be --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/utils/mpa/capping.go @@ -0,0 +1,466 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "fmt" + "math" + "math/big" + + core "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + mpa_types "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1alpha1" + vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/limitrange" + "k8s.io/klog/v2" + + vpa_api_util "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/vpa" + + apiv1 "k8s.io/api/core/v1" +) + +// NewCappingRecommendationProcessor constructs new RecommendationsProcessor that adjusts recommendation +// for given pod to obey MPA resources policy and container limits +func NewCappingRecommendationProcessor(limitsRangeCalculator limitrange.LimitRangeCalculator) RecommendationProcessor { + return &cappingRecommendationProcessor{limitsRangeCalculator: limitsRangeCalculator} +} + +type cappingRecommendationProcessor struct { + limitsRangeCalculator limitrange.LimitRangeCalculator +} + +type cappingAction string + +var ( + cappedToMinAllowed cappingAction = "capped to minAllowed" + cappedToMaxAllowed cappingAction = "capped to maxAllowed" + cappedToLimit cappingAction = "capped to container limit" + cappedProportionallyToMaxLimit cappingAction = "capped to fit Max in container LimitRange" + cappedProportionallyToMinLimit cappingAction = "capped to fit Min in container LimitRange" +) + +type roundingMode int + +const ( + noRounding roundingMode = iota + roundUpToFullUnit + roundDownToFullUnit +) + +// Apply returns a recommendation for the given pod, adjusted to obey policy and limits. +func (c *cappingRecommendationProcessor) Apply( + podRecommendation *vpa_types.RecommendedPodResources, + policy *vpa_types.PodResourcePolicy, + conditions []mpa_types.MultidimPodAutoscalerCondition, + pod *apiv1.Pod) (*vpa_types.RecommendedPodResources, ContainerToAnnotationsMap, error) { + // TODO: Annotate if request enforced by maintaining proportion with limit and allowed limit range is in conflict with policy. + + if podRecommendation == nil && policy == nil { + // If there is no recommendation and no policies have been defined then no recommendation can be computed. + return nil, nil, nil + } + if podRecommendation == nil { + // Policies have been specified. Create an empty recommendation so that the policies can be applied correctly. + podRecommendation = new(vpa_types.RecommendedPodResources) + } + updatedRecommendations := []vpa_types.RecommendedContainerResources{} + containerToAnnotationsMap := ContainerToAnnotationsMap{} + limitAdjustedRecommendation, err := c.capProportionallyToPodLimitRange(podRecommendation.ContainerRecommendations, pod) + if err != nil { + return nil, nil, err + } + for _, containerRecommendation := range limitAdjustedRecommendation { + container := getContainer(containerRecommendation.ContainerName, pod) + + if container == nil { + klog.V(2).Infof("no matching Container found for recommendation %s", containerRecommendation.ContainerName) + continue + } + + containerLimitRange, err := c.limitsRangeCalculator.GetContainerLimitRangeItem(pod.Namespace) + if err != nil { + klog.Warningf("failed to fetch LimitRange for %v namespace", pod.Namespace) + } + updatedContainerResources, containerAnnotations, err := getCappedRecommendationForContainer( + *container, &containerRecommendation, policy, containerLimitRange) + + if len(containerAnnotations) != 0 { + containerToAnnotationsMap[containerRecommendation.ContainerName] = containerAnnotations + } + + if err != nil { + return nil, nil, fmt.Errorf("cannot update recommendation for container name %v", container.Name) + } + updatedRecommendations = append(updatedRecommendations, *updatedContainerResources) + } + return &vpa_types.RecommendedPodResources{ContainerRecommendations: updatedRecommendations}, containerToAnnotationsMap, nil +} + +func applyPodLimitRange(resources []vpa_types.RecommendedContainerResources, + pod *apiv1.Pod, limitRange apiv1.LimitRangeItem, resourceName apiv1.ResourceName, + fieldGetter func(vpa_types.RecommendedContainerResources) *apiv1.ResourceList) []vpa_types.RecommendedContainerResources { + minLimit := limitRange.Min[resourceName] + maxLimit := limitRange.Max[resourceName] + defaultLimit := limitRange.Default[resourceName] + + var sumLimit, sumRecommendation resource.Quantity + for i, container := range pod.Spec.Containers { + if i >= len(resources) { + continue + } + limit := container.Resources.Limits[resourceName] + request := container.Resources.Requests[resourceName] + recommendation := (*fieldGetter(resources[i]))[resourceName] + containerLimit, _ := getProportionalResourceLimit(resourceName, &limit, &request, &recommendation, &defaultLimit) + if containerLimit != nil { + sumLimit.Add(*containerLimit) + } + sumRecommendation.Add(recommendation) + } + if minLimit.Cmp(sumLimit) <= 0 && minLimit.Cmp(sumRecommendation) <= 0 && (maxLimit.IsZero() || maxLimit.Cmp(sumLimit) >= 0) { + return resources + } + + if minLimit.Cmp(sumRecommendation) > 0 && !sumLimit.IsZero() { + for i := range pod.Spec.Containers { + request := (*fieldGetter(resources[i]))[resourceName] + var cappedContainerRequest *resource.Quantity + if resourceName == apiv1.ResourceMemory { + cappedContainerRequest, _ = scaleQuantityProportionallyMem(&request, &sumRecommendation, &minLimit, roundUpToFullUnit) + } else { + cappedContainerRequest, _ = scaleQuantityProportionallyCPU(&request, &sumRecommendation, &minLimit, noRounding) + } + (*fieldGetter(resources[i]))[resourceName] = *cappedContainerRequest + } + return resources + } + + if sumLimit.IsZero() { + return resources + } + + var targetTotalLimit resource.Quantity + if minLimit.Cmp(sumLimit) > 0 { + targetTotalLimit = minLimit + } + if !maxLimit.IsZero() && maxLimit.Cmp(sumLimit) < 0 { + targetTotalLimit = maxLimit + } + for i := range pod.Spec.Containers { + limit := (*fieldGetter(resources[i]))[resourceName] + var cappedContainerRequest *resource.Quantity + if resourceName == apiv1.ResourceMemory { + cappedContainerRequest, _ = scaleQuantityProportionallyMem(&limit, &sumLimit, &targetTotalLimit, roundDownToFullUnit) + } else { + cappedContainerRequest, _ = scaleQuantityProportionallyCPU(&limit, &sumLimit, &targetTotalLimit, noRounding) + } + (*fieldGetter(resources[i]))[resourceName] = *cappedContainerRequest + } + return resources +} + +func (c *cappingRecommendationProcessor) capProportionallyToPodLimitRange( + containerRecommendations []vpa_types.RecommendedContainerResources, pod *apiv1.Pod) ([]vpa_types.RecommendedContainerResources, error) { + podLimitRange, err := c.limitsRangeCalculator.GetPodLimitRangeItem(pod.Namespace) + if err != nil { + return nil, fmt.Errorf("error obtaining limit range: %s", err) + } + if podLimitRange == nil { + return containerRecommendations, nil + } + getTarget := func(rl vpa_types.RecommendedContainerResources) *apiv1.ResourceList { return &rl.Target } + getUpper := func(rl vpa_types.RecommendedContainerResources) *apiv1.ResourceList { return &rl.UpperBound } + getLower := func(rl vpa_types.RecommendedContainerResources) *apiv1.ResourceList { return &rl.LowerBound } + + containerRecommendations = applyPodLimitRange(containerRecommendations, pod, *podLimitRange, apiv1.ResourceCPU, getUpper) + containerRecommendations = applyPodLimitRange(containerRecommendations, pod, *podLimitRange, apiv1.ResourceMemory, getUpper) + + containerRecommendations = applyPodLimitRange(containerRecommendations, pod, *podLimitRange, apiv1.ResourceCPU, getTarget) + containerRecommendations = applyPodLimitRange(containerRecommendations, pod, *podLimitRange, apiv1.ResourceMemory, getTarget) + + containerRecommendations = applyPodLimitRange(containerRecommendations, pod, *podLimitRange, apiv1.ResourceCPU, getLower) + containerRecommendations = applyPodLimitRange(containerRecommendations, pod, *podLimitRange, apiv1.ResourceMemory, getLower) + return containerRecommendations, nil +} + +func getContainer(containerName string, pod *apiv1.Pod) *apiv1.Container { + for i, container := range pod.Spec.Containers { + if container.Name == containerName { + return &pod.Spec.Containers[i] + } + } + return nil +} + +// getCappedRecommendationForContainer returns a recommendation for the given container, adjusted to obey policy and limits. +func getCappedRecommendationForContainer( + container apiv1.Container, + containerRecommendation *vpa_types.RecommendedContainerResources, + policy *vpa_types.PodResourcePolicy, limitRange *apiv1.LimitRangeItem) (*vpa_types.RecommendedContainerResources, []string, error) { + if containerRecommendation == nil { + return nil, nil, fmt.Errorf("no recommendation available for container name %v", container.Name) + } + // containerPolicy can be nil (user does not have to configure it). + containerPolicy := vpa_api_util.GetContainerResourcePolicy(container.Name, policy) + containerControlledValues := vpa_api_util.GetContainerControlledValues(container.Name, policy) + + cappedRecommendations := containerRecommendation.DeepCopy() + + cappingAnnotations := make([]string, 0) + + process := func(recommendation apiv1.ResourceList, genAnnotations bool) { + // TODO: Add anotation if limitRange is conflicting with VPA policy. + limitAnnotations := applyContainerLimitRange(recommendation, container, limitRange) + annotations := applyMPAPolicy(recommendation, containerPolicy) + if genAnnotations { + cappingAnnotations = append(cappingAnnotations, limitAnnotations...) + cappingAnnotations = append(cappingAnnotations, annotations...) + } + // TODO: If limits and policy are conflicting, set some condition on the VPA. + if containerControlledValues == vpa_types.ContainerControlledValuesRequestsOnly { + annotations = capRecommendationToContainerLimit(recommendation, container) + if genAnnotations { + cappingAnnotations = append(cappingAnnotations, annotations...) + } + } + } + + process(cappedRecommendations.Target, true) + process(cappedRecommendations.LowerBound, false) + process(cappedRecommendations.UpperBound, false) + + return cappedRecommendations, cappingAnnotations, nil +} + +func getProportionalResourceLimit(resourceName core.ResourceName, originalLimit, originalRequest, recommendedRequest, defaultLimit *resource.Quantity) (*resource.Quantity, string) { + if originalLimit == nil || originalLimit.Value() == 0 && defaultLimit != nil { + originalLimit = defaultLimit + } + // originalLimit not set, don't set limit. + if originalLimit == nil || originalLimit.Value() == 0 { + return nil, "" + } + // recommendedRequest not set, don't set limit. + if recommendedRequest == nil || recommendedRequest.Value() == 0 { + return nil, "" + } + // originalLimit set but originalRequest not set - K8s will treat the pod as if they were equal, + // recommend limit equal to request + if originalRequest == nil || originalRequest.Value() == 0 { + result := *recommendedRequest + return &result, "" + } + // originalLimit and originalRequest are set. If they are equal recommend limit equal to request. + if originalRequest.MilliValue() == originalLimit.MilliValue() { + result := *recommendedRequest + return &result, "" + } + if resourceName == core.ResourceCPU { + result, capped := scaleQuantityProportionallyCPU( /*scaledQuantity=*/ originalLimit /*scaleBase=*/, originalRequest /*scaleResult=*/, recommendedRequest, noRounding) + if !capped { + return result, "" + } + return result, fmt.Sprintf( + "%v: failed to keep limit to request ratio; capping limit to int64", resourceName) + } + result, capped := scaleQuantityProportionallyMem( /*scaledQuantity=*/ originalLimit /*scaleBase=*/, originalRequest /*scaleResult=*/, recommendedRequest, noRounding) + if !capped { + return result, "" + } + return result, fmt.Sprintf( + "%v: failed to keep limit to request ratio; capping limit to int64", resourceName) +} + +// scaleQuantityProportionallyCPU returns a value in milliunits which has the same proportion to scaledQuantity as scaleResult has to scaleBase. +// It also returns a bool indicating if it had to cap result to MaxInt64 milliunits. +func scaleQuantityProportionallyCPU(scaledQuantity, scaleBase, scaleResult *resource.Quantity, rounding roundingMode) (*resource.Quantity, bool) { + originalMilli := big.NewInt(scaledQuantity.MilliValue()) + scaleBaseMilli := big.NewInt(scaleBase.MilliValue()) + scaleResultMilli := big.NewInt(scaleResult.MilliValue()) + var scaledOriginal big.Int + scaledOriginal.Mul(originalMilli, scaleResultMilli) + scaledOriginal.Div(&scaledOriginal, scaleBaseMilli) + if scaledOriginal.IsInt64() { + result := resource.NewMilliQuantity(scaledOriginal.Int64(), scaledQuantity.Format) + if rounding == roundUpToFullUnit { + result.RoundUp(resource.Scale(0)) + } + if rounding == roundDownToFullUnit { + result.Sub(*resource.NewMilliQuantity(999, result.Format)) + result.RoundUp(resource.Scale(0)) + } + return result, false + } + return resource.NewMilliQuantity(math.MaxInt64, scaledQuantity.Format), true +} + +// scaleQuantityProportionallyMem returns a value in whole units which has the same proportion to scaledQuantity as scaleResult has to scaleBase. +// It also returns a bool indicating if it had to cap result to MaxInt64 units. +func scaleQuantityProportionallyMem(scaledQuantity, scaleBase, scaleResult *resource.Quantity, rounding roundingMode) (*resource.Quantity, bool) { + originalValue := big.NewInt(scaledQuantity.Value()) + scaleBaseValue := big.NewInt(scaleBase.Value()) + scaleResultValue := big.NewInt(scaleResult.Value()) + var scaledOriginal big.Int + scaledOriginal.Mul(originalValue, scaleResultValue) + scaledOriginal.Div(&scaledOriginal, scaleBaseValue) + if scaledOriginal.IsInt64() { + result := resource.NewQuantity(scaledOriginal.Int64(), scaledQuantity.Format) + if rounding == roundUpToFullUnit { + result.RoundUp(resource.Scale(0)) + } + if rounding == roundDownToFullUnit { + result.Sub(*resource.NewMilliQuantity(999, result.Format)) + result.RoundUp(resource.Scale(0)) + } + return result, false + } + return resource.NewQuantity(math.MaxInt64, scaledQuantity.Format), true +} + +// applyContainerLimitRange updates recommendation if recommended resources are outside of limits defined in VPA resources policy +func applyContainerLimitRange(recommendation apiv1.ResourceList, container apiv1.Container, limitRange *apiv1.LimitRangeItem) []string { + annotations := make([]string, 0) + if limitRange == nil { + return annotations + } + maxAllowedRecommendation := getMaxAllowedRecommendation(recommendation, container, limitRange) + minAllowedRecommendation := getMinAllowedRecommendation(recommendation, container, limitRange) + for resourceName, recommended := range recommendation { + cappedToMin, isCapped := maybeCapToMin(recommended, resourceName, minAllowedRecommendation) + recommendation[resourceName] = cappedToMin + if isCapped { + annotations = append(annotations, toCappingAnnotation(resourceName, cappedProportionallyToMinLimit)) + } + cappedToMax, isCapped := maybeCapToMax(cappedToMin, resourceName, maxAllowedRecommendation) + recommendation[resourceName] = cappedToMax + if isCapped { + annotations = append(annotations, toCappingAnnotation(resourceName, cappedProportionallyToMaxLimit)) + } + } + return annotations +} + +func getMaxAllowedRecommendation(recommendation apiv1.ResourceList, container apiv1.Container, + podLimitRange *apiv1.LimitRangeItem) apiv1.ResourceList { + if podLimitRange == nil { + return apiv1.ResourceList{} + } + return getBoundaryRecommendation(recommendation, container, podLimitRange.Max, podLimitRange.Default) +} + +func getMinAllowedRecommendation(recommendation apiv1.ResourceList, container apiv1.Container, + podLimitRange *apiv1.LimitRangeItem) apiv1.ResourceList { + // Both limit and request must be higher than min set in the limit range: + // https://github.com/kubernetes/kubernetes/blob/016e9d5c06089774c6286fd825302cbae661a446/plugin/pkg/admission/limitranger/admission.go#L303 + if podLimitRange == nil { + return apiv1.ResourceList{} + } + minForLimit := getBoundaryRecommendation(recommendation, container, podLimitRange.Min, podLimitRange.Default) + minForRequest := podLimitRange.Min + if minForRequest == nil { + return minForLimit + } + result := minForLimit + if minForRequest.Cpu() != nil && minForRequest.Cpu().Cmp(*minForLimit.Cpu()) > 0 { + result[apiv1.ResourceCPU] = *minForRequest.Cpu() + } + if minForRequest.Memory() != nil && minForRequest.Memory().Cmp(*minForLimit.Memory()) > 0 { + result[apiv1.ResourceMemory] = *minForRequest.Memory() + } + return result +} + +func getBoundaryRecommendation(recommendation apiv1.ResourceList, container apiv1.Container, + boundaryLimit, defaultLimit apiv1.ResourceList) apiv1.ResourceList { + if boundaryLimit == nil { + return apiv1.ResourceList{} + } + boundaryCpu := vpa_api_util.GetBoundaryRequest(apiv1.ResourceCPU, container.Resources.Requests.Cpu(), container.Resources.Limits.Cpu(), boundaryLimit.Cpu(), defaultLimit.Cpu()) + boundaryMem := vpa_api_util.GetBoundaryRequest(apiv1.ResourceMemory, container.Resources.Requests.Memory(), container.Resources.Limits.Memory(), boundaryLimit.Memory(), defaultLimit.Memory()) + return apiv1.ResourceList{ + apiv1.ResourceCPU: *boundaryCpu, + apiv1.ResourceMemory: *boundaryMem, + } +} + +func maybeCapToPolicyMin(recommended resource.Quantity, resourceName apiv1.ResourceName, + containerPolicy *vpa_types.ContainerResourcePolicy) (resource.Quantity, bool) { + return maybeCapToMin(recommended, resourceName, containerPolicy.MinAllowed) +} + +func maybeCapToPolicyMax(recommended resource.Quantity, resourceName apiv1.ResourceName, + containerPolicy *vpa_types.ContainerResourcePolicy) (resource.Quantity, bool) { + return maybeCapToMax(recommended, resourceName, containerPolicy.MaxAllowed) +} + +func maybeCapToMin(recommended resource.Quantity, resourceName apiv1.ResourceName, + min apiv1.ResourceList) (resource.Quantity, bool) { + minResource, found := min[resourceName] + if found && !minResource.IsZero() && recommended.Cmp(minResource) < 0 { + return minResource, true + } + return recommended, false +} + +func maybeCapToMax(recommended resource.Quantity, resourceName apiv1.ResourceName, + max apiv1.ResourceList) (resource.Quantity, bool) { + maxResource, found := max[resourceName] + if found && !maxResource.IsZero() && recommended.Cmp(maxResource) > 0 { + return maxResource, true + } + return recommended, false +} + +func toCappingAnnotation(resourceName apiv1.ResourceName, action cappingAction) string { + return fmt.Sprintf("%s %s", resourceName, action) +} + +// applyMPAPolicy updates recommendation if recommended resources are outside of limits defined in VPA resources policy +func applyMPAPolicy(recommendation apiv1.ResourceList, policy *vpa_types.ContainerResourcePolicy) []string { + if policy == nil { + return nil + } + annotations := make([]string, 0) + for resourceName, recommended := range recommendation { + cappedToMin, isCapped := maybeCapToPolicyMin(recommended, resourceName, policy) + recommendation[resourceName] = cappedToMin + if isCapped { + annotations = append(annotations, toCappingAnnotation(resourceName, cappedToMinAllowed)) + } + cappedToMax, isCapped := maybeCapToPolicyMax(cappedToMin, resourceName, policy) + recommendation[resourceName] = cappedToMax + if isCapped { + annotations = append(annotations, toCappingAnnotation(resourceName, cappedToMaxAllowed)) + } + } + return annotations +} + +// capRecommendationToContainerLimit makes sure recommendation is not above current limit for the container. +// If this function makes adjustments appropriate annotations are returned. +func capRecommendationToContainerLimit(recommendation apiv1.ResourceList, container apiv1.Container) []string { + annotations := make([]string, 0) + // Iterate over limits set in the container. Unset means Infinite limit. + for resourceName, limit := range container.Resources.Limits { + recommendedValue, found := recommendation[resourceName] + if found && recommendedValue.MilliValue() > limit.MilliValue() { + recommendation[resourceName] = limit + annotations = append(annotations, toCappingAnnotation(resourceName, cappedToLimit)) + } + } + return annotations +} diff --git a/multidimensional-pod-autoscaler/pkg/utils/mpa/recommendation_processor.go b/multidimensional-pod-autoscaler/pkg/utils/mpa/recommendation_processor.go new file mode 100644 index 000000000000..abdcdf27937f --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/utils/mpa/recommendation_processor.go @@ -0,0 +1,37 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + v1 "k8s.io/api/core/v1" + mpa_types "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1alpha1" + vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" +) + +// ContainerToAnnotationsMap contains annotations per container. +type ContainerToAnnotationsMap = map[string][]string + +// RecommendationProcessor post-processes recommendation adjusting it to limits and environment context +type RecommendationProcessor interface { + // Apply processes and updates recommendation for given pod, based on container limits, + // VPA policy and possibly other internal RecommendationProcessor context. + // Must return a non-nil pointer to RecommendedPodResources or error. + Apply(podRecommendation *vpa_types.RecommendedPodResources, + policy *vpa_types.PodResourcePolicy, + conditions []mpa_types.MultidimPodAutoscalerCondition, + pod *v1.Pod) (*vpa_types.RecommendedPodResources, ContainerToAnnotationsMap, error) +} diff --git a/multidimensional-pod-autoscaler/pkg/utils/test/test_container.go b/multidimensional-pod-autoscaler/pkg/utils/test/test_container.go new file mode 100644 index 000000000000..fb0e4839b918 --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/utils/test/test_container.go @@ -0,0 +1,88 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package test + +import ( + apiv1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" +) + +type containerBuilder struct { + name string + cpuRequest *resource.Quantity + memRequest *resource.Quantity + cpuLimit *resource.Quantity + memLimit *resource.Quantity +} + +// Container returns object that helps build containers for tests. +func Container() *containerBuilder { + return &containerBuilder{} +} + +func (cb *containerBuilder) WithName(name string) *containerBuilder { + r := *cb + r.name = name + return &r +} + +func (cb *containerBuilder) WithCPURequest(cpuRequest resource.Quantity) *containerBuilder { + r := *cb + r.cpuRequest = &cpuRequest + return &r +} + +func (cb *containerBuilder) WithMemRequest(memRequest resource.Quantity) *containerBuilder { + r := *cb + r.memRequest = &memRequest + return &r +} + +func (cb *containerBuilder) WithCPULimit(cpuLimit resource.Quantity) *containerBuilder { + r := *cb + r.cpuLimit = &cpuLimit + return &r +} + +func (cb *containerBuilder) WithMemLimit(memLimit resource.Quantity) *containerBuilder { + r := *cb + r.memLimit = &memLimit + return &r +} + +func (cb *containerBuilder) Get() apiv1.Container { + container := apiv1.Container{ + Name: cb.name, + Resources: apiv1.ResourceRequirements{ + Requests: apiv1.ResourceList{}, + Limits: apiv1.ResourceList{}, + }, + } + if cb.cpuRequest != nil { + container.Resources.Requests[apiv1.ResourceCPU] = *cb.cpuRequest + } + if cb.memRequest != nil { + container.Resources.Requests[apiv1.ResourceMemory] = *cb.memRequest + } + if cb.cpuLimit != nil { + container.Resources.Limits[apiv1.ResourceCPU] = *cb.cpuLimit + } + if cb.memLimit != nil { + container.Resources.Limits[apiv1.ResourceMemory] = *cb.memLimit + } + return container +} diff --git a/multidimensional-pod-autoscaler/pkg/utils/test/test_mpa.go b/multidimensional-pod-autoscaler/pkg/utils/test/test_mpa.go new file mode 100644 index 000000000000..64ee5bcfaed0 --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/utils/test/test_mpa.go @@ -0,0 +1,289 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package test + +import ( + "time" + + autoscaling "k8s.io/api/autoscaling/v1" + core "k8s.io/api/core/v1" + meta "k8s.io/apimachinery/pkg/apis/meta/v1" + mpa_types "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1alpha1" + vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" +) + +// MultidimPodAutoscalerBuilder helps building test instances of MultidimPodAutoscaler. +type MultidimPodAutoscalerBuilder interface { + WithName(vpaName string) MultidimPodAutoscalerBuilder + WithContainer(containerName string) MultidimPodAutoscalerBuilder + WithNamespace(namespace string) MultidimPodAutoscalerBuilder + WithUpdateMode(updateMode vpa_types.UpdateMode) MultidimPodAutoscalerBuilder + WithCreationTimestamp(timestamp time.Time) MultidimPodAutoscalerBuilder + WithMinAllowed(containerName, cpu, memory string) MultidimPodAutoscalerBuilder + WithMaxAllowed(containerName, cpu, memory string) MultidimPodAutoscalerBuilder + WithControlledValues(containerName string, mode vpa_types.ContainerControlledValues) MultidimPodAutoscalerBuilder + WithScalingMode(containerName string, scalingMode vpa_types.ContainerScalingMode) MultidimPodAutoscalerBuilder + WithTarget(cpu, memory string) MultidimPodAutoscalerBuilder + WithTargetResource(resource core.ResourceName, value string) MultidimPodAutoscalerBuilder + WithLowerBound(cpu, memory string) MultidimPodAutoscalerBuilder + WithScaleTargetRef(targetRef *autoscaling.CrossVersionObjectReference) MultidimPodAutoscalerBuilder + WithUpperBound(cpu, memory string) MultidimPodAutoscalerBuilder + WithAnnotations(map[string]string) MultidimPodAutoscalerBuilder + WithRecommender(string2 string) MultidimPodAutoscalerBuilder + WithGroupVersion(gv meta.GroupVersion) MultidimPodAutoscalerBuilder + // WithEvictionRequirements([]*vpa_types.EvictionRequirement) MultidimPodAutoscalerBuilder // TODO: add this functionality + WithScalingConstraints(constraints mpa_types.ScalingConstraints) MultidimPodAutoscalerBuilder + WithHorizontalScalingConstraints(hconstraints mpa_types.HorizontalScalingConstraints) MultidimPodAutoscalerBuilder + AppendCondition(conditionType mpa_types.MultidimPodAutoscalerConditionType, + status core.ConditionStatus, reason, message string, lastTransitionTime time.Time) MultidimPodAutoscalerBuilder + AppendRecommendation(vpa_types.RecommendedContainerResources) MultidimPodAutoscalerBuilder + Get() *mpa_types.MultidimPodAutoscaler +} + +// MultidimPodAutoscaler returns a new MultidimPodAutoscalerBuilder. +func MultidimPodAutoscaler() MultidimPodAutoscalerBuilder { + return &multidimPodAutoscalerBuilder{ + groupVersion: meta.GroupVersion(mpa_types.SchemeGroupVersion), + recommendation: Recommendation(), + appendedRecommendations: []vpa_types.RecommendedContainerResources{}, + namespace: "default", + conditions: []mpa_types.MultidimPodAutoscalerCondition{}, + minAllowed: map[string]core.ResourceList{}, + maxAllowed: map[string]core.ResourceList{}, + controlledValues: map[string]*vpa_types.ContainerControlledValues{}, + scalingMode: map[string]*vpa_types.ContainerScalingMode{}, + scalingConstraints: mpa_types.ScalingConstraints{}, + } +} + +type multidimPodAutoscalerBuilder struct { + groupVersion meta.GroupVersion + mpaName string + containerNames []string + namespace string + updatePolicy *mpa_types.PodUpdatePolicy + creationTimestamp time.Time + minAllowed map[string]core.ResourceList + maxAllowed map[string]core.ResourceList + controlledValues map[string]*vpa_types.ContainerControlledValues + scalingMode map[string]*vpa_types.ContainerScalingMode + recommendation RecommendationBuilder + conditions []mpa_types.MultidimPodAutoscalerCondition + annotations map[string]string + scaleTargetRef *autoscaling.CrossVersionObjectReference + appendedRecommendations []vpa_types.RecommendedContainerResources + recommender string + scalingConstraints mpa_types.ScalingConstraints +} + +func (b *multidimPodAutoscalerBuilder) WithName(mpaName string) MultidimPodAutoscalerBuilder { + c := *b + c.mpaName = mpaName + return &c +} + +func (b *multidimPodAutoscalerBuilder) WithContainer(containerName string) MultidimPodAutoscalerBuilder { + c := *b + c.containerNames = append(c.containerNames, containerName) + return &c +} + +func (b *multidimPodAutoscalerBuilder) WithNamespace(namespace string) MultidimPodAutoscalerBuilder { + c := *b + c.namespace = namespace + return &c +} + +func (b *multidimPodAutoscalerBuilder) WithUpdateMode(updateMode vpa_types.UpdateMode) MultidimPodAutoscalerBuilder { + c := *b + if c.updatePolicy == nil { + c.updatePolicy = &mpa_types.PodUpdatePolicy{} + } + c.updatePolicy.UpdateMode = &updateMode + return &c +} + +func (b *multidimPodAutoscalerBuilder) WithCreationTimestamp(timestamp time.Time) MultidimPodAutoscalerBuilder { + c := *b + c.creationTimestamp = timestamp + return &c +} + +func (b *multidimPodAutoscalerBuilder) WithMinAllowed(containerName, cpu, memory string) MultidimPodAutoscalerBuilder { + c := *b + c.minAllowed[containerName] = Resources(cpu, memory) + return &c +} + +func (b *multidimPodAutoscalerBuilder) WithMaxAllowed(containerName, cpu, memory string) MultidimPodAutoscalerBuilder { + c := *b + c.maxAllowed[containerName] = Resources(cpu, memory) + return &c +} + +func (b *multidimPodAutoscalerBuilder) WithControlledValues(containerName string, mode vpa_types.ContainerControlledValues) MultidimPodAutoscalerBuilder { + c := *b + c.controlledValues[containerName] = &mode + return &c +} + +func (b *multidimPodAutoscalerBuilder) WithScalingMode(containerName string, scalingMode vpa_types.ContainerScalingMode) MultidimPodAutoscalerBuilder { + c := *b + c.scalingMode[containerName] = &scalingMode + return &c +} + +func (b *multidimPodAutoscalerBuilder) WithTarget(cpu, memory string) MultidimPodAutoscalerBuilder { + c := *b + c.recommendation = c.recommendation.WithTarget(cpu, memory) + return &c +} + +func (b *multidimPodAutoscalerBuilder) WithTargetResource(resource core.ResourceName, value string) MultidimPodAutoscalerBuilder { + c := *b + c.recommendation = c.recommendation.WithTargetResource(resource, value) + return &c +} + +func (b *multidimPodAutoscalerBuilder) WithLowerBound(cpu, memory string) MultidimPodAutoscalerBuilder { + c := *b + c.recommendation = c.recommendation.WithLowerBound(cpu, memory) + return &c +} + +func (b *multidimPodAutoscalerBuilder) WithUpperBound(cpu, memory string) MultidimPodAutoscalerBuilder { + c := *b + c.recommendation = c.recommendation.WithUpperBound(cpu, memory) + return &c +} + +func (b *multidimPodAutoscalerBuilder) WithScaleTargetRef(scaleTargetRef *autoscaling.CrossVersionObjectReference) MultidimPodAutoscalerBuilder { + c := *b + c.scaleTargetRef = scaleTargetRef + return &c +} + +func (b *multidimPodAutoscalerBuilder) WithAnnotations(annotations map[string]string) MultidimPodAutoscalerBuilder { + c := *b + c.annotations = annotations + return &c +} + +func (b *multidimPodAutoscalerBuilder) WithRecommender(recommender string) MultidimPodAutoscalerBuilder { + c := *b + c.recommender = recommender + return &c +} + +func (b *multidimPodAutoscalerBuilder) WithGroupVersion(gv meta.GroupVersion) MultidimPodAutoscalerBuilder { + c := *b + c.groupVersion = gv + return &c +} + +func (b *multidimPodAutoscalerBuilder) WithScalingConstraints(constraints mpa_types.ScalingConstraints) MultidimPodAutoscalerBuilder { + c := *b + c.scalingConstraints = constraints + return &c +} + +func (b *multidimPodAutoscalerBuilder) WithHorizontalScalingConstraints(constraints mpa_types.HorizontalScalingConstraints) MultidimPodAutoscalerBuilder { + c := *b + c.scalingConstraints.Global = &constraints + return &c +} + +// TODO: add this functionality +// func (b *multidimPodAutoscalerBuilder) WithEvictionRequirements(evictionRequirements []*vpa_types.EvictionRequirement) MultidimPodAutoscalerBuilder { +// updateModeAuto := vpa_types.UpdateModeAuto +// c := *b +// if c.updatePolicy == nil { +// c.updatePolicy = &mpa_types.PodUpdatePolicy{UpdateMode: &updateModeAuto} +// } +// c.updatePolicy.EvictionRequirements = evictionRequirements +// return &c +// } + +func (b *multidimPodAutoscalerBuilder) AppendCondition(conditionType mpa_types.MultidimPodAutoscalerConditionType, + status core.ConditionStatus, reason, message string, lastTransitionTime time.Time) MultidimPodAutoscalerBuilder { + c := *b + c.conditions = append(c.conditions, mpa_types.MultidimPodAutoscalerCondition{ + Type: conditionType, + Status: status, + Reason: reason, + Message: message, + LastTransitionTime: meta.NewTime(lastTransitionTime)}) + return &c +} + +func (b *multidimPodAutoscalerBuilder) AppendRecommendation(recommendation vpa_types.RecommendedContainerResources) MultidimPodAutoscalerBuilder { + c := *b + c.appendedRecommendations = append(c.appendedRecommendations, recommendation) + return &c +} + +func (b *multidimPodAutoscalerBuilder) Get() *mpa_types.MultidimPodAutoscaler { + if len(b.containerNames) == 0 { + panic("Must call WithContainer() before Get()") + } + var recommenders []*mpa_types.MultidimPodAutoscalerRecommenderSelector + if b.recommender != "" { + recommenders = []*mpa_types.MultidimPodAutoscalerRecommenderSelector{{Name: b.recommender}} + } + resourcePolicy := vpa_types.PodResourcePolicy{} + recommendation := &vpa_types.RecommendedPodResources{} + scalingModeAuto := vpa_types.ContainerScalingModeAuto + for _, containerName := range b.containerNames { + containerResourcePolicy := vpa_types.ContainerResourcePolicy{ + ContainerName: containerName, + MinAllowed: b.minAllowed[containerName], + MaxAllowed: b.maxAllowed[containerName], + ControlledValues: b.controlledValues[containerName], + Mode: &scalingModeAuto, + } + if scalingMode, ok := b.scalingMode[containerName]; ok { + containerResourcePolicy.Mode = scalingMode + } + resourcePolicy.ContainerPolicies = append(resourcePolicy.ContainerPolicies, containerResourcePolicy) + } + // MPAs with a single container may still use the old/implicit way of adding recommendations + r := b.recommendation.WithContainer(b.containerNames[0]).Get() + if r.ContainerRecommendations[0].Target != nil { + recommendation = r + } + recommendation.ContainerRecommendations = append(recommendation.ContainerRecommendations, b.appendedRecommendations...) + + return &mpa_types.MultidimPodAutoscaler{ + ObjectMeta: meta.ObjectMeta{ + Name: b.mpaName, + Namespace: b.namespace, + Annotations: b.annotations, + CreationTimestamp: meta.NewTime(b.creationTimestamp), + }, + Spec: mpa_types.MultidimPodAutoscalerSpec{ + Policy: b.updatePolicy, + ResourcePolicy: &resourcePolicy, + ScaleTargetRef: b.scaleTargetRef, + Recommenders: recommenders, + Constraints: &b.scalingConstraints, + }, + Status: mpa_types.MultidimPodAutoscalerStatus{ + Recommendation: recommendation, + Conditions: b.conditions, + }, + } +} diff --git a/multidimensional-pod-autoscaler/pkg/utils/test/test_recommendation.go b/multidimensional-pod-autoscaler/pkg/utils/test/test_recommendation.go new file mode 100644 index 000000000000..f696029d8ea8 --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/utils/test/test_recommendation.go @@ -0,0 +1,115 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package test + +import ( + apiv1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" +) + +// RecommendationBuilder helps building test instances of RecommendedPodResources. +type RecommendationBuilder interface { + WithContainer(containerName string) RecommendationBuilder + WithTarget(cpu, memory string) RecommendationBuilder + WithTargetResource(resource apiv1.ResourceName, value string) RecommendationBuilder + WithLowerBound(cpu, memory string) RecommendationBuilder + WithUpperBound(cpu, memory string) RecommendationBuilder + Get() *vpa_types.RecommendedPodResources + GetContainerResources() vpa_types.RecommendedContainerResources +} + +// Recommendation returns a new RecommendationBuilder. +func Recommendation() RecommendationBuilder { + return &recommendationBuilder{} +} + +type recommendationBuilder struct { + containerName string + target apiv1.ResourceList + lowerBound apiv1.ResourceList + upperBound apiv1.ResourceList +} + +func (b *recommendationBuilder) WithContainer(containerName string) RecommendationBuilder { + c := *b + c.containerName = containerName + return &c +} + +func (b *recommendationBuilder) WithTarget(cpu, memory string) RecommendationBuilder { + c := *b + c.target = Resources(cpu, memory) + return &c +} + +func (b *recommendationBuilder) WithTargetResource(resource apiv1.ResourceName, value string) RecommendationBuilder { + c := *b + if c.target == nil { + c.target = apiv1.ResourceList{} + } + addResource(c.target, resource, value) + return &c +} + +func (b *recommendationBuilder) WithLowerBound(cpu, memory string) RecommendationBuilder { + c := *b + c.lowerBound = Resources(cpu, memory) + return &c +} + +func (b *recommendationBuilder) WithUpperBound(cpu, memory string) RecommendationBuilder { + c := *b + c.upperBound = Resources(cpu, memory) + return &c +} + +func (b *recommendationBuilder) Get() *vpa_types.RecommendedPodResources { + if b.containerName == "" { + panic("Must call WithContainer() before Get()") + } + return &vpa_types.RecommendedPodResources{ + ContainerRecommendations: []vpa_types.RecommendedContainerResources{ + { + ContainerName: b.containerName, + Target: b.target, + UncappedTarget: b.target, + LowerBound: b.lowerBound, + UpperBound: b.upperBound, + }, + }} +} + +func (b *recommendationBuilder) GetContainerResources() vpa_types.RecommendedContainerResources { + return vpa_types.RecommendedContainerResources{ + ContainerName: b.containerName, + Target: b.target, + UncappedTarget: b.target, + LowerBound: b.lowerBound, + UpperBound: b.upperBound, + } +} + +// addResource add a resource to the given resource list +func addResource(rl apiv1.ResourceList, resourceName apiv1.ResourceName, value string) apiv1.ResourceList { + val, _ := resource.ParseQuantity(value) + if rl == nil { + rl = apiv1.ResourceList{} + } + rl[resourceName] = val + return rl +} diff --git a/multidimensional-pod-autoscaler/pkg/utils/test/test_utils.go b/multidimensional-pod-autoscaler/pkg/utils/test/test_utils.go new file mode 100644 index 000000000000..17e3e6c1878d --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/utils/test/test_utils.go @@ -0,0 +1,318 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package test + +import ( + "fmt" + "time" + + "github.com/stretchr/testify/mock" + apiv1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + mpa_types "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1alpha1" + mpa_lister "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/client/listers/autoscaling.k8s.io/v1alpha1" + vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" + vpa_types_v1beta1 "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1beta1" + vpa_lister "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/client/listers/autoscaling.k8s.io/v1" + vpa_lister_v1beta1 "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/client/listers/autoscaling.k8s.io/v1beta1" + v1 "k8s.io/client-go/listers/core/v1" + "k8s.io/client-go/tools/record" +) + +var ( + timeLayout = "2006-01-02 15:04:05" + testTimestamp, _ = time.Parse(timeLayout, "2017-04-18 17:35:05") +) + +// BuildTestContainer creates container with specified resources +func BuildTestContainer(containerName, cpu, mem string) apiv1.Container { + // TODO: Use builder directly, remove this function. + builder := Container().WithName(containerName) + + if len(cpu) > 0 { + cpuVal, _ := resource.ParseQuantity(cpu) + builder = builder.WithCPURequest(cpuVal) + } + if len(mem) > 0 { + memVal, _ := resource.ParseQuantity(mem) + builder = builder.WithMemRequest(memVal) + } + return builder.Get() +} + +// BuildTestPolicy creates ResourcesPolicy with specified constraints +func BuildTestPolicy(containerName, minCPU, maxCPU, minMemory, maxMemory string) *vpa_types.PodResourcePolicy { + minCPUVal, _ := resource.ParseQuantity(minCPU) + maxCPUVal, _ := resource.ParseQuantity(maxCPU) + minMemVal, _ := resource.ParseQuantity(minMemory) + maxMemVal, _ := resource.ParseQuantity(maxMemory) + return &vpa_types.PodResourcePolicy{ContainerPolicies: []vpa_types.ContainerResourcePolicy{{ + ContainerName: containerName, + MinAllowed: apiv1.ResourceList{ + apiv1.ResourceMemory: minMemVal, + apiv1.ResourceCPU: minCPUVal, + }, + MaxAllowed: apiv1.ResourceList{ + apiv1.ResourceMemory: maxMemVal, + apiv1.ResourceCPU: maxCPUVal, + }, + }, + }} +} + +// Resources creates a ResourceList with given amount of cpu and memory. +func Resources(cpu, mem string) apiv1.ResourceList { + result := make(apiv1.ResourceList) + if len(cpu) > 0 { + cpuVal, _ := resource.ParseQuantity(cpu) + result[apiv1.ResourceCPU] = cpuVal + } + if len(mem) > 0 { + memVal, _ := resource.ParseQuantity(mem) + result[apiv1.ResourceMemory] = memVal + } + return result +} + +// RecommenderAPIMock is a mock of RecommenderAPI +type RecommenderAPIMock struct { + mock.Mock +} + +// GetRecommendation is mock implementation of RecommenderAPI.GetRecommendation +func (m *RecommenderAPIMock) GetRecommendation(spec *apiv1.PodSpec) (*vpa_types.RecommendedPodResources, error) { + args := m.Called(spec) + var returnArg *vpa_types.RecommendedPodResources + if args.Get(0) != nil { + returnArg = args.Get(0).(*vpa_types.RecommendedPodResources) + } + return returnArg, args.Error(1) +} + +// RecommenderMock is a mock of Recommender +type RecommenderMock struct { + mock.Mock +} + +// Get is a mock implementation of Recommender.Get +func (m *RecommenderMock) Get(spec *apiv1.PodSpec) (*vpa_types.RecommendedPodResources, error) { + args := m.Called(spec) + var returnArg *vpa_types.RecommendedPodResources + if args.Get(0) != nil { + returnArg = args.Get(0).(*vpa_types.RecommendedPodResources) + } + return returnArg, args.Error(1) +} + +// PodsEvictionRestrictionMock is a mock of PodsEvictionRestriction +type PodsEvictionRestrictionMock struct { + mock.Mock +} + +// Evict is a mock implementation of PodsEvictionRestriction.Evict +func (m *PodsEvictionRestrictionMock) Evict(pod *apiv1.Pod, mpa *mpa_types.MultidimPodAutoscaler, eventRecorder record.EventRecorder) error { + args := m.Called(pod, eventRecorder) + return args.Error(0) +} + +// CanEvict is a mock implementation of PodsEvictionRestriction.CanEvict +func (m *PodsEvictionRestrictionMock) CanEvict(pod *apiv1.Pod) bool { + args := m.Called(pod) + return args.Bool(0) +} + +// PodListerMock is a mock of PodLister +type PodListerMock struct { + mock.Mock +} + +// Pods is a mock implementation of PodLister.Pods +func (m *PodListerMock) Pods(namespace string) v1.PodNamespaceLister { + args := m.Called(namespace) + var returnArg v1.PodNamespaceLister + if args.Get(0) != nil { + returnArg = args.Get(0).(v1.PodNamespaceLister) + } + return returnArg +} + +// List is a mock implementation of PodLister.List +func (m *PodListerMock) List(selector labels.Selector) (ret []*apiv1.Pod, err error) { + args := m.Called() + var returnArg []*apiv1.Pod + if args.Get(0) != nil { + returnArg = args.Get(0).([]*apiv1.Pod) + } + return returnArg, args.Error(1) +} + +// Get is not implemented for this mock +func (m *PodListerMock) Get(name string) (*apiv1.Pod, error) { + return nil, fmt.Errorf("unimplemented") +} + +// VerticalPodAutoscalerListerMock is a mock of VerticalPodAutoscalerLister or +// VerticalPodAutoscalerNamespaceLister - the crucial List method is the same. +type VerticalPodAutoscalerListerMock struct { + mock.Mock +} + +// List is a mock implementation of VerticalPodAutoscalerLister.List +func (m *VerticalPodAutoscalerListerMock) List(selector labels.Selector) (ret []*vpa_types.VerticalPodAutoscaler, err error) { + args := m.Called() + var returnArg []*vpa_types.VerticalPodAutoscaler + if args.Get(0) != nil { + returnArg = args.Get(0).([]*vpa_types.VerticalPodAutoscaler) + } + return returnArg, args.Error(1) +} + +// VerticalPodAutoscalers is a mock implementation of returning a lister for namespace. +func (m *VerticalPodAutoscalerListerMock) VerticalPodAutoscalers(namespace string) vpa_lister.VerticalPodAutoscalerNamespaceLister { + args := m.Called(namespace) + var returnArg vpa_lister.VerticalPodAutoscalerNamespaceLister + if args.Get(0) != nil { + returnArg = args.Get(0).(vpa_lister.VerticalPodAutoscalerNamespaceLister) + } + return returnArg +} + +// Get is not implemented for this mock +func (m *VerticalPodAutoscalerListerMock) Get(name string) (*vpa_types.VerticalPodAutoscaler, error) { + return nil, fmt.Errorf("unimplemented") +} + +// MultidimPodAutoscalerListerMock is a mock of MultidimPodAutoscalerLister or +// MultidimPodAutoscalerNamespaceLister - the crucial List method is the same. +type MultidimPodAutoscalerListerMock struct { + mock.Mock +} + +// List is a mock implementation of MultidimPodAutoscalerLister.List +func (m *MultidimPodAutoscalerListerMock) List(selector labels.Selector) (ret []*mpa_types.MultidimPodAutoscaler, err error) { + args := m.Called() + var returnArg []*mpa_types.MultidimPodAutoscaler + if args.Get(0) != nil { + returnArg = args.Get(0).([]*mpa_types.MultidimPodAutoscaler) + } + return returnArg, args.Error(1) +} + +// MultidimPodAutoscalers is a mock implementation of returning a lister for namespace. +func (m *MultidimPodAutoscalerListerMock) MultidimPodAutoscalers(namespace string) mpa_lister.MultidimPodAutoscalerNamespaceLister { + args := m.Called(namespace) + var returnArg mpa_lister.MultidimPodAutoscalerNamespaceLister + if args.Get(0) != nil { + returnArg = args.Get(0).(mpa_lister.MultidimPodAutoscalerNamespaceLister) + } + return returnArg +} + +// Get is not implemented for this mock +func (m *MultidimPodAutoscalerListerMock) Get(name string) (*mpa_types.MultidimPodAutoscaler, error) { + return nil, fmt.Errorf("unimplemented") +} + +// VerticalPodAutoscalerV1Beta1ListerMock is a mock of VerticalPodAutoscalerLister or +// VerticalPodAutoscalerNamespaceLister - the crucial List method is the same. +type VerticalPodAutoscalerV1Beta1ListerMock struct { + mock.Mock +} + +// List is a mock implementation of VerticalPodAutoscalerLister.List +func (m *VerticalPodAutoscalerV1Beta1ListerMock) List(selector labels.Selector) (ret []*vpa_types_v1beta1.VerticalPodAutoscaler, err error) { + args := m.Called() + var returnArg []*vpa_types_v1beta1.VerticalPodAutoscaler + if args.Get(0) != nil { + returnArg = args.Get(0).([]*vpa_types_v1beta1.VerticalPodAutoscaler) + } + return returnArg, args.Error(1) +} + +// VerticalPodAutoscalers is a mock implementation of returning a lister for namespace. +func (m *VerticalPodAutoscalerV1Beta1ListerMock) VerticalPodAutoscalers(namespace string) vpa_lister_v1beta1.VerticalPodAutoscalerNamespaceLister { + args := m.Called(namespace) + var returnArg vpa_lister_v1beta1.VerticalPodAutoscalerNamespaceLister + if args.Get(0) != nil { + returnArg = args.Get(0).(vpa_lister_v1beta1.VerticalPodAutoscalerNamespaceLister) + } + return returnArg +} + +// Get is not implemented for this mock +func (m *VerticalPodAutoscalerV1Beta1ListerMock) Get(name string) (*vpa_types_v1beta1.VerticalPodAutoscaler, error) { + return nil, fmt.Errorf("unimplemented") +} + +// RecommendationProcessorMock is mock implementation of RecommendationProcessor +type RecommendationProcessorMock struct { + mock.Mock +} + +// Apply is a mock implementation of RecommendationProcessor.Apply +func (m *RecommendationProcessorMock) Apply(podRecommendation *vpa_types.RecommendedPodResources, + policy *vpa_types.PodResourcePolicy, + conditions []mpa_types.MultidimPodAutoscalerCondition, + pod *apiv1.Pod) (*vpa_types.RecommendedPodResources, map[string][]string, error) { + args := m.Called() + var returnArg *vpa_types.RecommendedPodResources + if args.Get(0) != nil { + returnArg = args.Get(0).(*vpa_types.RecommendedPodResources) + } + var annotations map[string][]string + if args.Get(1) != nil { + annotations = args.Get(1).(map[string][]string) + } + return returnArg, annotations, args.Error(1) +} + +// FakeRecommendationProcessor is a dummy implementation of RecommendationProcessor +type FakeRecommendationProcessor struct{} + +// Apply is a dummy implementation of RecommendationProcessor.Apply which returns provided podRecommendation +func (f *FakeRecommendationProcessor) Apply(podRecommendation *vpa_types.RecommendedPodResources, + policy *vpa_types.PodResourcePolicy, + conditions []mpa_types.MultidimPodAutoscalerCondition, + pod *apiv1.Pod) (*vpa_types.RecommendedPodResources, map[string][]string, error) { + return podRecommendation, nil, nil +} + +// fakeEventRecorder is a dummy implementation of record.EventRecorder. +type fakeEventRecorder struct{} + +// Event is a dummy implementation of record.EventRecorder interface. +func (f *fakeEventRecorder) Event(object runtime.Object, eventtype, reason, message string) {} + +// Eventf is a dummy implementation of record.EventRecorder interface. +func (f *fakeEventRecorder) Eventf(object runtime.Object, eventtype, reason, messageFmt string, args ...interface{}) { +} + +// PastEventf is a dummy implementation of record.EventRecorder interface. +func (f *fakeEventRecorder) PastEventf(object runtime.Object, timestamp metav1.Time, eventtype, reason, messageFmt string, args ...interface{}) { +} + +// AnnotatedEventf is a dummy implementation of record.EventRecorder interface. +func (f *fakeEventRecorder) AnnotatedEventf(object runtime.Object, annotations map[string]string, eventtype, reason, messageFmt string, args ...interface{}) { +} + +// FakeEventRecorder returns a dummy implementation of record.EventRecorder. +func FakeEventRecorder() record.EventRecorder { + return &fakeEventRecorder{} +} diff --git a/vertical-pod-autoscaler/pkg/recommender/logic/recommender.go b/vertical-pod-autoscaler/pkg/recommender/logic/recommender.go index 8aaf9cbd5e91..a38bfe6dfb77 100644 --- a/vertical-pod-autoscaler/pkg/recommender/logic/recommender.go +++ b/vertical-pod-autoscaler/pkg/recommender/logic/recommender.go @@ -37,6 +37,11 @@ var ( humanizeMemory = flag.Bool("humanize-memory", false, "Convert memory values in recommendations to the highest appropriate SI unit with up to 2 decimal places for better readability.") ) +// GetHumanizeMemory returns the value of the HumanizeMemory flag. +func GetHumanizeMemory() bool { + return *humanizeMemory +} + // PodResourceRecommender computes resource recommendation for a Vpa object. type PodResourceRecommender interface { GetRecommendedPodResources(containerNameToAggregateStateMap model.ContainerNameToAggregateStateMap) RecommendedPodResources