diff --git a/README.md b/README.md index f0fc0b11..30af4b22 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,6 @@ [![Operating](https://github.com/KusionStack/operating/actions/workflows/release.yaml/badge.svg)](https://github.com/KusionStack/operating/actions/workflows/release.yaml) [![GitHub release](https://img.shields.io/github/release/KusionStack/operating.svg)](https://github.com/KusionStack/operating/releases) [![Go Report Card](https://goreportcard.com/badge/github.com/KusionStack/operating)](https://goreportcard.com/report/github.com/KusionStack/operating) -[![codecov](https://codecov.io/gh/KusionStack/operating/branch/master/graph/badge.svg)](https://codecov.io/gh/KusionStack/operating) KusionStack Operating ([official site](https://kusionstack.io/docs/operating/introduction/)) provides a set of workloads and operators built on Kubernetes Custom Resource Definitions (CRDs), with a primary aim of bridging the gap diff --git a/apis/apps/v1alpha1/default.go b/apis/apps/v1alpha1/default.go new file mode 100644 index 00000000..b5d46dd6 --- /dev/null +++ b/apis/apps/v1alpha1/default.go @@ -0,0 +1,253 @@ +/* +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + k8scorev1 "k8s.io/kubernetes/pkg/apis/core/v1" +) + +func SetDetaultCollaSet(cls *CollaSet) { + SetDefaultPodSpec(cls) + SetDefaultCollaSetUpdateStrategy(cls) +} + +func SetDefaultPodSpec(in *CollaSet) { + k8scorev1.SetDefaults_PodSpec(&in.Spec.Template.Spec) + for i := range in.Spec.Template.Spec.Volumes { + a := &in.Spec.Template.Spec.Volumes[i] + k8scorev1.SetDefaults_Volume(a) + if a.VolumeSource.HostPath != nil { + k8scorev1.SetDefaults_HostPathVolumeSource(a.VolumeSource.HostPath) + } + if a.VolumeSource.Secret != nil { + k8scorev1.SetDefaults_SecretVolumeSource(a.VolumeSource.Secret) + } + if a.VolumeSource.ISCSI != nil { + k8scorev1.SetDefaults_ISCSIVolumeSource(a.VolumeSource.ISCSI) + } + if a.VolumeSource.RBD != nil { + k8scorev1.SetDefaults_RBDVolumeSource(a.VolumeSource.RBD) + } + if a.VolumeSource.DownwardAPI != nil { + k8scorev1.SetDefaults_DownwardAPIVolumeSource(a.VolumeSource.DownwardAPI) + for j := range a.VolumeSource.DownwardAPI.Items { + b := &a.VolumeSource.DownwardAPI.Items[j] + if b.FieldRef != nil { + k8scorev1.SetDefaults_ObjectFieldSelector(b.FieldRef) + } + } + } + if a.VolumeSource.ConfigMap != nil { + k8scorev1.SetDefaults_ConfigMapVolumeSource(a.VolumeSource.ConfigMap) + } + if a.VolumeSource.AzureDisk != nil { + k8scorev1.SetDefaults_AzureDiskVolumeSource(a.VolumeSource.AzureDisk) + } + if a.VolumeSource.Projected != nil { + k8scorev1.SetDefaults_ProjectedVolumeSource(a.VolumeSource.Projected) + for j := range a.VolumeSource.Projected.Sources { + b := &a.VolumeSource.Projected.Sources[j] + if b.DownwardAPI != nil { + for k := range b.DownwardAPI.Items { + c := &b.DownwardAPI.Items[k] + if c.FieldRef != nil { + k8scorev1.SetDefaults_ObjectFieldSelector(c.FieldRef) + } + } + } + if b.ServiceAccountToken != nil { + k8scorev1.SetDefaults_ServiceAccountTokenProjection(b.ServiceAccountToken) + } + } + } + if a.VolumeSource.ScaleIO != nil { + k8scorev1.SetDefaults_ScaleIOVolumeSource(a.VolumeSource.ScaleIO) + } + if a.VolumeSource.Ephemeral != nil { + if a.VolumeSource.Ephemeral.VolumeClaimTemplate != nil { + k8scorev1.SetDefaults_PersistentVolumeClaimSpec(&a.VolumeSource.Ephemeral.VolumeClaimTemplate.Spec) + k8scorev1.SetDefaults_ResourceList(&a.VolumeSource.Ephemeral.VolumeClaimTemplate.Spec.Resources.Limits) + k8scorev1.SetDefaults_ResourceList(&a.VolumeSource.Ephemeral.VolumeClaimTemplate.Spec.Resources.Requests) + } + } + } + for i := range in.Spec.Template.Spec.InitContainers { + a := &in.Spec.Template.Spec.InitContainers[i] + k8scorev1.SetDefaults_Container(a) + for j := range a.Ports { + b := &a.Ports[j] + if b.Protocol == "" { + b.Protocol = "TCP" + } + } + for j := range a.Env { + b := &a.Env[j] + if b.ValueFrom != nil { + if b.ValueFrom.FieldRef != nil { + k8scorev1.SetDefaults_ObjectFieldSelector(b.ValueFrom.FieldRef) + } + } + } + k8scorev1.SetDefaults_ResourceList(&a.Resources.Limits) + k8scorev1.SetDefaults_ResourceList(&a.Resources.Requests) + if a.LivenessProbe != nil { + k8scorev1.SetDefaults_Probe(a.LivenessProbe) + if a.LivenessProbe.Handler.HTTPGet != nil { + k8scorev1.SetDefaults_HTTPGetAction(a.LivenessProbe.Handler.HTTPGet) + } + } + if a.ReadinessProbe != nil { + k8scorev1.SetDefaults_Probe(a.ReadinessProbe) + if a.ReadinessProbe.Handler.HTTPGet != nil { + k8scorev1.SetDefaults_HTTPGetAction(a.ReadinessProbe.Handler.HTTPGet) + } + } + if a.StartupProbe != nil { + k8scorev1.SetDefaults_Probe(a.StartupProbe) + if a.StartupProbe.Handler.HTTPGet != nil { + k8scorev1.SetDefaults_HTTPGetAction(a.StartupProbe.Handler.HTTPGet) + } + } + if a.Lifecycle != nil { + if a.Lifecycle.PostStart != nil { + if a.Lifecycle.PostStart.HTTPGet != nil { + k8scorev1.SetDefaults_HTTPGetAction(a.Lifecycle.PostStart.HTTPGet) + } + } + if a.Lifecycle.PreStop != nil { + if a.Lifecycle.PreStop.HTTPGet != nil { + k8scorev1.SetDefaults_HTTPGetAction(a.Lifecycle.PreStop.HTTPGet) + } + } + } + } + for i := range in.Spec.Template.Spec.Containers { + a := &in.Spec.Template.Spec.Containers[i] + k8scorev1.SetDefaults_Container(a) + for j := range a.Ports { + b := &a.Ports[j] + if b.Protocol == "" { + b.Protocol = "TCP" + } + } + for j := range a.Env { + b := &a.Env[j] + if b.ValueFrom != nil { + if b.ValueFrom.FieldRef != nil { + k8scorev1.SetDefaults_ObjectFieldSelector(b.ValueFrom.FieldRef) + } + } + } + k8scorev1.SetDefaults_ResourceList(&a.Resources.Limits) + k8scorev1.SetDefaults_ResourceList(&a.Resources.Requests) + if a.LivenessProbe != nil { + k8scorev1.SetDefaults_Probe(a.LivenessProbe) + if a.LivenessProbe.Handler.HTTPGet != nil { + k8scorev1.SetDefaults_HTTPGetAction(a.LivenessProbe.Handler.HTTPGet) + } + } + if a.ReadinessProbe != nil { + k8scorev1.SetDefaults_Probe(a.ReadinessProbe) + if a.ReadinessProbe.Handler.HTTPGet != nil { + k8scorev1.SetDefaults_HTTPGetAction(a.ReadinessProbe.Handler.HTTPGet) + } + } + if a.StartupProbe != nil { + k8scorev1.SetDefaults_Probe(a.StartupProbe) + if a.StartupProbe.Handler.HTTPGet != nil { + k8scorev1.SetDefaults_HTTPGetAction(a.StartupProbe.Handler.HTTPGet) + } + } + if a.Lifecycle != nil { + if a.Lifecycle.PostStart != nil { + if a.Lifecycle.PostStart.HTTPGet != nil { + k8scorev1.SetDefaults_HTTPGetAction(a.Lifecycle.PostStart.HTTPGet) + } + } + if a.Lifecycle.PreStop != nil { + if a.Lifecycle.PreStop.HTTPGet != nil { + k8scorev1.SetDefaults_HTTPGetAction(a.Lifecycle.PreStop.HTTPGet) + } + } + } + } + for i := range in.Spec.Template.Spec.EphemeralContainers { + a := &in.Spec.Template.Spec.EphemeralContainers[i] + k8scorev1.SetDefaults_EphemeralContainer(a) + for j := range a.EphemeralContainerCommon.Ports { + b := &a.EphemeralContainerCommon.Ports[j] + if b.Protocol == "" { + b.Protocol = "TCP" + } + } + for j := range a.EphemeralContainerCommon.Env { + b := &a.EphemeralContainerCommon.Env[j] + if b.ValueFrom != nil { + if b.ValueFrom.FieldRef != nil { + k8scorev1.SetDefaults_ObjectFieldSelector(b.ValueFrom.FieldRef) + } + } + } + k8scorev1.SetDefaults_ResourceList(&a.EphemeralContainerCommon.Resources.Limits) + k8scorev1.SetDefaults_ResourceList(&a.EphemeralContainerCommon.Resources.Requests) + if a.EphemeralContainerCommon.LivenessProbe != nil { + k8scorev1.SetDefaults_Probe(a.EphemeralContainerCommon.LivenessProbe) + if a.EphemeralContainerCommon.LivenessProbe.Handler.HTTPGet != nil { + k8scorev1.SetDefaults_HTTPGetAction(a.EphemeralContainerCommon.LivenessProbe.Handler.HTTPGet) + } + } + if a.EphemeralContainerCommon.ReadinessProbe != nil { + k8scorev1.SetDefaults_Probe(a.EphemeralContainerCommon.ReadinessProbe) + if a.EphemeralContainerCommon.ReadinessProbe.Handler.HTTPGet != nil { + k8scorev1.SetDefaults_HTTPGetAction(a.EphemeralContainerCommon.ReadinessProbe.Handler.HTTPGet) + } + } + if a.EphemeralContainerCommon.StartupProbe != nil { + k8scorev1.SetDefaults_Probe(a.EphemeralContainerCommon.StartupProbe) + if a.EphemeralContainerCommon.StartupProbe.Handler.HTTPGet != nil { + k8scorev1.SetDefaults_HTTPGetAction(a.EphemeralContainerCommon.StartupProbe.Handler.HTTPGet) + } + } + if a.EphemeralContainerCommon.Lifecycle != nil { + if a.EphemeralContainerCommon.Lifecycle.PostStart != nil { + if a.EphemeralContainerCommon.Lifecycle.PostStart.HTTPGet != nil { + k8scorev1.SetDefaults_HTTPGetAction(a.EphemeralContainerCommon.Lifecycle.PostStart.HTTPGet) + } + } + if a.EphemeralContainerCommon.Lifecycle.PreStop != nil { + if a.EphemeralContainerCommon.Lifecycle.PreStop.HTTPGet != nil { + k8scorev1.SetDefaults_HTTPGetAction(a.EphemeralContainerCommon.Lifecycle.PreStop.HTTPGet) + } + } + } + } + k8scorev1.SetDefaults_ResourceList(&in.Spec.Template.Spec.Overhead) +} + +func SetDefaultCollaSetUpdateStrategy(cls *CollaSet) { + if cls.Spec.UpdateStrategy.PodUpdatePolicy == "" { + cls.Spec.UpdateStrategy.PodUpdatePolicy = CollaSetInPlaceIfPossiblePodUpdateStrategyType + } + + if cls.Spec.UpdateStrategy.RollingUpdate == nil { + cls.Spec.UpdateStrategy.RollingUpdate = &RollingUpdateCollaSetStrategy{} + } + + if cls.Spec.UpdateStrategy.RollingUpdate.ByPartition == nil && cls.Spec.UpdateStrategy.RollingUpdate.ByLabel == nil { + cls.Spec.UpdateStrategy.RollingUpdate.ByPartition = &ByPartition{} + } +} diff --git a/pkg/controllers/collaset/collaset_controller.go b/pkg/controllers/collaset/collaset_controller.go index 8b049017..8bdd5634 100644 --- a/pkg/controllers/collaset/collaset_controller.go +++ b/pkg/controllers/collaset/collaset_controller.go @@ -210,7 +210,7 @@ func calculateStatus(instance *appsv1alpha1.CollaSet, newStatus *appsv1alpha1.Co replicas++ isUpdated := false - if isUpdated = controllerutils.IsPodUpdatedRevision(podWrapper.Pod, updatedRevision.Name); isUpdated { + if isUpdated = utils.IsPodUpdatedRevision(podWrapper.Pod, updatedRevision.Name); isUpdated { updatedReplicas++ } @@ -229,7 +229,7 @@ func calculateStatus(instance *appsv1alpha1.CollaSet, newStatus *appsv1alpha1.Co } } - if controllerutils.IsServiceAvailable(podWrapper.Pod) { + if controllerutils.IsPodServiceAvailable(podWrapper.Pod) { availableReplicas++ if isUpdated { updatedAvailableReplicas++ diff --git a/pkg/controllers/collaset/collaset_controller_test.go b/pkg/controllers/collaset/collaset_controller_test.go index 3d0774d4..835e33a2 100644 --- a/pkg/controllers/collaset/collaset_controller_test.go +++ b/pkg/controllers/collaset/collaset_controller_test.go @@ -117,7 +117,7 @@ var _ = Describe("collaset controller", func() { Eventually(func() bool { Expect(c.Get(context.TODO(), types.NamespacedName{Namespace: podToDelete.Namespace, Name: podToDelete.Name}, podToDelete)).Should(BeNil()) return podToDelete.DeletionTimestamp != nil - }, 500*time.Second, 1*time.Second).Should(BeTrue()) + }, 5*time.Second, 1*time.Second).Should(BeTrue()) // there should be 3 pods and one of them is terminating Eventually(func() bool { diff --git a/pkg/controllers/collaset/revision.go b/pkg/controllers/collaset/revision.go index f4cbdf27..8877fb47 100644 --- a/pkg/controllers/collaset/revision.go +++ b/pkg/controllers/collaset/revision.go @@ -22,6 +22,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" appsalphav1 "kusionstack.io/operating/apis/apps/v1alpha1" + "kusionstack.io/operating/pkg/controllers/utils/revision" ) func getCollaSetPatch(cls *appsalphav1.CollaSet) ([]byte, error) { @@ -52,6 +53,8 @@ func getCollaSetPatch(cls *appsalphav1.CollaSet) ([]byte, error) { return patch, err } +var _ revision.OwnerAdapter = &revisionOwnerAdapter{} + type revisionOwnerAdapter struct { } @@ -75,16 +78,6 @@ func (roa *revisionOwnerAdapter) GetPatch(obj metav1.Object) ([]byte, error) { return getCollaSetPatch(cs) } -func (roa *revisionOwnerAdapter) GetSelectorLabels(obj metav1.Object) map[string]string { - ips, _ := obj.(*appsalphav1.CollaSet) - labels := map[string]string{} - for k, v := range ips.Spec.Template.Labels { - labels[k] = v - } - - return labels -} - func (roa *revisionOwnerAdapter) GetCurrentRevision(obj metav1.Object) string { ips, _ := obj.(*appsalphav1.CollaSet) return ips.Status.CurrentRevision diff --git a/pkg/controllers/collaset/synccontrol/scale.go b/pkg/controllers/collaset/synccontrol/scale.go index 2cfc953c..50c09f65 100644 --- a/pkg/controllers/collaset/synccontrol/scale.go +++ b/pkg/controllers/collaset/synccontrol/scale.go @@ -20,7 +20,6 @@ import ( "sort" collasetutils "kusionstack.io/operating/pkg/controllers/collaset/utils" - controllerutils "kusionstack.io/operating/pkg/controllers/utils" "kusionstack.io/operating/pkg/controllers/utils/podopslifecycle" ) @@ -52,5 +51,5 @@ func (s ActivePodsForDeletion) Less(i, j int) bool { return false } - return controllerutils.ComparePod(l.Pod, r.Pod) + return collasetutils.ComparePod(l.Pod, r.Pod) } diff --git a/pkg/controllers/collaset/synccontrol/update.go b/pkg/controllers/collaset/synccontrol/update.go index d5f440df..7c11d4d5 100644 --- a/pkg/controllers/collaset/synccontrol/update.go +++ b/pkg/controllers/collaset/synccontrol/update.go @@ -25,10 +25,10 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/equality" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + appsv1alpha1 "kusionstack.io/operating/apis/apps/v1alpha1" "kusionstack.io/operating/pkg/controllers/collaset/utils" collasetutils "kusionstack.io/operating/pkg/controllers/collaset/utils" - controllerutils "kusionstack.io/operating/pkg/controllers/utils" "kusionstack.io/operating/pkg/controllers/utils/podopslifecycle" ) @@ -134,7 +134,7 @@ func (o orderByDefault) Less(i, j int) bool { return false } - return controllerutils.ComparePod(l.Pod, r.Pod) + return utils.ComparePod(l.Pod, r.Pod) } type PodUpdater interface { @@ -191,7 +191,7 @@ func (u *InPlaceIfPossibleUpdater) AnalyseAndGetUpdatedPod(cls *appsv1alpha1.Col } inPlaceUpdateSupport = true - updatedPod, err = controllerutils.PatchToPod(currentPod, updatedPod, podUpdateInfo.Pod) + updatedPod, err = utils.PatchToPod(currentPod, updatedPod, podUpdateInfo.Pod) if onlyMetadataChanged { if updatedPod.Annotations != nil { diff --git a/pkg/controllers/collaset/utils/pod.go b/pkg/controllers/collaset/utils/pod.go index 4e41ccc3..80826390 100644 --- a/pkg/controllers/collaset/utils/pod.go +++ b/pkg/controllers/collaset/utils/pod.go @@ -1,4 +1,5 @@ /* +Copyright 2014 The Kubernetes Authors. Copyright 2023 The KusionStack Authors. Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,15 +18,20 @@ limitations under the License. package utils import ( + "encoding/json" "fmt" "strconv" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + apimachineryvalidation "k8s.io/apimachinery/pkg/api/validation" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/strategicpatch" appsv1alpha1 "kusionstack.io/operating/apis/apps/v1alpha1" controllerutils "kusionstack.io/operating/pkg/controllers/utils" + revisionutils "kusionstack.io/operating/pkg/controllers/utils/revision" "kusionstack.io/operating/pkg/utils" ) @@ -64,11 +70,172 @@ func GetPodInstanceID(pod *corev1.Pod) (int, error) { } func NewPodFrom(owner metav1.Object, ownerRef *metav1.OwnerReference, revision *appsv1.ControllerRevision) (*corev1.Pod, error) { - pod, err := controllerutils.NewPodFrom(owner, ownerRef, revision) + pod, err := GetPodFromRevision(revision) if err != nil { - return nil, err + return pod, err } + pod.Namespace = owner.GetNamespace() + pod.GenerateName = GetPodsPrefix(owner.GetName()) + pod.OwnerReferences = append(pod.OwnerReferences, *ownerRef) + + pod.Labels[appsv1.ControllerRevisionHashLabelKey] = revision.Name + utils.ControllByKusionStack(pod) return pod, nil } + +func GetPodRevisionPatch(revision *appsv1.ControllerRevision) ([]byte, error) { + var raw map[string]interface{} + if err := json.Unmarshal([]byte(revision.Data.Raw), &raw); err != nil { + return nil, err + } + + spec := raw["spec"].(map[string]interface{}) + template := spec["template"].(map[string]interface{}) + patch, err := json.Marshal(template) + return patch, err +} + +func ApplyPatchFromRevision(pod *corev1.Pod, revision *appsv1.ControllerRevision) (*corev1.Pod, error) { + patch, err := GetPodRevisionPatch(revision) + if err != nil { + return nil, err + } + + clone := pod.DeepCopy() + patched, err := strategicpatch.StrategicMergePatch([]byte(runtime.EncodeOrDie(revisionutils.PodCodec, clone)), patch, clone) + if err != nil { + return nil, err + } + err = json.Unmarshal(patched, clone) + if err != nil { + return nil, err + } + return clone, nil +} + +// PatchToPod Use three way merge to get a updated pod. +func PatchToPod(currentRevisionPod, updateRevisionPod, currentPod *corev1.Pod) (*corev1.Pod, error) { + currentRevisionPodBytes, err := json.Marshal(currentRevisionPod) + if err != nil { + return nil, err + } + updateRevisionPodBytes, err := json.Marshal(updateRevisionPod) + + if err != nil { + return nil, err + } + + // 1. find the extra changes based on current revision + patch, err := strategicpatch.CreateTwoWayMergePatch(currentRevisionPodBytes, updateRevisionPodBytes, &corev1.Pod{}) + if err != nil { + return nil, err + } + + // 2. apply above changes to current pod + // We don't apply the diff between currentPod and currentRevisionPod to updateRevisionPod, + // because the PodTemplate changes should have the highest priority. + currentPodBytes, err := json.Marshal(currentPod) + if err != nil { + return nil, err + } + if updateRevisionPodBytes, err = strategicpatch.StrategicMergePatch(currentPodBytes, patch, &corev1.Pod{}); err != nil { + return nil, err + } + + newPod := &corev1.Pod{} + err = json.Unmarshal(updateRevisionPodBytes, newPod) + return newPod, err +} + +func GetPodFromRevision(revision *appsv1.ControllerRevision) (*corev1.Pod, error) { + pod, err := ApplyPatchFromRevision(&corev1.Pod{}, revision) + if err != nil { + return nil, err + } + + return pod, nil +} + +func GetPodsPrefix(controllerName string) string { + // use the dash (if the name isn't too long) to make the pod name a bit prettier + prefix := fmt.Sprintf("%s-", controllerName) + if len(apimachineryvalidation.NameIsDNSSubdomain(prefix, true)) != 0 { + prefix = controllerName + } + return prefix +} + +func ComparePod(l, r *corev1.Pod) bool { + // 1. Unassigned < assigned + // If only one of the pods is unassigned, the unassigned one is smaller + if l.Spec.NodeName != r.Spec.NodeName && (len(l.Spec.NodeName) == 0 || len(r.Spec.NodeName) == 0) { + return len(l.Spec.NodeName) == 0 + } + // 2. PodPending < PodUnknown < PodRunning + m := map[corev1.PodPhase]int{corev1.PodPending: 0, corev1.PodUnknown: 1, corev1.PodRunning: 2} + if m[l.Status.Phase] != m[r.Status.Phase] { + return m[l.Status.Phase] < m[r.Status.Phase] + } + // 3. Not ready < ready + // If only one of the pods is not ready, the not ready one is smaller + if controllerutils.IsPodReady(l) != controllerutils.IsPodReady(r) { + return !controllerutils.IsPodReady(l) + } + // TODO: take availability into account when we push minReadySeconds information from deployment into pods, + // see https://github.com/kubernetes/kubernetes/issues/22065 + // 4. Been ready for empty time < less time < more time + // If both pods are ready, the latest ready one is smaller + if controllerutils.IsPodReady(l) && controllerutils.IsPodReady(r) && !podReadyTime(l).Equal(podReadyTime(r)) { + return afterOrZero(podReadyTime(l), podReadyTime(r)) + } + // 5. Pods with containers with higher restart counts < lower restart counts + if maxContainerRestarts(l) != maxContainerRestarts(r) { + return maxContainerRestarts(l) > maxContainerRestarts(r) + } + // 6. Empty creation time pods < newer pods < older pods + if !l.CreationTimestamp.Equal(&r.CreationTimestamp) { + return afterOrZero(&l.CreationTimestamp, &r.CreationTimestamp) + } + return false +} + +func maxContainerRestarts(pod *corev1.Pod) int { + var maxRestarts int32 + for _, c := range pod.Status.ContainerStatuses { + if c.RestartCount > maxRestarts { + maxRestarts = c.RestartCount + } + } + return int(maxRestarts) +} + +func podReadyTime(pod *corev1.Pod) *metav1.Time { + if controllerutils.IsPodReady(pod) { + for _, c := range pod.Status.Conditions { + // we only care about pod ready conditions + if c.Type == corev1.PodReady && c.Status == corev1.ConditionTrue { + return &c.LastTransitionTime + } + } + } + return &metav1.Time{} +} + +// afterOrZero checks if time t1 is after time t2; if one of them +// is zero, the zero time is seen as after non-zero time. +func afterOrZero(t1, t2 *metav1.Time) bool { + if t1.Time.IsZero() || t2.Time.IsZero() { + return t1.Time.IsZero() + } + return t1.After(t2.Time) +} + +func IsPodUpdatedRevision(pod *corev1.Pod, revision string) bool { + if pod.Labels == nil { + return false + } + + return pod.Labels[appsv1.ControllerRevisionHashLabelKey] == revision +} diff --git a/pkg/controllers/podopslifecycle/podopslifecycle_controller.go b/pkg/controllers/podopslifecycle/podopslifecycle_controller.go index 4bac2547..2d504ac4 100644 --- a/pkg/controllers/podopslifecycle/podopslifecycle_controller.go +++ b/pkg/controllers/podopslifecycle/podopslifecycle_controller.go @@ -193,7 +193,7 @@ func (r *ReconcilePodOpsLifecycle) addServiceAvailable(pod *corev1.Pod) (bool, e return false, nil } - satisfied, _, err := controllerutils.SatisfyExpectedFinalizers(pod) // whether all expected finalizers are satisfied + satisfied, _, err := controllerutils.IsExpectedFinalizerSatisfied(pod) // whether all expected finalizers are satisfied if err != nil || !satisfied { return false, err } @@ -362,7 +362,7 @@ func (r *ReconcilePodOpsLifecycle) initPodTransitionRuleManager() { return labels != nil && labelHasPrefix(labels, v1alpha1.PodPostCheckLabelPrefix) }) podtransitionrule.AddUnAvailableFunc(func(po *corev1.Pod) (bool, *int64) { - return !controllerutils.IsServiceAvailable(po), nil + return !controllerutils.IsPodServiceAvailable(po), nil }) } diff --git a/pkg/controllers/utils/expectations/active_expectation_test.go b/pkg/controllers/utils/expectations/active_expectation_test.go index c995fca9..b7fd46e0 100644 --- a/pkg/controllers/utils/expectations/active_expectation_test.go +++ b/pkg/controllers/utils/expectations/active_expectation_test.go @@ -17,6 +17,7 @@ limitations under the License. package expectations import ( + "fmt" "testing" "time" @@ -95,11 +96,18 @@ func TestActiveExpectations(t *testing.T) { }, } g.Expect(client.Create(context.TODO(), podB)).Should(gomega.BeNil()) - time.Sleep(3 * time.Second) - - sa, err = exp.IsSatisfied(pod) - g.Expect(err).Should(gomega.BeNil()) - g.Expect(sa).Should(gomega.BeTrue()) + g.Eventually(func() error { + sa, err = exp.IsSatisfied(pod) + if err != nil { + return err + } + + if !sa { + return fmt.Errorf("expectation unsatisfied") + } + + return nil + }, 5*time.Second, 1*time.Second).Should(gomega.BeNil()) expectation, err = exp.GetExpectation(pod.Namespace, pod.Name) g.Expect(err).Should(gomega.BeNil()) g.Expect(expectation).Should(gomega.BeNil()) @@ -109,12 +117,25 @@ func TestActiveExpectations(t *testing.T) { "test": "test", } g.Expect(client.Update(context.TODO(), podB)).Should(gomega.BeNil()) - exp.ExpectUpdate(pod, Pod, pod.Name, pod.ResourceVersion) - time.Sleep(3 * time.Second) - - sa, err = exp.IsSatisfied(pod) - g.Expect(err).Should(gomega.BeNil()) - g.Expect(sa).Should(gomega.BeTrue()) + g.Expect(exp.ExpectUpdate(pod, Pod, pod.Name, pod.ResourceVersion)).Should(gomega.BeNil()) + // update twice + podB.Labels = map[string]string{ + "test": "foo", + } + g.Expect(client.Update(context.TODO(), podB)).Should(gomega.BeNil()) + g.Expect(exp.ExpectUpdate(pod, Pod, pod.Name, pod.ResourceVersion)).Should(gomega.BeNil()) + g.Eventually(func() error { + sa, err = exp.IsSatisfied(pod) + if err != nil { + return err + } + + if !sa { + return fmt.Errorf("expectation unsatisfied") + } + + return nil + }, 5*time.Second, 1*time.Second).Should(gomega.BeNil()) expectation, err = exp.GetExpectation(pod.Namespace, pod.Name) g.Expect(err).Should(gomega.BeNil()) g.Expect(expectation).Should(gomega.BeNil()) @@ -136,11 +157,18 @@ func TestActiveExpectations(t *testing.T) { g.Expect(len(expectation.items.List())).Should(gomega.BeEquivalentTo(1)) g.Expect(client.Delete(context.TODO(), podB)).Should(gomega.BeNil()) - time.Sleep(3 * time.Second) - - sa, err = exp.IsSatisfied(pod) - g.Expect(err).Should(gomega.BeNil()) - g.Expect(sa).Should(gomega.BeTrue()) + g.Eventually(func() error { + sa, err = exp.IsSatisfied(pod) + if err != nil { + return err + } + + if !sa { + return fmt.Errorf("expectation unsatisfied") + } + + return nil + }, 5*time.Second, 1*time.Second).Should(gomega.BeNil()) expectation, err = exp.GetExpectation(pod.Namespace, pod.Name) g.Expect(err).Should(gomega.BeNil()) g.Expect(expectation).Should(gomega.BeNil()) @@ -150,4 +178,66 @@ func TestActiveExpectations(t *testing.T) { expectation, err = exp.GetExpectation(pod.Namespace, pod.Name) g.Expect(err).Should(gomega.BeNil()) g.Expect(expectation).Should(gomega.BeNil()) + + g.Expect(exp.ExpectCreate(pod, Pod, podA.Name)).Should(gomega.BeNil()) + g.Expect(exp.IsSatisfied(pod)).Should(gomega.BeFalse()) + g.Expect(exp.DeleteItem(pod, Pod, podA.Name)).Should(gomega.BeNil()) + g.Expect(exp.IsSatisfied(pod)).Should(gomega.BeTrue()) + // delete no existing item should not get error + g.Expect(exp.DeleteItem(pod, Pod, podA.Name)).Should(gomega.BeNil()) +} + +func TestActiveExpectationsForAllKinds(t *testing.T) { + g := gomega.NewGomegaWithT(t) + for kind, newFn := range ResourceInitializers { + g.Expect(newFn()).ShouldNot(gomega.BeNil(), fmt.Sprintf("check initializer for kind %s", kind)) + } +} + +func TestActiveExpectationsValidationForPanics(t *testing.T) { + _, _, _, client, stopFunc := Setup(t) + defer func() { + stopFunc() + }() + + exp := NewActiveExpectations(client) + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "parent", + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "test", + Image: "image:v1", + }, + }, + }, + } + assertPanic(t, func() { + exp.expect(pod, "no-existing", pod.Name, Create) + }) + + assertPanic(t, func() { + exp.expect(pod, Pod, pod.Name, Update) + }) + + assertPanic(t, func() { + exp.ExpectUpdate(pod, Pod, pod.Name, "string-type") + }) + + assertPanic(t, func() { + exp.ExpectUpdate(pod, "no-existing", pod.Name, "1") + }) +} + +func assertPanic(t *testing.T, fn func()) { + defer func() { + if r := recover(); r == nil { + t.Errorf("expected panic, got nil") + } + }() + + fn() } diff --git a/pkg/controllers/utils/expectations/expectation_test.go b/pkg/controllers/utils/expectations/expectation_test.go new file mode 100644 index 00000000..8faa6290 --- /dev/null +++ b/pkg/controllers/utils/expectations/expectation_test.go @@ -0,0 +1,107 @@ +/* +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package expectations + +import ( + "testing" + + "github.com/onsi/gomega" +) + +func TestExpectation(t *testing.T) { + g := gomega.NewGomegaWithT(t) + key := "test/foo" + exp := NewControllerExpectations("test") + ex, exist, err := exp.GetExpectations(key) + g.Expect(err).Should(gomega.BeNil()) + g.Expect(exist).Should(gomega.BeFalse()) + g.Expect(ex).Should(gomega.BeNil()) + + g.Expect(exp.InitExpectations(key)).Should(gomega.BeNil()) + ex, exist, err = exp.GetExpectations(key) + g.Expect(err).Should(gomega.BeNil()) + g.Expect(exist).Should(gomega.BeTrue()) + g.Expect(ex).ShouldNot(gomega.BeNil()) + add, del := ex.GetExpectations() + g.Expect(add).Should(gomega.BeEquivalentTo(0)) + g.Expect(del).Should(gomega.BeEquivalentTo(0)) + + testcases := []struct { + add int + del int + expectedAdd int + expectedDel int + }{ + { + add: 1, + del: 1, + expectedAdd: 1, + expectedDel: 1, + }, + { + add: -1, + del: -1, + expectedAdd: 0, + expectedDel: 0, + }, + { + add: -1, + del: -1, + expectedAdd: -1, + expectedDel: -1, + }, + } + + for _, testcase := range testcases { + exp.RaiseExpectations(key, testcase.add, testcase.del) + ex, exist, err = exp.GetExpectations(key) + g.Expect(err).Should(gomega.BeNil()) + g.Expect(exist).Should(gomega.BeTrue()) + g.Expect(ex).ShouldNot(gomega.BeNil()) + add, del = ex.GetExpectations() + g.Expect(add).Should(gomega.BeEquivalentTo(testcase.expectedAdd)) + g.Expect(del).Should(gomega.BeEquivalentTo(testcase.expectedDel)) + } + + exp.DeleteExpectations(key) + ex, exist, err = exp.GetExpectations(key) + g.Expect(err).Should(gomega.BeNil()) + g.Expect(exist).Should(gomega.BeFalse()) + g.Expect(ex).Should(gomega.BeNil()) + g.Expect(exp.SatisfiedExpectations(key)).Should(gomega.BeTrue()) +} + +func TestExpectationSatisfied(t *testing.T) { + g := gomega.NewGomegaWithT(t) + key := "test/foo" + exp := NewControllerExpectations("test") + g.Expect(exp.InitExpectations(key)).Should(gomega.BeNil()) + + g.Expect(exp.ExpectCreations(key, 2)).Should(gomega.BeNil()) + exp.CreationObserved(key) + g.Expect(exp.SatisfiedExpectations(key)).Should(gomega.BeFalse()) + exp.CreationObserved(key) + g.Expect(exp.SatisfiedExpectations(key)).Should(gomega.BeTrue()) + + g.Expect(exp.ExpectDeletions(key, 2)).Should(gomega.BeNil()) + exp.CreationObserved(key) + g.Expect(exp.SatisfiedExpectations(key)).Should(gomega.BeFalse()) + exp.DeletionObserved(key) + g.Expect(exp.SatisfiedExpectations(key)).Should(gomega.BeFalse()) + exp.DeletionObserved(key) + g.Expect(exp.SatisfiedExpectations(key)).Should(gomega.BeTrue()) +} diff --git a/pkg/controllers/utils/expectations/resourceversion_expectation_test.go b/pkg/controllers/utils/expectations/resourceversion_expectation_test.go index 297a3078..f997dba9 100644 --- a/pkg/controllers/utils/expectations/resourceversion_expectation_test.go +++ b/pkg/controllers/utils/expectations/resourceversion_expectation_test.go @@ -24,30 +24,32 @@ import ( func TestResourceVersionExpectation(t *testing.T) { g := gomega.NewGomegaWithT(t) + key := "test" exp := NewResourceVersionExpectation() - exp.SetExpectations("test", "1") - item, ok, err := exp.GetExpectations("test") + exp.SetExpectations(key, "1") + item, ok, err := exp.GetExpectations(key) g.Expect(err).Should(gomega.BeNil()) g.Expect(ok).Should(gomega.BeTrue()) g.Expect(item.resourceVersion).Should(gomega.Equal(int64(1))) - exp.ExpectUpdate("test", "2") - item, ok, err = exp.GetExpectations("test") + exp.ExpectUpdate(key, "2") + item, ok, err = exp.GetExpectations(key) g.Expect(err).Should(gomega.BeNil()) g.Expect(ok).Should(gomega.BeTrue()) g.Expect(item.resourceVersion).Should(gomega.Equal(int64(2))) - ok = exp.SatisfiedExpectations("test", "1") + ok = exp.SatisfiedExpectations(key, "1") g.Expect(ok).Should(gomega.BeFalse()) - ok = exp.SatisfiedExpectations("test", "2") + ok = exp.SatisfiedExpectations(key, "2") g.Expect(ok).Should(gomega.BeFalse()) - ok = exp.SatisfiedExpectations("test", "3") + ok = exp.SatisfiedExpectations(key, "3") g.Expect(ok).Should(gomega.BeTrue()) - exp.DeleteExpectations("test") - item, ok, err = exp.GetExpectations("test") + exp.DeleteExpectations(key) + item, ok, err = exp.GetExpectations(key) g.Expect(err).Should(gomega.BeNil()) g.Expect(ok).Should(gomega.BeFalse()) g.Expect(item).Should(gomega.BeNil()) + g.Expect(exp.SatisfiedExpectations(key, "1")).Should(gomega.BeTrue()) } diff --git a/pkg/controllers/utils/patch_test.go b/pkg/controllers/utils/patch_test.go index 0c4b4bf8..b4722cb9 100644 --- a/pkg/controllers/utils/patch_test.go +++ b/pkg/controllers/utils/patch_test.go @@ -17,11 +17,12 @@ package utils import ( - "fmt" "testing" ) func TestPatch(t *testing.T) { val := GetLabelAnnoPatchBytes(nil, nil, nil, map[string]string{"detailAnno": "newDetail"}) - fmt.Println(string(val)) + if "{\"metadata\":{\"annotations\":{\"detailAnno\":\"newDetail\"}}}" != string(val) { + t.Fatalf("get unexpected patch: %s", val) + } } diff --git a/pkg/controllers/utils/pod_utils.go b/pkg/controllers/utils/pod_utils.go index 17ea9aac..ab00b934 100644 --- a/pkg/controllers/utils/pod_utils.go +++ b/pkg/controllers/utils/pod_utils.go @@ -1,5 +1,4 @@ /* -Copyright 2014 The Kubernetes Authors. Copyright 2023 The KusionStack Authors. Licensed under the Apache License, Version 2.0 (the "License"); @@ -19,183 +18,14 @@ package utils import ( "encoding/json" - "fmt" - appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" - apimachineryvalidation "k8s.io/apimachinery/pkg/api/validation" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/apimachinery/pkg/util/strategicpatch" "kusionstack.io/operating/apis/apps/v1alpha1" appsv1alpha1 "kusionstack.io/operating/apis/apps/v1alpha1" - revisionutils "kusionstack.io/operating/pkg/controllers/utils/revision" ) -func GetPodRevisionPatch(revision *appsv1.ControllerRevision) ([]byte, error) { - var raw map[string]interface{} - if err := json.Unmarshal([]byte(revision.Data.Raw), &raw); err != nil { - return nil, err - } - - spec := raw["spec"].(map[string]interface{}) - template := spec["template"].(map[string]interface{}) - patch, err := json.Marshal(template) - return patch, err -} - -func ApplyPatchFromRevision(pod *corev1.Pod, revision *appsv1.ControllerRevision) (*corev1.Pod, error) { - patch, err := GetPodRevisionPatch(revision) - if err != nil { - return nil, err - } - - clone := pod.DeepCopy() - patched, err := strategicpatch.StrategicMergePatch([]byte(runtime.EncodeOrDie(revisionutils.PodCodec, clone)), patch, clone) - if err != nil { - return nil, err - } - err = json.Unmarshal(patched, clone) - if err != nil { - return nil, err - } - return clone, nil -} - -// PatchToPod Use three way merge to get a updated pod. -func PatchToPod(currentRevisionPod, updateRevisionPod, currentPod *corev1.Pod) (*corev1.Pod, error) { - currentRevisionPodBytes, err := json.Marshal(currentRevisionPod) - if err != nil { - return nil, err - } - updateRevisionPodBytes, err := json.Marshal(updateRevisionPod) - - if err != nil { - return nil, err - } - - // 1. find the extra changes based on current revision - patch, err := strategicpatch.CreateTwoWayMergePatch(currentRevisionPodBytes, updateRevisionPodBytes, &corev1.Pod{}) - if err != nil { - return nil, err - } - - // 2. apply above changes to current pod - // We don't apply the diff between currentPod and currentRevisionPod to updateRevisionPod, - // because the PodTemplate changes should have the highest priority. - currentPodBytes, err := json.Marshal(currentPod) - if err != nil { - return nil, err - } - if updateRevisionPodBytes, err = strategicpatch.StrategicMergePatch(currentPodBytes, patch, &corev1.Pod{}); err != nil { - return nil, err - } - - newPod := &corev1.Pod{} - err = json.Unmarshal(updateRevisionPodBytes, newPod) - return newPod, err -} - -func NewPodFrom(owner metav1.Object, ownerRef *metav1.OwnerReference, revision *appsv1.ControllerRevision) (*corev1.Pod, error) { - pod, err := GetPodFromRevision(revision) - if err != nil { - return pod, err - } - - pod.Namespace = owner.GetNamespace() - pod.GenerateName = GetPodsPrefix(owner.GetName()) - pod.OwnerReferences = append(pod.OwnerReferences, *ownerRef) - - pod.Labels[appsv1.ControllerRevisionHashLabelKey] = revision.Name - - return pod, nil -} - -func GetPodFromRevision(revision *appsv1.ControllerRevision) (*corev1.Pod, error) { - pod, err := ApplyPatchFromRevision(&corev1.Pod{}, revision) - if err != nil { - return nil, err - } - - return pod, nil -} - -func GetPodsPrefix(controllerName string) string { - // use the dash (if the name isn't too long) to make the pod name a bit prettier - prefix := fmt.Sprintf("%s-", controllerName) - if len(apimachineryvalidation.NameIsDNSSubdomain(prefix, true)) != 0 { - prefix = controllerName - } - return prefix -} - -func ComparePod(l, r *corev1.Pod) bool { - // 1. Unassigned < assigned - // If only one of the pods is unassigned, the unassigned one is smaller - if l.Spec.NodeName != r.Spec.NodeName && (len(l.Spec.NodeName) == 0 || len(r.Spec.NodeName) == 0) { - return len(l.Spec.NodeName) == 0 - } - // 2. PodPending < PodUnknown < PodRunning - m := map[corev1.PodPhase]int{corev1.PodPending: 0, corev1.PodUnknown: 1, corev1.PodRunning: 2} - if m[l.Status.Phase] != m[r.Status.Phase] { - return m[l.Status.Phase] < m[r.Status.Phase] - } - // 3. Not ready < ready - // If only one of the pods is not ready, the not ready one is smaller - if IsPodReady(l) != IsPodReady(r) { - return !IsPodReady(l) - } - // TODO: take availability into account when we push minReadySeconds information from deployment into pods, - // see https://github.com/kubernetes/kubernetes/issues/22065 - // 4. Been ready for empty time < less time < more time - // If both pods are ready, the latest ready one is smaller - if IsPodReady(l) && IsPodReady(r) && !podReadyTime(l).Equal(podReadyTime(r)) { - return afterOrZero(podReadyTime(l), podReadyTime(r)) - } - // 5. Pods with containers with higher restart counts < lower restart counts - if maxContainerRestarts(l) != maxContainerRestarts(r) { - return maxContainerRestarts(l) > maxContainerRestarts(r) - } - // 6. Empty creation time pods < newer pods < older pods - if !l.CreationTimestamp.Equal(&r.CreationTimestamp) { - return afterOrZero(&l.CreationTimestamp, &r.CreationTimestamp) - } - return false -} - -func maxContainerRestarts(pod *corev1.Pod) int { - var maxRestarts int32 - for _, c := range pod.Status.ContainerStatuses { - if c.RestartCount > maxRestarts { - maxRestarts = c.RestartCount - } - } - return int(maxRestarts) -} - -func podReadyTime(pod *corev1.Pod) *metav1.Time { - if IsPodReady(pod) { - for _, c := range pod.Status.Conditions { - // we only care about pod ready conditions - if c.Type == corev1.PodReady && c.Status == corev1.ConditionTrue { - return &c.LastTransitionTime - } - } - } - return &metav1.Time{} -} - -// afterOrZero checks if time t1 is after time t2; if one of them -// is zero, the zero time is seen as after non-zero time. -func afterOrZero(t1, t2 *metav1.Time) bool { - if t1.Time.IsZero() || t2.Time.IsZero() { - return t1.Time.IsZero() - } - return t1.After(t2.Time) -} - func IsPodScheduled(pod *corev1.Pod) bool { return IsPodScheduledConditionTrue(pod.Status) } @@ -261,7 +91,7 @@ func GetPodConditionFromList(conditions []corev1.PodCondition, conditionType cor return -1, nil } -func IsServiceAvailable(pod *corev1.Pod) bool { +func IsPodServiceAvailable(pod *corev1.Pod) bool { if pod.Labels == nil { return false } @@ -270,15 +100,7 @@ func IsServiceAvailable(pod *corev1.Pod) bool { return exist } -func IsPodUpdatedRevision(pod *corev1.Pod, revision string) bool { - if pod.Labels == nil { - return false - } - - return pod.Labels[appsv1.ControllerRevisionHashLabelKey] == revision -} - -func SatisfyExpectedFinalizers(pod *corev1.Pod) (bool, []string, error) { +func IsExpectedFinalizerSatisfied(pod *corev1.Pod) (bool, []string, error) { satisfied := true var expectedFinalizers []string // expected finalizers that are not satisfied diff --git a/pkg/controllers/utils/pod_utils_test.go b/pkg/controllers/utils/pod_utils_test.go new file mode 100644 index 00000000..7cf36e9f --- /dev/null +++ b/pkg/controllers/utils/pod_utils_test.go @@ -0,0 +1,337 @@ +/* +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "encoding/json" + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + appsv1alpha1 "kusionstack.io/operating/apis/apps/v1alpha1" +) + +func TestIsPodScheduled(t *testing.T) { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + }, + Spec: corev1.PodSpec{}, + } + + if IsPodScheduled(pod) { + t.Fatalf("expected failure") + } + + pod = &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + }, + Spec: corev1.PodSpec{}, + Status: corev1.PodStatus{ + Conditions: []corev1.PodCondition{}, + }, + } + + if IsPodScheduled(pod) { + t.Fatalf("expected failure") + } + + pod = &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + }, + Spec: corev1.PodSpec{}, + Status: corev1.PodStatus{ + Conditions: []corev1.PodCondition{ + { + Type: corev1.PodScheduled, + Status: corev1.ConditionFalse, + }, + }, + }, + } + + if IsPodScheduled(pod) { + t.Fatalf("expected failure") + } + + pod = &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + }, + Spec: corev1.PodSpec{}, + Status: corev1.PodStatus{ + Conditions: []corev1.PodCondition{ + { + Type: corev1.PodScheduled, + Status: corev1.ConditionTrue, + }, + }, + }, + } + + if !IsPodScheduled(pod) { + t.Fatalf("expected failure") + } +} + +func TestIsPodReady(t *testing.T) { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + }, + Spec: corev1.PodSpec{}, + } + + if IsPodReady(pod) { + t.Fatalf("expected failure") + } + + pod = &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + }, + Spec: corev1.PodSpec{}, + Status: corev1.PodStatus{ + Conditions: []corev1.PodCondition{ + { + Type: corev1.PodReady, + Status: corev1.ConditionFalse, + }, + }, + }, + } + + if IsPodReady(pod) { + t.Fatalf("expected failure") + } + + pod = &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + }, + Spec: corev1.PodSpec{}, + Status: corev1.PodStatus{ + Conditions: []corev1.PodCondition{ + { + Type: corev1.PodReady, + Status: corev1.ConditionTrue, + }, + }, + }, + } + + if !IsPodReady(pod) { + t.Fatalf("expected failure") + } +} + +func TestIsPodTerminal(t *testing.T) { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + }, + Spec: corev1.PodSpec{}, + Status: corev1.PodStatus{ + Phase: corev1.PodFailed, + }, + } + + if !IsPodTerminal(pod) { + t.Fatalf("expected failure") + } + + pod = &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + }, + Spec: corev1.PodSpec{}, + Status: corev1.PodStatus{ + Phase: corev1.PodSucceeded, + }, + } + + if !IsPodTerminal(pod) { + t.Fatalf("expected failure") + } + + pod = &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + }, + Spec: corev1.PodSpec{}, + Status: corev1.PodStatus{ + Phase: corev1.PodPending, + }, + } + + if IsPodTerminal(pod) { + t.Fatalf("expected failure") + } + + pod = &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + }, + Spec: corev1.PodSpec{}, + Status: corev1.PodStatus{ + Phase: corev1.PodRunning, + }, + } + + if IsPodTerminal(pod) { + t.Fatalf("expected failure") + } + + pod = &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + }, + Spec: corev1.PodSpec{}, + Status: corev1.PodStatus{ + Phase: corev1.PodUnknown, + }, + } + + if IsPodTerminal(pod) { + t.Fatalf("expected failure") + } +} + +func TestIsPodServiceAvailable(t *testing.T) { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + }, + Spec: corev1.PodSpec{}, + } + + if IsPodServiceAvailable(pod) { + t.Fatalf("expected failure") + } + + pod.Labels = map[string]string{ + appsv1alpha1.PodServiceAvailableLabel: "true", + } + + if !IsPodServiceAvailable(pod) { + t.Fatalf("expected failure") + } +} + +func TestIsPodExpectedFinalizerSatisfied(t *testing.T) { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + }, + Spec: corev1.PodSpec{}, + } + + if satisfied, _, err := IsExpectedFinalizerSatisfied(pod); err != nil || !satisfied { + t.Fatalf("expected failure") + } + + pod = &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Annotations: map[string]string{}, + }, + Spec: corev1.PodSpec{}, + } + + if satisfied, _, err := IsExpectedFinalizerSatisfied(pod); err != nil || !satisfied { + t.Fatalf("expected failure") + } + + pod = &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Annotations: map[string]string{ + appsv1alpha1.PodAvailableConditionsAnnotation: "invalid", + }, + }, + Spec: corev1.PodSpec{}, + } + + if _, _, err := IsExpectedFinalizerSatisfied(pod); err == nil { + t.Fatalf("expected failure") + } + + condFinalizer1 := "test/cond1" + condFinalizer2 := "test/cond2" + cond := &appsv1alpha1.PodAvailableConditions{ + ExpectedFinalizers: map[string]string{ + "cond1": condFinalizer1, + "cond2": condFinalizer2, + }, + } + s, err := json.Marshal(cond) + if err != nil { + t.Fatalf("unexpected err") + } + + pod = &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Annotations: map[string]string{ + appsv1alpha1.PodAvailableConditionsAnnotation: string(s), + }, + }, + Spec: corev1.PodSpec{}, + } + + if satisfied, _, err := IsExpectedFinalizerSatisfied(pod); err != nil || satisfied { + t.Fatalf("expected failure") + } + + pod = &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Annotations: map[string]string{ + appsv1alpha1.PodAvailableConditionsAnnotation: string(s), + }, + Finalizers: []string{ + condFinalizer1, + }, + }, + Spec: corev1.PodSpec{}, + } + + if satisfied, _, err := IsExpectedFinalizerSatisfied(pod); err != nil || satisfied { + t.Fatalf("expected failure") + } + + pod = &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Annotations: map[string]string{ + appsv1alpha1.PodAvailableConditionsAnnotation: string(s), + }, + Finalizers: []string{ + condFinalizer1, + condFinalizer2, + }, + }, + Spec: corev1.PodSpec{}, + } + + if satisfied, _, err := IsExpectedFinalizerSatisfied(pod); err != nil || !satisfied { + t.Fatalf("expected failure") + } +} diff --git a/pkg/controllers/utils/refmanager/ref_manager.go b/pkg/controllers/utils/refmanager/ref_manager.go index eca5d5a4..3d054a9c 100644 --- a/pkg/controllers/utils/refmanager/ref_manager.go +++ b/pkg/controllers/utils/refmanager/ref_manager.go @@ -98,17 +98,14 @@ func (mgr *RefManager) adopt(obj client.Object) error { } if mgr.schema == nil { - return nil + return fmt.Errorf("schema should not be nil") } if err := controllerutil.SetControllerReference(mgr.owner, obj, mgr.schema); err != nil { return fmt.Errorf("can't set Object %v/%v (%v) owner reference: %v", obj.GetNamespace(), obj.GetName(), obj.GetUID(), err) } - if err := mgr.client.Update(context.TODO(), obj); err != nil { - return fmt.Errorf("can't update Object %v/%v (%v) owner reference: %v", obj.GetNamespace(), obj.GetName(), obj.GetUID(), err) - } - return nil + return mgr.client.Update(context.TODO(), obj) } func (mgr *RefManager) release(obj client.Object) error { @@ -122,8 +119,7 @@ func (mgr *RefManager) release(obj client.Object) error { if idx > -1 { obj.SetOwnerReferences(append(obj.GetOwnerReferences()[:idx], obj.GetOwnerReferences()[idx+1:]...)) if err := mgr.client.Update(context.TODO(), obj); err != nil { - return fmt.Errorf("can't remove Pod %v/%v (%v) owner reference %v/%v (%v): %v", - obj.GetNamespace(), obj.GetName(), obj.GetUID(), obj.GetNamespace(), obj.GetName(), mgr.owner.GetUID(), err) + return err } } diff --git a/pkg/controllers/utils/refmanager/ref_manager_test.go b/pkg/controllers/utils/refmanager/ref_manager_test.go index 24cb0df7..c0928ee1 100644 --- a/pkg/controllers/utils/refmanager/ref_manager_test.go +++ b/pkg/controllers/utils/refmanager/ref_manager_test.go @@ -19,6 +19,7 @@ package utils import ( "context" + "fmt" "testing" "github.com/onsi/gomega" @@ -27,11 +28,416 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" appsv1alpha1 "kusionstack.io/operating/apis/apps/v1alpha1" ) -func Test(t *testing.T) { +func TestAdopt(t *testing.T) { + g := gomega.NewGomegaWithT(t) + + scheme := runtime.NewScheme() + appsv1alpha1.AddToScheme(scheme) + + testName := "ref-manager" + testcases := []struct { + name string + isSelectorMatch bool + isPodExist bool + isOwnerInTerminating bool + isPodInTerminating bool + isConflict bool + expectErr bool + expectAdopted bool + }{ + { + name: "happy pass", + isSelectorMatch: true, + isPodExist: true, + isOwnerInTerminating: false, + isPodInTerminating: false, + isConflict: false, + expectErr: false, + expectAdopted: true, + }, + { + name: "update conflict", + isSelectorMatch: true, + isPodExist: true, + isOwnerInTerminating: false, + isPodInTerminating: false, + isConflict: true, + expectErr: true, + expectAdopted: false, + }, + { + name: "not match", + isSelectorMatch: false, + isPodExist: true, + isOwnerInTerminating: false, + isPodInTerminating: false, + isConflict: false, + expectErr: false, + expectAdopted: false, + }, + { + name: "pod not exist", + isSelectorMatch: true, + isPodExist: false, + isOwnerInTerminating: false, + isPodInTerminating: false, + isConflict: false, + expectErr: false, + expectAdopted: false, + }, + { + name: "owner in terminating", + isSelectorMatch: true, + isPodExist: true, + isOwnerInTerminating: true, + isPodInTerminating: false, + isConflict: false, + expectErr: false, + expectAdopted: false, + }, + { + name: "pod in terminating", + isSelectorMatch: true, + isPodExist: true, + isOwnerInTerminating: false, + isPodInTerminating: true, + isConflict: false, + expectErr: false, + expectAdopted: false, + }, + } + + for _, testcase := range testcases { + cs := &appsv1alpha1.CollaSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: "default", + UID: types.UID("fake-uid"), + }, + Spec: appsv1alpha1.CollaSetSpec{ + Replicas: int32Pointer(2), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "foo", + "case": testName, + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "foo", + "case": testName, + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "nginx", + Image: "nginx:v1", + }, + }, + }, + }, + }, + } + + objs := []client.Object{ + &appsv1alpha1.ResourceContext{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + Namespace: "default", + Labels: map[string]string{ + "app": "foo", + "case": testName, + }, + }, + }, + } + + c := fake.NewClientBuilder().WithScheme(scheme).Build() + ref, err := NewRefManager(c, cs.Spec.Selector, cs, scheme) + g.Expect(err).Should(gomega.BeNil()) + + if !testcase.isSelectorMatch { + for _, obj := range objs { + obj.GetLabels()["app"] = "foo1" + } + } + + if testcase.isPodExist { + for _, obj := range objs { + g.Expect(c.Create(context.TODO(), obj)).Should(gomega.BeNil()) + } + } + + now := metav1.Now() + if testcase.isOwnerInTerminating { + cs.DeletionTimestamp = &now + } + + if testcase.isPodInTerminating { + for _, obj := range objs { + obj.SetDeletionTimestamp(&now) + } + } + + if testcase.isConflict { + for _, obj := range objs { + obj.SetResourceVersion("100") + } + } + + podObjects, err := ref.ClaimOwned(objs) + if testcase.expectErr { + g.Expect(err).ShouldNot(gomega.BeNil(), fmt.Sprintf("case: %s", testcase.name)) + } else { + g.Expect(err).Should(gomega.BeNil(), fmt.Sprintf("case: %s", testcase.name)) + } + + if testcase.expectAdopted { + g.Expect(len(podObjects)).Should(gomega.BeEquivalentTo(1), fmt.Sprintf("case: %s", testcase.name)) + } else { + g.Expect(len(podObjects)).Should(gomega.BeEquivalentTo(0), fmt.Sprintf("case: %s", testcase.name)) + } + } +} + +func TestRelease(t *testing.T) { + g := gomega.NewGomegaWithT(t) + + scheme := runtime.NewScheme() + appsv1alpha1.AddToScheme(scheme) + + testName := "ref-manager" + testcases := []struct { + name string + isOwnedByIt bool + isSelectorMatch bool + isPodExist bool + isOwnerInTerminating bool + isPodInTerminating bool + isConflict bool + expectErr bool + expectReleased bool + }{ + { + name: "happy pass", + isOwnedByIt: true, + isSelectorMatch: true, + isPodExist: true, + isOwnerInTerminating: false, + isPodInTerminating: false, + isConflict: false, + expectErr: false, + expectReleased: false, + }, + { + name: "update conflict", + isOwnedByIt: true, + isSelectorMatch: false, + isPodExist: true, + isOwnerInTerminating: false, + isPodInTerminating: false, + isConflict: true, + expectErr: true, + expectReleased: false, + }, + { + name: "not own pod", + isOwnedByIt: false, + isSelectorMatch: true, + isPodExist: true, + isOwnerInTerminating: false, + isPodInTerminating: false, + isConflict: false, + expectErr: false, + expectReleased: true, + }, + { + name: "not match", + isOwnedByIt: true, + isSelectorMatch: false, + isPodExist: true, + isOwnerInTerminating: false, + isPodInTerminating: false, + isConflict: false, + expectErr: false, + expectReleased: true, + }, + { + name: "pod not exist", + isOwnedByIt: true, + isSelectorMatch: false, + isPodExist: false, + isOwnerInTerminating: false, + isPodInTerminating: false, + isConflict: false, + expectErr: false, + expectReleased: true, + }, + { + name: "owner in terminating", + isOwnedByIt: true, + isSelectorMatch: false, + isPodExist: true, + isOwnerInTerminating: true, + isPodInTerminating: false, + isConflict: false, + expectErr: false, + expectReleased: false, + }, + { + name: "pod in terminating", + isOwnedByIt: true, + isSelectorMatch: false, + isPodExist: true, + isOwnerInTerminating: false, + isPodInTerminating: true, + isConflict: false, + expectErr: false, + expectReleased: true, + }, + } + + for _, testcase := range testcases { + cs := &appsv1alpha1.CollaSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: "default", + UID: types.UID("fake-uid"), + }, + Spec: appsv1alpha1.CollaSetSpec{ + Replicas: int32Pointer(2), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "foo", + "case": testName, + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "foo", + "case": testName, + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "nginx", + Image: "nginx:v1", + }, + }, + }, + }, + }, + } + + objs := []client.Object{ + &appsv1alpha1.ResourceContext{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + Namespace: "default", + Labels: map[string]string{ + "app": "foo", + "case": testName, + }, + }, + }, + } + + c := fake.NewClientBuilder().WithScheme(scheme).Build() + ref, err := NewRefManager(c, cs.Spec.Selector, cs, scheme) + g.Expect(err).Should(gomega.BeNil()) + for _, obj := range objs { + if !testcase.isOwnedByIt { + boolean := true + obj.SetOwnerReferences([]metav1.OwnerReference{ + { + UID: "other-uid", + Controller: &boolean, + }, + }) + } + g.Expect(c.Create(context.TODO(), obj)).Should(gomega.BeNil()) + } + + if testcase.isOwnedByIt { + podObjects, err := ref.ClaimOwned(objs) + g.Expect(err).Should(gomega.BeNil()) + g.Expect(len(podObjects)).Should(gomega.BeEquivalentTo(len(objs))) + } + + if !testcase.isSelectorMatch { + for _, obj := range objs { + obj.GetLabels()["app"] = "foo1" + } + } + + if !testcase.isPodExist { + for _, obj := range objs { + g.Expect(c.Delete(context.TODO(), obj)).Should(gomega.BeNil()) + } + } + + now := metav1.Now() + if testcase.isOwnerInTerminating { + cs.DeletionTimestamp = &now + } + + if testcase.isPodInTerminating { + for _, obj := range objs { + obj.SetDeletionTimestamp(&now) + } + } + + if testcase.isConflict { + for _, obj := range objs { + obj.SetResourceVersion("100") + } + } + + _, err = ref.ClaimOwned(objs) + rcList := &appsv1alpha1.ResourceContextList{} + g.Expect(c.List(context.TODO(), rcList, client.InNamespace(cs.Namespace))).Should(gomega.BeNil()) + if testcase.expectErr { + g.Expect(err).ShouldNot(gomega.BeNil(), fmt.Sprintf("case: %s", testcase.name)) + } else { + g.Expect(err).Should(gomega.BeNil(), fmt.Sprintf("case: %s", testcase.name)) + } + + if testcase.expectReleased { + g.Expect(len(filterOwnedObj(rcList.Items, cs))).Should(gomega.BeEquivalentTo(0), fmt.Sprintf("case: %s", testcase.name)) + } else { + g.Expect(len(filterOwnedObj(rcList.Items, cs))).Should(gomega.BeEquivalentTo(1), fmt.Sprintf("case: %s", testcase.name)) + } + } +} + +func filterOwnedObj(objs []appsv1alpha1.ResourceContext, owner client.Object) []client.Object { + var res []client.Object + for _, obj := range objs { + ownerRef := metav1.GetControllerOf(&obj) + if ownerRef == nil { + continue + } + + if ownerRef.UID != owner.GetUID() { + continue + } + + res = append(res, &obj) + } + + return res +} + +func TestCoverCornerCase(t *testing.T) { g := gomega.NewGomegaWithT(t) scheme := runtime.NewScheme() @@ -71,8 +477,8 @@ func Test(t *testing.T) { }, } - pods := []corev1.Pod{ - { + objs := []client.Object{ + &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "pod1", Namespace: "default", @@ -92,55 +498,28 @@ func Test(t *testing.T) { }, } - ref, err := NewRefManager(&MockClient{}, cs.Spec.Selector, cs, scheme) + c := fake.NewClientBuilder().Build() + ref, err := NewRefManager(c, cs.Spec.Selector, cs, nil) g.Expect(err).Should(gomega.BeNil()) + g.Expect(ref.adopt(objs[0])).ShouldNot(gomega.BeNil()) - podObjects := make([]client.Object, len(pods)) - for i := range pods { - podObjects[i] = &pods[i] - } - - // adopt orphan pod - podObjects, err = ref.ClaimOwned(podObjects) + ref, err = NewRefManager(c, cs.Spec.Selector, cs, runtime.NewScheme()) g.Expect(err).Should(gomega.BeNil()) - g.Expect(len(podObjects)).Should(gomega.BeEquivalentTo(1)) + g.Expect(ref.adopt(objs[0])).ShouldNot(gomega.BeNil()) - ownerRef := metav1.GetControllerOf(podObjects[0]) - g.Expect(ownerRef).ShouldNot(gomega.BeNil()) - g.Expect(ownerRef.UID).Should(gomega.BeEquivalentTo(cs.UID)) + ref, err = NewRefManager(c, &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "_1": "invalid", + }, + }, cs, scheme) + g.Expect(err).ShouldNot(gomega.BeNil()) - // release pod not selected - labels := podObjects[0].GetLabels() - delete(labels, "app") - podObjects[0].SetLabels(labels) - claimedPodObjects, err := ref.ClaimOwned(podObjects) + ref, err = NewRefManager(c, cs.Spec.Selector, cs, scheme) g.Expect(err).Should(gomega.BeNil()) - g.Expect(len(claimedPodObjects)).Should(gomega.BeEquivalentTo(0)) - ownerRef = metav1.GetControllerOf(podObjects[0]) - g.Expect(ownerRef).Should(gomega.BeNil()) -} - -type MockClient struct { -} - -func (c *MockClient) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { - return nil -} - -func (c *MockClient) Delete(ctx context.Context, obj client.Object, opts ...client.DeleteOption) error { - return nil -} - -func (c *MockClient) Update(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error { - return nil -} - -func (c *MockClient) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { - return nil -} -func (c *MockClient) DeleteAllOf(ctx context.Context, obj client.Object, opts ...client.DeleteAllOfOption) error { - return nil + now := metav1.Now() + cs.DeletionTimestamp = &now + g.Expect(ref.adopt(objs[0])).ShouldNot(gomega.BeNil()) } func int32Pointer(val int32) *int32 { diff --git a/pkg/controllers/utils/revision/hash.go b/pkg/controllers/utils/revision/hash.go deleted file mode 100644 index 7eb7b8e2..00000000 --- a/pkg/controllers/utils/revision/hash.go +++ /dev/null @@ -1,45 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. -Copyright 2023 The KusionStack Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package revision - -import ( - "hash" - - "github.com/davecgh/go-spew/spew" - corev1 "k8s.io/api/core/v1" - "k8s.io/client-go/kubernetes/scheme" - "k8s.io/klog/v2" -) - -var PodCodec = scheme.Codecs.LegacyCodec(corev1.SchemeGroupVersion) - -// DeepHashObject writes specified object to hash using the spew library -// which follows pointers and prints actual values of the nested objects -// ensuring the hash does not change when a pointer changes. -func DeepHashObject(hasher hash.Hash, objectToWrite interface{}) { - hasher.Reset() - printer := spew.ConfigState{ - Indent: " ", - SortKeys: true, - DisableMethods: true, - SpewKeys: true, - } - if _, err := printer.Fprintf(hasher, "%#v", objectToWrite); err != nil { - klog.Error("fail to deep hash: %s", err) - } -} diff --git a/pkg/controllers/utils/revision/revision_manager.go b/pkg/controllers/utils/revision/revision_manager.go index 8dda30b8..e2fb0881 100644 --- a/pkg/controllers/utils/revision/revision_manager.go +++ b/pkg/controllers/utils/revision/revision_manager.go @@ -26,6 +26,7 @@ import ( "strconv" apps "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -33,6 +34,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/rand" + "k8s.io/client-go/kubernetes/scheme" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/apiutil" @@ -41,12 +43,13 @@ import ( const ControllerRevisionHashLabel = "controller.kubernetes.io/hash" +var PodCodec = scheme.Codecs.LegacyCodec(corev1.SchemeGroupVersion) + type OwnerAdapter interface { GetSelector(obj metav1.Object) *metav1.LabelSelector GetCollisionCount(obj metav1.Object) *int32 GetHistoryLimit(obj metav1.Object) int32 GetPatch(obj metav1.Object) ([]byte, error) - GetSelectorLabels(obj metav1.Object) map[string]string GetCurrentRevision(obj metav1.Object) string IsInUsed(obj metav1.Object, controllerRevision string) bool } @@ -163,6 +166,7 @@ func (rm *RevisionManager) ConstructRevisions(set client.Object, dryRun bool) (* } } + revisions = append(revisions, updateRevision) createNewRevision = true } @@ -181,11 +185,8 @@ func (rm *RevisionManager) ConstructRevisions(set client.Object, dryRun bool) (* return currentRevision, updateRevision, revisions, collisionCount, createNewRevision, nil } -func (rm *RevisionManager) cleanExpiredRevision(cd metav1.Object, sortedRevisions *[]*apps.ControllerRevision) (*[]*apps.ControllerRevision, error) { - limit := int(rm.ownerGetter.GetHistoryLimit(cd)) - if limit <= 0 { - limit = 10 - } +func (rm *RevisionManager) cleanExpiredRevision(set metav1.Object, sortedRevisions *[]*apps.ControllerRevision) (*[]*apps.ControllerRevision, error) { + limit := int(rm.ownerGetter.GetHistoryLimit(set)) // reserve 2 extra unused revisions for diagnose exceedNum := len(*sortedRevisions) - limit - 2 @@ -198,7 +199,7 @@ func (rm *RevisionManager) cleanExpiredRevision(cd metav1.Object, sortedRevision break } - if rm.ownerGetter.IsInUsed(cd, revision.Name) { + if rm.ownerGetter.IsInUsed(set, revision.Name) { continue } @@ -261,9 +262,6 @@ func hashControllerRevision(revision *apps.ControllerRevision, probe *int32) str if len(revision.Data.Raw) > 0 { hf.Write(revision.Data.Raw) } - if revision.Data.Object != nil { - DeepHashObject(hf, revision.Data.Object) - } if probe != nil { hf.Write([]byte(strconv.FormatInt(int64(*probe), 10))) } @@ -289,11 +287,7 @@ func (rm *RevisionManager) newRevision(set metav1.Object, revision int64, collis return nil, err } - revisionLabels := rm.ownerGetter.GetSelectorLabels(set) - if revisionLabels == nil { - revisionLabels = map[string]string{} - } - + revisionLabels := map[string]string{} if selector := rm.ownerGetter.GetSelector(set); selector != nil { for k, v := range selector.MatchLabels { revisionLabels[k] = v @@ -311,16 +305,6 @@ func (rm *RevisionManager) newRevision(set metav1.Object, revision int64, collis } cr.Namespace = set.GetNamespace() - if cr.ObjectMeta.Annotations == nil { - cr.ObjectMeta.Annotations = make(map[string]string) - } - for key, value := range set.GetAnnotations() { - cr.ObjectMeta.Annotations[key] = value - } - - if cr.ObjectMeta.Labels == nil { - cr.ObjectMeta.Labels = make(map[string]string) - } return cr, nil } diff --git a/pkg/controllers/utils/revision/revision_manager_test.go b/pkg/controllers/utils/revision/revision_manager_test.go new file mode 100644 index 00000000..2dffce23 --- /dev/null +++ b/pkg/controllers/utils/revision/revision_manager_test.go @@ -0,0 +1,336 @@ +/* +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package revision + +import ( + "context" + "encoding/json" + "fmt" + "testing" + "time" + + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +var ( + c client.Client + g *WithT + + selectedLabels = map[string]string{"test": "foo"} + selector, _ = metav1.ParseToLabelSelector("test=foo") +) + +func TestRevisionConstruction(t *testing.T) { + g = NewGomegaWithT(t) + testcase := "test-revision-construction" + schema := runtime.NewScheme() + corev1.AddToScheme(schema) + appsv1.AddToScheme(schema) + c = fake.NewClientBuilder().WithScheme(schema).Build() + g.Expect(createNamespace(c, testcase)).Should(BeNil()) + + deploy := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: testcase, + Name: testcase, + }, + Spec: appsv1.DeploymentSpec{ + Selector: selector, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: selectedLabels, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "test", + Image: "nginx:v1", + }, + }, + }, + }, + }, + } + g.Expect(c.Create(context.TODO(), deploy)).Should(BeNil()) + + adapter := &OwnerAdapterImpl{ + name: testcase, + selector: selector, + selectedLabels: selectedLabels, + collisionCount: 0, + historyLimit: 2, + currentRevision: "test", + inUsed: true, + } + + revisionManager := NewRevisionManager(c, schema, adapter) + currentRevision, updatedRevision, revisionList, collisionCount, createNewRevision, err := revisionManager.ConstructRevisions(deploy, false) + g.Expect(err).Should(BeNil()) + g.Expect(createNewRevision).Should(BeTrue()) + g.Expect(collisionCount).ShouldNot(BeNil()) + g.Expect(*collisionCount).Should(BeEquivalentTo(0)) + g.Expect(len(revisionList)).Should(BeEquivalentTo(1)) + g.Expect(len(currentRevision.Name)).ShouldNot(BeEquivalentTo(0)) + g.Expect(updatedRevision.Name).Should(BeEquivalentTo(currentRevision.Name)) + v1RevisionName := updatedRevision.Name + waitingCacheUpdate(deploy.Namespace, 1) + adapter.currentRevision = updatedRevision.Name + + // updating deploy spec should construct a new updated CcontrollerRevision + deploy.Spec.Template.Spec.Containers[0].Image = "nginx:v2" + currentRevision, updatedRevision, revisionList, collisionCount, createNewRevision, err = revisionManager.ConstructRevisions(deploy, false) + g.Expect(err).Should(BeNil()) + g.Expect(createNewRevision).Should(BeTrue()) + g.Expect(collisionCount).ShouldNot(BeNil()) + g.Expect(*collisionCount).Should(BeEquivalentTo(0)) + g.Expect(len(revisionList)).Should(BeEquivalentTo(2)) + g.Expect(len(currentRevision.Name)).ShouldNot(BeEquivalentTo(0)) + g.Expect(len(currentRevision.Name)).ShouldNot(BeEquivalentTo(0)) + g.Expect(updatedRevision.Name).ShouldNot(BeEquivalentTo(currentRevision.Name)) + waitingCacheUpdate(deploy.Namespace, 2) + + // reconcile with same spec with current revision is updated to updated revision + adapter.currentRevision = updatedRevision.Name + currentRevision, updatedRevision, revisionList, collisionCount, createNewRevision, err = revisionManager.ConstructRevisions(deploy, false) + g.Expect(err).Should(BeNil()) + g.Expect(createNewRevision).Should(BeFalse()) + g.Expect(collisionCount).ShouldNot(BeNil()) + g.Expect(*collisionCount).Should(BeEquivalentTo(0)) + g.Expect(len(revisionList)).Should(BeEquivalentTo(2)) + g.Expect(len(currentRevision.Name)).ShouldNot(BeEquivalentTo(0)) + g.Expect(len(currentRevision.Name)).ShouldNot(BeEquivalentTo(0)) + g.Expect(updatedRevision.Name).Should(BeEquivalentTo(currentRevision.Name)) + waitingCacheUpdate(deploy.Namespace, 2) + + // updating deploy spec to old version should not construct a new updated CcontrollerRevision + deploy.Spec.Template.Spec.Containers[0].Image = "nginx:v1" + currentRevision, updatedRevision, revisionList, collisionCount, createNewRevision, err = revisionManager.ConstructRevisions(deploy, false) + g.Expect(err).Should(BeNil()) + g.Expect(createNewRevision).Should(BeFalse()) + g.Expect(collisionCount).ShouldNot(BeNil()) + g.Expect(*collisionCount).Should(BeEquivalentTo(0)) + g.Expect(len(revisionList)).Should(BeEquivalentTo(2)) + g.Expect(len(currentRevision.Name)).ShouldNot(BeEquivalentTo(0)) + g.Expect(len(currentRevision.Name)).ShouldNot(BeEquivalentTo(0)) + g.Expect(updatedRevision.Name).ShouldNot(BeEquivalentTo(currentRevision.Name)) + g.Expect(updatedRevision.Name).Should(BeEquivalentTo(v1RevisionName)) +} + +func TestRevisionCleanUp(t *testing.T) { + g = NewGomegaWithT(t) + testcase := "test-revision-cleanup" + schema := runtime.NewScheme() + corev1.AddToScheme(schema) + appsv1.AddToScheme(schema) + c = fake.NewClientBuilder().WithScheme(schema).Build() + g.Expect(createNamespace(c, testcase)).Should(BeNil()) + + deploy := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: testcase, + Name: testcase, + }, + Spec: appsv1.DeploymentSpec{ + Selector: selector, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: selectedLabels, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "test", + Image: "nginx:v1", + }, + }, + }, + }, + }, + } + g.Expect(c.Create(context.TODO(), deploy)).Should(BeNil()) + + adapter := &OwnerAdapterImpl{ + name: testcase, + selector: selector, + selectedLabels: selectedLabels, + collisionCount: 0, + historyLimit: 0, // we at lease reserve 2 extra controller revisions + currentRevision: "test", + inUsed: true, + } + + revisionManager := NewRevisionManager(c, schema, adapter) + revisionManager.ConstructRevisions(deploy, false) + waitingCacheUpdate(deploy.Namespace, 1) + + deploy.Spec.Template.Spec.Containers[0].Image = "nginx:v2" + revisionManager.ConstructRevisions(deploy, false) + waitingCacheUpdate(deploy.Namespace, 2) + + deploy.Spec.Template.Spec.Containers[0].Image = "nginx:v3" + revisionManager.ConstructRevisions(deploy, false) + waitingCacheUpdate(deploy.Namespace, 3) + + revisionManager.ConstructRevisions(deploy, false) + waitingCacheUpdate(deploy.Namespace, 3) + + adapter.inUsed = false + revisionManager.ConstructRevisions(deploy, false) + waitingCacheUpdate(deploy.Namespace, 2) +} + +func TestRevisionCreation(t *testing.T) { + g = NewGomegaWithT(t) + testcase := "test-revision-creation" + schema := runtime.NewScheme() + corev1.AddToScheme(schema) + appsv1.AddToScheme(schema) + c = fake.NewClientBuilder().WithScheme(schema).Build() + g.Expect(createNamespace(c, testcase)).Should(BeNil()) + + deploy := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: testcase, + Name: testcase, + }, + Spec: appsv1.DeploymentSpec{ + Selector: selector, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: selectedLabels, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "test", + Image: "nginx:v1", + }, + }, + }, + }, + }, + } + g.Expect(c.Create(context.TODO(), deploy)).Should(BeNil()) + + adapter := &OwnerAdapterImpl{ + name: testcase, + selector: selector, + selectedLabels: selectedLabels, + collisionCount: 0, + historyLimit: 0, // we at lease reserve 2 extra controller revisions + currentRevision: "test", + inUsed: true, + } + + revisionManager := NewRevisionManager(c, schema, adapter) + currentRevision, _, _, _, _, err := revisionManager.ConstructRevisions(deploy, false) + g.Expect(err).Should(BeNil()) + waitingCacheUpdate(deploy.Namespace, 1) + + _, err = revisionManager.createControllerRevision(context.TODO(), deploy, currentRevision, nil) + g.Expect(err).ShouldNot(BeNil()) + + var collisionCount int32 = 0 + // if new revision conflict with existing revision and their contents are equals, then will reuse the existing one + newRevision, err := revisionManager.newRevision(deploy, currentRevision.Revision+1, &collisionCount) + g.Expect(err).Should(BeNil()) + newRevision, err = revisionManager.createControllerRevision(context.TODO(), deploy, newRevision, &collisionCount) + g.Expect(err).Should(BeNil()) + g.Expect(newRevision.Name).Should(BeEquivalentTo(currentRevision.Name)) + + // change the data of existing revision + deployClone := deploy.DeepCopy() + deployClone.Spec.Template.Labels["foo"] = "foo" + currentRevision.Data.Raw, _ = revisionManager.ownerGetter.GetPatch(deployClone) + g.Expect(c.Update(context.TODO(), currentRevision)).Should(BeNil()) + // if their contents are not equals, it should regenerate a new name + newRevision, err = revisionManager.newRevision(deploy, currentRevision.Revision+1, &collisionCount) + g.Expect(err).Should(BeNil()) + newRevision, err = revisionManager.createControllerRevision(context.TODO(), deploy, newRevision, &collisionCount) + g.Expect(err).Should(BeNil()) + g.Expect(newRevision.Name).ShouldNot(BeEquivalentTo(currentRevision.Name)) +} + +func waitingCacheUpdate(namespace string, expectedRevisionCount int) { + g.Eventually(func() error { + revisionList := &appsv1.ControllerRevisionList{} + if err := c.List(context.TODO(), revisionList, &client.ListOptions{Namespace: namespace}); err != nil { + return err + } + + if len(revisionList.Items) != expectedRevisionCount { + return fmt.Errorf("expected %d, got %d\n", expectedRevisionCount, len(revisionList.Items)) + } + + return nil + }, 5*time.Second, 1*time.Second).Should(BeNil()) +} + +type OwnerAdapterImpl struct { + name string + selector *metav1.LabelSelector + selectedLabels map[string]string + collisionCount int32 + historyLimit int32 + currentRevision string + inUsed bool +} + +func (a OwnerAdapterImpl) GetSelector(obj metav1.Object) *metav1.LabelSelector { + return a.selector +} + +func (a OwnerAdapterImpl) GetCollisionCount(obj metav1.Object) *int32 { + return &a.collisionCount +} + +func (a OwnerAdapterImpl) GetHistoryLimit(obj metav1.Object) int32 { + return a.historyLimit +} + +func (a OwnerAdapterImpl) GetPatch(obj metav1.Object) ([]byte, error) { + // mock patch + return json.Marshal(obj) +} + +func (a OwnerAdapterImpl) GetCurrentRevision(obj metav1.Object) string { + return a.currentRevision +} + +func (a OwnerAdapterImpl) IsInUsed(obj metav1.Object, controllerRevision string) bool { + return a.inUsed +} + +func createNamespace(c client.Client, namespaceName string) error { + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: namespaceName, + }, + } + + return c.Create(context.TODO(), ns) +} + +func int32Pointer(val int32) *int32 { + return &val +} diff --git a/pkg/controllers/utils/slow_start.go b/pkg/controllers/utils/slow_start.go index 27199517..085aae58 100644 --- a/pkg/controllers/utils/slow_start.go +++ b/pkg/controllers/utils/slow_start.go @@ -1,4 +1,5 @@ /* +Copyright 2016 The Kubernetes Authors. Copyright 2023 The KusionStack Authors. Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/pkg/controllers/utils/slow_start_test.go b/pkg/controllers/utils/slow_start_test.go new file mode 100644 index 00000000..95adebc8 --- /dev/null +++ b/pkg/controllers/utils/slow_start_test.go @@ -0,0 +1,107 @@ +/* +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "fmt" + "sync" + "testing" +) + +const expectedErrMsg = "expected error" + +func TestSlowStartBatch(t *testing.T) { + testcases := []struct { + name string + count int + errorIndex *int + shortcut bool + + expectedCallCount int + expectedSuccessCount int + }{ + { + name: "happy pass", + count: 10, + errorIndex: nil, + shortcut: false, + expectedCallCount: 10, + expectedSuccessCount: 10, + }, + { + name: "failed without shortcut", + count: 10, + errorIndex: intPointer(5), + shortcut: false, + expectedCallCount: 10, + expectedSuccessCount: 9, + }, + { + name: "failed without shortcut", + count: 10, + errorIndex: intPointer(5), + shortcut: true, + expectedCallCount: 7, // 1 count in batch 1 + 2 counts in batch 2 + 4 counts in batch 3 + expectedSuccessCount: 6, + }, + } + + for _, testcase := range testcases { + callCounter := intPointer(0) + successCount, err := SlowStartBatch(testcase.count, 1, testcase.shortcut, buildFn(callCounter, testcase.errorIndex)) + if testcase.errorIndex == nil { + if err != nil { + t.Fatalf("case %s has unexpected err: %s", testcase.name, err) + } + } else { + if err == nil { + t.Fatalf("case %s has no expected err", testcase.name) + } else if err.Error() != expectedErrMsg { + t.Fatalf("case %s has expected err with unexpected message: %s", testcase.name, err) + } + } + + if successCount != testcase.expectedSuccessCount { + t.Fatalf("case %s gets unexpected success count: expected %d, got %d", testcase.name, testcase.expectedSuccessCount, successCount) + } + + if *callCounter != testcase.expectedCallCount { + t.Fatalf("case %s gets unexpected call count: expected %d, got %d", testcase.name, testcase.expectedCallCount, *callCounter) + } + } +} + +func buildFn(callCounter, errIdx *int) func(int, error) error { + lock := sync.Mutex{} + return func(i int, err error) error { + lock.Lock() + defer func() { + *callCounter++ + lock.Unlock() + }() + + if errIdx != nil && *callCounter == *errIdx { + return fmt.Errorf(expectedErrMsg) + } + + return nil + } +} + +func intPointer(val int) *int { + return &val +} diff --git a/pkg/webhook/server/generic/collaset/collaset_mutating_handler.go b/pkg/webhook/server/generic/collaset/collaset_mutating_handler.go index eb3f9d85..116cbb80 100644 --- a/pkg/webhook/server/generic/collaset/collaset_mutating_handler.go +++ b/pkg/webhook/server/generic/collaset/collaset_mutating_handler.go @@ -22,7 +22,6 @@ import ( "net/http" admissionv1 "k8s.io/api/admission/v1" - k8scorev1 "k8s.io/kubernetes/pkg/apis/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/runtime/inject" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" @@ -57,7 +56,7 @@ func (h *MutatingHandler) Handle(ctx context.Context, req admission.Request) (re logger.Error(err, "failed to decode collaset") return admission.Errored(http.StatusBadRequest, err) } - h.setDetaultCollaSet(cls) + appsv1alpha1.SetDetaultCollaSet(cls) marshalled, err := json.Marshal(cls) if err != nil { logger.Error(err, "failed to marshal collaset to json") @@ -67,238 +66,6 @@ func (h *MutatingHandler) Handle(ctx context.Context, req admission.Request) (re return admission.PatchResponseFromRaw(req.AdmissionRequest.Object.Raw, marshalled) } -func (h *MutatingHandler) setDetaultCollaSet(cls *appsv1alpha1.CollaSet) { - h.setDefaultPodSpec(cls) - h.setDefaultCollaSetUpdateStrategy(cls) -} - -func (h *MutatingHandler) setDefaultPodSpec(in *appsv1alpha1.CollaSet) { - k8scorev1.SetDefaults_PodSpec(&in.Spec.Template.Spec) - for i := range in.Spec.Template.Spec.Volumes { - a := &in.Spec.Template.Spec.Volumes[i] - k8scorev1.SetDefaults_Volume(a) - if a.VolumeSource.HostPath != nil { - k8scorev1.SetDefaults_HostPathVolumeSource(a.VolumeSource.HostPath) - } - if a.VolumeSource.Secret != nil { - k8scorev1.SetDefaults_SecretVolumeSource(a.VolumeSource.Secret) - } - if a.VolumeSource.ISCSI != nil { - k8scorev1.SetDefaults_ISCSIVolumeSource(a.VolumeSource.ISCSI) - } - if a.VolumeSource.RBD != nil { - k8scorev1.SetDefaults_RBDVolumeSource(a.VolumeSource.RBD) - } - if a.VolumeSource.DownwardAPI != nil { - k8scorev1.SetDefaults_DownwardAPIVolumeSource(a.VolumeSource.DownwardAPI) - for j := range a.VolumeSource.DownwardAPI.Items { - b := &a.VolumeSource.DownwardAPI.Items[j] - if b.FieldRef != nil { - k8scorev1.SetDefaults_ObjectFieldSelector(b.FieldRef) - } - } - } - if a.VolumeSource.ConfigMap != nil { - k8scorev1.SetDefaults_ConfigMapVolumeSource(a.VolumeSource.ConfigMap) - } - if a.VolumeSource.AzureDisk != nil { - k8scorev1.SetDefaults_AzureDiskVolumeSource(a.VolumeSource.AzureDisk) - } - if a.VolumeSource.Projected != nil { - k8scorev1.SetDefaults_ProjectedVolumeSource(a.VolumeSource.Projected) - for j := range a.VolumeSource.Projected.Sources { - b := &a.VolumeSource.Projected.Sources[j] - if b.DownwardAPI != nil { - for k := range b.DownwardAPI.Items { - c := &b.DownwardAPI.Items[k] - if c.FieldRef != nil { - k8scorev1.SetDefaults_ObjectFieldSelector(c.FieldRef) - } - } - } - if b.ServiceAccountToken != nil { - k8scorev1.SetDefaults_ServiceAccountTokenProjection(b.ServiceAccountToken) - } - } - } - if a.VolumeSource.ScaleIO != nil { - k8scorev1.SetDefaults_ScaleIOVolumeSource(a.VolumeSource.ScaleIO) - } - if a.VolumeSource.Ephemeral != nil { - if a.VolumeSource.Ephemeral.VolumeClaimTemplate != nil { - k8scorev1.SetDefaults_PersistentVolumeClaimSpec(&a.VolumeSource.Ephemeral.VolumeClaimTemplate.Spec) - k8scorev1.SetDefaults_ResourceList(&a.VolumeSource.Ephemeral.VolumeClaimTemplate.Spec.Resources.Limits) - k8scorev1.SetDefaults_ResourceList(&a.VolumeSource.Ephemeral.VolumeClaimTemplate.Spec.Resources.Requests) - } - } - } - for i := range in.Spec.Template.Spec.InitContainers { - a := &in.Spec.Template.Spec.InitContainers[i] - k8scorev1.SetDefaults_Container(a) - for j := range a.Ports { - b := &a.Ports[j] - if b.Protocol == "" { - b.Protocol = "TCP" - } - } - for j := range a.Env { - b := &a.Env[j] - if b.ValueFrom != nil { - if b.ValueFrom.FieldRef != nil { - k8scorev1.SetDefaults_ObjectFieldSelector(b.ValueFrom.FieldRef) - } - } - } - k8scorev1.SetDefaults_ResourceList(&a.Resources.Limits) - k8scorev1.SetDefaults_ResourceList(&a.Resources.Requests) - if a.LivenessProbe != nil { - k8scorev1.SetDefaults_Probe(a.LivenessProbe) - if a.LivenessProbe.Handler.HTTPGet != nil { - k8scorev1.SetDefaults_HTTPGetAction(a.LivenessProbe.Handler.HTTPGet) - } - } - if a.ReadinessProbe != nil { - k8scorev1.SetDefaults_Probe(a.ReadinessProbe) - if a.ReadinessProbe.Handler.HTTPGet != nil { - k8scorev1.SetDefaults_HTTPGetAction(a.ReadinessProbe.Handler.HTTPGet) - } - } - if a.StartupProbe != nil { - k8scorev1.SetDefaults_Probe(a.StartupProbe) - if a.StartupProbe.Handler.HTTPGet != nil { - k8scorev1.SetDefaults_HTTPGetAction(a.StartupProbe.Handler.HTTPGet) - } - } - if a.Lifecycle != nil { - if a.Lifecycle.PostStart != nil { - if a.Lifecycle.PostStart.HTTPGet != nil { - k8scorev1.SetDefaults_HTTPGetAction(a.Lifecycle.PostStart.HTTPGet) - } - } - if a.Lifecycle.PreStop != nil { - if a.Lifecycle.PreStop.HTTPGet != nil { - k8scorev1.SetDefaults_HTTPGetAction(a.Lifecycle.PreStop.HTTPGet) - } - } - } - } - for i := range in.Spec.Template.Spec.Containers { - a := &in.Spec.Template.Spec.Containers[i] - k8scorev1.SetDefaults_Container(a) - for j := range a.Ports { - b := &a.Ports[j] - if b.Protocol == "" { - b.Protocol = "TCP" - } - } - for j := range a.Env { - b := &a.Env[j] - if b.ValueFrom != nil { - if b.ValueFrom.FieldRef != nil { - k8scorev1.SetDefaults_ObjectFieldSelector(b.ValueFrom.FieldRef) - } - } - } - k8scorev1.SetDefaults_ResourceList(&a.Resources.Limits) - k8scorev1.SetDefaults_ResourceList(&a.Resources.Requests) - if a.LivenessProbe != nil { - k8scorev1.SetDefaults_Probe(a.LivenessProbe) - if a.LivenessProbe.Handler.HTTPGet != nil { - k8scorev1.SetDefaults_HTTPGetAction(a.LivenessProbe.Handler.HTTPGet) - } - } - if a.ReadinessProbe != nil { - k8scorev1.SetDefaults_Probe(a.ReadinessProbe) - if a.ReadinessProbe.Handler.HTTPGet != nil { - k8scorev1.SetDefaults_HTTPGetAction(a.ReadinessProbe.Handler.HTTPGet) - } - } - if a.StartupProbe != nil { - k8scorev1.SetDefaults_Probe(a.StartupProbe) - if a.StartupProbe.Handler.HTTPGet != nil { - k8scorev1.SetDefaults_HTTPGetAction(a.StartupProbe.Handler.HTTPGet) - } - } - if a.Lifecycle != nil { - if a.Lifecycle.PostStart != nil { - if a.Lifecycle.PostStart.HTTPGet != nil { - k8scorev1.SetDefaults_HTTPGetAction(a.Lifecycle.PostStart.HTTPGet) - } - } - if a.Lifecycle.PreStop != nil { - if a.Lifecycle.PreStop.HTTPGet != nil { - k8scorev1.SetDefaults_HTTPGetAction(a.Lifecycle.PreStop.HTTPGet) - } - } - } - } - for i := range in.Spec.Template.Spec.EphemeralContainers { - a := &in.Spec.Template.Spec.EphemeralContainers[i] - k8scorev1.SetDefaults_EphemeralContainer(a) - for j := range a.EphemeralContainerCommon.Ports { - b := &a.EphemeralContainerCommon.Ports[j] - if b.Protocol == "" { - b.Protocol = "TCP" - } - } - for j := range a.EphemeralContainerCommon.Env { - b := &a.EphemeralContainerCommon.Env[j] - if b.ValueFrom != nil { - if b.ValueFrom.FieldRef != nil { - k8scorev1.SetDefaults_ObjectFieldSelector(b.ValueFrom.FieldRef) - } - } - } - k8scorev1.SetDefaults_ResourceList(&a.EphemeralContainerCommon.Resources.Limits) - k8scorev1.SetDefaults_ResourceList(&a.EphemeralContainerCommon.Resources.Requests) - if a.EphemeralContainerCommon.LivenessProbe != nil { - k8scorev1.SetDefaults_Probe(a.EphemeralContainerCommon.LivenessProbe) - if a.EphemeralContainerCommon.LivenessProbe.Handler.HTTPGet != nil { - k8scorev1.SetDefaults_HTTPGetAction(a.EphemeralContainerCommon.LivenessProbe.Handler.HTTPGet) - } - } - if a.EphemeralContainerCommon.ReadinessProbe != nil { - k8scorev1.SetDefaults_Probe(a.EphemeralContainerCommon.ReadinessProbe) - if a.EphemeralContainerCommon.ReadinessProbe.Handler.HTTPGet != nil { - k8scorev1.SetDefaults_HTTPGetAction(a.EphemeralContainerCommon.ReadinessProbe.Handler.HTTPGet) - } - } - if a.EphemeralContainerCommon.StartupProbe != nil { - k8scorev1.SetDefaults_Probe(a.EphemeralContainerCommon.StartupProbe) - if a.EphemeralContainerCommon.StartupProbe.Handler.HTTPGet != nil { - k8scorev1.SetDefaults_HTTPGetAction(a.EphemeralContainerCommon.StartupProbe.Handler.HTTPGet) - } - } - if a.EphemeralContainerCommon.Lifecycle != nil { - if a.EphemeralContainerCommon.Lifecycle.PostStart != nil { - if a.EphemeralContainerCommon.Lifecycle.PostStart.HTTPGet != nil { - k8scorev1.SetDefaults_HTTPGetAction(a.EphemeralContainerCommon.Lifecycle.PostStart.HTTPGet) - } - } - if a.EphemeralContainerCommon.Lifecycle.PreStop != nil { - if a.EphemeralContainerCommon.Lifecycle.PreStop.HTTPGet != nil { - k8scorev1.SetDefaults_HTTPGetAction(a.EphemeralContainerCommon.Lifecycle.PreStop.HTTPGet) - } - } - } - } - k8scorev1.SetDefaults_ResourceList(&in.Spec.Template.Spec.Overhead) -} - -func (h *MutatingHandler) setDefaultCollaSetUpdateStrategy(cls *appsv1alpha1.CollaSet) { - if cls.Spec.UpdateStrategy.PodUpdatePolicy == "" { - cls.Spec.UpdateStrategy.PodUpdatePolicy = appsv1alpha1.CollaSetInPlaceIfPossiblePodUpdateStrategyType - } - - if cls.Spec.UpdateStrategy.RollingUpdate == nil { - cls.Spec.UpdateStrategy.RollingUpdate = &appsv1alpha1.RollingUpdateCollaSetStrategy{} - } - - if cls.Spec.UpdateStrategy.RollingUpdate.ByPartition == nil && cls.Spec.UpdateStrategy.RollingUpdate.ByLabel == nil { - cls.Spec.UpdateStrategy.RollingUpdate.ByPartition = &appsv1alpha1.ByPartition{} - } -} - var _ inject.Client = &MutatingHandler{} func (h *MutatingHandler) InjectClient(c client.Client) error { diff --git a/pkg/webhook/server/generic/collaset/collaset_mutating_handler_test.go b/pkg/webhook/server/generic/collaset/collaset_mutating_handler_test.go index 3a13efe2..888fe2a0 100644 --- a/pkg/webhook/server/generic/collaset/collaset_mutating_handler_test.go +++ b/pkg/webhook/server/generic/collaset/collaset_mutating_handler_test.go @@ -32,8 +32,7 @@ func TestMutatingCollaSet(t *testing.T) { Spec: appsv1alpha1.CollaSetSpec{}, } - mutatingHandler := NewMutatingHandler() - mutatingHandler.setDetaultCollaSet(cls) + appsv1alpha1.SetDetaultCollaSet(cls) if cls.Spec.UpdateStrategy.PodUpdatePolicy != appsv1alpha1.CollaSetInPlaceIfPossiblePodUpdateStrategyType { t.Fatalf("expected default value is %s, got %s", appsv1alpha1.CollaSetInPlaceIfPossiblePodUpdateStrategyType, diff --git a/pkg/webhook/server/generic/collaset/collaset_validating_handler_test.go b/pkg/webhook/server/generic/collaset/collaset_validating_handler_test.go index e66b4368..88e27454 100644 --- a/pkg/webhook/server/generic/collaset/collaset_validating_handler_test.go +++ b/pkg/webhook/server/generic/collaset/collaset_validating_handler_test.go @@ -66,10 +66,9 @@ func TestValidatingCollaSet(t *testing.T) { } validatingHandler := NewValidatingHandler() - mutatingHandler := NewMutatingHandler() for i, tc := range successCases { - mutatingHandler.setDetaultCollaSet(tc.cls) + appsv1alpha1.SetDetaultCollaSet(tc.cls) if err := validatingHandler.validate(tc.cls, tc.old); err != nil { t.Fatalf("got unexpected err for %d case: %s", i, err) } @@ -397,7 +396,7 @@ func TestValidatingCollaSet(t *testing.T) { } for key, tc := range failureCases { - mutatingHandler.setDetaultCollaSet(tc.cls) + appsv1alpha1.SetDetaultCollaSet(tc.cls) err := validatingHandler.validate(tc.cls, tc.old) if err == nil { t.Fatalf("expected err, got nil in case %s", key) diff --git a/pkg/webhook/server/generic/pod/opslifecycle/mutating.go b/pkg/webhook/server/generic/pod/opslifecycle/mutating.go index 02321f07..3822f62d 100644 --- a/pkg/webhook/server/generic/pod/opslifecycle/mutating.go +++ b/pkg/webhook/server/generic/pod/opslifecycle/mutating.go @@ -126,7 +126,7 @@ func (lc *OpsLifecycle) Mutating(ctx context.Context, c client.Client, oldPod, n } if completeCount == numOfIDs { // all operations are completed - satisfied, expectedFinalizers, err := controllerutils.SatisfyExpectedFinalizers(newPod) // whether all expected finalizers are satisfied + satisfied, expectedFinalizers, err := controllerutils.IsExpectedFinalizerSatisfied(newPod) // whether all expected finalizers are satisfied if err != nil || !satisfied { klog.Infof("pod: %s/%s, satisfied: %v, expectedFinalizer: %v, err: %v", newPod.Namespace, newPod.Name, satisfied, expectedFinalizers, err) return err