diff --git a/pkg/controllers/collaset/collaset_controller.go b/pkg/controllers/collaset/collaset_controller.go index 8b049017..8bdd5634 100644 --- a/pkg/controllers/collaset/collaset_controller.go +++ b/pkg/controllers/collaset/collaset_controller.go @@ -210,7 +210,7 @@ func calculateStatus(instance *appsv1alpha1.CollaSet, newStatus *appsv1alpha1.Co replicas++ isUpdated := false - if isUpdated = controllerutils.IsPodUpdatedRevision(podWrapper.Pod, updatedRevision.Name); isUpdated { + if isUpdated = utils.IsPodUpdatedRevision(podWrapper.Pod, updatedRevision.Name); isUpdated { updatedReplicas++ } @@ -229,7 +229,7 @@ func calculateStatus(instance *appsv1alpha1.CollaSet, newStatus *appsv1alpha1.Co } } - if controllerutils.IsServiceAvailable(podWrapper.Pod) { + if controllerutils.IsPodServiceAvailable(podWrapper.Pod) { availableReplicas++ if isUpdated { updatedAvailableReplicas++ diff --git a/pkg/controllers/collaset/collaset_controller_test.go b/pkg/controllers/collaset/collaset_controller_test.go index 3d0774d4..835e33a2 100644 --- a/pkg/controllers/collaset/collaset_controller_test.go +++ b/pkg/controllers/collaset/collaset_controller_test.go @@ -117,7 +117,7 @@ var _ = Describe("collaset controller", func() { Eventually(func() bool { Expect(c.Get(context.TODO(), types.NamespacedName{Namespace: podToDelete.Namespace, Name: podToDelete.Name}, podToDelete)).Should(BeNil()) return podToDelete.DeletionTimestamp != nil - }, 500*time.Second, 1*time.Second).Should(BeTrue()) + }, 5*time.Second, 1*time.Second).Should(BeTrue()) // there should be 3 pods and one of them is terminating Eventually(func() bool { diff --git a/pkg/controllers/collaset/revision.go b/pkg/controllers/collaset/revision.go index f4cbdf27..3f58c730 100644 --- a/pkg/controllers/collaset/revision.go +++ b/pkg/controllers/collaset/revision.go @@ -22,6 +22,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" appsalphav1 "kusionstack.io/operating/apis/apps/v1alpha1" + "kusionstack.io/operating/pkg/controllers/utils/revision" ) func getCollaSetPatch(cls *appsalphav1.CollaSet) ([]byte, error) { @@ -52,6 +53,8 @@ func getCollaSetPatch(cls *appsalphav1.CollaSet) ([]byte, error) { return patch, err } +var _ revision.OwnerAdapter = &revisionOwnerAdapter{} + type revisionOwnerAdapter struct { } diff --git a/pkg/controllers/collaset/synccontrol/scale.go b/pkg/controllers/collaset/synccontrol/scale.go index 2cfc953c..50c09f65 100644 --- a/pkg/controllers/collaset/synccontrol/scale.go +++ b/pkg/controllers/collaset/synccontrol/scale.go @@ -20,7 +20,6 @@ import ( "sort" collasetutils "kusionstack.io/operating/pkg/controllers/collaset/utils" - controllerutils "kusionstack.io/operating/pkg/controllers/utils" "kusionstack.io/operating/pkg/controllers/utils/podopslifecycle" ) @@ -52,5 +51,5 @@ func (s ActivePodsForDeletion) Less(i, j int) bool { return false } - return controllerutils.ComparePod(l.Pod, r.Pod) + return collasetutils.ComparePod(l.Pod, r.Pod) } diff --git a/pkg/controllers/collaset/synccontrol/update.go b/pkg/controllers/collaset/synccontrol/update.go index d5f440df..7c11d4d5 100644 --- a/pkg/controllers/collaset/synccontrol/update.go +++ b/pkg/controllers/collaset/synccontrol/update.go @@ -25,10 +25,10 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/equality" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + appsv1alpha1 "kusionstack.io/operating/apis/apps/v1alpha1" "kusionstack.io/operating/pkg/controllers/collaset/utils" collasetutils "kusionstack.io/operating/pkg/controllers/collaset/utils" - controllerutils "kusionstack.io/operating/pkg/controllers/utils" "kusionstack.io/operating/pkg/controllers/utils/podopslifecycle" ) @@ -134,7 +134,7 @@ func (o orderByDefault) Less(i, j int) bool { return false } - return controllerutils.ComparePod(l.Pod, r.Pod) + return utils.ComparePod(l.Pod, r.Pod) } type PodUpdater interface { @@ -191,7 +191,7 @@ func (u *InPlaceIfPossibleUpdater) AnalyseAndGetUpdatedPod(cls *appsv1alpha1.Col } inPlaceUpdateSupport = true - updatedPod, err = controllerutils.PatchToPod(currentPod, updatedPod, podUpdateInfo.Pod) + updatedPod, err = utils.PatchToPod(currentPod, updatedPod, podUpdateInfo.Pod) if onlyMetadataChanged { if updatedPod.Annotations != nil { diff --git a/pkg/controllers/collaset/utils/pod.go b/pkg/controllers/collaset/utils/pod.go index 4e41ccc3..80826390 100644 --- a/pkg/controllers/collaset/utils/pod.go +++ b/pkg/controllers/collaset/utils/pod.go @@ -1,4 +1,5 @@ /* +Copyright 2014 The Kubernetes Authors. Copyright 2023 The KusionStack Authors. Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,15 +18,20 @@ limitations under the License. package utils import ( + "encoding/json" "fmt" "strconv" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + apimachineryvalidation "k8s.io/apimachinery/pkg/api/validation" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/strategicpatch" appsv1alpha1 "kusionstack.io/operating/apis/apps/v1alpha1" controllerutils "kusionstack.io/operating/pkg/controllers/utils" + revisionutils "kusionstack.io/operating/pkg/controllers/utils/revision" "kusionstack.io/operating/pkg/utils" ) @@ -64,11 +70,172 @@ func GetPodInstanceID(pod *corev1.Pod) (int, error) { } func NewPodFrom(owner metav1.Object, ownerRef *metav1.OwnerReference, revision *appsv1.ControllerRevision) (*corev1.Pod, error) { - pod, err := controllerutils.NewPodFrom(owner, ownerRef, revision) + pod, err := GetPodFromRevision(revision) if err != nil { - return nil, err + return pod, err } + pod.Namespace = owner.GetNamespace() + pod.GenerateName = GetPodsPrefix(owner.GetName()) + pod.OwnerReferences = append(pod.OwnerReferences, *ownerRef) + + pod.Labels[appsv1.ControllerRevisionHashLabelKey] = revision.Name + utils.ControllByKusionStack(pod) return pod, nil } + +func GetPodRevisionPatch(revision *appsv1.ControllerRevision) ([]byte, error) { + var raw map[string]interface{} + if err := json.Unmarshal([]byte(revision.Data.Raw), &raw); err != nil { + return nil, err + } + + spec := raw["spec"].(map[string]interface{}) + template := spec["template"].(map[string]interface{}) + patch, err := json.Marshal(template) + return patch, err +} + +func ApplyPatchFromRevision(pod *corev1.Pod, revision *appsv1.ControllerRevision) (*corev1.Pod, error) { + patch, err := GetPodRevisionPatch(revision) + if err != nil { + return nil, err + } + + clone := pod.DeepCopy() + patched, err := strategicpatch.StrategicMergePatch([]byte(runtime.EncodeOrDie(revisionutils.PodCodec, clone)), patch, clone) + if err != nil { + return nil, err + } + err = json.Unmarshal(patched, clone) + if err != nil { + return nil, err + } + return clone, nil +} + +// PatchToPod Use three way merge to get a updated pod. +func PatchToPod(currentRevisionPod, updateRevisionPod, currentPod *corev1.Pod) (*corev1.Pod, error) { + currentRevisionPodBytes, err := json.Marshal(currentRevisionPod) + if err != nil { + return nil, err + } + updateRevisionPodBytes, err := json.Marshal(updateRevisionPod) + + if err != nil { + return nil, err + } + + // 1. find the extra changes based on current revision + patch, err := strategicpatch.CreateTwoWayMergePatch(currentRevisionPodBytes, updateRevisionPodBytes, &corev1.Pod{}) + if err != nil { + return nil, err + } + + // 2. apply above changes to current pod + // We don't apply the diff between currentPod and currentRevisionPod to updateRevisionPod, + // because the PodTemplate changes should have the highest priority. + currentPodBytes, err := json.Marshal(currentPod) + if err != nil { + return nil, err + } + if updateRevisionPodBytes, err = strategicpatch.StrategicMergePatch(currentPodBytes, patch, &corev1.Pod{}); err != nil { + return nil, err + } + + newPod := &corev1.Pod{} + err = json.Unmarshal(updateRevisionPodBytes, newPod) + return newPod, err +} + +func GetPodFromRevision(revision *appsv1.ControllerRevision) (*corev1.Pod, error) { + pod, err := ApplyPatchFromRevision(&corev1.Pod{}, revision) + if err != nil { + return nil, err + } + + return pod, nil +} + +func GetPodsPrefix(controllerName string) string { + // use the dash (if the name isn't too long) to make the pod name a bit prettier + prefix := fmt.Sprintf("%s-", controllerName) + if len(apimachineryvalidation.NameIsDNSSubdomain(prefix, true)) != 0 { + prefix = controllerName + } + return prefix +} + +func ComparePod(l, r *corev1.Pod) bool { + // 1. Unassigned < assigned + // If only one of the pods is unassigned, the unassigned one is smaller + if l.Spec.NodeName != r.Spec.NodeName && (len(l.Spec.NodeName) == 0 || len(r.Spec.NodeName) == 0) { + return len(l.Spec.NodeName) == 0 + } + // 2. PodPending < PodUnknown < PodRunning + m := map[corev1.PodPhase]int{corev1.PodPending: 0, corev1.PodUnknown: 1, corev1.PodRunning: 2} + if m[l.Status.Phase] != m[r.Status.Phase] { + return m[l.Status.Phase] < m[r.Status.Phase] + } + // 3. Not ready < ready + // If only one of the pods is not ready, the not ready one is smaller + if controllerutils.IsPodReady(l) != controllerutils.IsPodReady(r) { + return !controllerutils.IsPodReady(l) + } + // TODO: take availability into account when we push minReadySeconds information from deployment into pods, + // see https://github.com/kubernetes/kubernetes/issues/22065 + // 4. Been ready for empty time < less time < more time + // If both pods are ready, the latest ready one is smaller + if controllerutils.IsPodReady(l) && controllerutils.IsPodReady(r) && !podReadyTime(l).Equal(podReadyTime(r)) { + return afterOrZero(podReadyTime(l), podReadyTime(r)) + } + // 5. Pods with containers with higher restart counts < lower restart counts + if maxContainerRestarts(l) != maxContainerRestarts(r) { + return maxContainerRestarts(l) > maxContainerRestarts(r) + } + // 6. Empty creation time pods < newer pods < older pods + if !l.CreationTimestamp.Equal(&r.CreationTimestamp) { + return afterOrZero(&l.CreationTimestamp, &r.CreationTimestamp) + } + return false +} + +func maxContainerRestarts(pod *corev1.Pod) int { + var maxRestarts int32 + for _, c := range pod.Status.ContainerStatuses { + if c.RestartCount > maxRestarts { + maxRestarts = c.RestartCount + } + } + return int(maxRestarts) +} + +func podReadyTime(pod *corev1.Pod) *metav1.Time { + if controllerutils.IsPodReady(pod) { + for _, c := range pod.Status.Conditions { + // we only care about pod ready conditions + if c.Type == corev1.PodReady && c.Status == corev1.ConditionTrue { + return &c.LastTransitionTime + } + } + } + return &metav1.Time{} +} + +// afterOrZero checks if time t1 is after time t2; if one of them +// is zero, the zero time is seen as after non-zero time. +func afterOrZero(t1, t2 *metav1.Time) bool { + if t1.Time.IsZero() || t2.Time.IsZero() { + return t1.Time.IsZero() + } + return t1.After(t2.Time) +} + +func IsPodUpdatedRevision(pod *corev1.Pod, revision string) bool { + if pod.Labels == nil { + return false + } + + return pod.Labels[appsv1.ControllerRevisionHashLabelKey] == revision +} diff --git a/pkg/controllers/podopslifecycle/podopslifecycle_controller.go b/pkg/controllers/podopslifecycle/podopslifecycle_controller.go index 4bac2547..2d504ac4 100644 --- a/pkg/controllers/podopslifecycle/podopslifecycle_controller.go +++ b/pkg/controllers/podopslifecycle/podopslifecycle_controller.go @@ -193,7 +193,7 @@ func (r *ReconcilePodOpsLifecycle) addServiceAvailable(pod *corev1.Pod) (bool, e return false, nil } - satisfied, _, err := controllerutils.SatisfyExpectedFinalizers(pod) // whether all expected finalizers are satisfied + satisfied, _, err := controllerutils.IsExpectedFinalizerSatisfied(pod) // whether all expected finalizers are satisfied if err != nil || !satisfied { return false, err } @@ -362,7 +362,7 @@ func (r *ReconcilePodOpsLifecycle) initPodTransitionRuleManager() { return labels != nil && labelHasPrefix(labels, v1alpha1.PodPostCheckLabelPrefix) }) podtransitionrule.AddUnAvailableFunc(func(po *corev1.Pod) (bool, *int64) { - return !controllerutils.IsServiceAvailable(po), nil + return !controllerutils.IsPodServiceAvailable(po), nil }) } diff --git a/pkg/controllers/utils/patch_test.go b/pkg/controllers/utils/patch_test.go index 0c4b4bf8..b4722cb9 100644 --- a/pkg/controllers/utils/patch_test.go +++ b/pkg/controllers/utils/patch_test.go @@ -17,11 +17,12 @@ package utils import ( - "fmt" "testing" ) func TestPatch(t *testing.T) { val := GetLabelAnnoPatchBytes(nil, nil, nil, map[string]string{"detailAnno": "newDetail"}) - fmt.Println(string(val)) + if "{\"metadata\":{\"annotations\":{\"detailAnno\":\"newDetail\"}}}" != string(val) { + t.Fatalf("get unexpected patch: %s", val) + } } diff --git a/pkg/controllers/utils/pod_utils.go b/pkg/controllers/utils/pod_utils.go index 17ea9aac..ab00b934 100644 --- a/pkg/controllers/utils/pod_utils.go +++ b/pkg/controllers/utils/pod_utils.go @@ -1,5 +1,4 @@ /* -Copyright 2014 The Kubernetes Authors. Copyright 2023 The KusionStack Authors. Licensed under the Apache License, Version 2.0 (the "License"); @@ -19,183 +18,14 @@ package utils import ( "encoding/json" - "fmt" - appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" - apimachineryvalidation "k8s.io/apimachinery/pkg/api/validation" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/apimachinery/pkg/util/strategicpatch" "kusionstack.io/operating/apis/apps/v1alpha1" appsv1alpha1 "kusionstack.io/operating/apis/apps/v1alpha1" - revisionutils "kusionstack.io/operating/pkg/controllers/utils/revision" ) -func GetPodRevisionPatch(revision *appsv1.ControllerRevision) ([]byte, error) { - var raw map[string]interface{} - if err := json.Unmarshal([]byte(revision.Data.Raw), &raw); err != nil { - return nil, err - } - - spec := raw["spec"].(map[string]interface{}) - template := spec["template"].(map[string]interface{}) - patch, err := json.Marshal(template) - return patch, err -} - -func ApplyPatchFromRevision(pod *corev1.Pod, revision *appsv1.ControllerRevision) (*corev1.Pod, error) { - patch, err := GetPodRevisionPatch(revision) - if err != nil { - return nil, err - } - - clone := pod.DeepCopy() - patched, err := strategicpatch.StrategicMergePatch([]byte(runtime.EncodeOrDie(revisionutils.PodCodec, clone)), patch, clone) - if err != nil { - return nil, err - } - err = json.Unmarshal(patched, clone) - if err != nil { - return nil, err - } - return clone, nil -} - -// PatchToPod Use three way merge to get a updated pod. -func PatchToPod(currentRevisionPod, updateRevisionPod, currentPod *corev1.Pod) (*corev1.Pod, error) { - currentRevisionPodBytes, err := json.Marshal(currentRevisionPod) - if err != nil { - return nil, err - } - updateRevisionPodBytes, err := json.Marshal(updateRevisionPod) - - if err != nil { - return nil, err - } - - // 1. find the extra changes based on current revision - patch, err := strategicpatch.CreateTwoWayMergePatch(currentRevisionPodBytes, updateRevisionPodBytes, &corev1.Pod{}) - if err != nil { - return nil, err - } - - // 2. apply above changes to current pod - // We don't apply the diff between currentPod and currentRevisionPod to updateRevisionPod, - // because the PodTemplate changes should have the highest priority. - currentPodBytes, err := json.Marshal(currentPod) - if err != nil { - return nil, err - } - if updateRevisionPodBytes, err = strategicpatch.StrategicMergePatch(currentPodBytes, patch, &corev1.Pod{}); err != nil { - return nil, err - } - - newPod := &corev1.Pod{} - err = json.Unmarshal(updateRevisionPodBytes, newPod) - return newPod, err -} - -func NewPodFrom(owner metav1.Object, ownerRef *metav1.OwnerReference, revision *appsv1.ControllerRevision) (*corev1.Pod, error) { - pod, err := GetPodFromRevision(revision) - if err != nil { - return pod, err - } - - pod.Namespace = owner.GetNamespace() - pod.GenerateName = GetPodsPrefix(owner.GetName()) - pod.OwnerReferences = append(pod.OwnerReferences, *ownerRef) - - pod.Labels[appsv1.ControllerRevisionHashLabelKey] = revision.Name - - return pod, nil -} - -func GetPodFromRevision(revision *appsv1.ControllerRevision) (*corev1.Pod, error) { - pod, err := ApplyPatchFromRevision(&corev1.Pod{}, revision) - if err != nil { - return nil, err - } - - return pod, nil -} - -func GetPodsPrefix(controllerName string) string { - // use the dash (if the name isn't too long) to make the pod name a bit prettier - prefix := fmt.Sprintf("%s-", controllerName) - if len(apimachineryvalidation.NameIsDNSSubdomain(prefix, true)) != 0 { - prefix = controllerName - } - return prefix -} - -func ComparePod(l, r *corev1.Pod) bool { - // 1. Unassigned < assigned - // If only one of the pods is unassigned, the unassigned one is smaller - if l.Spec.NodeName != r.Spec.NodeName && (len(l.Spec.NodeName) == 0 || len(r.Spec.NodeName) == 0) { - return len(l.Spec.NodeName) == 0 - } - // 2. PodPending < PodUnknown < PodRunning - m := map[corev1.PodPhase]int{corev1.PodPending: 0, corev1.PodUnknown: 1, corev1.PodRunning: 2} - if m[l.Status.Phase] != m[r.Status.Phase] { - return m[l.Status.Phase] < m[r.Status.Phase] - } - // 3. Not ready < ready - // If only one of the pods is not ready, the not ready one is smaller - if IsPodReady(l) != IsPodReady(r) { - return !IsPodReady(l) - } - // TODO: take availability into account when we push minReadySeconds information from deployment into pods, - // see https://github.com/kubernetes/kubernetes/issues/22065 - // 4. Been ready for empty time < less time < more time - // If both pods are ready, the latest ready one is smaller - if IsPodReady(l) && IsPodReady(r) && !podReadyTime(l).Equal(podReadyTime(r)) { - return afterOrZero(podReadyTime(l), podReadyTime(r)) - } - // 5. Pods with containers with higher restart counts < lower restart counts - if maxContainerRestarts(l) != maxContainerRestarts(r) { - return maxContainerRestarts(l) > maxContainerRestarts(r) - } - // 6. Empty creation time pods < newer pods < older pods - if !l.CreationTimestamp.Equal(&r.CreationTimestamp) { - return afterOrZero(&l.CreationTimestamp, &r.CreationTimestamp) - } - return false -} - -func maxContainerRestarts(pod *corev1.Pod) int { - var maxRestarts int32 - for _, c := range pod.Status.ContainerStatuses { - if c.RestartCount > maxRestarts { - maxRestarts = c.RestartCount - } - } - return int(maxRestarts) -} - -func podReadyTime(pod *corev1.Pod) *metav1.Time { - if IsPodReady(pod) { - for _, c := range pod.Status.Conditions { - // we only care about pod ready conditions - if c.Type == corev1.PodReady && c.Status == corev1.ConditionTrue { - return &c.LastTransitionTime - } - } - } - return &metav1.Time{} -} - -// afterOrZero checks if time t1 is after time t2; if one of them -// is zero, the zero time is seen as after non-zero time. -func afterOrZero(t1, t2 *metav1.Time) bool { - if t1.Time.IsZero() || t2.Time.IsZero() { - return t1.Time.IsZero() - } - return t1.After(t2.Time) -} - func IsPodScheduled(pod *corev1.Pod) bool { return IsPodScheduledConditionTrue(pod.Status) } @@ -261,7 +91,7 @@ func GetPodConditionFromList(conditions []corev1.PodCondition, conditionType cor return -1, nil } -func IsServiceAvailable(pod *corev1.Pod) bool { +func IsPodServiceAvailable(pod *corev1.Pod) bool { if pod.Labels == nil { return false } @@ -270,15 +100,7 @@ func IsServiceAvailable(pod *corev1.Pod) bool { return exist } -func IsPodUpdatedRevision(pod *corev1.Pod, revision string) bool { - if pod.Labels == nil { - return false - } - - return pod.Labels[appsv1.ControllerRevisionHashLabelKey] == revision -} - -func SatisfyExpectedFinalizers(pod *corev1.Pod) (bool, []string, error) { +func IsExpectedFinalizerSatisfied(pod *corev1.Pod) (bool, []string, error) { satisfied := true var expectedFinalizers []string // expected finalizers that are not satisfied diff --git a/pkg/controllers/utils/pod_utils_test.go b/pkg/controllers/utils/pod_utils_test.go new file mode 100644 index 00000000..7cf36e9f --- /dev/null +++ b/pkg/controllers/utils/pod_utils_test.go @@ -0,0 +1,337 @@ +/* +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "encoding/json" + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + appsv1alpha1 "kusionstack.io/operating/apis/apps/v1alpha1" +) + +func TestIsPodScheduled(t *testing.T) { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + }, + Spec: corev1.PodSpec{}, + } + + if IsPodScheduled(pod) { + t.Fatalf("expected failure") + } + + pod = &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + }, + Spec: corev1.PodSpec{}, + Status: corev1.PodStatus{ + Conditions: []corev1.PodCondition{}, + }, + } + + if IsPodScheduled(pod) { + t.Fatalf("expected failure") + } + + pod = &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + }, + Spec: corev1.PodSpec{}, + Status: corev1.PodStatus{ + Conditions: []corev1.PodCondition{ + { + Type: corev1.PodScheduled, + Status: corev1.ConditionFalse, + }, + }, + }, + } + + if IsPodScheduled(pod) { + t.Fatalf("expected failure") + } + + pod = &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + }, + Spec: corev1.PodSpec{}, + Status: corev1.PodStatus{ + Conditions: []corev1.PodCondition{ + { + Type: corev1.PodScheduled, + Status: corev1.ConditionTrue, + }, + }, + }, + } + + if !IsPodScheduled(pod) { + t.Fatalf("expected failure") + } +} + +func TestIsPodReady(t *testing.T) { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + }, + Spec: corev1.PodSpec{}, + } + + if IsPodReady(pod) { + t.Fatalf("expected failure") + } + + pod = &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + }, + Spec: corev1.PodSpec{}, + Status: corev1.PodStatus{ + Conditions: []corev1.PodCondition{ + { + Type: corev1.PodReady, + Status: corev1.ConditionFalse, + }, + }, + }, + } + + if IsPodReady(pod) { + t.Fatalf("expected failure") + } + + pod = &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + }, + Spec: corev1.PodSpec{}, + Status: corev1.PodStatus{ + Conditions: []corev1.PodCondition{ + { + Type: corev1.PodReady, + Status: corev1.ConditionTrue, + }, + }, + }, + } + + if !IsPodReady(pod) { + t.Fatalf("expected failure") + } +} + +func TestIsPodTerminal(t *testing.T) { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + }, + Spec: corev1.PodSpec{}, + Status: corev1.PodStatus{ + Phase: corev1.PodFailed, + }, + } + + if !IsPodTerminal(pod) { + t.Fatalf("expected failure") + } + + pod = &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + }, + Spec: corev1.PodSpec{}, + Status: corev1.PodStatus{ + Phase: corev1.PodSucceeded, + }, + } + + if !IsPodTerminal(pod) { + t.Fatalf("expected failure") + } + + pod = &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + }, + Spec: corev1.PodSpec{}, + Status: corev1.PodStatus{ + Phase: corev1.PodPending, + }, + } + + if IsPodTerminal(pod) { + t.Fatalf("expected failure") + } + + pod = &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + }, + Spec: corev1.PodSpec{}, + Status: corev1.PodStatus{ + Phase: corev1.PodRunning, + }, + } + + if IsPodTerminal(pod) { + t.Fatalf("expected failure") + } + + pod = &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + }, + Spec: corev1.PodSpec{}, + Status: corev1.PodStatus{ + Phase: corev1.PodUnknown, + }, + } + + if IsPodTerminal(pod) { + t.Fatalf("expected failure") + } +} + +func TestIsPodServiceAvailable(t *testing.T) { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + }, + Spec: corev1.PodSpec{}, + } + + if IsPodServiceAvailable(pod) { + t.Fatalf("expected failure") + } + + pod.Labels = map[string]string{ + appsv1alpha1.PodServiceAvailableLabel: "true", + } + + if !IsPodServiceAvailable(pod) { + t.Fatalf("expected failure") + } +} + +func TestIsPodExpectedFinalizerSatisfied(t *testing.T) { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + }, + Spec: corev1.PodSpec{}, + } + + if satisfied, _, err := IsExpectedFinalizerSatisfied(pod); err != nil || !satisfied { + t.Fatalf("expected failure") + } + + pod = &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Annotations: map[string]string{}, + }, + Spec: corev1.PodSpec{}, + } + + if satisfied, _, err := IsExpectedFinalizerSatisfied(pod); err != nil || !satisfied { + t.Fatalf("expected failure") + } + + pod = &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Annotations: map[string]string{ + appsv1alpha1.PodAvailableConditionsAnnotation: "invalid", + }, + }, + Spec: corev1.PodSpec{}, + } + + if _, _, err := IsExpectedFinalizerSatisfied(pod); err == nil { + t.Fatalf("expected failure") + } + + condFinalizer1 := "test/cond1" + condFinalizer2 := "test/cond2" + cond := &appsv1alpha1.PodAvailableConditions{ + ExpectedFinalizers: map[string]string{ + "cond1": condFinalizer1, + "cond2": condFinalizer2, + }, + } + s, err := json.Marshal(cond) + if err != nil { + t.Fatalf("unexpected err") + } + + pod = &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Annotations: map[string]string{ + appsv1alpha1.PodAvailableConditionsAnnotation: string(s), + }, + }, + Spec: corev1.PodSpec{}, + } + + if satisfied, _, err := IsExpectedFinalizerSatisfied(pod); err != nil || satisfied { + t.Fatalf("expected failure") + } + + pod = &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Annotations: map[string]string{ + appsv1alpha1.PodAvailableConditionsAnnotation: string(s), + }, + Finalizers: []string{ + condFinalizer1, + }, + }, + Spec: corev1.PodSpec{}, + } + + if satisfied, _, err := IsExpectedFinalizerSatisfied(pod); err != nil || satisfied { + t.Fatalf("expected failure") + } + + pod = &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Annotations: map[string]string{ + appsv1alpha1.PodAvailableConditionsAnnotation: string(s), + }, + Finalizers: []string{ + condFinalizer1, + condFinalizer2, + }, + }, + Spec: corev1.PodSpec{}, + } + + if satisfied, _, err := IsExpectedFinalizerSatisfied(pod); err != nil || !satisfied { + t.Fatalf("expected failure") + } +} diff --git a/pkg/controllers/utils/revision/revision_manager.go b/pkg/controllers/utils/revision/revision_manager.go index 8dda30b8..7a79c630 100644 --- a/pkg/controllers/utils/revision/revision_manager.go +++ b/pkg/controllers/utils/revision/revision_manager.go @@ -163,6 +163,7 @@ func (rm *RevisionManager) ConstructRevisions(set client.Object, dryRun bool) (* } } + revisions = append(revisions, updateRevision) createNewRevision = true } @@ -181,11 +182,8 @@ func (rm *RevisionManager) ConstructRevisions(set client.Object, dryRun bool) (* return currentRevision, updateRevision, revisions, collisionCount, createNewRevision, nil } -func (rm *RevisionManager) cleanExpiredRevision(cd metav1.Object, sortedRevisions *[]*apps.ControllerRevision) (*[]*apps.ControllerRevision, error) { - limit := int(rm.ownerGetter.GetHistoryLimit(cd)) - if limit <= 0 { - limit = 10 - } +func (rm *RevisionManager) cleanExpiredRevision(set metav1.Object, sortedRevisions *[]*apps.ControllerRevision) (*[]*apps.ControllerRevision, error) { + limit := int(rm.ownerGetter.GetHistoryLimit(set)) // reserve 2 extra unused revisions for diagnose exceedNum := len(*sortedRevisions) - limit - 2 @@ -198,7 +196,7 @@ func (rm *RevisionManager) cleanExpiredRevision(cd metav1.Object, sortedRevision break } - if rm.ownerGetter.IsInUsed(cd, revision.Name) { + if rm.ownerGetter.IsInUsed(set, revision.Name) { continue } diff --git a/pkg/controllers/utils/revision/revision_manager_test.go b/pkg/controllers/utils/revision/revision_manager_test.go new file mode 100644 index 00000000..ce12b896 --- /dev/null +++ b/pkg/controllers/utils/revision/revision_manager_test.go @@ -0,0 +1,340 @@ +/* +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package revision + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "kusionstack.io/operating/apis" + "kusionstack.io/operating/pkg/utils/inject" +) + +var ( + env *envtest.Environment + mgr manager.Manager + request chan reconcile.Request + + ctx context.Context + cancel context.CancelFunc + c client.Client +) + +var _ = Describe("revision manager", func() { + + selectedLabels := map[string]string{"test": "foo"} + selector, err := metav1.ParseToLabelSelector("test=foo") + Expect(err).Should(BeNil()) + + It("revision construction", func() { + testcase := "test-revision-construction" + Expect(createNamespace(c, testcase)).Should(BeNil()) + + deploy := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: testcase, + Name: testcase, + }, + Spec: appsv1.DeploymentSpec{ + Selector: selector, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: selectedLabels, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "test", + Image: "nginx:v1", + }, + }, + }, + }, + }, + } + Expect(mgr.GetClient().Create(context.TODO(), deploy)).Should(BeNil()) + + adapter := &OwnerAdapterImpl{ + name: testcase, + selector: selector, + selectedLabels: selectedLabels, + collisionCount: 0, + historyLimit: 2, + currentRevision: "test", + inUsed: true, + } + + revisionManager := NewRevisionManager(mgr.GetClient(), mgr.GetScheme(), adapter) + currentRevision, updatedRevision, revisionList, collisionCount, createNewRevision, err := revisionManager.ConstructRevisions(deploy, false) + Expect(err).Should(BeNil()) + Expect(createNewRevision).Should(BeTrue()) + Expect(collisionCount).ShouldNot(BeNil()) + Expect(*collisionCount).Should(BeEquivalentTo(0)) + Expect(len(revisionList)).Should(BeEquivalentTo(1)) + Expect(len(currentRevision.Name)).ShouldNot(BeEquivalentTo(0)) + Expect(updatedRevision.Name).Should(BeEquivalentTo(currentRevision.Name)) + v1RevisionName := updatedRevision.Name + waitingCacheUpdate(deploy.Namespace, 1) + adapter.currentRevision = updatedRevision.Name + + // updating deploy spec should construct a new updated CcontrollerRevision + deploy.Spec.Template.Spec.Containers[0].Image = "nginx:v2" + currentRevision, updatedRevision, revisionList, collisionCount, createNewRevision, err = revisionManager.ConstructRevisions(deploy, false) + Expect(err).Should(BeNil()) + Expect(createNewRevision).Should(BeTrue()) + Expect(collisionCount).ShouldNot(BeNil()) + Expect(*collisionCount).Should(BeEquivalentTo(0)) + Expect(len(revisionList)).Should(BeEquivalentTo(2)) + Expect(len(currentRevision.Name)).ShouldNot(BeEquivalentTo(0)) + Expect(len(currentRevision.Name)).ShouldNot(BeEquivalentTo(0)) + Expect(updatedRevision.Name).ShouldNot(BeEquivalentTo(currentRevision.Name)) + waitingCacheUpdate(deploy.Namespace, 2) + + // reconcile with same spec with current revision is updated to updated revision + adapter.currentRevision = updatedRevision.Name + currentRevision, updatedRevision, revisionList, collisionCount, createNewRevision, err = revisionManager.ConstructRevisions(deploy, false) + Expect(err).Should(BeNil()) + Expect(createNewRevision).Should(BeFalse()) + Expect(collisionCount).ShouldNot(BeNil()) + Expect(*collisionCount).Should(BeEquivalentTo(0)) + Expect(len(revisionList)).Should(BeEquivalentTo(2)) + Expect(len(currentRevision.Name)).ShouldNot(BeEquivalentTo(0)) + Expect(len(currentRevision.Name)).ShouldNot(BeEquivalentTo(0)) + Expect(updatedRevision.Name).Should(BeEquivalentTo(currentRevision.Name)) + waitingCacheUpdate(deploy.Namespace, 2) + + // updating deploy spec to old version should not construct a new updated CcontrollerRevision + deploy.Spec.Template.Spec.Containers[0].Image = "nginx:v1" + currentRevision, updatedRevision, revisionList, collisionCount, createNewRevision, err = revisionManager.ConstructRevisions(deploy, false) + Expect(err).Should(BeNil()) + Expect(createNewRevision).Should(BeFalse()) + Expect(collisionCount).ShouldNot(BeNil()) + Expect(*collisionCount).Should(BeEquivalentTo(0)) + Expect(len(revisionList)).Should(BeEquivalentTo(2)) + Expect(len(currentRevision.Name)).ShouldNot(BeEquivalentTo(0)) + Expect(len(currentRevision.Name)).ShouldNot(BeEquivalentTo(0)) + Expect(updatedRevision.Name).ShouldNot(BeEquivalentTo(currentRevision.Name)) + Expect(updatedRevision.Name).Should(BeEquivalentTo(v1RevisionName)) + }) + + It("revision cleanup", func() { + testcase := "test-revision-cleanup" + Expect(createNamespace(c, testcase)).Should(BeNil()) + + deploy := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: testcase, + Name: testcase, + }, + Spec: appsv1.DeploymentSpec{ + Selector: selector, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: selectedLabels, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "test", + Image: "nginx:v1", + }, + }, + }, + }, + }, + } + Expect(mgr.GetClient().Create(context.TODO(), deploy)).Should(BeNil()) + + adapter := &OwnerAdapterImpl{ + name: testcase, + selector: selector, + selectedLabels: selectedLabels, + collisionCount: 0, + historyLimit: 0, // we at lease reserve 2 extra controller revisions + currentRevision: "test", + inUsed: false, + } + + revisionManager := NewRevisionManager(mgr.GetClient(), mgr.GetScheme(), adapter) + revisionManager.ConstructRevisions(deploy, false) + waitingCacheUpdate(deploy.Namespace, 1) + + deploy.Spec.Template.Spec.Containers[0].Image = "nginx:v2" + revisionManager.ConstructRevisions(deploy, false) + waitingCacheUpdate(deploy.Namespace, 2) + + deploy.Spec.Template.Spec.Containers[0].Image = "nginx:v3" + revisionManager.ConstructRevisions(deploy, false) + waitingCacheUpdate(deploy.Namespace, 3) + + revisionManager.ConstructRevisions(deploy, false) + waitingCacheUpdate(deploy.Namespace, 2) + }) + +}) + +func waitingCacheUpdate(namespace string, expectedRevisionCount int) { + Eventually(func() error { + revisionList := &appsv1.ControllerRevisionList{} + if err := c.List(context.TODO(), revisionList, &client.ListOptions{Namespace: namespace}); err != nil { + return err + } + + if len(revisionList.Items) != expectedRevisionCount { + return fmt.Errorf("expected %d, got %d\n", expectedRevisionCount, len(revisionList.Items)) + } + + return nil + }, 5*time.Second, 1*time.Second).Should(BeNil()) +} + +type OwnerAdapterImpl struct { + name string + selector *metav1.LabelSelector + selectedLabels map[string]string + collisionCount int32 + historyLimit int32 + currentRevision string + inUsed bool +} + +func (a OwnerAdapterImpl) GetSelector(obj metav1.Object) *metav1.LabelSelector { + return a.selector +} + +func (a OwnerAdapterImpl) GetCollisionCount(obj metav1.Object) *int32 { + return &a.collisionCount +} + +func (a OwnerAdapterImpl) GetHistoryLimit(obj metav1.Object) int32 { + return a.historyLimit +} + +func (a OwnerAdapterImpl) GetPatch(obj metav1.Object) ([]byte, error) { + // mock patch + return json.Marshal(obj) +} + +func (a OwnerAdapterImpl) GetSelectorLabels(obj metav1.Object) map[string]string { + return a.selectedLabels +} + +func (a OwnerAdapterImpl) GetCurrentRevision(obj metav1.Object) string { + return a.currentRevision +} + +func (a OwnerAdapterImpl) IsInUsed(obj metav1.Object, controllerRevision string) bool { + return a.inUsed +} + +func TestCollaSetController(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "CollaSetController Test Suite") +} + +var _ = BeforeSuite(func() { + By("bootstrapping test environment") + + ctx, cancel = context.WithCancel(context.TODO()) + logf.SetLogger(zap.New(zap.WriteTo(os.Stdout), zap.UseDevMode(true))) + + env = &envtest.Environment{ + CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "config", "crd", "bases")}, + } + + config, err := env.Start() + Expect(err).NotTo(HaveOccurred()) + Expect(config).NotTo(BeNil()) + + mgr, err = manager.New(config, manager.Options{ + MetricsBindAddress: "0", + NewCache: inject.NewCacheWithFieldIndex, + }) + Expect(err).NotTo(HaveOccurred()) + + scheme := mgr.GetScheme() + err = appsv1.SchemeBuilder.AddToScheme(scheme) + Expect(err).NotTo(HaveOccurred()) + err = apis.AddToScheme(scheme) + Expect(err).NotTo(HaveOccurred()) + + c = mgr.GetClient() + + go func() { + err = mgr.Start(ctx) + Expect(err).NotTo(HaveOccurred()) + }() +}) + +var _ = AfterSuite(func() { + By("tearing down the test environment") + + cancel() + + err := env.Stop() + Expect(err).NotTo(HaveOccurred()) +}) + +var _ = AfterEach(func() { + crList := &appsv1.ControllerRevisionList{} + Expect(mgr.GetClient().List(context.Background(), crList)).Should(BeNil()) + + for i := range crList.Items { + Expect(mgr.GetClient().Delete(context.TODO(), &crList.Items[i])).Should(BeNil()) + } + + nsList := &corev1.NamespaceList{} + Expect(mgr.GetClient().List(context.Background(), nsList)).Should(BeNil()) + + for i := range nsList.Items { + if strings.HasPrefix(nsList.Items[i].Name, "test-") { + mgr.GetClient().Delete(context.TODO(), &nsList.Items[i]) + } + } +}) + +func createNamespace(c client.Client, namespaceName string) error { + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: namespaceName, + }, + } + + return c.Create(context.TODO(), ns) +} + +func int32Pointer(val int32) *int32 { + return &val +} diff --git a/pkg/controllers/utils/slow_start.go b/pkg/controllers/utils/slow_start.go index 27199517..085aae58 100644 --- a/pkg/controllers/utils/slow_start.go +++ b/pkg/controllers/utils/slow_start.go @@ -1,4 +1,5 @@ /* +Copyright 2016 The Kubernetes Authors. Copyright 2023 The KusionStack Authors. Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/pkg/controllers/utils/slow_start_test.go b/pkg/controllers/utils/slow_start_test.go new file mode 100644 index 00000000..95adebc8 --- /dev/null +++ b/pkg/controllers/utils/slow_start_test.go @@ -0,0 +1,107 @@ +/* +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "fmt" + "sync" + "testing" +) + +const expectedErrMsg = "expected error" + +func TestSlowStartBatch(t *testing.T) { + testcases := []struct { + name string + count int + errorIndex *int + shortcut bool + + expectedCallCount int + expectedSuccessCount int + }{ + { + name: "happy pass", + count: 10, + errorIndex: nil, + shortcut: false, + expectedCallCount: 10, + expectedSuccessCount: 10, + }, + { + name: "failed without shortcut", + count: 10, + errorIndex: intPointer(5), + shortcut: false, + expectedCallCount: 10, + expectedSuccessCount: 9, + }, + { + name: "failed without shortcut", + count: 10, + errorIndex: intPointer(5), + shortcut: true, + expectedCallCount: 7, // 1 count in batch 1 + 2 counts in batch 2 + 4 counts in batch 3 + expectedSuccessCount: 6, + }, + } + + for _, testcase := range testcases { + callCounter := intPointer(0) + successCount, err := SlowStartBatch(testcase.count, 1, testcase.shortcut, buildFn(callCounter, testcase.errorIndex)) + if testcase.errorIndex == nil { + if err != nil { + t.Fatalf("case %s has unexpected err: %s", testcase.name, err) + } + } else { + if err == nil { + t.Fatalf("case %s has no expected err", testcase.name) + } else if err.Error() != expectedErrMsg { + t.Fatalf("case %s has expected err with unexpected message: %s", testcase.name, err) + } + } + + if successCount != testcase.expectedSuccessCount { + t.Fatalf("case %s gets unexpected success count: expected %d, got %d", testcase.name, testcase.expectedSuccessCount, successCount) + } + + if *callCounter != testcase.expectedCallCount { + t.Fatalf("case %s gets unexpected call count: expected %d, got %d", testcase.name, testcase.expectedCallCount, *callCounter) + } + } +} + +func buildFn(callCounter, errIdx *int) func(int, error) error { + lock := sync.Mutex{} + return func(i int, err error) error { + lock.Lock() + defer func() { + *callCounter++ + lock.Unlock() + }() + + if errIdx != nil && *callCounter == *errIdx { + return fmt.Errorf(expectedErrMsg) + } + + return nil + } +} + +func intPointer(val int) *int { + return &val +} diff --git a/pkg/webhook/server/generic/pod/opslifecycle/mutating.go b/pkg/webhook/server/generic/pod/opslifecycle/mutating.go index 02321f07..3822f62d 100644 --- a/pkg/webhook/server/generic/pod/opslifecycle/mutating.go +++ b/pkg/webhook/server/generic/pod/opslifecycle/mutating.go @@ -126,7 +126,7 @@ func (lc *OpsLifecycle) Mutating(ctx context.Context, c client.Client, oldPod, n } if completeCount == numOfIDs { // all operations are completed - satisfied, expectedFinalizers, err := controllerutils.SatisfyExpectedFinalizers(newPod) // whether all expected finalizers are satisfied + satisfied, expectedFinalizers, err := controllerutils.IsExpectedFinalizerSatisfied(newPod) // whether all expected finalizers are satisfied if err != nil || !satisfied { klog.Infof("pod: %s/%s, satisfied: %v, expectedFinalizer: %v, err: %v", newPod.Namespace, newPod.Name, satisfied, expectedFinalizers, err) return err