Merge pull request #1771 from shysank/v1beta1_upgrade_tests

v1beta1 cluster upgrade tests (using clusterctl upgrade)
kubernetes-sigs · Oct 28, 2021 · df6c654 · df6c654
2 parents 4482f7f + aefd581
commit df6c654
Show file tree

Hide file tree

Showing 9 changed files with 474 additions and 65 deletions.
diff --git a/api/v1beta1/azurecluster_webhook.go b/api/v1beta1/azurecluster_webhook.go
@@ -85,10 +85,20 @@ func (c *AzureCluster) ValidateUpdate(oldRaw runtime.Object) error {
 	}
 
 	if !reflect.DeepEqual(c.Spec.AzureEnvironment, old.Spec.AzureEnvironment) {
-		allErrs = append(allErrs,
-			field.Invalid(field.NewPath("spec", "AzureEnvironment"),
-				c.Spec.AzureEnvironment, "field is immutable"),
-		)
+		// The equality failure could be because of default mismatch between v1alpha3 and v1beta1. This happens because
+		// the new object `r` will have run through the default webhooks but the old object `old` would not have so.
+		// This means if the old object was in v1alpha3, it would not get the new defaults set in v1beta1 resulting
+		// in object inequality. To workaround this, we set the v1beta1 defaults here so that the old object also gets
+		// the new defaults.
+		old.setAzureEnvironmentDefault()
+
+		// if it's still not equal, return error.
+		if !reflect.DeepEqual(c.Spec.AzureEnvironment, old.Spec.AzureEnvironment) {
+			allErrs = append(allErrs,
+				field.Invalid(field.NewPath("spec", "AzureEnvironment"),
+					c.Spec.AzureEnvironment, "field is immutable"),
+			)
+		}
 	}
 
 	if !reflect.DeepEqual(c.Spec.NetworkSpec.PrivateDNSZoneName, old.Spec.NetworkSpec.PrivateDNSZoneName) {

diff --git a/api/v1beta1/azurecluster_webhook_test.go b/api/v1beta1/azurecluster_webhook_test.go
@@ -234,6 +234,16 @@ func TestAzureCluster_ValidateUpdate(t *testing.T) {
 			},
 			wantErr: true,
 		},
+		{
+			name:       "azurecluster azureEnvironment default mismatch",
+			oldCluster: createValidCluster(),
+			cluster: func() *AzureCluster {
+				cluster := createValidCluster()
+				cluster.Spec.AzureEnvironment = "AzurePublicCloud"
+				return cluster
+			}(),
+			wantErr: false,
+		},
 		{
 			name: "control plane outbound lb is immutable",
 			oldCluster: &AzureCluster{

diff --git a/api/v1beta1/azuremachinetemplate_webhook.go b/api/v1beta1/azuremachinetemplate_webhook.go
@@ -64,9 +64,19 @@ func (r *AzureMachineTemplate) ValidateUpdate(oldRaw runtime.Object) error {
 	old := oldRaw.(*AzureMachineTemplate)
 
 	if !reflect.DeepEqual(r.Spec.Template.Spec, old.Spec.Template.Spec) {
-		allErrs = append(allErrs,
-			field.Invalid(field.NewPath("AzureMachineTemplate", "spec", "template", "spec"), r, AzureMachineTemplateImmutableMsg),
-		)
+		// The equality failure could be because of default mismatch between v1alpha3 and v1beta1. This happens because
+		// the new object `r` will have run through the default webhooks but the old object `old` would not have so.
+		// This means if the old object was in v1alpha3, it would not get the new defaults set in v1beta1 resulting
+		// in object inequality. To workaround this, we set the v1beta1 defaults here so that the old object also gets
+		// the new defaults.
+		old.Default()
+
+		// if it's still not equal, return error.
+		if !reflect.DeepEqual(r.Spec.Template.Spec, old.Spec.Template.Spec) {
+			allErrs = append(allErrs,
+				field.Invalid(field.NewPath("AzureMachineTemplate", "spec", "template", "spec"), r, AzureMachineTemplateImmutableMsg),
+			)
+		}
 	}
 
 	if len(allErrs) == 0 {
@@ -83,5 +93,7 @@ func (r *AzureMachineTemplate) ValidateDelete() error {
 // Default implements webhookutil.defaulter so a webhook will be registered for the type.
 func (r *AzureMachineTemplate) Default() {
 	machinetemplatelog.Info("default", "name", r.Name)
-	r.Spec.Template.Spec.SetDefaults(machinetemplatelog)
+	r.Spec.Template.Spec.SetDefaultCachingType()
+	r.Spec.Template.Spec.SetDataDisksDefaults()
+	r.Spec.Template.Spec.SetIdentityDefaults()
 }
diff --git a/api/v1beta1/azuremachinetemplate_webhook_test.go b/api/v1beta1/azuremachinetemplate_webhook_test.go
@@ -226,6 +226,49 @@ func TestAzureMachineTemplate_ValidateUpdate(t *testing.T) {
 			},
 			wantErr: false,
 		},
+		{
+			name: "AzureMachineTemplate with default mismatch",
+			oldTemplate: &AzureMachineTemplate{
+				Spec: AzureMachineTemplateSpec{
+					Template: AzureMachineTemplateResource{
+						Spec: AzureMachineSpec{
+							VMSize:        "size",
+							FailureDomain: &failureDomain,
+							OSDisk: OSDisk{
+								OSType:      "type",
+								DiskSizeGB:  to.Int32Ptr(11),
+								CachingType: "",
+							},
+							DataDisks:    []DataDisk{},
+							SSHPublicKey: "",
+						},
+					},
+				},
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "OldTemplate",
+				},
+			},
+			template: &AzureMachineTemplate{
+				Spec: AzureMachineTemplateSpec{
+					Template: AzureMachineTemplateResource{
+						Spec: AzureMachineSpec{
+							VMSize:        "size",
+							FailureDomain: &failureDomain,
+							OSDisk: OSDisk{
+								OSType:      "type",
+								DiskSizeGB:  to.Int32Ptr(11),
+								CachingType: "None",
+							},
+							DataDisks: []DataDisk{},
+						},
+					},
+				},
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "NewTemplate",
+				},
+			},
+			wantErr: false,
+		},
 	}
 
 	for _, amt := range tests {

diff --git a/test/e2e/capi_test.go b/test/e2e/capi_test.go
@@ -20,13 +20,13 @@ package e2e
 
 import (
 	"context"
+	"encoding/base64"
 	"fmt"
-	"os"
-
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"os"
 	e2e_namespace "sigs.k8s.io/cluster-api-provider-azure/test/e2e/kubernetes/namespace"
 	clusterctl "sigs.k8s.io/cluster-api/cmd/clusterctl/api/v1alpha3"
 	capi_e2e "sigs.k8s.io/cluster-api/test/e2e"
@@ -228,4 +228,34 @@ var _ = Describe("Running the Cluster API E2E tests", func() {
 			}
 		})
 	})
+
+	if os.Getenv("LOCAL_ONLY") != "true" {
+		Context("upgrade from v1alpha3 to v1beta1, and scale workload clusters created in v1alpha3", func() {
+			BeforeEach(func() {
+				// Unset resource group and vnet env variables, since we capi test creates 2 clusters,
+				// and will result in both the clusters using the same vnet and resource group.
+				Expect(os.Unsetenv(AzureResourceGroup)).To(Succeed())
+				Expect(os.Unsetenv(AzureVNetName)).To(Succeed())
+
+				// Set base64 encoded values for v1alpha3 cluster.
+				Expect(os.Setenv("AZURE_CLIENT_ID_B64", base64.StdEncoding.EncodeToString([]byte(os.Getenv(AzureClientId))))).To(Succeed())
+				Expect(os.Setenv("AZURE_CLIENT_SECRET_B64", base64.StdEncoding.EncodeToString([]byte(os.Getenv(AzureClientSecret))))).To(Succeed())
+				Expect(os.Setenv("AZURE_SUBSCRIPTION_ID_B64", base64.StdEncoding.EncodeToString([]byte(os.Getenv("AZURE_SUBSCRIPTION_ID"))))).To(Succeed())
+				Expect(os.Setenv("AZURE_TENANT_ID_B64", base64.StdEncoding.EncodeToString([]byte(os.Getenv("AZURE_TENANT_ID"))))).To(Succeed())
+
+				// Unset windows specific variables
+				Expect(os.Unsetenv("WINDOWS_WORKER_MACHINE_COUNT")).To(Succeed())
+				Expect(os.Unsetenv("K8S_FEATURE_GATES")).To(Succeed())
+			})
+			capi_e2e.ClusterctlUpgradeSpec(ctx, func() capi_e2e.ClusterctlUpgradeSpecInput {
+				return capi_e2e.ClusterctlUpgradeSpecInput{
+					E2EConfig:             e2eConfig,
+					ClusterctlConfigPath:  clusterctlConfigPath,
+					BootstrapClusterProxy: bootstrapClusterProxy,
+					ArtifactFolder:        artifactFolder,
+					SkipCleanup:           skipCleanup,
+				}
+			})
+		})
+	}
 })
diff --git a/test/e2e/config/azure-dev.yaml b/test/e2e/config/azure-dev.yaml
@@ -8,6 +8,15 @@ providers:
   - name: cluster-api
     type: CoreProvider
     versions:
+    - name: v0.3.23 # latest published release in the v1alpha3 series; this is used for v1alpha3 --> v1beta1 clusterctl upgrades test only.
+      value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.3.23/core-components.yaml
+      type: url
+      contract: v1alpha3
+      files:
+        - sourcePath: "../data/shared/v1alpha3/metadata.yaml"
+      replacements:
+        - old: "imagePullPolicy: Always"
+          new: "imagePullPolicy: IfNotPresent"
     - name: v1.0.0
       value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.0.0/core-components.yaml
       type: url
@@ -16,11 +25,20 @@ providers:
       replacements:
       - old: "imagePullPolicy: Always"
         new: "imagePullPolicy: IfNotPresent"
-      - old: "--leader-elect"
-        new: "--leader-elect=false"
+
+
   - name: kubeadm
     type: BootstrapProvider
     versions:
+    - name: v0.3.23 # latest published release in the v1alpha3 series; this is used for v1alpha3 --> v1beta1 clusterctl upgrades test only.
+      value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.3.23/bootstrap-components.yaml
+      type: url
+      contract: v1alpha3
+      files:
+        - sourcePath: "../data/shared/v1alpha3/metadata.yaml"
+      replacements:
+        - old: "imagePullPolicy: Always"
+          new: "imagePullPolicy: IfNotPresent"
     - name: v1.0.0
       value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.0.0/bootstrap-components.yaml
       type: url
@@ -29,11 +47,19 @@ providers:
       replacements:
       - old: "imagePullPolicy: Always"
         new: "imagePullPolicy: IfNotPresent"
-      - old: "--leader-elect"
-        new: "--leader-elect=false"
+
   - name: kubeadm
     type: ControlPlaneProvider
     versions:
+    - name: v0.3.23 # latest published release in the v1alpha3 series; this is used for v1alpha3 --> v1beta1 clusterctl upgrades test only.
+      value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.3.23/control-plane-components.yaml
+      type: url
+      contract: v1alpha3
+      files:
+        - sourcePath: "../data/shared/v1alpha3/metadata.yaml"
+      replacements:
+        - old: "imagePullPolicy: Always"
+          new: "imagePullPolicy: IfNotPresent"
     - name: v1.0.0
       value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.0.0/control-plane-components.yaml
       type: url
@@ -42,66 +68,74 @@ providers:
       replacements:
       - old: "imagePullPolicy: Always"
         new: "imagePullPolicy: IfNotPresent"
-      - old: "--leader-elect"
-        new: "--leader-elect=false"
+
   - name: azure
     type: InfrastructureProvider
     versions:
-    - name: v1.0.0
+    - name: v0.4.15 # latest published release in the v1alpha3 series; this is used for v1alpha3 --> v1beta1 clusterctl upgrades test only.
+      value: https://github.com/kubernetes-sigs/cluster-api-provider-azure/releases/download/v0.4.15/infrastructure-components.yaml
+      type: url
+      contract: v1alpha3
+      files:
+      - sourcePath: "../data/shared/v1alpha3_provider/metadata.yaml"
+      - sourcePath: "../data/infrastructure-azure/v1alpha3/cluster-template-prow.yaml"
+        targetName: "cluster-template.yaml"
+      replacements:
+      - old: "imagePullPolicy: Always"
+        new: "imagePullPolicy: IfNotPresent"
+    - name: v1.0.99 # next; use manifest from source files
       value: "${PWD}/config/default"
       files:
       - sourcePath: "../data/shared/v1beta1_provider/metadata.yaml"
+      - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow.yaml"
+        targetName: "cluster-template.yaml"
+      - sourcePath: "../data/infrastructure-azure/v1beta1/cluster-template.yaml"
+        targetName: "cluster-template-management.yaml"
+      - sourcePath: "../data/infrastructure-azure/v1beta1/cluster-template-kcp-adoption.yaml"
+        targetName: "cluster-template-kcp-adoption.yaml"
+      - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-ipv6.yaml"
+        targetName: "cluster-template-ipv6.yaml"
+      - sourcePath: "../data/infrastructure-azure/v1beta1/cluster-template-md-remediation.yaml"
+        targetName: "cluster-template-md-remediation.yaml"
+      - sourcePath: "../data/infrastructure-azure/v1beta1/cluster-template-kcp-remediation.yaml"
+        targetName: "cluster-template-kcp-remediation.yaml"
+      - sourcePath: "../data/infrastructure-azure/v1beta1/cluster-template-kcp-scale-in.yaml"
+        targetName: "cluster-template-kcp-scale-in.yaml"
+      - sourcePath: "../data/infrastructure-azure/v1beta1/cluster-template-node-drain.yaml"
+        targetName: "cluster-template-node-drain.yaml"
+      - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-machine-pool.yaml"
+        targetName: "cluster-template-machine-pool.yaml"
+      - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml"
+        targetName: "cluster-template-nvidia-gpu.yaml"
+      - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-private.yaml"
+        targetName: "cluster-template-private.yaml"
+      - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-ci-version.yaml"
+        targetName: "cluster-template-conformance-ci-artifacts.yaml"
+      - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-ci-version-windows.yaml"
+        targetName: "cluster-template-conformance-ci-artifacts-windows.yaml"
+      - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-ci-version-windows-containerd-2022.yaml"
+        targetName: "cluster-template-conformance-ci-artifacts-windows-containerd-2022.yaml"
+      - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-ci-version.yaml"
+        targetName: "cluster-template-conformance-ci-artifacts-windows-containerd.yaml"
+      - sourcePath: "${PWD}/templates/test/dev/cluster-template-custom-builds.yaml"
+        targetName: "cluster-template-conformance-presubmit-artifacts.yaml"
+      - sourcePath: "${PWD}/templates/test/dev/cluster-template-custom-builds.yaml"
+        targetName: "cluster-template-conformance-presubmit-artifacts-windows-containerd.yaml"
+      - sourcePath: "${PWD}/templates/test/dev/cluster-template-custom-builds-windows.yaml"
+        targetName: "cluster-template-conformance-presubmit-artifacts-windows.yaml"
+      - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-windows.yaml"
+        targetName: "cluster-template-windows.yaml"
+      - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-machine-pool-windows.yaml"
+        targetName: "cluster-template-machine-pool-windows.yaml"
+      - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-external-cloud-provider.yaml"
+        targetName: "cluster-template-external-cloud-provider.yaml"
+      - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-aks-multi-tenancy.yaml"
+        targetName: "cluster-template-aks-multi-tenancy.yaml"
+      - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-custom-vnet.yaml"
+        targetName: "cluster-template-custom-vnet.yaml"
       replacements:
       - old: "--v=0"
         new: "--v=2"
-    files:
-    - sourcePath: "../data/shared/v1beta1/metadata.yaml"
-    - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow.yaml"
-      targetName: "cluster-template.yaml"
-    - sourcePath: "../data/infrastructure-azure/v1beta1/cluster-template.yaml"
-      targetName: "cluster-template-management.yaml"
-    - sourcePath: "../data/infrastructure-azure/v1beta1/cluster-template-kcp-adoption.yaml"
-      targetName: "cluster-template-kcp-adoption.yaml"
-    - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-ipv6.yaml"
-      targetName: "cluster-template-ipv6.yaml"
-    - sourcePath: "../data/infrastructure-azure/v1beta1/cluster-template-md-remediation.yaml"
-      targetName: "cluster-template-md-remediation.yaml"
-    - sourcePath: "../data/infrastructure-azure/v1beta1/cluster-template-kcp-remediation.yaml"
-      targetName: "cluster-template-kcp-remediation.yaml"
-    - sourcePath: "../data/infrastructure-azure/v1beta1/cluster-template-kcp-scale-in.yaml"
-      targetName: "cluster-template-kcp-scale-in.yaml"
-    - sourcePath: "../data/infrastructure-azure/v1beta1/cluster-template-node-drain.yaml"
-      targetName: "cluster-template-node-drain.yaml"
-    - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-machine-pool.yaml"
-      targetName: "cluster-template-machine-pool.yaml"
-    - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml"
-      targetName: "cluster-template-nvidia-gpu.yaml"
-    - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-private.yaml"
-      targetName: "cluster-template-private.yaml"
-    - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-ci-version.yaml"
-      targetName: "cluster-template-conformance-ci-artifacts.yaml"
-    - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-ci-version-windows.yaml"
-      targetName: "cluster-template-conformance-ci-artifacts-windows.yaml"
-    - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-ci-version-windows-containerd-2022.yaml"
-      targetName: "cluster-template-conformance-ci-artifacts-windows-containerd-2022.yaml"
-    - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-ci-version.yaml"
-      targetName: "cluster-template-conformance-ci-artifacts-windows-containerd.yaml"
-    - sourcePath: "${PWD}/templates/test/dev/cluster-template-custom-builds.yaml"
-      targetName: "cluster-template-conformance-presubmit-artifacts.yaml"
-    - sourcePath: "${PWD}/templates/test/dev/cluster-template-custom-builds.yaml"
-      targetName: "cluster-template-conformance-presubmit-artifacts-windows-containerd.yaml"
-    - sourcePath: "${PWD}/templates/test/dev/cluster-template-custom-builds-windows.yaml"
-      targetName: "cluster-template-conformance-presubmit-artifacts-windows.yaml"
-    - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-windows.yaml"
-      targetName: "cluster-template-windows.yaml"
-    - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-machine-pool-windows.yaml"
-      targetName: "cluster-template-machine-pool-windows.yaml"
-    - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-external-cloud-provider.yaml"
-      targetName: "cluster-template-external-cloud-provider.yaml"
-    - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-aks-multi-tenancy.yaml"
-      targetName: "cluster-template-aks-multi-tenancy.yaml"
-    - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-custom-vnet.yaml"
-      targetName: "cluster-template-custom-vnet.yaml"
 
 variables:
   KUBERNETES_VERSION: "${KUBERNETES_VERSION:-v1.22.1}"
@@ -125,6 +159,11 @@ variables:
   CLUSTER_IDENTITY_NAME: "cluster-identity"
   NODE_DRAIN_TIMEOUT: "60s"
   CI_VERSION: ""
+  # NOTE: INIT_WITH_BINARY and INIT_WITH_KUBERNETES_VERSION are only used by the clusterctl upgrade test to initialize
+  # the management cluster to be upgraded.
+  INIT_WITH_BINARY: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.3.23/clusterctl-{OS}-{ARCH}"
+  INIT_WITH_PROVIDERS_CONTRACT: "v1alpha3"
+  INIT_WITH_KUBERNETES_VERSION: "v1.21.2"
 
 intervals:
   default/wait-controllers: ["3m", "10s"]