diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index bdba163d..9fd8c3cb 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -61,6 +61,9 @@ jobs: make configuration env: TERRAFORM_BACKEND_NAMESPACE: terraform + - name: dump controller logs + if: ${{ always() }} + run: kubectl logs deploy/terraform-controller -n terraform - name: Upload coverage report uses: codecov/codecov-action@v2 diff --git a/Makefile b/Makefile index 1b00f876..965fd7e6 100644 --- a/Makefile +++ b/Makefile @@ -267,7 +267,8 @@ custom: custom-credentials custom-provider configuration: - go test -coverprofile=e2e-coverage1.xml -v ./e2e/... -count=1 + go test -coverprofile=e2e-coverage1.xml -v $(go list ./e2e/...|grep -v controllernamespace) -count=1 + go test -v ./e2e/controllernamespace/... e2e-setup: install-chart alibaba diff --git a/controllers/configuration/backend/backend.go b/controllers/configuration/backend/backend.go index 6c8b1258..9cce0bf8 100644 --- a/controllers/configuration/backend/backend.go +++ b/controllers/configuration/backend/backend.go @@ -56,7 +56,7 @@ var backendInitFuncMap = map[string]backendInitFunc{ } // ParseConfigurationBackend parses backend Conf from the v1beta2.Configuration -func ParseConfigurationBackend(configuration *v1beta2.Configuration, k8sClient client.Client, credentials map[string]string) (Backend, error) { +func ParseConfigurationBackend(configuration *v1beta2.Configuration, k8sClient client.Client, credentials map[string]string, controllerNSSpecified bool) (Backend, error) { backend := configuration.Spec.Backend var ( @@ -68,7 +68,7 @@ func ParseConfigurationBackend(configuration *v1beta2.Configuration, k8sClient c switch { case backend == nil || (backend.Inline == "" && backend.BackendType == ""): // use the default k8s backend - return handleDefaultBackend(configuration, k8sClient) + return handleDefaultBackend(configuration, k8sClient, controllerNSSpecified) case backend.Inline != "" && backend.BackendType != "": return nil, errors.New("it's not allowed to set `spec.backend.inline` and `spec.backend.backendType` at the same time") @@ -79,6 +79,8 @@ func ParseConfigurationBackend(configuration *v1beta2.Configuration, k8sClient c case backend.BackendType != "": // In this case, use the explicit custom backend + // we don't change backend secret suffix to UID of configuration here. + // If backend specified, it's user's responsibility to set the right secret suffix, to avoid conflict. backendType, backendConf, err = handleExplicitBackend(backend) } if err != nil { @@ -92,7 +94,7 @@ func ParseConfigurationBackend(configuration *v1beta2.Configuration, k8sClient c return initFunc(k8sClient, backendConf, credentials) } -func handleDefaultBackend(configuration *v1beta2.Configuration, k8sClient client.Client) (Backend, error) { +func handleDefaultBackend(configuration *v1beta2.Configuration, k8sClient client.Client, controllerNSSpecified bool) (Backend, error) { if configuration.Spec.Backend != nil { if configuration.Spec.Backend.SecretSuffix == "" { configuration.Spec.Backend.SecretSuffix = configuration.Name @@ -104,7 +106,7 @@ func handleDefaultBackend(configuration *v1beta2.Configuration, k8sClient client InClusterConfig: true, } } - return newDefaultK8SBackend(configuration.Spec.Backend.SecretSuffix, k8sClient, configuration.Namespace), nil + return newDefaultK8SBackend(configuration, k8sClient, controllerNSSpecified), nil } func handleInlineBackendHCL(hclCode string) (string, interface{}, error) { diff --git a/controllers/configuration/backend/backend_test.go b/controllers/configuration/backend/backend_test.go index 444bb910..d0dd3d49 100644 --- a/controllers/configuration/backend/backend_test.go +++ b/controllers/configuration/backend/backend_test.go @@ -13,8 +13,9 @@ import ( func TestParseConfigurationBackend(t *testing.T) { type args struct { - configuration *v1beta2.Configuration - credentials map[string]string + configuration *v1beta2.Configuration + credentials map[string]string + controllerNSSpecified bool } type want struct { backend Backend @@ -299,17 +300,46 @@ terraform { errMsg: "it's not allowed to set `spec.backend.inline` and `spec.backend.backendType` at the same time", }, }, + { + name: "backend is nil, specify controller namespace, generate backend with legacy secret suffix", + args: args{ + configuration: &v1beta2.Configuration{ + ObjectMeta: metav1.ObjectMeta{Name: "name", Namespace: "ns", UID: "xxxx-xxxx"}, + Spec: v1beta2.ConfigurationSpec{ + Backend: nil, + }, + }, + controllerNSSpecified: true, + }, + want: want{ + backend: &K8SBackend{ + LegacySecretSuffix: "name", + SecretNS: "ns", + SecretSuffix: "xxxx-xxxx", + Client: k8sClient, + HCLCode: ` +terraform { + backend "kubernetes" { + secret_suffix = "xxxx-xxxx" + in_cluster_config = true + namespace = "ns" + } +} +`, + }, + }, + }, } for _, tc := range testcases { t.Run(tc.name, func(t *testing.T) { - got, err := ParseConfigurationBackend(tc.args.configuration, k8sClient, tc.args.credentials) + got, err := ParseConfigurationBackend(tc.args.configuration, k8sClient, tc.args.credentials, tc.args.controllerNSSpecified) if tc.want.errMsg != "" && !strings.Contains(err.Error(), tc.want.errMsg) { t.Errorf("ValidConfigurationObject() error = %v, wantErr %v", err, tc.want.errMsg) return } if !reflect.DeepEqual(tc.want.backend, got) { - t.Errorf("got %#v, want %#v", got, tc.want.backend) + t.Errorf("\ngot %#v,\nwant %#v", got, tc.want.backend) } }) } diff --git a/controllers/configuration/backend/kubernetes.go b/controllers/configuration/backend/kubernetes.go index 17505d95..e89bbedd 100644 --- a/controllers/configuration/backend/kubernetes.go +++ b/controllers/configuration/backend/kubernetes.go @@ -21,12 +21,14 @@ import ( "fmt" "os" - "github.com/oam-dev/terraform-controller/api/v1beta2" - "github.com/oam-dev/terraform-controller/controllers/util" "github.com/pkg/errors" v1 "k8s.io/api/core/v1" "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/oam-dev/terraform-controller/api/v1beta2" + "github.com/oam-dev/terraform-controller/controllers/util" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) const ( @@ -47,19 +49,31 @@ type K8SBackend struct { SecretSuffix string // SecretNS is the namespace of the Terraform backend secret SecretNS string + // LegacySecretSuffix is the same as SecretSuffix, but only used when `--controller-namespace` is specified + LegacySecretSuffix string } -func newDefaultK8SBackend(suffix string, client client.Client, namespace string) *K8SBackend { +func newDefaultK8SBackend(configuration *v1beta2.Configuration, client client.Client, controllerNSSpecified bool) *K8SBackend { ns := os.Getenv("TERRAFORM_BACKEND_NAMESPACE") if ns == "" { - ns = namespace + ns = configuration.GetNamespace() + } + + var ( + suffix = configuration.Spec.Backend.SecretSuffix + legacySuffix string + ) + if controllerNSSpecified { + legacySuffix = suffix + suffix = string(configuration.GetUID()) } hcl := renderK8SBackendHCL(suffix, ns) return &K8SBackend{ - Client: client, - HCLCode: hcl, - SecretSuffix: suffix, - SecretNS: ns, + Client: client, + HCLCode: hcl, + SecretSuffix: suffix, + SecretNS: ns, + LegacySecretSuffix: legacySuffix, } } @@ -95,15 +109,23 @@ terraform { return fmt.Sprintf(fmtStr, suffix, ns) } -func (k K8SBackend) secretName() string { +func (k *K8SBackend) secretName() string { return fmt.Sprintf(TFBackendSecret, terraformWorkspace, k.SecretSuffix) } +func (k *K8SBackend) legacySecretName() string { + return fmt.Sprintf(TFBackendSecret, terraformWorkspace, k.LegacySecretSuffix) +} + // GetTFStateJSON gets Terraform state json from the Terraform kubernetes backend func (k *K8SBackend) GetTFStateJSON(ctx context.Context) ([]byte, error) { var s = v1.Secret{} - if err := k.Client.Get(ctx, client.ObjectKey{Name: k.secretName(), Namespace: k.SecretNS}, &s); err != nil { - return nil, errors.Wrap(err, "terraform state file backend secret is not generated") + // Try to get legacy secret first, if it doesn't exist, try to get new secret + err := k.Client.Get(ctx, client.ObjectKey{Name: k.legacySecretName(), Namespace: k.SecretNS}, &s) + if err != nil { + if err = k.Client.Get(ctx, client.ObjectKey{Name: k.secretName(), Namespace: k.SecretNS}, &s); err != nil { + return nil, errors.Wrap(err, "terraform state file backend secret is not generated") + } } tfStateData, ok := s.Data[TerraformStateNameInSecret] if !ok { @@ -119,8 +141,15 @@ func (k *K8SBackend) GetTFStateJSON(ctx context.Context) ([]byte, error) { // CleanUp will delete the Terraform kubernetes backend secret when deleting the configuration object func (k *K8SBackend) CleanUp(ctx context.Context) error { - klog.InfoS("Deleting the secret which stores Kubernetes backend", "Name", k.secretName()) + klog.InfoS("Deleting the legacy secret which stores Kubernetes backend", "Name", k.legacySecretName()) var kubernetesBackendSecret v1.Secret + if err := k.Client.Get(ctx, client.ObjectKey{Name: k.legacySecretName(), Namespace: k.SecretNS}, &kubernetesBackendSecret); err == nil { + if err := k.Client.Delete(ctx, &kubernetesBackendSecret); err != nil { + return err + } + } + + klog.InfoS("Deleting the secret which stores Kubernetes backend", "Name", k.secretName()) if err := k.Client.Get(ctx, client.ObjectKey{Name: k.secretName(), Namespace: k.SecretNS}, &kubernetesBackendSecret); err == nil { if err := k.Client.Delete(ctx, &kubernetesBackendSecret); err != nil { return err @@ -131,8 +160,40 @@ func (k *K8SBackend) CleanUp(ctx context.Context) error { // HCL returns the backend hcl code string func (k *K8SBackend) HCL() string { + if k.LegacySecretSuffix != "" { + err := k.migrateLegacySecret() + if err != nil { + klog.ErrorS(err, "Failed to migrate legacy secret") + } + } + if k.HCLCode == "" { k.HCLCode = renderK8SBackendHCL(k.SecretSuffix, k.SecretNS) } return k.HCLCode } + +// migrateLegacySecret will migrate the legacy secret to the new secret if the legacy secret exists +// This is needed when the --controller-namespace is specified and restart the controller +func (k *K8SBackend) migrateLegacySecret() error { + ctx := context.TODO() + s := v1.Secret{} + if err := k.Client.Get(ctx, client.ObjectKey{Name: k.legacySecretName(), Namespace: k.SecretNS}, &s); err == nil { + klog.InfoS("Migrating legacy secret to new secret", "LegacyName", k.legacySecretName(), "NewName", k.secretName(), "Namespace", k.SecretNS) + newSecret := v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: k.secretName(), + Namespace: k.SecretNS, + }, + Data: s.Data, + } + err = k.Client.Create(ctx, &newSecret) + if err != nil { + return errors.Wrapf(err, "Fail to create new secret, Name: %s, Namespace: %s", k.secretName(), k.SecretNS) + } else if err = k.Client.Delete(ctx, &s); err != nil { + // Only delete the legacy secret if the new secret is successfully created + return errors.Wrapf(err, "Fail to delete legacy secret, Name: %s, Namespace: %s", k.legacySecretName(), k.SecretNS) + } + } + return nil +} diff --git a/controllers/configuration/backend/kubernetes_test.go b/controllers/configuration/backend/kubernetes_test.go index 2fd45516..f57a430d 100644 --- a/controllers/configuration/backend/kubernetes_test.go +++ b/controllers/configuration/backend/kubernetes_test.go @@ -3,6 +3,7 @@ package backend import ( "context" "encoding/base64" + "gotest.tools/assert" "reflect" "testing" @@ -82,17 +83,19 @@ terraform { } func TestK8SBackend_GetTFStateJSON(t *testing.T) { + const UID = "xxxx-xxxx" type fields struct { - Client client.Client - HCLCode string - SecretSuffix string - SecretNS string + Client client.Client + HCLCode string + SecretSuffix string + SecretNS string + LegacySecretSuffix string } type args struct { ctx context.Context } tfStateData, _ := base64.StdEncoding.DecodeString("H4sIAAAAAAAA/4SQzarbMBCF934KoXUdPKNf+1VKCWNp5AocO8hyaSl592KlcBd3cZfnHPHpY/52QshfXI68b3IS+tuVK5dCaS+P+8ci4TbcULb94JJplZPAFte8MS18PQrKBO8Q+xk59SHa1AMA9M4YmoN3FGJ8M/azPs96yElcCkLIsG+V8sblnqOc3uXlRuvZ0GxSSuiCRUYbw2gGHRFGPxitEgJYQDQ0a68I2ChNo1cAZJ2bR20UtW8bsv55NuJRS94W2erXe5X5QQs3A/FZ4fhJaOwUgZTVMRjto1HGpSGSQuuD955hdDDPcR6NY1ZpQJ/YwagTRAvBpsi8LXn7Pa1U+ahfWHX/zWThYz9L4Otg3390r+5fAAAA//8hmcuNuQEAAA==") - secret := &v1.Secret{ + baseSecret := &v1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: "tfstate-default-a", Namespace: "default", @@ -102,7 +105,8 @@ func TestK8SBackend_GetTFStateJSON(t *testing.T) { TerraformStateNameInSecret: tfStateData, }, } - k8sClient := fake.NewClientBuilder().WithObjects(secret).Build() + + k8sClient := fake.NewClientBuilder().WithObjects(baseSecret).Build() k8sClient2 := fake.NewClientBuilder().Build() tests := []struct { name string @@ -120,24 +124,7 @@ func TestK8SBackend_GetTFStateJSON(t *testing.T) { SecretNS: "default", }, args: args{ctx: context.Background()}, - want: []byte(`{ - "version": 4, - "terraform_version": "1.0.2", - "serial": 2, - "lineage": "c35c8722-b2ef-cd6f-1111-755abc87acdd", - "outputs": { - "container_id":{ - "value": "e5fff27c62e26dc9504d21980543f21161225ab483a1e534a98311a677b9453a", - "type": "string" - }, - "image_id": { - "value": "sha256:d1a364dc548d5357f0da3268c888e1971bbdb957ee3f028fe7194f1d61c6fdeenginx:latest", - "type": "string" - } - }, - "resources": [] -} -`), + want: tfStateJson, }, { name: "secret doesn't exist", @@ -151,14 +138,26 @@ func TestK8SBackend_GetTFStateJSON(t *testing.T) { want: nil, wantErr: true, }, + { + name: "got a legacy secret", + fields: fields{ + Client: k8sClient, + HCLCode: "", + LegacySecretSuffix: "a", + SecretNS: "default", + SecretSuffix: UID, + }, + want: tfStateJson, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { k := &K8SBackend{ - Client: tt.fields.Client, - HCLCode: tt.fields.HCLCode, - SecretSuffix: tt.fields.SecretSuffix, - SecretNS: tt.fields.SecretNS, + Client: tt.fields.Client, + HCLCode: tt.fields.HCLCode, + SecretSuffix: tt.fields.SecretSuffix, + SecretNS: tt.fields.SecretNS, + LegacySecretSuffix: tt.fields.LegacySecretSuffix, } got, err := k.GetTFStateJSON(tt.args.ctx) if (err != nil) != tt.wantErr { @@ -221,3 +220,54 @@ func TestK8SBackend_CleanUp(t *testing.T) { }) } } + +func TestMigrateLegacySecret(t *testing.T) { + secretNS := "default" + secret := v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tfstate-default-a", + Namespace: secretNS, + }, + Type: v1.SecretTypeOpaque, + } + k8sClient := fake.NewClientBuilder().WithObjects(&secret).Build() + fakeUID := "xxxx-xxxx" + k := &K8SBackend{ + Client: k8sClient, + HCLCode: "", + SecretSuffix: fakeUID, + SecretNS: secretNS, + LegacySecretSuffix: "a", + } + err := k.migrateLegacySecret() + assert.NilError(t, err) + NoLegacySecK8sClient := fake.NewClientBuilder().Build() + k = &K8SBackend{ + Client: NoLegacySecK8sClient, + HCLCode: "", + SecretSuffix: fakeUID, + SecretNS: secretNS, + LegacySecretSuffix: "a", + } + err = k.migrateLegacySecret() + assert.NilError(t, err) +} + +var tfStateJson = []byte(`{ + "version": 4, + "terraform_version": "1.0.2", + "serial": 2, + "lineage": "c35c8722-b2ef-cd6f-1111-755abc87acdd", + "outputs": { + "container_id":{ + "value": "e5fff27c62e26dc9504d21980543f21161225ab483a1e534a98311a677b9453a", + "type": "string" + }, + "image_id": { + "value": "sha256:d1a364dc548d5357f0da3268c888e1971bbdb957ee3f028fe7194f1d61c6fdeenginx:latest", + "type": "string" + } + }, + "resources": [] +} +`) diff --git a/controllers/configuration/configuration.go b/controllers/configuration/configuration.go index 3720b1e7..1668d75f 100644 --- a/controllers/configuration/configuration.go +++ b/controllers/configuration/configuration.go @@ -6,7 +6,6 @@ import ( "strconv" "strings" - "github.com/oam-dev/terraform-controller/controllers/configuration/backend" "github.com/pkg/errors" kerrors "k8s.io/apimachinery/pkg/api/errors" apitypes "k8s.io/apimachinery/pkg/types" @@ -50,25 +49,6 @@ func ValidConfigurationObject(configuration *v1beta2.Configuration) (types.Confi return "", nil } -// RenderConfiguration will compose the Terraform configuration with hcl/json and backend -func RenderConfiguration(configuration *v1beta2.Configuration, k8sClient client.Client, configurationType types.ConfigurationType, credentials map[string]string) (string, backend.Backend, error) { - backendInterface, err := backend.ParseConfigurationBackend(configuration, k8sClient, credentials) - if err != nil { - return "", nil, errors.Wrap(err, "failed to prepare Terraform backend configuration") - } - - switch configurationType { - case types.ConfigurationHCL: - completedConfiguration := configuration.Spec.HCL - completedConfiguration += "\n" + backendInterface.HCL() - return completedConfiguration, backendInterface, nil - case types.ConfigurationRemote: - return backendInterface.HCL(), backendInterface, nil - default: - return "", nil, errors.New("Unsupported Configuration Type") - } -} - // SetRegion will set the region for Configuration func SetRegion(ctx context.Context, k8sClient client.Client, namespace, name string, providerObj *v1beta1.Provider) (string, error) { configuration, err := Get(ctx, k8sClient, apitypes.NamespacedName{Namespace: namespace, Name: name}) diff --git a/controllers/configuration/configuration_test.go b/controllers/configuration/configuration_test.go index f1734790..0f30a933 100644 --- a/controllers/configuration/configuration_test.go +++ b/controllers/configuration/configuration_test.go @@ -2,11 +2,9 @@ package configuration import ( "context" - "reflect" "strings" "testing" - "github.com/oam-dev/terraform-controller/controllers/configuration/backend" "github.com/stretchr/testify/assert" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -103,134 +101,6 @@ func TestValidConfigurationObject(t *testing.T) { } -func TestRenderConfiguration(t *testing.T) { - type args struct { - configuration *v1beta2.Configuration - configurationType types.ConfigurationType - credentials map[string]string - } - type want struct { - cfg string - backendInterface backend.Backend - errMsg string - } - - k8sClient := fake.NewClientBuilder().Build() - - testcases := []struct { - name string - args args - want want - }{ - { - name: "backend is not nil, configuration is hcl", - args: args{ - configuration: &v1beta2.Configuration{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: "n1", - }, - Spec: v1beta2.ConfigurationSpec{ - Backend: &v1beta2.Backend{}, - HCL: "image_id=123", - }, - }, - configurationType: types.ConfigurationHCL, - }, - want: want{ - cfg: `image_id=123 - -terraform { - backend "kubernetes" { - secret_suffix = "" - in_cluster_config = true - namespace = "n1" - } -} -`, - backendInterface: &backend.K8SBackend{ - Client: k8sClient, - HCLCode: ` -terraform { - backend "kubernetes" { - secret_suffix = "" - in_cluster_config = true - namespace = "n1" - } -} -`, - SecretSuffix: "", - SecretNS: "n1", - }, - }, - }, - { - name: "backend is nil, configuration is remote", - args: args{ - configuration: &v1beta2.Configuration{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: "n2", - }, - Spec: v1beta2.ConfigurationSpec{ - Remote: "https://github.com/a/b.git", - }, - }, - configurationType: types.ConfigurationRemote, - }, - want: want{ - cfg: ` -terraform { - backend "kubernetes" { - secret_suffix = "" - in_cluster_config = true - namespace = "n2" - } -} -`, - backendInterface: &backend.K8SBackend{ - Client: k8sClient, - HCLCode: ` -terraform { - backend "kubernetes" { - secret_suffix = "" - in_cluster_config = true - namespace = "n2" - } -} -`, - SecretSuffix: "", - SecretNS: "n2", - }, - }, - }, - { - name: "backend is nil, configuration is not supported", - args: args{ - configuration: &v1beta2.Configuration{ - Spec: v1beta2.ConfigurationSpec{}, - }, - }, - want: want{ - errMsg: "Unsupported Configuration Type", - }, - }, - } - - for _, tc := range testcases { - t.Run(tc.name, func(t *testing.T) { - got, backendConf, err := RenderConfiguration(tc.args.configuration, k8sClient, tc.args.configurationType, tc.args.credentials) - if tc.want.errMsg != "" && !strings.Contains(err.Error(), tc.want.errMsg) { - t.Errorf("ValidConfigurationObject() error = %v, wantErr %v", err, tc.want.errMsg) - return - } - assert.Equal(t, tc.want.cfg, got) - - if !reflect.DeepEqual(tc.want.backendInterface, backendConf) { - t.Errorf("backendInterface is not equal.\n got %#v\n, want %#v", backendConf, tc.want.backendInterface) - } - }) - } -} - func TestReplaceTerraformSource(t *testing.T) { testcases := []struct { remote string diff --git a/controllers/configuration_controller.go b/controllers/configuration_controller.go index 8cfad819..d7844a5a 100644 --- a/controllers/configuration_controller.go +++ b/controllers/configuration_controller.go @@ -113,7 +113,7 @@ func (r *ConfigurationReconciler) Reconcile(ctx context.Context, req ctrl.Reques return ctrl.Result{}, client.IgnoreNotFound(err) } - meta := initTFConfigurationMeta(req, configuration) + meta := initTFConfigurationMeta(req, configuration, r.Client) if r.ControllerNamespace != "" { uid := string(configuration.GetUID()) // @step: since we are using a single namespace to run these, we must ensure the names @@ -124,11 +124,7 @@ func (r *ConfigurationReconciler) Reconcile(ctx context.Context, req ctrl.Reques meta.ConfigurationCMName = fmt.Sprintf(TFInputConfigMapName, uid) meta.VariableSecretName = fmt.Sprintf(TFVariableSecret, uid) meta.ControllerNamespace = r.ControllerNamespace - - configuration.Spec.Backend = &v1beta2.Backend{ - InClusterConfig: true, - SecretSuffix: uid, - } + meta.ControllerNSSpecified = true } // add finalizer @@ -147,15 +143,6 @@ func (r *ConfigurationReconciler) Reconcile(ctx context.Context, req ctrl.Reques return ctrl.Result{}, err } - var tfExecutionJob = &batchv1.Job{} - if err := r.Client.Get(ctx, client.ObjectKey{Name: meta.ApplyJobName, Namespace: meta.ControllerNamespace}, tfExecutionJob); err == nil { - if !meta.EnvChanged && tfExecutionJob.Status.Succeeded == int32(1) { - if err := meta.updateApplyStatus(ctx, r.Client, types.Available, types.MessageCloudResourceDeployed); err != nil { - return ctrl.Result{}, err - } - } - } - if isDeleting { // terraform destroy klog.InfoS("performing Configuration Destroy", "Namespace", req.Namespace, "Name", req.Name, "JobName", meta.DestroyJobName) @@ -188,6 +175,14 @@ func (r *ConfigurationReconciler) Reconcile(ctx context.Context, req ctrl.Reques return ctrl.Result{}, nil } + var tfExecutionJob = &batchv1.Job{} + if err := meta.getApplyJob(ctx, r.Client, tfExecutionJob); err == nil { + if !meta.EnvChanged && tfExecutionJob.Status.Succeeded == int32(1) { + err = meta.updateApplyStatus(ctx, r.Client, types.Available, types.MessageCloudResourceDeployed) + return ctrl.Result{}, err + } + } + // Terraform apply (create or update) klog.InfoS("performing Terraform Apply (cloud resource create/update)", "Namespace", req.Namespace, "Name", req.Name) if err := r.terraformApply(ctx, configuration, meta); err != nil { @@ -220,6 +215,17 @@ type LegacySubResources struct { VariableSecretName string } +type ResourceQuota struct { + ResourcesLimitsCPU string + ResourcesLimitsCPUQuantity resource.Quantity + ResourcesLimitsMemory string + ResourcesLimitsMemoryQuantity resource.Quantity + ResourcesRequestsCPU string + ResourcesRequestsCPUQuantity resource.Quantity + ResourcesRequestsMemory string + ResourcesRequestsMemoryQuantity resource.Quantity +} + // TFConfigurationMeta is all the metadata of a Configuration type TFConfigurationMeta struct { Name string @@ -251,20 +257,16 @@ type TFConfigurationMeta struct { BusyboxImage string GitImage string - // Resources series Variables are for Setting Compute Resources required by this container - ResourcesLimitsCPU string - ResourcesLimitsCPUQuantity resource.Quantity - ResourcesLimitsMemory string - ResourcesLimitsMemoryQuantity resource.Quantity - ResourcesRequestsCPU string - ResourcesRequestsCPUQuantity resource.Quantity - ResourcesRequestsMemory string - ResourcesRequestsMemoryQuantity resource.Quantity + // ResourceQuota series Variables are for Setting Compute Resources required by this container + ResourceQuota ResourceQuota + + LegacySubResources LegacySubResources + ControllerNSSpecified bool - LegacySubResources LegacySubResources + K8sClient client.Client } -func initTFConfigurationMeta(req ctrl.Request, configuration v1beta2.Configuration) *TFConfigurationMeta { +func initTFConfigurationMeta(req ctrl.Request, configuration v1beta2.Configuration, k8sClient client.Client) *TFConfigurationMeta { var meta = &TFConfigurationMeta{ ControllerNamespace: req.Namespace, Namespace: req.Namespace, @@ -273,6 +275,7 @@ func initTFConfigurationMeta(req ctrl.Request, configuration v1beta2.Configurati VariableSecretName: fmt.Sprintf(TFVariableSecret, req.Name), ApplyJobName: req.Name + "-" + string(TerraformApply), DestroyJobName: req.Name + "-" + string(TerraformDestroy), + K8sClient: k8sClient, } jobNodeSelectorStr := os.Getenv("JOB_NODE_SELECTOR") @@ -368,7 +371,7 @@ func (r *ConfigurationReconciler) terraformDestroy(ctx context.Context, configur } } if err := meta.updateTerraformJobIfNeeded(ctx, k8sClient, destroyJob); err != nil { - klog.ErrorS(err, types.ErrUpdateTerraformApplyJob, "Name", meta.ApplyJobName) + klog.ErrorS(err, types.ErrUpdateTerraformApplyJob, "Name", meta.DestroyJobName) return errors.Wrap(err, types.ErrUpdateTerraformApplyJob) } } @@ -439,45 +442,45 @@ func (r *ConfigurationReconciler) cleanUpSubResources(ctx context.Context, confi func (r *ConfigurationReconciler) preCheckResourcesSetting(meta *TFConfigurationMeta) error { - meta.ResourcesLimitsCPU = os.Getenv("RESOURCES_LIMITS_CPU") - if meta.ResourcesLimitsCPU != "" { - limitsCPU, err := resource.ParseQuantity(meta.ResourcesLimitsCPU) + meta.ResourceQuota.ResourcesLimitsCPU = os.Getenv("RESOURCES_LIMITS_CPU") + if meta.ResourceQuota.ResourcesLimitsCPU != "" { + limitsCPU, err := resource.ParseQuantity(meta.ResourceQuota.ResourcesLimitsCPU) if err != nil { errMsg := "failed to parse env variable RESOURCES_LIMITS_CPU into resource.Quantity" klog.ErrorS(err, errMsg) return errors.Wrap(err, errMsg) } - meta.ResourcesLimitsCPUQuantity = limitsCPU + meta.ResourceQuota.ResourcesLimitsCPUQuantity = limitsCPU } - meta.ResourcesLimitsMemory = os.Getenv("RESOURCES_LIMITS_MEMORY") - if meta.ResourcesLimitsMemory != "" { - limitsMemory, err := resource.ParseQuantity(meta.ResourcesLimitsMemory) + meta.ResourceQuota.ResourcesLimitsMemory = os.Getenv("RESOURCES_LIMITS_MEMORY") + if meta.ResourceQuota.ResourcesLimitsMemory != "" { + limitsMemory, err := resource.ParseQuantity(meta.ResourceQuota.ResourcesLimitsMemory) if err != nil { errMsg := "failed to parse env variable RESOURCES_LIMITS_MEMORY into resource.Quantity" klog.ErrorS(err, errMsg) return errors.Wrap(err, errMsg) } - meta.ResourcesLimitsMemoryQuantity = limitsMemory + meta.ResourceQuota.ResourcesLimitsMemoryQuantity = limitsMemory } - meta.ResourcesRequestsCPU = os.Getenv("RESOURCES_REQUESTS_CPU") - if meta.ResourcesRequestsCPU != "" { - requestsCPU, err := resource.ParseQuantity(meta.ResourcesRequestsCPU) + meta.ResourceQuota.ResourcesRequestsCPU = os.Getenv("RESOURCES_REQUESTS_CPU") + if meta.ResourceQuota.ResourcesRequestsCPU != "" { + requestsCPU, err := resource.ParseQuantity(meta.ResourceQuota.ResourcesRequestsCPU) if err != nil { errMsg := "failed to parse env variable RESOURCES_REQUESTS_CPU into resource.Quantity" klog.ErrorS(err, errMsg) return errors.Wrap(err, errMsg) } - meta.ResourcesRequestsCPUQuantity = requestsCPU + meta.ResourceQuota.ResourcesRequestsCPUQuantity = requestsCPU } - meta.ResourcesRequestsMemory = os.Getenv("RESOURCES_REQUESTS_MEMORY") - if meta.ResourcesRequestsMemory != "" { - requestsMemory, err := resource.ParseQuantity(meta.ResourcesRequestsMemory) + meta.ResourceQuota.ResourcesRequestsMemory = os.Getenv("RESOURCES_REQUESTS_MEMORY") + if meta.ResourceQuota.ResourcesRequestsMemory != "" { + requestsMemory, err := resource.ParseQuantity(meta.ResourceQuota.ResourcesRequestsMemory) if err != nil { errMsg := "failed to parse env variable RESOURCES_REQUESTS_MEMORY into resource.Quantity" klog.ErrorS(err, errMsg) return errors.Wrap(err, errMsg) } - meta.ResourcesRequestsMemoryQuantity = requestsMemory + meta.ResourceQuota.ResourcesRequestsMemoryQuantity = requestsMemory } return nil } @@ -533,7 +536,7 @@ func (r *ConfigurationReconciler) preCheck(ctx context.Context, configuration *v } // Render configuration with backend - completeConfiguration, backendConf, err := tfcfg.RenderConfiguration(configuration, r.Client, configurationType, meta.Credentials) + completeConfiguration, backendConf, err := meta.RenderConfiguration(configuration, configurationType) if err != nil { return err } @@ -610,6 +613,7 @@ func (meta *TFConfigurationMeta) updateApplyStatus(ctx context.Context, k8sClien if state == types.Available { outputs, err := meta.getTFOutputs(ctx, k8sClient, configuration) if err != nil { + klog.InfoS("Failed to get outputs", "error", err) configuration.Status.Apply = v1beta2.ConfigurationApplyStatus{ State: types.GeneratingOutputs, Message: types.ErrGenerateOutputs + ": " + err.Error(), @@ -650,7 +654,7 @@ func (meta *TFConfigurationMeta) assembleAndTriggerJob(ctx context.Context, k8sC return k8sClient.Create(ctx, job) } -// updateTerraformJob will set deletion finalizer to the Terraform job if its envs are changed, which will result in +// updateTerraformJobIfNeeded will set deletion finalizer to the Terraform job if its envs are changed, which will result in // deleting the job. Finally, a new Terraform job will be generated func (meta *TFConfigurationMeta) updateTerraformJobIfNeeded(ctx context.Context, k8sClient client.Client, job batchv1.Job) error { // if either one changes, delete the job @@ -768,25 +772,25 @@ func (meta *TFConfigurationMeta) assembleTerraformJob(executionType TerraformExe Env: meta.Envs, } - if meta.ResourcesLimitsCPU != "" || meta.ResourcesLimitsMemory != "" || - meta.ResourcesRequestsCPU != "" || meta.ResourcesRequestsMemory != "" { + if meta.ResourceQuota.ResourcesLimitsCPU != "" || meta.ResourceQuota.ResourcesLimitsMemory != "" || + meta.ResourceQuota.ResourcesRequestsCPU != "" || meta.ResourceQuota.ResourcesRequestsMemory != "" { resourceRequirements := v1.ResourceRequirements{} - if meta.ResourcesLimitsCPU != "" || meta.ResourcesLimitsMemory != "" { + if meta.ResourceQuota.ResourcesLimitsCPU != "" || meta.ResourceQuota.ResourcesLimitsMemory != "" { resourceRequirements.Limits = map[v1.ResourceName]resource.Quantity{} - if meta.ResourcesLimitsCPU != "" { - resourceRequirements.Limits["cpu"] = meta.ResourcesLimitsCPUQuantity + if meta.ResourceQuota.ResourcesLimitsCPU != "" { + resourceRequirements.Limits["cpu"] = meta.ResourceQuota.ResourcesLimitsCPUQuantity } - if meta.ResourcesLimitsMemory != "" { - resourceRequirements.Limits["memory"] = meta.ResourcesLimitsMemoryQuantity + if meta.ResourceQuota.ResourcesLimitsMemory != "" { + resourceRequirements.Limits["memory"] = meta.ResourceQuota.ResourcesLimitsMemoryQuantity } } - if meta.ResourcesRequestsCPU != "" || meta.ResourcesLimitsMemory != "" { + if meta.ResourceQuota.ResourcesRequestsCPU != "" || meta.ResourceQuota.ResourcesLimitsMemory != "" { resourceRequirements.Requests = map[v1.ResourceName]resource.Quantity{} - if meta.ResourcesRequestsCPU != "" { - resourceRequirements.Requests["cpu"] = meta.ResourcesRequestsCPUQuantity + if meta.ResourceQuota.ResourcesRequestsCPU != "" { + resourceRequirements.Requests["cpu"] = meta.ResourceQuota.ResourcesRequestsCPUQuantity } - if meta.ResourcesRequestsMemory != "" { - resourceRequirements.Requests["memory"] = meta.ResourcesRequestsMemoryQuantity + if meta.ResourceQuota.ResourcesRequestsMemory != "" { + resourceRequirements.Requests["memory"] = meta.ResourceQuota.ResourcesRequestsMemoryQuantity } } container.Resources = resourceRequirements @@ -960,6 +964,7 @@ func (meta *TFConfigurationMeta) getTFOutputs(ctx context.Context, k8sClient cli gotSecret.Namespace, name, ownerNamespace, ownerName, ) + klog.ErrorS(err, "fail to update backend secret") return nil, errors.New(errMsg) } gotSecret.Data = data @@ -1232,8 +1237,29 @@ func (meta *TFConfigurationMeta) KeepLegacySubResourceMetas() { func (meta *TFConfigurationMeta) getApplyJob(ctx context.Context, k8sClient client.Client, job *batchv1.Job) error { if err := k8sClient.Get(ctx, client.ObjectKey{Name: meta.LegacySubResources.ApplyJobName, Namespace: meta.LegacySubResources.Namespace}, job); err == nil { + klog.InfoS("Found legacy apply job", "Configuration", fmt.Sprintf("%s/%s", meta.Name, meta.Namespace), + "Job", fmt.Sprintf("%s/%s", meta.LegacySubResources.Namespace, meta.LegacySubResources.ApplyJobName)) return nil } err := k8sClient.Get(ctx, client.ObjectKey{Name: meta.ApplyJobName, Namespace: meta.ControllerNamespace}, job) return err } + +// RenderConfiguration will compose the Terraform configuration with hcl/json and backend +func (meta *TFConfigurationMeta) RenderConfiguration(configuration *v1beta2.Configuration, configurationType types.ConfigurationType) (string, backend.Backend, error) { + backendInterface, err := backend.ParseConfigurationBackend(configuration, meta.K8sClient, meta.Credentials, meta.ControllerNSSpecified) + if err != nil { + return "", nil, errors.Wrap(err, "failed to prepare Terraform backend configuration") + } + + switch configurationType { + case types.ConfigurationHCL: + completedConfiguration := configuration.Spec.HCL + completedConfiguration += "\n" + backendInterface.HCL() + return completedConfiguration, backendInterface, nil + case types.ConfigurationRemote: + return backendInterface.HCL(), backendInterface, nil + default: + return "", nil, errors.New("Unsupported Configuration Type") + } +} diff --git a/controllers/configuration_controller_test.go b/controllers/configuration_controller_test.go index de24c18a..d6ee6bae 100644 --- a/controllers/configuration_controller_test.go +++ b/controllers/configuration_controller_test.go @@ -114,7 +114,7 @@ func TestInitTFConfigurationMeta(t *testing.T) { for _, tc := range testcases { t.Run(tc.name, func(t *testing.T) { - meta := initTFConfigurationMeta(req, tc.configuration) + meta := initTFConfigurationMeta(req, tc.configuration, nil) if !reflect.DeepEqual(meta.Name, tc.want.Name) { t.Errorf("initTFConfigurationMeta = %v, want %v", meta, tc.want) } @@ -176,7 +176,7 @@ func TestInitTFConfigurationMetaWithDeleteResource(t *testing.T) { } for _, tc := range testcases { t.Run(tc.name, func(t *testing.T) { - meta := initTFConfigurationMeta(req, tc.configuration) + meta := initTFConfigurationMeta(req, tc.configuration, nil) if !reflect.DeepEqual(meta.DeleteResource, tc.meta.DeleteResource) { t.Errorf("initTFConfigurationMeta = %v, want %v", meta, tc.meta) } @@ -197,7 +197,7 @@ func TestInitTFConfigurationMetaWithJobNodeSelector(t *testing.T) { }, Spec: v1beta2.ConfigurationSpec{}, } - meta := initTFConfigurationMeta(req, configuration) + meta := initTFConfigurationMeta(req, configuration, nil) assert.Equal(t, meta.JobNodeSelector, map[string]string{"ssd": "true"}) } @@ -1437,14 +1437,16 @@ func TestAssembleTerraformJobWithResourcesSetting(t *testing.T) { TerraformImage: "f", RemoteGit: "g", - ResourcesLimitsCPU: "10m", - ResourcesLimitsCPUQuantity: quantityLimitsCPU, - ResourcesLimitsMemory: "10Mi", - ResourcesLimitsMemoryQuantity: quantityLimitsMemory, - ResourcesRequestsCPU: "100m", - ResourcesRequestsCPUQuantity: quantityRequestsCPU, - ResourcesRequestsMemory: "5Gi", - ResourcesRequestsMemoryQuantity: quantityRequestsMemory, + ResourceQuota: ResourceQuota{ + ResourcesLimitsCPU: "10m", + ResourcesLimitsCPUQuantity: quantityLimitsCPU, + ResourcesLimitsMemory: "10Mi", + ResourcesLimitsMemoryQuantity: quantityLimitsMemory, + ResourcesRequestsCPU: "100m", + ResourcesRequestsCPUQuantity: quantityRequestsCPU, + ResourcesRequestsMemory: "5Gi", + ResourcesRequestsMemoryQuantity: quantityRequestsMemory, + }, } job := meta.assembleTerraformJob(TerraformApply) @@ -2174,3 +2176,183 @@ func TestGetApplyJob(t *testing.T) { }) } } + +func TestRenderConfiguration(t *testing.T) { + type args struct { + configuration *v1beta2.Configuration + configurationType types.ConfigurationType + credentials map[string]string + controllerNSSpecified bool + } + type want struct { + cfg string + backendInterface backend.Backend + errMsg string + } + + k8sClient := fake.NewClientBuilder().Build() + baseMeta := TFConfigurationMeta{ + K8sClient: k8sClient, + } + + testcases := []struct { + name string + args args + want want + }{ + { + name: "backend is not nil, configuration is hcl", + args: args{ + configuration: &v1beta2.Configuration{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "n1", + }, + Spec: v1beta2.ConfigurationSpec{ + Backend: &v1beta2.Backend{}, + HCL: "image_id=123", + }, + }, + configurationType: types.ConfigurationHCL, + }, + want: want{ + cfg: `image_id=123 + +terraform { + backend "kubernetes" { + secret_suffix = "" + in_cluster_config = true + namespace = "n1" + } +} +`, + backendInterface: &backend.K8SBackend{ + Client: k8sClient, + HCLCode: ` +terraform { + backend "kubernetes" { + secret_suffix = "" + in_cluster_config = true + namespace = "n1" + } +} +`, + SecretSuffix: "", + SecretNS: "n1", + }, + }, + }, + { + name: "backend is nil, configuration is remote", + args: args{ + configuration: &v1beta2.Configuration{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "n2", + }, + Spec: v1beta2.ConfigurationSpec{ + Remote: "https://github.com/a/b.git", + }, + }, + configurationType: types.ConfigurationRemote, + }, + want: want{ + cfg: ` +terraform { + backend "kubernetes" { + secret_suffix = "" + in_cluster_config = true + namespace = "n2" + } +} +`, + backendInterface: &backend.K8SBackend{ + Client: k8sClient, + HCLCode: ` +terraform { + backend "kubernetes" { + secret_suffix = "" + in_cluster_config = true + namespace = "n2" + } +} +`, + SecretSuffix: "", + SecretNS: "n2", + }, + }, + }, + { + name: "backend is nil, configuration is not supported", + args: args{ + configuration: &v1beta2.Configuration{ + Spec: v1beta2.ConfigurationSpec{}, + }, + }, + want: want{ + errMsg: "Unsupported Configuration Type", + }, + }, + { + name: "controller-namespace specified, backend should have legacy secret suffix", + args: args{ + configuration: &v1beta2.Configuration{ + Spec: v1beta2.ConfigurationSpec{ + Backend: nil, + }, + ObjectMeta: metav1.ObjectMeta{ + UID: "xxxx-xxxx", + Namespace: "n2", + Name: "name", + }, + }, + controllerNSSpecified: true, + configurationType: types.ConfigurationRemote, + }, + want: want{ + cfg: ` +terraform { + backend "kubernetes" { + secret_suffix = "xxxx-xxxx" + in_cluster_config = true + namespace = "n2" + } +} +`, + backendInterface: &backend.K8SBackend{ + Client: k8sClient, + HCLCode: ` +terraform { + backend "kubernetes" { + secret_suffix = "xxxx-xxxx" + in_cluster_config = true + namespace = "n2" + } +} +`, + SecretSuffix: "xxxx-xxxx", + SecretNS: "n2", + LegacySecretSuffix: "name", + }}, + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + meta := baseMeta + meta.ControllerNSSpecified = tc.args.controllerNSSpecified + got, backendConf, err := meta.RenderConfiguration(tc.args.configuration, tc.args.configurationType) + if tc.want.errMsg != "" && !strings.Contains(err.Error(), tc.want.errMsg) { + t.Errorf("ValidConfigurationObject() error = %v, wantErr %v", err, tc.want.errMsg) + return + } + if tc.want.errMsg == "" && err != nil { + t.Errorf("ValidConfigurationObject() error = %v, wantErr nil", err) + return + } + assert.Equal(t, tc.want.cfg, got) + + if !reflect.DeepEqual(tc.want.backendInterface, backendConf) { + t.Errorf("backendInterface is not equal.\n got %#v\n, want %#v", backendConf, tc.want.backendInterface) + } + }) + } +} diff --git a/e2e/controllernamespace/controllernamespace_test.go b/e2e/controllernamespace/controllernamespace_test.go new file mode 100644 index 00000000..4c4c5f18 --- /dev/null +++ b/e2e/controllernamespace/controllernamespace_test.go @@ -0,0 +1,126 @@ +package controllernamespace + +import ( + "context" + types2 "github.com/oam-dev/terraform-controller/api/types" + "strings" + "time" + + crossplane "github.com/oam-dev/terraform-controller/api/types/crossplane-runtime" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/pkg/errors" + appv1 "k8s.io/api/apps/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + pkgClient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/config" + + "github.com/oam-dev/terraform-controller/api/v1beta2" +) + +var _ = Describe("Restart with controller-namespace", func() { + const ( + defaultNamespace = "default" + controllerNamespace = "terraform" + chartNamespace = "terraform" + ) + var ( + controllerDeployMeta = types.NamespacedName{Name: "terraform-controller", Namespace: chartNamespace} + ) + ctx := context.Background() + + // create k8s rest config + restConf, err := config.GetConfig() + Expect(err).NotTo(HaveOccurred()) + k8sClient, err := pkgClient.New(restConf, pkgClient.Options{}) + s := k8sClient.Scheme() + _ = v1beta2.AddToScheme(s) + Expect(err).NotTo(HaveOccurred()) + + configuration := &v1beta2.Configuration{ + ObjectMeta: v1.ObjectMeta{ + Name: "e2e-for-ctrl-ns", + Namespace: defaultNamespace, + }, + Spec: v1beta2.ConfigurationSpec{ + HCL: ` +resource "random_id" "server" { + byte_length = 8 +} + +output "random_id" { + value = random_id.server.hex +}`, + InlineCredentials: true, + WriteConnectionSecretToReference: &crossplane.SecretReference{ + Name: "some-conn", + Namespace: defaultNamespace, + }, + }, + } + AfterEach(func() { + _ = k8sClient.Delete(ctx, configuration) + }) + It("Restart with controller namespace", func() { + By("apply configuration without --controller-namespace", func() { + err = k8sClient.Create(ctx, configuration) + Expect(err).NotTo(HaveOccurred()) + var cfg = &v1beta2.Configuration{} + Eventually(func() error { + err = k8sClient.Get(ctx, types.NamespacedName{Name: configuration.Name, Namespace: configuration.Namespace}, cfg) + if err != nil { + return err + } + if cfg.Status.Apply.State != types2.Available { + return errors.Errorf("configuration is not available, status now: %s", cfg.Status.Apply.State) + } + return nil + }, time.Second*60, time.Second*5).Should(Succeed()) + }) + By("restart controller with --controller-namespace", func() { + ctrlDeploy := appv1.Deployment{} + err = k8sClient.Get(ctx, controllerDeployMeta, &ctrlDeploy) + Expect(err).NotTo(HaveOccurred()) + ctrlDeploy.Spec.Template.Spec.Containers[0].Args = append(ctrlDeploy.Spec.Template.Spec.Containers[0].Args, "--controller-namespace="+controllerNamespace) + err := k8sClient.Update(ctx, &ctrlDeploy) + Expect(err).NotTo(HaveOccurred()) + + Eventually(func() error { + err := k8sClient.Get(ctx, controllerDeployMeta, &ctrlDeploy) + if err != nil { + return err + } + if ctrlDeploy.Status.UnavailableReplicas == 1 { + return errors.New("controller is not updated") + } + return nil + }, time.Second*60, time.Second*5).Should(Succeed()) + + }) + By("configuration should be still available", func() { + // wait about half minute to check configuration's state isn't changed + for i := 0; i < 30; i++ { + err := k8sClient.Get(ctx, types.NamespacedName{ + Name: configuration.Name, Namespace: configuration.Namespace, + }, configuration) + Expect(err).NotTo(HaveOccurred()) + time.Sleep(time.Second) + } + }) + By("restore controller", func() { + ctrlDeploy := appv1.Deployment{} + err = k8sClient.Get(ctx, controllerDeployMeta, &ctrlDeploy) + Expect(err).NotTo(HaveOccurred()) + cmds := make([]string, 0) + for _, cmd := range ctrlDeploy.Spec.Template.Spec.Containers[0].Args { + if !strings.HasPrefix(cmd, "--controller-namespace") { + cmds = append(cmds, cmd) + } + } + ctrlDeploy.Spec.Template.Spec.Containers[0].Args = cmds + err := k8sClient.Update(ctx, &ctrlDeploy) + Expect(err).NotTo(HaveOccurred()) + }) + }) +}) diff --git a/e2e/controllernamespace/e2e_suite_test.go b/e2e/controllernamespace/e2e_suite_test.go new file mode 100644 index 00000000..a1d09c50 --- /dev/null +++ b/e2e/controllernamespace/e2e_suite_test.go @@ -0,0 +1,14 @@ +package controllernamespace_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestE2e(t *testing.T) { + RegisterFailHandler(Fail) + defer GinkgoRecover() + RunSpecs(t, "E2e Suite") +} diff --git a/e2e/configuration_test.go b/e2e/normal/configuration_test.go similarity index 99% rename from e2e/configuration_test.go rename to e2e/normal/configuration_test.go index f5c5927c..b6f0575f 100644 --- a/e2e/configuration_test.go +++ b/e2e/normal/configuration_test.go @@ -1,6 +1,7 @@ -package e2e +package normal import ( + "context" "fmt" "os" "os/exec" @@ -9,7 +10,6 @@ import ( "testing" "time" - "golang.org/x/net/context" "gotest.tools/assert" kerrors "k8s.io/apimachinery/pkg/api/errors" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -29,6 +29,8 @@ var ( "examples/alibaba/oss/configuration_hcl_bucket.yaml", } testConfigurationsForceDelete = "examples/random/configuration_force_delete.yaml" + + chartNamespace = "terraform" ) type ConfigurationAttr struct { diff --git a/e2e/regression.go b/e2e/normal/regression.go similarity index 99% rename from e2e/regression.go rename to e2e/normal/regression.go index 3dad6c04..8bf4afe9 100644 --- a/e2e/regression.go +++ b/e2e/normal/regression.go @@ -1,4 +1,4 @@ -package e2e +package normal import ( "fmt" diff --git a/gitee/gitee_configuration_test.go b/gitee/gitee_configuration_test.go index d9112a97..6751c35d 100644 --- a/gitee/gitee_configuration_test.go +++ b/gitee/gitee_configuration_test.go @@ -1,9 +1,8 @@ package gitee import ( + "github.com/oam-dev/terraform-controller/e2e/normal" "testing" - - "github.com/oam-dev/terraform-controller/e2e" ) var ( @@ -16,5 +15,5 @@ var ( func TestGiteeConfigurationRegression(t *testing.T) { var retryTimes = 240 - e2e.Regression(t, giteeConfigurationsRegression, retryTimes) + normal.Regression(t, giteeConfigurationsRegression, retryTimes) } diff --git a/go.mod b/go.mod index 73c64d92..f000bcbb 100644 --- a/go.mod +++ b/go.mod @@ -12,10 +12,9 @@ require ( github.com/hashicorp/hcl/v2 v2.12.0 github.com/jinzhu/copier v0.3.5 github.com/onsi/ginkgo v1.16.4 - github.com/onsi/gomega v1.14.0 + github.com/onsi/gomega v1.19.0 github.com/pkg/errors v0.9.1 github.com/stretchr/testify v1.7.0 - golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd gopkg.in/yaml.v2 v2.4.0 gotest.tools v2.2.0+incompatible k8s.io/api v0.21.3 @@ -60,8 +59,9 @@ require ( github.com/prometheus/procfs v0.6.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/zclconf/go-cty v1.10.0 // indirect + golang.org/x/net v0.0.0-20220225172249-27dd8689420f // indirect golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d // indirect - golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e // indirect + golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8 // indirect golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect golang.org/x/text v0.3.7 // indirect golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect diff --git a/go.sum b/go.sum index df2d0613..3b0fa0da 100644 --- a/go.sum +++ b/go.sum @@ -199,6 +199,7 @@ github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OI github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -246,6 +247,7 @@ github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2p github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= @@ -337,12 +339,16 @@ github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+ github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= +github.com/onsi/ginkgo/v2 v2.1.3 h1:e/3Cwtogj0HA+25nMP1jCMDIf8RtRYbGwGGuBIFztkc= +github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.14.0 h1:ep6kpPVwmr/nTbklSx2nrLNSIO62DoYAhnPNIMhK8gI= github.com/onsi/gomega v1.14.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0= +github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= +github.com/onsi/gomega v1.19.0 h1:4ieX6qQjPP/BfC3mpsAtIGGlxTWPeA3Inl/7DtXw1tw= +github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= @@ -531,8 +537,9 @@ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= -golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f h1:oA4XRj0qtSt8Yo1Zms0CUlsT3KG69V2UGQWPBxujDmc= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -594,8 +601,9 @@ golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8 h1:OH54vjqzRWmbJ62fjuhxy7AxFFgoHN0/DPc/UrL8cAs= +golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=