diff --git a/chart/templates/terraform_controller.yaml b/chart/templates/terraform_controller.yaml index 0de30f7f..dd94bbba 100644 --- a/chart/templates/terraform_controller.yaml +++ b/chart/templates/terraform_controller.yaml @@ -35,4 +35,20 @@ spec: value: {{ .Values.gitImage}} - name: GITHUB_BLOCKED value: {{ .Values.githubBlocked }} + {{ if .Values.resources.limits.cpu }} + - name: RESOURCES_LIMITS_CPU + value: {{ .Values.resources.limits.cpu }} + {{ end }} + {{ if .Values.resources.limits.memory }} + - name: RESOURCES_LIMITS_MEMORY + value: {{ .Values.resources.limits.memory }} + {{ end }} + {{ if .Values.resources.requests.cpu }} + - name: RESOURCES_REQUESTS_CPU + value: {{ .Values.resources.requests.cpu }} + {{ end }} + {{ if .Values.resources.requests.memory }} + - name: RESOURCES_REQUESTS_MEMORY + value: {{ .Values.resources.requests.memory }} + {{ end }} serviceAccountName: tf-controller-service-account diff --git a/chart/values.yaml b/chart/values.yaml index 4cdcfbdb..8d3e8b15 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -9,6 +9,14 @@ gitImage: alpine/git:latest busyboxImage: busybox:latest terraformImage: oamdev/docker-terraform:1.1.2 +resources: + limits: + cpu: "1000m" + memory: "2Gi" + requests: + cpu: "1000m" + memory: "2Gi" + backend: namespace: vela-system diff --git a/controllers/configuration_controller.go b/controllers/configuration_controller.go index 27b12e3c..395efdaa 100644 --- a/controllers/configuration_controller.go +++ b/controllers/configuration_controller.go @@ -32,6 +32,7 @@ import ( batchv1 "k8s.io/api/batch/v1" v1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/klog/v2" @@ -219,6 +220,16 @@ type TFConfigurationMeta struct { TerraformBackendNamespace string BusyboxImage string GitImage string + + // Resources series Variables are for Setting Compute Resources required by this container + ResourcesLimitsCPU string + ResourcesLimitsCPUQuantity resource.Quantity + ResourcesLimitsMemory string + ResourcesLimitsMemoryQuantity resource.Quantity + ResourcesRequestsCPU string + ResourcesRequestsCPUQuantity resource.Quantity + ResourcesRequestsMemory string + ResourcesRequestsMemoryQuantity resource.Quantity } func initTFConfigurationMeta(req ctrl.Request, configuration v1beta2.Configuration) *TFConfigurationMeta { @@ -388,6 +399,51 @@ func (r *ConfigurationReconciler) terraformDestroy(ctx context.Context, namespac return errors.New(types.MessageDestroyJobNotCompleted) } +func (r *ConfigurationReconciler) preCheckResourcesSetting(meta *TFConfigurationMeta) error { + + meta.ResourcesLimitsCPU = os.Getenv("RESOURCES_LIMITS_CPU") + if meta.ResourcesLimitsCPU != "" { + limitsCPU, err := resource.ParseQuantity(meta.ResourcesLimitsCPU) + if err != nil { + errMsg := "failed to parse env variable RESOURCES_LIMITS_CPU into resource.Quantity" + klog.ErrorS(err, errMsg) + return errors.Wrap(err, errMsg) + } + meta.ResourcesLimitsCPUQuantity = limitsCPU + } + meta.ResourcesLimitsMemory = os.Getenv("RESOURCES_LIMITS_MEMORY") + if meta.ResourcesLimitsMemory != "" { + limitsMemory, err := resource.ParseQuantity(meta.ResourcesLimitsMemory) + if err != nil { + errMsg := "failed to parse env variable RESOURCES_LIMITS_MEMORY into resource.Quantity" + klog.ErrorS(err, errMsg) + return errors.Wrap(err, errMsg) + } + meta.ResourcesLimitsMemoryQuantity = limitsMemory + } + meta.ResourcesRequestsCPU = os.Getenv("RESOURCES_REQUESTS_CPU") + if meta.ResourcesRequestsCPU != "" { + requestsCPU, err := resource.ParseQuantity(meta.ResourcesRequestsCPU) + if err != nil { + errMsg := "failed to parse env variable RESOURCES_REQUESTS_CPU into resource.Quantity" + klog.ErrorS(err, errMsg) + return errors.Wrap(err, errMsg) + } + meta.ResourcesRequestsCPUQuantity = requestsCPU + } + meta.ResourcesRequestsMemory = os.Getenv("RESOURCES_REQUESTS_MEMORY") + if meta.ResourcesRequestsMemory != "" { + requestsMemory, err := resource.ParseQuantity(meta.ResourcesRequestsMemory) + if err != nil { + errMsg := "failed to parse env variable RESOURCES_REQUESTS_MEMORY into resource.Quantity" + klog.ErrorS(err, errMsg) + return errors.Wrap(err, errMsg) + } + meta.ResourcesRequestsMemoryQuantity = requestsMemory + } + return nil +} + func (r *ConfigurationReconciler) preCheck(ctx context.Context, configuration *v1beta2.Configuration, meta *TFConfigurationMeta) error { var k8sClient = r.Client @@ -410,6 +466,10 @@ func (r *ConfigurationReconciler) preCheck(ctx context.Context, configuration *v meta.GitImage = "alpine/git:latest" } + if err := r.preCheckResourcesSetting(meta); err != nil { + return err + } + // Validation: 1) validate Configuration itself configurationType, err := tfcfg.ValidConfigurationObject(configuration) if err != nil { @@ -649,6 +709,52 @@ func (meta *TFConfigurationMeta) assembleTerraformJob(executionType TerraformExe } initContainers = append(initContainers, tfPreApplyInitContainer) + container := v1.Container{ + Name: terraformContainerName, + Image: meta.TerraformImage, + ImagePullPolicy: v1.PullIfNotPresent, + Command: []string{ + "bash", + "-c", + fmt.Sprintf("terraform %s -lock=false -auto-approve", executionType), + }, + VolumeMounts: []v1.VolumeMount{ + { + Name: meta.Name, + MountPath: WorkingVolumeMountPath, + }, + { + Name: InputTFConfigurationVolumeName, + MountPath: InputTFConfigurationVolumeMountPath, + }, + }, + Env: meta.Envs, + } + + if meta.ResourcesLimitsCPU != "" || meta.ResourcesLimitsMemory != "" || + meta.ResourcesRequestsCPU != "" || meta.ResourcesRequestsMemory != "" { + resourceRequirements := v1.ResourceRequirements{} + if meta.ResourcesLimitsCPU != "" || meta.ResourcesLimitsMemory != "" { + resourceRequirements.Limits = v1.ResourceList(map[v1.ResourceName]resource.Quantity{}) + if meta.ResourcesLimitsCPU != "" { + resourceRequirements.Limits["cpu"] = meta.ResourcesLimitsCPUQuantity + } + if meta.ResourcesLimitsMemory != "" { + resourceRequirements.Limits["memory"] = meta.ResourcesLimitsMemoryQuantity + } + } + if meta.ResourcesRequestsCPU != "" || meta.ResourcesLimitsMemory != "" { + resourceRequirements.Requests = v1.ResourceList(map[v1.ResourceName]resource.Quantity{}) + if meta.ResourcesRequestsCPU != "" { + resourceRequirements.Requests["cpu"] = meta.ResourcesRequestsCPUQuantity + } + if meta.ResourcesRequestsMemory != "" { + resourceRequirements.Requests["memory"] = meta.ResourcesRequestsMemoryQuantity + } + } + container.Resources = resourceRequirements + } + return &batchv1.Job{ TypeMeta: metav1.TypeMeta{ Kind: "Job", @@ -676,29 +782,8 @@ func (meta *TFConfigurationMeta) assembleTerraformJob(executionType TerraformExe // state file directory in advance InitContainers: initContainers, // Container terraform-executor will first copy predefined terraform.d to working directory, and - // then run terraform apply/destroy. - Containers: []v1.Container{{ - Name: terraformContainerName, - Image: meta.TerraformImage, - ImagePullPolicy: v1.PullIfNotPresent, - Command: []string{ - "bash", - "-c", - fmt.Sprintf("terraform %s -lock=false -auto-approve", executionType), - }, - VolumeMounts: []v1.VolumeMount{ - { - Name: meta.Name, - MountPath: WorkingVolumeMountPath, - }, - { - Name: InputTFConfigurationVolumeName, - MountPath: InputTFConfigurationVolumeMountPath, - }, - }, - Env: meta.Envs, - }, - }, + // then run terraform init/apply. + Containers: []v1.Container{container}, ServiceAccountName: ServiceAccountName, Volumes: executorVolumes, RestartPolicy: v1.RestartPolicyOnFailure, diff --git a/controllers/configuration_controller_test.go b/controllers/configuration_controller_test.go index 9581f451..a9ab7c49 100644 --- a/controllers/configuration_controller_test.go +++ b/controllers/configuration_controller_test.go @@ -16,7 +16,10 @@ import ( batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" + + "k8s.io/apimachinery/pkg/api/resource" kerrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -462,6 +465,193 @@ func TestConfigurationReconcile(t *testing.T) { } } +func TestPreCheckResourcesSetting(t *testing.T) { + r := &ConfigurationReconciler{} + s := runtime.NewScheme() + v1beta1.AddToScheme(s) + v1beta2.AddToScheme(s) + corev1.AddToScheme(s) + corev1.AddToScheme(s) + provider := &v1beta1.Provider{ + ObjectMeta: v1.ObjectMeta{ + Name: "default", + Namespace: "default", + }, + Status: v1beta1.ProviderStatus{ + State: types.ProviderIsNotReady, + }, + } + r.Client = fake.NewClientBuilder().WithScheme(s).WithObjects(provider).Build() + + type args struct { + r *ConfigurationReconciler + configuration *v1beta2.Configuration + meta *TFConfigurationMeta + } + + type want struct { + errMsg string + } + + type prepare func(*testing.T) + + testcases := []struct { + name string + prepare + args args + want want + }{ + { + name: "wrong value in environment variable RESOURCES_LIMITS_CPU", + prepare: func(t *testing.T) { + t.Setenv("RESOURCES_LIMITS_CPU", "abcde") + }, + args: args{ + r: r, + configuration: &v1beta2.Configuration{ + ObjectMeta: v1.ObjectMeta{ + Name: "abc", + }, + Spec: v1beta2.ConfigurationSpec{ + HCL: "bbb", + }, + }, + meta: &TFConfigurationMeta{ + ConfigurationCMName: "abc", + ProviderReference: &crossplane.Reference{ + Namespace: "default", + Name: "default", + }, + }, + }, + want: want{ + errMsg: "failed to parse env variable RESOURCES_LIMITS_CPU into resource.Quantity", + }, + }, + { + name: "wrong value in environment variable RESOURCES_LIMITS_MEMORY", + prepare: func(t *testing.T) { + t.Setenv("RESOURCES_LIMITS_MEMORY", "xxxx5Gi") + }, + args: args{ + r: r, + configuration: &v1beta2.Configuration{ + ObjectMeta: v1.ObjectMeta{ + Name: "abc", + }, + Spec: v1beta2.ConfigurationSpec{ + HCL: "bbb", + }, + }, + meta: &TFConfigurationMeta{ + ConfigurationCMName: "abc", + ProviderReference: &crossplane.Reference{ + Namespace: "default", + Name: "default", + }, + }, + }, + want: want{ + errMsg: "failed to parse env variable RESOURCES_LIMITS_MEMORY into resource.Quantity", + }, + }, + { + name: "wrong value in environment variable RESOURCES_REQUESTS_CPU", + prepare: func(t *testing.T) { + t.Setenv("RESOURCES_REQUESTS_CPU", "ekiadasdflksas") + }, + args: args{ + r: r, + configuration: &v1beta2.Configuration{ + ObjectMeta: v1.ObjectMeta{ + Name: "abc", + }, + Spec: v1beta2.ConfigurationSpec{ + HCL: "bbb", + }, + }, + meta: &TFConfigurationMeta{ + ConfigurationCMName: "abc", + ProviderReference: &crossplane.Reference{ + Namespace: "default", + Name: "default", + }, + }, + }, + want: want{ + errMsg: "failed to parse env variable RESOURCES_REQUESTS_CPU into resource.Quantity", + }, + }, + { + name: "wrong value in environment variable RESOURCES_REQUESTS_MEMORY", + prepare: func(t *testing.T) { + t.Setenv("RESOURCES_REQUESTS_MEMORY", "123x456") + }, + args: args{ + r: r, + configuration: &v1beta2.Configuration{ + ObjectMeta: v1.ObjectMeta{ + Name: "abc", + }, + Spec: v1beta2.ConfigurationSpec{ + HCL: "bbb", + }, + }, + meta: &TFConfigurationMeta{ + ConfigurationCMName: "abc", + ProviderReference: &crossplane.Reference{ + Namespace: "default", + Name: "default", + }, + }, + }, + want: want{ + errMsg: "failed to parse env variable RESOURCES_REQUESTS_MEMORY into resource.Quantity", + }, + }, + { + name: "correct value of resources setting in environment variable", + prepare: func(t *testing.T) { + t.Setenv("RESOURCES_LIMITS_CPU", "10m") + t.Setenv("RESOURCES_LIMITS_MEMORY", "10Mi") + t.Setenv("RESOURCES_REQUESTS_CPU", "100") + t.Setenv("RESOURCES_REQUESTS_MEMORY", "5Gi") + }, + args: args{ + r: r, + configuration: &v1beta2.Configuration{ + ObjectMeta: v1.ObjectMeta{ + Name: "abc", + }, + Spec: v1beta2.ConfigurationSpec{ + HCL: "bbb", + }, + }, + meta: &TFConfigurationMeta{ + ConfigurationCMName: "abc", + ProviderReference: &crossplane.Reference{ + Namespace: "default", + Name: "default", + }, + }, + }, + want: want{}, + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + if tc.prepare != nil { + tc.prepare(t) + } + if err := tc.args.r.preCheckResourcesSetting(tc.args.meta); (tc.want.errMsg != "") && + !strings.Contains(err.Error(), tc.want.errMsg) { + t.Errorf("preCheckResourcesSetting() error = %v, wantErr %v", err, tc.want.errMsg) + } + }) + } +} + func TestPreCheck(t *testing.T) { r := &ConfigurationReconciler{} ctx := context.Background() @@ -491,8 +681,11 @@ func TestPreCheck(t *testing.T) { errMsg string } + type prepare func(*testing.T) + testcases := []struct { name string + prepare args args want want }{ @@ -583,10 +776,150 @@ func TestPreCheck(t *testing.T) { errMsg: "provider not found", }, }, + { + name: "wrong value in environment variable RESOURCES_LIMITS_CPU", + prepare: func(t *testing.T) { + t.Setenv("RESOURCES_LIMITS_CPU", "abcde") + }, + args: args{ + r: r, + configuration: &v1beta2.Configuration{ + ObjectMeta: v1.ObjectMeta{ + Name: "abc", + }, + Spec: v1beta2.ConfigurationSpec{ + HCL: "bbb", + }, + }, + meta: &TFConfigurationMeta{ + ConfigurationCMName: "abc", + ProviderReference: &crossplane.Reference{ + Namespace: "default", + Name: "default", + }, + }, + }, + want: want{ + errMsg: "failed to parse env variable RESOURCES_LIMITS_CPU into resource.Quantity", + }, + }, + { + name: "wrong value in environment variable RESOURCES_LIMITS_MEMORY", + prepare: func(t *testing.T) { + t.Setenv("RESOURCES_LIMITS_MEMORY", "xxxx5Gi") + }, + args: args{ + r: r, + configuration: &v1beta2.Configuration{ + ObjectMeta: v1.ObjectMeta{ + Name: "abc", + }, + Spec: v1beta2.ConfigurationSpec{ + HCL: "bbb", + }, + }, + meta: &TFConfigurationMeta{ + ConfigurationCMName: "abc", + ProviderReference: &crossplane.Reference{ + Namespace: "default", + Name: "default", + }, + }, + }, + want: want{ + errMsg: "failed to parse env variable RESOURCES_LIMITS_MEMORY into resource.Quantity", + }, + }, + { + name: "wrong value in environment variable RESOURCES_REQUESTS_CPU", + prepare: func(t *testing.T) { + t.Setenv("RESOURCES_REQUESTS_CPU", "ekiadasdflksas") + }, + args: args{ + r: r, + configuration: &v1beta2.Configuration{ + ObjectMeta: v1.ObjectMeta{ + Name: "abc", + }, + Spec: v1beta2.ConfigurationSpec{ + HCL: "bbb", + }, + }, + meta: &TFConfigurationMeta{ + ConfigurationCMName: "abc", + ProviderReference: &crossplane.Reference{ + Namespace: "default", + Name: "default", + }, + }, + }, + want: want{ + errMsg: "failed to parse env variable RESOURCES_REQUESTS_CPU into resource.Quantity", + }, + }, + { + name: "wrong value in environment variable RESOURCES_REQUESTS_MEMORY", + prepare: func(t *testing.T) { + t.Setenv("RESOURCES_REQUESTS_MEMORY", "123x456") + }, + args: args{ + r: r, + configuration: &v1beta2.Configuration{ + ObjectMeta: v1.ObjectMeta{ + Name: "abc", + }, + Spec: v1beta2.ConfigurationSpec{ + HCL: "bbb", + }, + }, + meta: &TFConfigurationMeta{ + ConfigurationCMName: "abc", + ProviderReference: &crossplane.Reference{ + Namespace: "default", + Name: "default", + }, + }, + }, + want: want{ + errMsg: "failed to parse env variable RESOURCES_REQUESTS_MEMORY into resource.Quantity", + }, + }, + { + name: "correct value of resources setting in environment variable", + prepare: func(t *testing.T) { + t.Setenv("RESOURCES_LIMITS_CPU", "10m") + t.Setenv("RESOURCES_LIMITS_MEMORY", "10Mi") + t.Setenv("RESOURCES_REQUESTS_CPU", "100") + t.Setenv("RESOURCES_REQUESTS_MEMORY", "5Gi") + }, + args: args{ + r: r, + configuration: &v1beta2.Configuration{ + ObjectMeta: v1.ObjectMeta{ + Name: "abc", + }, + Spec: v1beta2.ConfigurationSpec{ + HCL: "bbb", + }, + }, + meta: &TFConfigurationMeta{ + ConfigurationCMName: "abc", + ProviderReference: &crossplane.Reference{ + Namespace: "default", + Name: "default", + }, + }, + }, + want: want{}, + }, } for _, tc := range testcases { t.Run(tc.name, func(t *testing.T) { + if tc.prepare != nil { + tc.prepare(t) + } + err := tc.args.r.preCheck(ctx, tc.args.configuration, tc.args.meta) if tc.want.errMsg != "" || err != nil { if !strings.Contains(err.Error(), tc.want.errMsg) { @@ -859,6 +1192,46 @@ func TestAssembleTerraformJob(t *testing.T) { assert.Equal(t, containers[1].Image, "d") } +func TestAssembleTerraformJobWithResourcesSetting(t *testing.T) { + quantityLimitsCPU, _ := resource.ParseQuantity("10m") + quantityLimitsMemory, _ := resource.ParseQuantity("10Mi") + quantityRequestsCPU, _ := resource.ParseQuantity("100m") + quantityRequestsMemory, _ := resource.ParseQuantity("5Gi") + meta := &TFConfigurationMeta{ + Name: "a", + ConfigurationCMName: "b", + BusyboxImage: "c", + GitImage: "d", + Namespace: "e", + TerraformImage: "f", + RemoteGit: "g", + + ResourcesLimitsCPU: "10m", + ResourcesLimitsCPUQuantity: quantityLimitsCPU, + ResourcesLimitsMemory: "10Mi", + ResourcesLimitsMemoryQuantity: quantityLimitsMemory, + ResourcesRequestsCPU: "100m", + ResourcesRequestsCPUQuantity: quantityRequestsCPU, + ResourcesRequestsMemory: "5Gi", + ResourcesRequestsMemoryQuantity: quantityRequestsMemory, + } + + job := meta.assembleTerraformJob(TerraformApply) + initContainers := job.Spec.Template.Spec.InitContainers + assert.Equal(t, initContainers[0].Image, "c") + assert.Equal(t, initContainers[1].Image, "d") + + container := job.Spec.Template.Spec.Containers[0] + limitsCPU := container.Resources.Limits["cpu"] + limitsMemory := container.Resources.Limits["memory"] + requestsCPU := container.Resources.Requests["cpu"] + requestsMemory := container.Resources.Requests["memory"] + assert.Equal(t, "10m", limitsCPU.String()) + assert.Equal(t, "10Mi", limitsMemory.String()) + assert.Equal(t, "100m", requestsCPU.String()) + assert.Equal(t, "5Gi", requestsMemory.String()) +} + func TestTfStatePropertyToToProperty(t *testing.T) { testcases := []TfStateProperty{ {