Add varsFromEnvFiles config option (#132)

.goreleaser.yml

@@ -38,7 +38,7 @@ brews:
       name: craftypath-ci-bot
       email: [email protected]
     folder: Formula
-    homepage:  https://github.com/craftypath/gotf/
+    homepage: https://github.com/craftypath/gotf/
     description: Handling multiple environments with Terraform made easy
     install: |
       bin.install "gotf"

README.md

@@ -101,6 +101,31 @@ Variables which are added to the Terraform environment via `TF_VAR_<var>=value`
 Module-specific variables which are added to the Terraform environment if the corresponding module is run via `TF_VAR_<var>=value` for commands that support them.
 Module-specific variables override global ones.
 
+#### `varsFromEnvFiles`
+
+Allows variables to be configured as env files (`name=value` per line) which can also be sourced from a shell script.
+`gotf` interprets these files and passes each entry via `TF_VAR_` environment variable.
+Names are automatically lower-cased to match the common Terraform style.
+This feature can be quite convenient if you create parts of your infrastructure via Terraform and other parts via shell scripts
+but want to have a common source for shared variables.
+This is also a workaround for getting rid of Terraform warnings in case a variable is not declared, which might happen if you use global var files for different modules.
+Comments and variable expansion are supported.
+
+```shell
+# comment for FOO
+FOO=foo
+
+# comment for BAR
+BAR="bar is just a $FOO"
+```
+
+This would then be set as follows:
+
+```shell
+TF_VAR_foo=foo
+TV_VAR_bar="bar is just a foo"
+```
+
 #### `envs`
 
 Environment variables to be added to the Terraform process.
@@ -159,6 +184,9 @@ moduleVars:
   02_compute:
     myvar: value for compute
 
+varsFromEnvFiles:
+  - '{{ .Params.environment }}.env'
+
 envs:
   BAR: barvalue
   TEMPLATED_ENV: "{{ .Params.param }}"
@@ -254,31 +282,32 @@ For example, the integration test in [cmd/gotf/gotf_test.go](cmd/gotf/gotf_test.
 ```console
 gotf> Loading config file: testdata/test-config.yaml
 gotf> Processing var files...
-gotf> Processing global var files...
+gotf> File testdata/global-does-not-exists.tfvars does not exist. Ignoring it.
 gotf> Processing module var files...
+gotf> Processing vars from env files...
 gotf> Processing global vars...
 gotf> Processing module vars...
 gotf> Processing envs...
 gotf> Processing backend configs...
-gotf> Using Terraform version 1.0.9
-gotf> Terraform version 1.0.9 already installed.
-gotf> Terraform binary: /Users/myuser/Library/Caches/gotf/terraform/1.0.9/terraform
+gotf> Using Terraform version 1.1.5
+gotf> Terraform version 1.1.5 already installed.
+gotf> Terraform binary: /Users/myuser/Library/Caches/gotf/terraform/1.1.5/terraform
 gotf>
 gotf> Terraform command-line:
 gotf> -----------------------
-gotf> /Users/myuser/Library/Caches/gotf/terraform/1.0.9/terraform init -no-color
+gotf> /Users/myuser/Library/Caches/gotf/terraform/1.1.5/terraform apply -auto-approve -no-color
 gotf>
 gotf> Terraform environment:
 gotf> ----------------------
-gotf> TF_CLI_ARGS_destroy=-var-file="../global-prod.tfvars" -var-file="../global.tfvars" -var-file="prod.tfvars"
-gotf> TF_VAR_templated_var=myval
-gotf> BAR=barvalue
-gotf> TF_CLI_ARGS_apply=-var-file="../global-prod.tfvars" -var-file="../global.tfvars" -var-file="prod.tfvars"
-gotf> TF_VAR_state_key=networking
-gotf> TF_VAR_myvar=value for networking
-gotf> TF_CLI_ARGS_plan=-var-file="../global-prod.tfvars" -var-file="../global.tfvars" -var-file="prod.tfvars"
 gotf> TF_CLI_ARGS_import=-var-file="../global-prod.tfvars" -var-file="../global.tfvars" -var-file="prod.tfvars"
+gotf> TF_CLI_ARGS_init=-backend-config=path=".terraform/terraform-networking-prod.tfstate"
+gotf> TEMPLATED_ENV=myval
+gotf> TF_CLI_ARGS_plan=-var-file="../global-prod.tfvars" -var-file="../global.tfvars" -var-file="prod.tfvars"
+gotf> TF_VAR_myvar=value for networking
+gotf> TF_CLI_ARGS_apply=-var-file="../global-prod.tfvars" -var-file="../global.tfvars" -var-file="prod.tfvars"
+gotf> TF_CLI_ARGS_refresh=-var-file="../global-prod.tfvars" -var-file="../global.tfvars" -var-file="prod.tfvars"
 gotf> TF_VAR_foo=42
+gotf> TF_VAR_var_from_env_file=prod-env
 gotf> TF_VAR_mapvar={
   entry1 = {
     value1 = testvalue1
@@ -289,8 +318,9 @@ gotf> TF_VAR_mapvar={
     value2 = false
   }
 }
+gotf> TF_VAR_state_key=networking
+gotf> BAR=barvalue
+gotf> TF_CLI_ARGS_destroy=-var-file="../global-prod.tfvars" -var-file="../global.tfvars" -var-file="prod.tfvars"
+gotf> TF_VAR_templated_var=myval
 gotf> TF_VAR_module_dir=01_networking
-gotf> TF_CLI_ARGS_init=-backend-config=path=".terraform/terraform-networking-prod.tfstate"
-gotf> TEMPLATED_ENV=myval
-gotf> TF_CLI_ARGS_refresh=-var-file="../global-prod.tfvars" -var-file="../global.tfvars" -var-file="prod.tfvars"
 ```

cmd/gotf/gotf_test.go

@@ -54,9 +54,9 @@ func TestExecute(t *testing.T) {
 					want: []string{
 						".terraform/terraform-networking-prod.tfstate",
 						`bar = "module1_prod"
-envSpecificVar = "prodvalue"
+env_specific_var = "prodvalue"
 foo = "42"
-globalVar = "globalvalue"
+global_var = "globalvalue"
 mapvar = <<EOT
 {
   entry1 = {
@@ -70,6 +70,7 @@ mapvar = <<EOT
 }
 EOT
 myvar = "value for networking"
+var_from_env_file = "prod-env"
 `},
 				},
 			},
@@ -93,9 +94,9 @@ myvar = "value for networking"
 					want: []string{
 						".terraform/terraform-compute-dev.tfstate",
 						`bar = "module2_dev"
-envSpecificVar = "devvalue"
+env_specific_var = "devvalue"
 foo = "42"
-globalVar = "globalvalue"
+global_var = "globalvalue"
 mapvar = <<EOT
 {
   entry1 = {
@@ -109,6 +110,7 @@ mapvar = <<EOT
 }
 EOT
 myvar = "value for compute"
+var_from_env_file = "dev-env"
 `},
 				},
 			},

cmd/gotf/testdata/01_networking/test.tf

@@ -10,9 +10,11 @@ variable "mapvar" {}
 
 variable "myvar" {}
 
-variable "globalVar" {}
+variable "global_var" {}
 
-variable "envSpecificVar" {}
+variable "env_specific_var" {}
+
+variable "var_from_env_file" {}
 
 output "bar" {
   value = var.bar
@@ -23,19 +25,23 @@ output "foo" {
 }
 
 output "mapvar" {
-  value   = var.mapvar
+  value = var.mapvar
+}
+
+output "global_var" {
+  value = var.global_var
 }
 
-output "globalVar" {
-  value   = var.globalVar
+output "env_specific_var" {
+  value = var.env_specific_var
 }
 
-output "envSpecificVar" {
-  value   = var.envSpecificVar
+output "var_from_env_file" {
+  value = var.var_from_env_file
 }
 
 output "myvar" {
-  value   = var.myvar
+  value = var.myvar
 }
 
 resource "null_resource" "echo" {

cmd/gotf/testdata/02_compute/test.tf

@@ -10,9 +10,11 @@ variable "mapvar" {}
 
 variable "myvar" {}
 
-variable "globalVar" {}
+variable "global_var" {}
 
-variable "envSpecificVar" {}
+variable "env_specific_var" {}
+
+variable "var_from_env_file" {}
 
 output "bar" {
   value = var.bar
@@ -23,19 +25,23 @@ output "foo" {
 }
 
 output "mapvar" {
-  value   = var.mapvar
+  value = var.mapvar
+}
+
+output "global_var" {
+  value = var.global_var
 }
 
-output "globalVar" {
-  value   = var.globalVar
+output "env_specific_var" {
+  value = var.env_specific_var
 }
 
-output "envSpecificVar" {
-  value   = var.envSpecificVar
+output "var_from_env_file" {
+  value = var.var_from_env_file
 }
 
 output "myvar" {
-  value   = var.myvar
+  value = var.myvar
 }
 
 resource "null_resource" "echo" {

cmd/gotf/testdata/dev.env

@@ -0,0 +1 @@
+VAR_FROM_ENV_FILE=dev-env

cmd/gotf/testdata/global-dev.tfvars

@@ -1 +1 @@
-envSpecificVar = "devvalue"
+env_specific_var = "devvalue"

cmd/gotf/testdata/global-prod.tfvars

@@ -1 +1 @@
-envSpecificVar = "prodvalue"
+env_specific_var = "prodvalue"

cmd/gotf/testdata/global.tfvars

@@ -1 +1 @@
-globalVar = "globalvalue"
+global_var = "globalvalue"

cmd/gotf/testdata/prod.env

@@ -0,0 +1 @@
+VAR_FROM_ENV_FILE=prod-env

cmd/gotf/testdata/test-config.yaml

@@ -45,6 +45,9 @@ moduleVars:
   02_compute:
     myvar: value for compute
 
+varsFromEnvFiles:
+  - '{{ .Params.environment }}.env'
+
 envs:
   BAR: barvalue
   TEMPLATED_ENV: "{{ .Params.param }}"

demo/01_first/variables.tf

@@ -10,5 +10,5 @@ variable "use_special_chars" {
 
 variable "module_specific_messages" {
   description = "Messages to print to the console"
-  type = list(string)
+  type        = list(string)
 }

go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/golangci/golangci-lint v1.44.0
 	github.com/goreleaser/goreleaser v1.4.1
 	github.com/hashicorp/go-multierror v1.1.1
+	github.com/joho/godotenv v1.4.0
 	github.com/magefile/mage v1.12.1
 	github.com/mholt/archiver/v3 v3.5.1
 	github.com/spf13/cobra v1.3.0

go.sum

@@ -780,6 +780,8 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGw
 github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
 github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
+github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg=
+github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/jonboulle/clockwork v0.2.0/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8=
 github.com/josharian/txtarfs v0.0.0-20210218200122-0702f000015a/go.mod h1:izVPOvVRsHiKkeGCT6tYBNWyDVuzj9wAaBb5R9qamfw=