feat: Download gcloud directly in Terraform and allow skipping comple…

…tely (#41)

BREAKING CHANGE: gcloud is now downloaded as part of Terraform execution, meaning your Terraform runner needs access to the internet. Alternatively, you can install gcloud out-of-band and set `skip_download` to true.

Former-commit-id: 41fe46b

.gitignore

@@ -46,3 +46,7 @@ google-cloud-sdk/
 google-cloud-sdk.staging/
 credentials.json
 tmp
+
+# Ignore cache directory
+cache/*
+!cache/README.md

Makefile

@@ -16,10 +16,8 @@
 # Please make sure to contribute relevant changes upstream!
 
 # Make will use bash instead of sh
-SHELL := /usr/bin/env bash
+SHELL := /usr/bin/env bash -O extglob
 
-GCLOUD_SDK_VERSION := $(shell cat SDK_VERSION)
-JQ_VERSION := 1.6
 DOCKER_TAG_VERSION_DEVELOPER_TOOLS := 0
 DOCKER_IMAGE_DEVELOPER_TOOLS := cft/developer-tools
 REGISTRY_URL := gcr.io/cloud-foundation-cicd
@@ -66,7 +64,6 @@ docker_test_integration:
 		$(REGISTRY_URL)/${DOCKER_IMAGE_DEVELOPER_TOOLS}:${DOCKER_TAG_VERSION_DEVELOPER_TOOLS} \
 		/usr/local/bin/test_integration.sh
 
-
 # Execute lint tests within the docker container
 .PHONY: docker_test_lint
 docker_test_lint:
@@ -88,54 +85,6 @@ docker_generate_docs:
 .PHONY: generate_docs
 generate_docs: docker_generate_docs
 
-.PHONY: all
-all: reset
-all:
-	$(MAKE) gcloud.darwin
-	$(MAKE) gcloud.linux
-	$(MAKE) jq.download
-
-.PHONY: gcloud.darwin
-gcloud.darwin: OS_ARCH=darwin
-gcloud.darwin: gcloud.download
-
-.PHONY: gcloud.linux
-gcloud.linux: OS_ARCH=linux
-gcloud.linux: gcloud.download
-
-.PHONY: gcloud.download
-gcloud.download:
-	mkdir -p cache/${OS_ARCH}/
-	cd cache/${OS_ARCH}/ && \
-		curl -sL -o google-cloud-sdk.tar.gz https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-${GCLOUD_SDK_VERSION}-${OS_ARCH}-x86_64.tar.gz
-
-.PHONY: jq.download
-jq.download:
-	mkdir -p cache/darwin/
-	cd cache/darwin/ && \
-		curl -sL -o jq https://github.com/stedolan/jq/releases/download/jq-${JQ_VERSION}/jq-osx-amd64 && \
-		chmod +x jq
-	mkdir -p cache/linux/
-	cd cache/linux/ && \
-		curl -sL -o jq https://github.com/stedolan/jq/releases/download/jq-${JQ_VERSION}/jq-linux64 && \
-		chmod +x jq
-
 .PHONY: clean
-clean: ## Clean caches of decompressed SDKs
-	rm -rf cache/darwin/google-cloud-sdk/
-	rm -rf cache/linux/google-cloud-sdk/
-	rm -rf cache/darwin/google-cloud-sdk.staging/
-	rm -rf cache/linux/google-cloud-sdk.staging/
-	rm -rf cache/darwin/jq
-	rm -rf cache/linux/jq
-
-.PHONY: reset
-reset:
-	rm -rf cache
-
-.PHONY: update-gcloud-version
-update-gcloud-version:
-	mkdir -p tmp && cd tmp && \
-		curl -sL -o google-cloud-sdk.tar.gz https://dl.google.com/dl/cloudsdk/channels/rapid/google-cloud-sdk.tar.gz && \
-		tar -xzf google-cloud-sdk.tar.gz -C . && \
-		cp google-cloud-sdk/VERSION ../SDK_VERSION
+clean: ## Clean caches
+	rm -rf cache/!(README.md)

README.md

@@ -40,6 +40,10 @@ The [jq](https://stedolan.github.io/jq/) binary is also included in this module
 | destroy\_cmd\_body | On destroy, the command body you'd like to run with your entrypoint. | string | `"info"` | no |
 | destroy\_cmd\_entrypoint | On destroy, the command entrypoint you'd like to use.  Can also be set to a custom script. Module's bin directory will be prepended to path. | string | `"gcloud"` | no |
 | enabled | Flag to optionally disable usage of this module. | bool | `"true"` | no |
+| gcloud\_download\_url | Custom gcloud download url. Optional. | string | `""` | no |
+| gcloud\_sdk\_version | The gcloud sdk version to download. | string | `"281.0.0"` | no |
+| jq\_download\_url | Custom jq download url. Optional. | string | `""` | no |
+| jq\_version | The jq version to download. | string | `"1.6"` | no |
 | module\_depends\_on | List of modules or resources this module depends on. | list | `<list>` | no |
 | platform | Platform CLI will run on. Defaults to linux. Valid values: linux, darwin | string | `"linux"` | no |
 | service\_account\_key\_file | Path to service account key file to run `gcloud auth activate-service-account` with. Optional. | string | `""` | no |
@@ -68,6 +72,7 @@ The following dependencies must be available:
 
 - [Terraform][terraform] v0.12
 - [Terraform Provider for GCP][terraform-provider-gcp] plugin v2.0
+- [curl][curl]
 
 ### Service Account
 
@@ -92,4 +97,4 @@ information on contributing to this module.
 [project-factory-module]: https://registry.terraform.io/modules/terraform-google-modules/project-factory/google
 [terraform-provider-gcp]: https://www.terraform.io/docs/providers/google/index.html
 [terraform]: https://www.terraform.io/downloads.html
-
+[curl]: https://curl.haxx.se

cache/README.md

@@ -0,0 +1 @@
+Cache directory.

main.tf

@@ -16,14 +16,16 @@
 
 locals {
   tmp_credentials_path = "${path.module}/terraform-google-credentials.json"
-  original_path        = "${path.module}/cache/${var.platform}"
   cache_path           = "${path.module}/cache/${random_id.cache.hex}"
   gcloud_tar_path      = "${local.cache_path}/google-cloud-sdk.tar.gz"
   gcloud_bin_path      = "${local.cache_path}/google-cloud-sdk/bin"
   gcloud_bin_abs_path  = abspath(local.gcloud_bin_path)
   components           = join(" ", var.additional_components)
 
-  gcloud = var.skip_download ? "gcloud" : "${local.gcloud_bin_path}/gcloud"
+  gcloud              = var.skip_download ? "gcloud" : "${local.gcloud_bin_path}/gcloud"
+  gcloud_download_url = var.gcloud_download_url != "" ? var.gcloud_download_url : "https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-${var.gcloud_sdk_version}-${var.platform}-x86_64.tar.gz"
+  jq_platform         = var.platform == "darwin" ? "osx-amd" : var.platform
+  jq_download_url     = var.jq_download_url != "" ? var.jq_download_url : "https://github.com/stedolan/jq/releases/download/jq-${var.jq_version}/jq-${local.jq_platform}64"
 
   create_cmd_bin  = var.skip_download ? var.create_cmd_entrypoint : "${local.gcloud_bin_path}/${var.create_cmd_entrypoint}"
   destroy_cmd_bin = var.skip_download ? var.destroy_cmd_entrypoint : "${local.gcloud_bin_path}/${var.destroy_cmd_entrypoint}"
@@ -33,7 +35,9 @@ locals {
     ) + length(null_resource.gcloud_auth_google_credentials.*.triggers,
   ) + length(null_resource.run_command.*.triggers)
 
-  copy_command                                 = "cp -R ${local.original_path} ${local.cache_path}"
+  prepare_cache_command                        = "mkdir ${local.cache_path}"
+  download_gcloud_command                      = "curl -sL -o ${local.cache_path}/google-cloud-sdk.tar.gz ${local.gcloud_download_url}"
+  download_jq_command                          = "curl -sL -o ${local.cache_path}/jq ${local.jq_download_url} && chmod +x ${local.cache_path}/jq"
   decompress_command                           = "tar -xzf ${local.gcloud_tar_path} -C ${local.cache_path} && cp ${local.cache_path}/jq ${local.cache_path}/google-cloud-sdk/bin/"
   upgrade_command                              = "${local.gcloud} components update --quiet"
   additional_components_command                = "${local.gcloud} components install ${local.components} --quiet"
@@ -57,38 +61,74 @@ resource "null_resource" "module_depends_on" {
   }
 }
 
-resource "null_resource" "copy" {
+resource "null_resource" "prepare_cache" {
   count = (var.enabled && ! var.skip_download) ? 1 : 0
 
   triggers = merge({
-    md5          = md5(var.create_cmd_entrypoint)
-    arguments    = md5(var.create_cmd_body)
-    copy_command = local.copy_command
+    md5                   = md5(var.create_cmd_entrypoint)
+    arguments             = md5(var.create_cmd_body)
+    prepare_cache_command = local.prepare_cache_command
   }, var.create_cmd_triggers)
 
   provisioner "local-exec" {
     when    = create
-    command = self.triggers.copy_command
+    command = self.triggers.prepare_cache_command
   }
 
   depends_on = [null_resource.module_depends_on]
 }
 
+resource "null_resource" "download_gcloud" {
+  count = (var.enabled && ! var.skip_download) ? 1 : 0
+
+  triggers = merge({
+    md5                     = md5(var.create_cmd_entrypoint)
+    arguments               = md5(var.create_cmd_body)
+    download_gcloud_command = local.download_gcloud_command
+  }, var.create_cmd_triggers)
+
+  provisioner "local-exec" {
+    when    = create
+    command = self.triggers.download_gcloud_command
+  }
+
+  depends_on = [null_resource.prepare_cache]
+}
+
+resource "null_resource" "download_jq" {
+  count = (var.enabled && ! var.skip_download) ? 1 : 0
+
+  triggers = merge({
+    md5                 = md5(var.create_cmd_entrypoint)
+    arguments           = md5(var.create_cmd_body)
+    download_jq_command = local.download_jq_command
+  }, var.create_cmd_triggers)
+
+  provisioner "local-exec" {
+    when    = create
+    command = self.triggers.download_jq_command
+  }
+
+  depends_on = [null_resource.prepare_cache]
+}
+
 resource "null_resource" "decompress" {
   count = (var.enabled && ! var.skip_download) ? 1 : 0
 
   triggers = merge({
-    md5                = md5(var.create_cmd_entrypoint)
-    arguments          = md5(var.create_cmd_body)
-    decompress_command = local.decompress_command
+    md5                     = md5(var.create_cmd_entrypoint)
+    arguments               = md5(var.create_cmd_body)
+    decompress_command      = local.decompress_command
+    download_gcloud_command = local.download_gcloud_command
+    download_jq_command     = local.download_jq_command
   }, var.create_cmd_triggers)
 
   provisioner "local-exec" {
     when    = create
     command = self.triggers.decompress_command
   }
 
-  depends_on = [null_resource.copy]
+  depends_on = [null_resource.download_gcloud, null_resource.download_jq]
 }
 
 resource "null_resource" "upgrade" {

variables.tf

@@ -83,3 +83,25 @@ variable "use_tf_google_credentials_env_var" {
   description = "Use GOOGLE_CREDENTIALS environment variable to run `gcloud auth activate-service-account` with. Optional."
   default     = false
 }
+
+variable "jq_version" {
+  description = "The jq version to download."
+  default     = "1.6"
+}
+
+variable "gcloud_sdk_version" {
+  description = "The gcloud sdk version to download."
+  default     = "281.0.0"
+}
+
+variable "gcloud_download_url" {
+  description = "Custom gcloud download url. Optional."
+  type        = string
+  default     = ""
+}
+
+variable "jq_download_url" {
+  description = "Custom jq download url. Optional."
+  type        = string
+  default     = ""
+}