From c2603c03534e364e4fb27daf4afbee54db50aaa5 Mon Sep 17 00:00:00 2001
From: Mike VanDenburgh <michael.vandenburgh@kitware.com>
Date: Thu, 24 Oct 2024 19:25:03 -0400
Subject: [PATCH 1/4] Move `binary_mirror` TF module up a directory

---
 .../modules => }/binary_mirror/binary_mirror_bucket.tf        | 0
 .../{spack_aws_k8s/modules => }/binary_mirror/cloudfront.tf   | 0
 .../modules => }/binary_mirror/logging_bucket.tf              | 0
 .../modules/{spack_aws_k8s/modules => }/binary_mirror/main.tf | 0
 .../{spack_aws_k8s/modules => }/binary_mirror/outputs.tf      | 0
 .../{spack_aws_k8s/modules => }/binary_mirror/variables.tf    | 0
 terraform/modules/spack_aws_k8s/binary_mirrors.tf             | 4 ++--
 7 files changed, 2 insertions(+), 2 deletions(-)
 rename terraform/modules/{spack_aws_k8s/modules => }/binary_mirror/binary_mirror_bucket.tf (100%)
 rename terraform/modules/{spack_aws_k8s/modules => }/binary_mirror/cloudfront.tf (100%)
 rename terraform/modules/{spack_aws_k8s/modules => }/binary_mirror/logging_bucket.tf (100%)
 rename terraform/modules/{spack_aws_k8s/modules => }/binary_mirror/main.tf (100%)
 rename terraform/modules/{spack_aws_k8s/modules => }/binary_mirror/outputs.tf (100%)
 rename terraform/modules/{spack_aws_k8s/modules => }/binary_mirror/variables.tf (100%)

diff --git a/terraform/modules/spack_aws_k8s/modules/binary_mirror/binary_mirror_bucket.tf b/terraform/modules/binary_mirror/binary_mirror_bucket.tf
similarity index 100%
rename from terraform/modules/spack_aws_k8s/modules/binary_mirror/binary_mirror_bucket.tf
rename to terraform/modules/binary_mirror/binary_mirror_bucket.tf
diff --git a/terraform/modules/spack_aws_k8s/modules/binary_mirror/cloudfront.tf b/terraform/modules/binary_mirror/cloudfront.tf
similarity index 100%
rename from terraform/modules/spack_aws_k8s/modules/binary_mirror/cloudfront.tf
rename to terraform/modules/binary_mirror/cloudfront.tf
diff --git a/terraform/modules/spack_aws_k8s/modules/binary_mirror/logging_bucket.tf b/terraform/modules/binary_mirror/logging_bucket.tf
similarity index 100%
rename from terraform/modules/spack_aws_k8s/modules/binary_mirror/logging_bucket.tf
rename to terraform/modules/binary_mirror/logging_bucket.tf
diff --git a/terraform/modules/spack_aws_k8s/modules/binary_mirror/main.tf b/terraform/modules/binary_mirror/main.tf
similarity index 100%
rename from terraform/modules/spack_aws_k8s/modules/binary_mirror/main.tf
rename to terraform/modules/binary_mirror/main.tf
diff --git a/terraform/modules/spack_aws_k8s/modules/binary_mirror/outputs.tf b/terraform/modules/binary_mirror/outputs.tf
similarity index 100%
rename from terraform/modules/spack_aws_k8s/modules/binary_mirror/outputs.tf
rename to terraform/modules/binary_mirror/outputs.tf
diff --git a/terraform/modules/spack_aws_k8s/modules/binary_mirror/variables.tf b/terraform/modules/binary_mirror/variables.tf
similarity index 100%
rename from terraform/modules/spack_aws_k8s/modules/binary_mirror/variables.tf
rename to terraform/modules/binary_mirror/variables.tf
diff --git a/terraform/modules/spack_aws_k8s/binary_mirrors.tf b/terraform/modules/spack_aws_k8s/binary_mirrors.tf
index 3934be4a4..c23aa40a8 100644
--- a/terraform/modules/spack_aws_k8s/binary_mirrors.tf
+++ b/terraform/modules/spack_aws_k8s/binary_mirrors.tf
@@ -25,7 +25,7 @@ resource "aws_cloudfront_cache_policy" "min_ttl_zero" {
 }
 
 module "pr_binary_mirror" {
-  source = "./modules/binary_mirror"
+  source = "../binary_mirror"
 
   bucket_iam_username = "pull-requests-binary-mirror${local.bucket_name_suffix}"
   bucket_name         = "spack-binaries-prs${local.bucket_name_suffix}"
@@ -38,7 +38,7 @@ module "pr_binary_mirror" {
 }
 
 module "protected_binary_mirror" {
-  source = "./modules/binary_mirror"
+  source = "../binary_mirror"
 
   bucket_iam_username = "protected-binary-mirror${local.bucket_name_suffix}"
   bucket_name         = "spack-binaries${local.bucket_name_suffix}"

From b1b663bcf328a04fd4ab7114fdfb5e0bccf679c7 Mon Sep 17 00:00:00 2001
From: Mike VanDenburgh <michael.vandenburgh@kitware.com>
Date: Tue, 22 Oct 2024 16:47:47 -0400
Subject: [PATCH 2/4] Move gitlab runner configuration to a separate terraform
 module

---
 .../modules/spack_gitlab/dev_projects.tf      |   8 +
 .../developer_project/binary_mirrors.tf       |  33 ++++
 .../modules/developer_project/gitlab.tf       |  15 ++
 .../modules/developer_project/variables.tf    |  11 ++
 .../modules/developer_project/versions.tf     |  10 ++
 .../gitlab_runner_configuration/gitlab.tf     |  32 ++++
 .../gitlab_runner_configuration/iam.tf        | 115 ++++++++++++++
 .../gitlab_runner_configuration/variables.tf  |  21 +++
 .../gitlab_runner_configuration/versions.tf   |  10 ++
 terraform/modules/spack_gitlab/runner_iam.tf  | 146 +-----------------
 10 files changed, 262 insertions(+), 139 deletions(-)
 create mode 100644 terraform/modules/spack_gitlab/dev_projects.tf
 create mode 100644 terraform/modules/spack_gitlab/modules/developer_project/binary_mirrors.tf
 create mode 100644 terraform/modules/spack_gitlab/modules/developer_project/gitlab.tf
 create mode 100644 terraform/modules/spack_gitlab/modules/developer_project/variables.tf
 create mode 100644 terraform/modules/spack_gitlab/modules/developer_project/versions.tf
 create mode 100644 terraform/modules/spack_gitlab/modules/gitlab_runner_configuration/gitlab.tf
 create mode 100644 terraform/modules/spack_gitlab/modules/gitlab_runner_configuration/iam.tf
 create mode 100644 terraform/modules/spack_gitlab/modules/gitlab_runner_configuration/variables.tf
 create mode 100644 terraform/modules/spack_gitlab/modules/gitlab_runner_configuration/versions.tf

diff --git a/terraform/modules/spack_gitlab/dev_projects.tf b/terraform/modules/spack_gitlab/dev_projects.tf
new file mode 100644
index 000000000..724545452
--- /dev/null
+++ b/terraform/modules/spack_gitlab/dev_projects.tf
@@ -0,0 +1,8 @@
+module "mike_dev_project" {
+  source = "./modules/developer_project"
+
+  deployment_name  = var.deployment_name
+  deployment_stage = var.deployment_stage
+
+  gitlab_repo = "mvandenburgh/spack"
+}
diff --git a/terraform/modules/spack_gitlab/modules/developer_project/binary_mirrors.tf b/terraform/modules/spack_gitlab/modules/developer_project/binary_mirrors.tf
new file mode 100644
index 000000000..26bde286d
--- /dev/null
+++ b/terraform/modules/spack_gitlab/modules/developer_project/binary_mirrors.tf
@@ -0,0 +1,33 @@
+locals {
+  bucket_name_suffix = "-${replace(data.gitlab_project.this.path_with_namespace, "/", "-")}${var.deployment_name == "prod" ? "" : "-${var.deployment_name}"}-${var.deployment_stage}"
+}
+
+data "aws_cloudfront_cache_policy" "min_ttl_zero" {
+  # Same cache policy that is used for production buckets
+  name = "CachingAllowNoCache${"${var.deployment_name != "prod" ? "-${var.deployment_name}" : ""}-${var.deployment_stage}"}"
+}
+
+module "pr_binary_mirror" {
+  source = "../../../binary_mirror"
+
+  bucket_iam_username = "pull-requests-binary-mirror${local.bucket_name_suffix}"
+  bucket_name         = "spack-binaries-prs${local.bucket_name_suffix}"
+
+  enable_logging      = true
+  logging_bucket_name = "spack-logs${local.bucket_name_suffix}"
+
+  cdn_domain      = "binaries-prs${local.bucket_name_suffix}.spack.io"
+  cache_policy_id = data.aws_cloudfront_cache_policy.min_ttl_zero.id
+}
+
+module "protected_binary_mirror" {
+  source = "../../../binary_mirror"
+
+  bucket_iam_username = "protected-binary-mirror${local.bucket_name_suffix}"
+  bucket_name         = "spack-binaries${local.bucket_name_suffix}"
+
+  enable_logging = false
+
+  cdn_domain      = "binaries${local.bucket_name_suffix}.spack.io"
+  cache_policy_id = data.aws_cloudfront_cache_policy.min_ttl_zero.id
+}
diff --git a/terraform/modules/spack_gitlab/modules/developer_project/gitlab.tf b/terraform/modules/spack_gitlab/modules/developer_project/gitlab.tf
new file mode 100644
index 000000000..ee826d10e
--- /dev/null
+++ b/terraform/modules/spack_gitlab/modules/developer_project/gitlab.tf
@@ -0,0 +1,15 @@
+data "gitlab_project" "this" {
+  path_with_namespace = var.gitlab_repo
+}
+
+module "gitlab_runner_configuration" {
+  source = "../gitlab_runner_configuration"
+
+  deployment_name  = var.deployment_name
+  deployment_stage = var.deployment_stage
+
+  pr_binary_bucket_arn        = module.pr_binary_mirror.bucket_arn
+  protected_binary_bucket_arn = module.protected_binary_mirror.bucket_arn
+
+  gitlab_repo = data.gitlab_project.this.path_with_namespace
+}
diff --git a/terraform/modules/spack_gitlab/modules/developer_project/variables.tf b/terraform/modules/spack_gitlab/modules/developer_project/variables.tf
new file mode 100644
index 000000000..d64671ad2
--- /dev/null
+++ b/terraform/modules/spack_gitlab/modules/developer_project/variables.tf
@@ -0,0 +1,11 @@
+variable "deployment_name" {
+  type = string
+}
+
+variable "deployment_stage" {
+  type = string
+}
+
+variable "gitlab_repo" {
+  type = string
+}
diff --git a/terraform/modules/spack_gitlab/modules/developer_project/versions.tf b/terraform/modules/spack_gitlab/modules/developer_project/versions.tf
new file mode 100644
index 000000000..9ffb9bfac
--- /dev/null
+++ b/terraform/modules/spack_gitlab/modules/developer_project/versions.tf
@@ -0,0 +1,10 @@
+terraform {
+  required_providers {
+    aws = {
+      source = "hashicorp/aws"
+    }
+    gitlab = {
+      source = "gitlabhq/gitlab"
+    }
+  }
+}
diff --git a/terraform/modules/spack_gitlab/modules/gitlab_runner_configuration/gitlab.tf b/terraform/modules/spack_gitlab/modules/gitlab_runner_configuration/gitlab.tf
new file mode 100644
index 000000000..cebfab011
--- /dev/null
+++ b/terraform/modules/spack_gitlab/modules/gitlab_runner_configuration/gitlab.tf
@@ -0,0 +1,32 @@
+data "gitlab_project" "this" {
+  path_with_namespace = var.gitlab_repo
+}
+
+resource "gitlab_project_variable" "binary_mirror_role_arn" {
+  for_each = resource.aws_iam_role.gitlab_runner
+
+  project = data.gitlab_project.this.id
+  key     = local.mirror_roles[each.key].role_arn_ci_var_name
+  value   = each.value.arn
+}
+
+# pre_build.py needs access to this to request PR prefix scoped permissions
+resource "gitlab_project_variable" "pr_binary_mirror_bucket_arn" {
+  project = data.gitlab_project.this.id
+  key     = "PR_BINARY_MIRROR_BUCKET_ARN"
+  value   = var.pr_binary_bucket_arn
+}
+
+# Configure retries
+resource "gitlab_project_variable" "retries" {
+  for_each = toset([
+    # Enable retries for artifact downloads, source fetching, and cache restoration in CI jobs
+    "ARTIFACT_DOWNLOAD_ATTEMPTS",
+    "GET_SOURCES_ATTEMPTS",
+    "RESTORE_CACHE_ATTEMPTS",
+  ])
+
+  project = data.gitlab_project.this.id
+  key     = each.value
+  value   = "3"
+}
diff --git a/terraform/modules/spack_gitlab/modules/gitlab_runner_configuration/iam.tf b/terraform/modules/spack_gitlab/modules/gitlab_runner_configuration/iam.tf
new file mode 100644
index 000000000..ec12063b6
--- /dev/null
+++ b/terraform/modules/spack_gitlab/modules/gitlab_runner_configuration/iam.tf
@@ -0,0 +1,115 @@
+locals {
+  gitlab_domain = "gitlab${var.deployment_name == "prod" ? "" : ".${var.deployment_name}"}.spack.io"
+  suffix        = var.deployment_name != "prod" ? "-${var.deployment_name}" : ""
+
+  mirror_roles = {
+    "pr_binary_mirror" = {
+      "role_name_suffix"     = "PRBinaryMirror${var.deployment_name == "prod" ? "" : "-${var.deployment_name}"}-${var.deployment_stage}",
+      "role_arn_ci_var_name" = "PR_BINARY_MIRROR_ROLE_ARN",
+      "conditions"           = ["project_path:${data.gitlab_project.this.path_with_namespace}:ref_type:branch:ref:pr*"],
+    },
+    "protected_binary_mirror" = {
+      "role_name_suffix"     = "ProtectedBinaryMirror${var.deployment_name == "prod" ? "" : "-${var.deployment_name}"}-${var.deployment_stage}",
+      "role_arn_ci_var_name" = "PROTECTED_BINARY_MIRROR_ROLE_ARN",
+      "conditions" = [
+        "project_path:${data.gitlab_project.this.path_with_namespace}:ref_type:branch:ref:develop",
+        "project_path:${data.gitlab_project.this.path_with_namespace}:ref_type:branch:ref:releases/v*",
+        "project_path:${data.gitlab_project.this.path_with_namespace}:ref_type:tag:ref:develop-*",
+        "project_path:${data.gitlab_project.this.path_with_namespace}:ref_type:tag:ref:v*"
+      ],
+    }
+  }
+}
+
+data "aws_caller_identity" "current" {}
+
+data "tls_certificate" "gitlab" {
+  url = "https://${local.gitlab_domain}"
+}
+
+resource "aws_iam_openid_connect_provider" "gitlab" {
+  url            = "https://${local.gitlab_domain}"
+  client_id_list = keys(local.mirror_roles)
+
+  # Only use the last item in the list, since the first certificate is the root CA, and we don't want to use that.
+  thumbprint_list = [data.tls_certificate.gitlab.certificates[length(data.tls_certificate.gitlab.certificates) - 1].sha1_fingerprint]
+}
+
+data "aws_iam_policy_document" "gitlab_oidc_assume_role" {
+  for_each = local.mirror_roles
+
+  statement {
+    effect  = "Allow"
+    actions = ["sts:AssumeRoleWithWebIdentity"]
+
+    principals {
+      type        = "Federated"
+      identifiers = ["arn:aws:iam::${data.aws_caller_identity.current.account_id}:oidc-provider/${local.gitlab_domain}"]
+    }
+
+    condition {
+      test     = "StringEquals"
+      variable = "${local.gitlab_domain}:aud"
+      values   = [each.key]
+    }
+
+    condition {
+      test     = "StringLike"
+      variable = "${local.gitlab_domain}:sub"
+      values   = each.value.conditions
+    }
+  }
+}
+
+resource "aws_iam_role" "gitlab_runner" {
+  for_each = data.aws_iam_policy_document.gitlab_oidc_assume_role
+
+  name                 = "GitLabRunner${local.mirror_roles[each.key].role_name_suffix}"
+  assume_role_policy   = each.value.json
+  max_session_duration = 3600 * 6 # only allow a max of 6 hours for a session to be active
+}
+
+data "aws_iam_policy_document" "gitlab_runner" {
+  for_each = var.deployment_name != "prod" ? local.mirror_roles : {}
+
+  statement {
+    effect  = "Allow"
+    actions = ["s3:PutObject", "s3:DeleteObject"]
+
+    resources = [
+      each.key == "protected_binary_mirror" ? "${var.protected_binary_bucket_arn}/*" : "${var.pr_binary_bucket_arn}/*",
+    ]
+  }
+}
+
+resource "aws_iam_policy" "gitlab_runner" {
+  for_each = data.aws_iam_policy_document.gitlab_runner
+
+  name        = "WriteBinariesTo${local.mirror_roles[each.key].role_name_suffix}"
+  description = "Managed by Terraform. IAM Policy that provides access to S3 buckets for binary mirrors."
+  policy      = each.value.json
+}
+
+resource "aws_iam_role_policy_attachment" "gitlab_runner" {
+  for_each = aws_iam_policy.gitlab_runner
+
+  role       = aws_iam_role.gitlab_runner[each.key].name
+  policy_arn = each.value.arn
+}
+
+# attachments for the pre-existing hardcoded policies in production
+resource "aws_iam_role_policy_attachment" "legacy_gitlab_runner_pr_binary_mirror" {
+  for_each = var.deployment_name == "prod" ? toset(["arn:aws:iam::588562868276:policy/DeleteObjectsFromBucketSpackBinariesPRs",
+  "arn:aws:iam::588562868276:policy/PutObjectsInBucketSpackBinariesPRs"]) : []
+
+  role       = aws_iam_role.gitlab_runner["pr_binary_mirror"].name
+  policy_arn = each.value
+}
+
+resource "aws_iam_role_policy_attachment" "legacy_gitlab_runner_protected_binary_mirror" {
+  for_each = var.deployment_name == "prod" ? toset(["arn:aws:iam::588562868276:policy/DeleteObjectsFromBucketSpackBinaries",
+  "arn:aws:iam::588562868276:policy/PutObjectsInBucketSpackBinaries"]) : []
+
+  role       = aws_iam_role.gitlab_runner["protected_binary_mirror"].name
+  policy_arn = each.value
+}
diff --git a/terraform/modules/spack_gitlab/modules/gitlab_runner_configuration/variables.tf b/terraform/modules/spack_gitlab/modules/gitlab_runner_configuration/variables.tf
new file mode 100644
index 000000000..1dfb15bf9
--- /dev/null
+++ b/terraform/modules/spack_gitlab/modules/gitlab_runner_configuration/variables.tf
@@ -0,0 +1,21 @@
+variable "deployment_name" {
+  type = string
+}
+
+variable "deployment_stage" {
+  type = string
+}
+
+variable "protected_binary_bucket_arn" {
+  description = "The ARN of the S3 bucket that contains protected binaries."
+  type        = string
+}
+
+variable "pr_binary_bucket_arn" {
+  description = "The ARN of the S3 bucket that contains PR binaries."
+  type        = string
+}
+
+variable "gitlab_repo" {
+  type = string
+}
diff --git a/terraform/modules/spack_gitlab/modules/gitlab_runner_configuration/versions.tf b/terraform/modules/spack_gitlab/modules/gitlab_runner_configuration/versions.tf
new file mode 100644
index 000000000..9ffb9bfac
--- /dev/null
+++ b/terraform/modules/spack_gitlab/modules/gitlab_runner_configuration/versions.tf
@@ -0,0 +1,10 @@
+terraform {
+  required_providers {
+    aws = {
+      source = "hashicorp/aws"
+    }
+    gitlab = {
+      source = "gitlabhq/gitlab"
+    }
+  }
+}
diff --git a/terraform/modules/spack_gitlab/runner_iam.tf b/terraform/modules/spack_gitlab/runner_iam.tf
index ecc02158d..73b469c93 100644
--- a/terraform/modules/spack_gitlab/runner_iam.tf
+++ b/terraform/modules/spack_gitlab/runner_iam.tf
@@ -1,31 +1,3 @@
-locals {
-  gitlab_domain = "gitlab${var.deployment_name == "prod" ? "" : ".${var.deployment_name}"}.spack.io"
-
-  mirror_roles = {
-    "pr_binary_mirror" = {
-      "role_name_suffix"     = "PRBinaryMirror${var.deployment_name == "prod" ? "" : "-${var.deployment_name}"}-${var.deployment_stage}",
-      "role_arn_ci_var_name" = "PR_BINARY_MIRROR_ROLE_ARN",
-      "conditions"           = ["project_path:${data.gitlab_project.spack.path_with_namespace}:ref_type:branch:ref:pr*"],
-    },
-    "protected_binary_mirror" = {
-      "role_name_suffix"     = "ProtectedBinaryMirror${var.deployment_name == "prod" ? "" : "-${var.deployment_name}"}-${var.deployment_stage}",
-      "role_arn_ci_var_name" = "PROTECTED_BINARY_MIRROR_ROLE_ARN",
-      "conditions" = [
-        "project_path:${data.gitlab_project.spack.path_with_namespace}:ref_type:branch:ref:develop",
-        "project_path:${data.gitlab_project.spack.path_with_namespace}:ref_type:branch:ref:releases/v*",
-        "project_path:${data.gitlab_project.spack.path_with_namespace}:ref_type:tag:ref:develop-*",
-        "project_path:${data.gitlab_project.spack.path_with_namespace}:ref_type:tag:ref:v*"
-      ],
-    }
-  }
-}
-
-data "aws_caller_identity" "current" {}
-
-data "tls_certificate" "gitlab" {
-  url = "https://${local.gitlab_domain}"
-}
-
 data "aws_s3_bucket" "protected_mirror" {
   bucket = "spack-binaries${local.suffix}"
 }
@@ -34,118 +6,14 @@ data "aws_s3_bucket" "pr_mirror" {
   bucket = "spack-binaries-prs${local.suffix}"
 }
 
-resource "aws_iam_openid_connect_provider" "gitlab" {
-  url            = "https://${local.gitlab_domain}"
-  client_id_list = keys(local.mirror_roles)
-
-  # Only use the last item in the list, since the first certificate is the root CA, and we don't want to use that.
-  thumbprint_list = [data.tls_certificate.gitlab.certificates[length(data.tls_certificate.gitlab.certificates) - 1].sha1_fingerprint]
-}
-
-data "aws_iam_policy_document" "gitlab_oidc_assume_role" {
-  for_each = local.mirror_roles
-
-  statement {
-    effect  = "Allow"
-    actions = ["sts:AssumeRoleWithWebIdentity"]
-
-    principals {
-      type        = "Federated"
-      identifiers = ["arn:aws:iam::${data.aws_caller_identity.current.account_id}:oidc-provider/${local.gitlab_domain}"]
-    }
-
-    condition {
-      test     = "StringEquals"
-      variable = "${local.gitlab_domain}:aud"
-      values   = [each.key]
-    }
-
-    condition {
-      test     = "StringLike"
-      variable = "${local.gitlab_domain}:sub"
-      values   = each.value.conditions
-    }
-  }
-}
-
-resource "aws_iam_role" "gitlab_runner" {
-  for_each = data.aws_iam_policy_document.gitlab_oidc_assume_role
-
-  name                 = "GitLabRunner${local.mirror_roles[each.key].role_name_suffix}"
-  assume_role_policy   = each.value.json
-  max_session_duration = 3600 * 6 # only allow a max of 6 hours for a session to be active
-}
-
-data "aws_iam_policy_document" "gitlab_runner" {
-  for_each = var.deployment_name != "prod" ? local.mirror_roles : {}
-
-  statement {
-    effect  = "Allow"
-    actions = ["s3:PutObject", "s3:DeleteObject"]
+module "spack_project_runner_configuration" {
+  source = "./modules/gitlab_runner_configuration"
 
-    resources = [
-      each.key == "protected_binary_mirror" ? "${data.aws_s3_bucket.protected_mirror.arn}/*" : "${data.aws_s3_bucket.pr_mirror.arn}/*",
-    ]
-  }
-}
-
-resource "aws_iam_policy" "gitlab_runner" {
-  for_each = data.aws_iam_policy_document.gitlab_runner
-
-  name        = "WriteBinariesTo${local.mirror_roles[each.key].role_name_suffix}"
-  description = "Managed by Terraform. IAM Policy that provides access to S3 buckets for binary mirrors."
-  policy      = each.value.json
-}
-
-resource "aws_iam_role_policy_attachment" "gitlab_runner" {
-  for_each = aws_iam_policy.gitlab_runner
-
-  role       = aws_iam_role.gitlab_runner[each.key].name
-  policy_arn = each.value.arn
-}
-
-resource "gitlab_project_variable" "binary_mirror_role_arn" {
-  for_each = resource.aws_iam_role.gitlab_runner
-
-  project = data.gitlab_project.spack.id
-  key     = local.mirror_roles[each.key].role_arn_ci_var_name
-  value   = each.value.arn
-}
-
-# pre_build.py needs access to this to request PR prefix scoped permissions
-resource "gitlab_project_variable" "pr_binary_mirror_bucket_arn" {
-  project = data.gitlab_project.spack.id
-  key     = "PR_BINARY_MIRROR_BUCKET_ARN"
-  value   = data.aws_s3_bucket.pr_mirror.arn
-}
-
-# attachments for the pre-existing hardcoded policies in production
-resource "aws_iam_role_policy_attachment" "legacy_gitlab_runner_pr_binary_mirror" {
-  for_each = var.deployment_name == "prod" ? toset(["arn:aws:iam::588562868276:policy/DeleteObjectsFromBucketSpackBinariesPRs",
-  "arn:aws:iam::588562868276:policy/PutObjectsInBucketSpackBinariesPRs"]) : []
-
-  role       = aws_iam_role.gitlab_runner["pr_binary_mirror"].name
-  policy_arn = each.value
-}
-
-resource "aws_iam_role_policy_attachment" "legacy_gitlab_runner_protected_binary_mirror" {
-  for_each = var.deployment_name == "prod" ? toset(["arn:aws:iam::588562868276:policy/DeleteObjectsFromBucketSpackBinaries",
-  "arn:aws:iam::588562868276:policy/PutObjectsInBucketSpackBinaries"]) : []
-
-  role       = aws_iam_role.gitlab_runner["protected_binary_mirror"].name
-  policy_arn = each.value
-}
+  deployment_name  = var.deployment_name
+  deployment_stage = var.deployment_stage
 
-# Configure retries
-resource "gitlab_project_variable" "retries" {
-  for_each = toset([
-    # Enable retries for artifact downloads, source fetching, and cache restoration in CI jobs
-    "ARTIFACT_DOWNLOAD_ATTEMPTS",
-    "GET_SOURCES_ATTEMPTS",
-    "RESTORE_CACHE_ATTEMPTS",
-  ])
+  pr_binary_bucket_arn        = data.aws_s3_bucket.pr_mirror.arn
+  protected_binary_bucket_arn = data.aws_s3_bucket.protected_mirror.arn
 
-  project = data.gitlab_project.spack.id
-  key     = each.value
-  value   = "3"
+  gitlab_repo = data.gitlab_project.spack.path_with_namespace
 }

From b6e2492d3aa5ecf5bda82188e512dd824104de43 Mon Sep 17 00:00:00 2001
From: Mike VanDenburgh <michael.vandenburgh@kitware.com>
Date: Wed, 26 Jun 2024 18:33:41 -0400
Subject: [PATCH 3/4] Migrate to AWS Load Balancer Controller

Resource:
https://aws.amazon.com/blogs/containers/exposing-kubernetes-applications-part-3-nginx-ingress-controller/
---
 .../aws-load-balancer-controller/release.yaml |  40 +++
 k8s/production/sealed-secrets/release.yaml    |   2 +-
 terraform/modules/spack_aws_k8s/eks.tf        | 293 ++++++++++++++++++
 3 files changed, 334 insertions(+), 1 deletion(-)
 create mode 100644 k8s/production/aws-load-balancer-controller/release.yaml

diff --git a/k8s/production/aws-load-balancer-controller/release.yaml b/k8s/production/aws-load-balancer-controller/release.yaml
new file mode 100644
index 000000000..6b4856152
--- /dev/null
+++ b/k8s/production/aws-load-balancer-controller/release.yaml
@@ -0,0 +1,40 @@
+---
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: aws-load-balancer-controller
+  namespace: kube-system
+spec:
+  interval: 10m
+  url: https://aws.github.io/eks-charts
+
+---
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: aws-load-balancer-controller
+  namespace: kube-system
+spec:
+  interval: 10m
+  chart:
+    spec:
+      chart: aws-load-balancer-controller
+      version: 1.9.0  # aws-load-balancer-controller@v2.9.0
+      sourceRef:
+        kind: HelmRepository
+        name: aws-load-balancer-controller
+  install:
+    crds: CreateReplace
+  upgrade:
+    crds: CreateReplace
+  valuesFrom:
+    # See terraform/modules/spack/eks.tf
+    - kind: ConfigMap
+      name: aws-lb-controller-config
+      valuesKey: values.yaml
+  values:
+    tolerations:
+      # The LB Controller should be scheduled on the initial managed nodegroup pods
+      # so that it is not dependent on Karpenter to be scheduled.
+      - key: "CriticalAddonsOnly"
+        operator: "Exists"
diff --git a/k8s/production/sealed-secrets/release.yaml b/k8s/production/sealed-secrets/release.yaml
index 6f607b9d7..de5d7c526 100644
--- a/k8s/production/sealed-secrets/release.yaml
+++ b/k8s/production/sealed-secrets/release.yaml
@@ -28,5 +28,5 @@ spec:
       spack.io/node-pool: base
     tolerations:
       - key: CriticalAddonsOnly
-        operator: Equal
+        operator: Exists
         effect: NoSchedule
diff --git a/terraform/modules/spack_aws_k8s/eks.tf b/terraform/modules/spack_aws_k8s/eks.tf
index 9a1785709..2f852e32d 100644
--- a/terraform/modules/spack_aws_k8s/eks.tf
+++ b/terraform/modules/spack_aws_k8s/eks.tf
@@ -296,3 +296,296 @@ resource "kubectl_manifest" "cluster_name_config_map" {
         cluster-name: ${module.eks.cluster_name}
   YAML
 }
+
+
+resource "aws_iam_role" "load_balancer_controller" {
+  name = "AmazonEKSLoadBalancerControllerRole-${var.deployment_name}-${var.deployment_stage}"
+  assume_role_policy = jsonencode({
+    "Version" : "2012-10-17",
+    "Statement" : [
+      {
+        "Effect" : "Allow",
+        "Principal" : {
+          "Federated" : module.eks.oidc_provider_arn,
+        },
+        "Action" : "sts:AssumeRoleWithWebIdentity",
+        "Condition" : {
+          "StringEquals" : {
+            "${module.eks.oidc_provider}:aud" : "sts.amazonaws.com",
+            "${module.eks.oidc_provider}:sub" : "system:serviceaccount:kube-system:aws-load-balancer-controller"
+          }
+        }
+      }
+    ]
+  })
+}
+
+resource "aws_iam_role_policy" "load_balancer_controller" {
+  name = "AmazonEKSLoadBalancerControllerPolicy-${var.deployment_name}-${var.deployment_stage}"
+  role = aws_iam_role.load_balancer_controller.id
+  # Copied from https://docs.aws.amazon.com/eks/latest/userguide/lbc-manifest.html#lbc-iam
+  policy = jsonencode({
+    "Version" : "2012-10-17",
+    "Statement" : [
+      {
+        "Effect" : "Allow",
+        "Action" : [
+          "iam:CreateServiceLinkedRole"
+        ],
+        "Resource" : "*",
+        "Condition" : {
+          "StringEquals" : {
+            "iam:AWSServiceName" : "elasticloadbalancing.amazonaws.com"
+          }
+        }
+      },
+      {
+        "Effect" : "Allow",
+        "Action" : [
+          "ec2:DescribeAccountAttributes",
+          "ec2:DescribeAddresses",
+          "ec2:DescribeAvailabilityZones",
+          "ec2:DescribeInternetGateways",
+          "ec2:DescribeVpcs",
+          "ec2:DescribeVpcPeeringConnections",
+          "ec2:DescribeSubnets",
+          "ec2:DescribeSecurityGroups",
+          "ec2:DescribeInstances",
+          "ec2:DescribeNetworkInterfaces",
+          "ec2:DescribeTags",
+          "ec2:GetCoipPoolUsage",
+          "ec2:DescribeCoipPools",
+          "elasticloadbalancing:DescribeLoadBalancers",
+          "elasticloadbalancing:DescribeLoadBalancerAttributes",
+          "elasticloadbalancing:DescribeListeners",
+          "elasticloadbalancing:DescribeListenerCertificates",
+          "elasticloadbalancing:DescribeSSLPolicies",
+          "elasticloadbalancing:DescribeRules",
+          "elasticloadbalancing:DescribeTargetGroups",
+          "elasticloadbalancing:DescribeTargetGroupAttributes",
+          "elasticloadbalancing:DescribeTargetHealth",
+          "elasticloadbalancing:DescribeTags",
+          "elasticloadbalancing:DescribeTrustStores",
+          "elasticloadbalancing:DescribeListenerAttributes"
+        ],
+        "Resource" : "*"
+      },
+      {
+        "Effect" : "Allow",
+        "Action" : [
+          "cognito-idp:DescribeUserPoolClient",
+          "acm:ListCertificates",
+          "acm:DescribeCertificate",
+          "iam:ListServerCertificates",
+          "iam:GetServerCertificate",
+          "waf-regional:GetWebACL",
+          "waf-regional:GetWebACLForResource",
+          "waf-regional:AssociateWebACL",
+          "waf-regional:DisassociateWebACL",
+          "wafv2:GetWebACL",
+          "wafv2:GetWebACLForResource",
+          "wafv2:AssociateWebACL",
+          "wafv2:DisassociateWebACL",
+          "shield:GetSubscriptionState",
+          "shield:DescribeProtection",
+          "shield:CreateProtection",
+          "shield:DeleteProtection"
+        ],
+        "Resource" : "*"
+      },
+      {
+        "Effect" : "Allow",
+        "Action" : [
+          "ec2:AuthorizeSecurityGroupIngress",
+          "ec2:RevokeSecurityGroupIngress"
+        ],
+        "Resource" : "*"
+      },
+      {
+        "Effect" : "Allow",
+        "Action" : [
+          "ec2:CreateSecurityGroup"
+        ],
+        "Resource" : "*"
+      },
+      {
+        "Effect" : "Allow",
+        "Action" : [
+          "ec2:CreateTags"
+        ],
+        "Resource" : "arn:aws:ec2:*:*:security-group/*",
+        "Condition" : {
+          "StringEquals" : {
+            "ec2:CreateAction" : "CreateSecurityGroup"
+          },
+          "Null" : {
+            "aws:RequestTag/elbv2.k8s.aws/cluster" : "false"
+          }
+        }
+      },
+      {
+        "Effect" : "Allow",
+        "Action" : [
+          "ec2:CreateTags",
+          "ec2:DeleteTags"
+        ],
+        "Resource" : "arn:aws:ec2:*:*:security-group/*",
+        "Condition" : {
+          "Null" : {
+            "aws:RequestTag/elbv2.k8s.aws/cluster" : "true",
+            "aws:ResourceTag/elbv2.k8s.aws/cluster" : "false"
+          }
+        }
+      },
+      {
+        "Effect" : "Allow",
+        "Action" : [
+          "ec2:AuthorizeSecurityGroupIngress",
+          "ec2:RevokeSecurityGroupIngress",
+          "ec2:DeleteSecurityGroup"
+        ],
+        "Resource" : "*",
+        "Condition" : {
+          "Null" : {
+            "aws:ResourceTag/elbv2.k8s.aws/cluster" : "false"
+          }
+        }
+      },
+      {
+        "Effect" : "Allow",
+        "Action" : [
+          "elasticloadbalancing:CreateLoadBalancer",
+          "elasticloadbalancing:CreateTargetGroup"
+        ],
+        "Resource" : "*",
+        "Condition" : {
+          "Null" : {
+            "aws:RequestTag/elbv2.k8s.aws/cluster" : "false"
+          }
+        }
+      },
+      {
+        "Effect" : "Allow",
+        "Action" : [
+          "elasticloadbalancing:CreateListener",
+          "elasticloadbalancing:DeleteListener",
+          "elasticloadbalancing:CreateRule",
+          "elasticloadbalancing:DeleteRule"
+        ],
+        "Resource" : "*"
+      },
+      {
+        "Effect" : "Allow",
+        "Action" : [
+          "elasticloadbalancing:AddTags",
+          "elasticloadbalancing:RemoveTags"
+        ],
+        "Resource" : [
+          "arn:aws:elasticloadbalancing:*:*:targetgroup/*/*",
+          "arn:aws:elasticloadbalancing:*:*:loadbalancer/net/*/*",
+          "arn:aws:elasticloadbalancing:*:*:loadbalancer/app/*/*"
+        ],
+        "Condition" : {
+          "Null" : {
+            "aws:RequestTag/elbv2.k8s.aws/cluster" : "true",
+            "aws:ResourceTag/elbv2.k8s.aws/cluster" : "false"
+          }
+        }
+      },
+      {
+        "Effect" : "Allow",
+        "Action" : [
+          "elasticloadbalancing:AddTags",
+          "elasticloadbalancing:RemoveTags"
+        ],
+        "Resource" : [
+          "arn:aws:elasticloadbalancing:*:*:listener/net/*/*/*",
+          "arn:aws:elasticloadbalancing:*:*:listener/app/*/*/*",
+          "arn:aws:elasticloadbalancing:*:*:listener-rule/net/*/*/*",
+          "arn:aws:elasticloadbalancing:*:*:listener-rule/app/*/*/*"
+        ]
+      },
+      {
+        "Effect" : "Allow",
+        "Action" : [
+          "elasticloadbalancing:ModifyLoadBalancerAttributes",
+          "elasticloadbalancing:SetIpAddressType",
+          "elasticloadbalancing:SetSecurityGroups",
+          "elasticloadbalancing:SetSubnets",
+          "elasticloadbalancing:DeleteLoadBalancer",
+          "elasticloadbalancing:ModifyTargetGroup",
+          "elasticloadbalancing:ModifyTargetGroupAttributes",
+          "elasticloadbalancing:DeleteTargetGroup",
+          "elasticloadbalancing:ModifyListenerAttributes"
+        ],
+        "Resource" : "*",
+        "Condition" : {
+          "Null" : {
+            "aws:ResourceTag/elbv2.k8s.aws/cluster" : "false"
+          }
+        }
+      },
+      {
+        "Effect" : "Allow",
+        "Action" : [
+          "elasticloadbalancing:AddTags"
+        ],
+        "Resource" : [
+          "arn:aws:elasticloadbalancing:*:*:targetgroup/*/*",
+          "arn:aws:elasticloadbalancing:*:*:loadbalancer/net/*/*",
+          "arn:aws:elasticloadbalancing:*:*:loadbalancer/app/*/*"
+        ],
+        "Condition" : {
+          "StringEquals" : {
+            "elasticloadbalancing:CreateAction" : [
+              "CreateTargetGroup",
+              "CreateLoadBalancer"
+            ]
+          },
+          "Null" : {
+            "aws:RequestTag/elbv2.k8s.aws/cluster" : "false"
+          }
+        }
+      },
+      {
+        "Effect" : "Allow",
+        "Action" : [
+          "elasticloadbalancing:RegisterTargets",
+          "elasticloadbalancing:DeregisterTargets"
+        ],
+        "Resource" : "arn:aws:elasticloadbalancing:*:*:targetgroup/*/*"
+      },
+      {
+        "Effect" : "Allow",
+        "Action" : [
+          "elasticloadbalancing:SetWebAcl",
+          "elasticloadbalancing:ModifyListener",
+          "elasticloadbalancing:AddListenerCertificates",
+          "elasticloadbalancing:RemoveListenerCertificates",
+          "elasticloadbalancing:ModifyRule"
+        ],
+        "Resource" : "*"
+      }
+    ]
+  })
+}
+
+resource "kubectl_manifest" "load_balancer_controller" {
+  yaml_body = <<-YAML
+    apiVersion: v1
+    kind: ConfigMap
+    metadata:
+      name: aws-lb-controller-config
+      namespace: kube-system
+    data:
+      values.yaml: |
+        clusterName: ${module.eks.cluster_name}
+        serviceAccount:
+          create: true
+          name: aws-load-balancer-controller
+          annotations:
+            eks.amazonaws.com/role-arn: ${aws_iam_role.load_balancer_controller.arn}
+        region: ${data.aws_region.current.name}
+        vpcId: ${module.vpc.vpc_id}
+  YAML
+}

From 45c476602053f5fb6f700525b4362bf725cb5231 Mon Sep 17 00:00:00 2001
From: Mike VanDenburgh <michael.vandenburgh@kitware.com>
Date: Tue, 29 Oct 2024 21:25:30 -0400
Subject: [PATCH 4/4] wip

---
 k8s/production/ingress-nginx/release.yaml | 26 +++++++++++++++++++----
 1 file changed, 22 insertions(+), 4 deletions(-)

diff --git a/k8s/production/ingress-nginx/release.yaml b/k8s/production/ingress-nginx/release.yaml
index 54a72215d..16fde86ab 100644
--- a/k8s/production/ingress-nginx/release.yaml
+++ b/k8s/production/ingress-nginx/release.yaml
@@ -43,7 +43,12 @@ spec:
       #   configMapNamespace: ""
 
       nodeSelector:
-        spack.io/node-pool: beefy
+        spack.io/node-pool: nginx
+
+      tolerations:
+        - key: "nginx"
+          operator: "Exists"
+          effect: "NoSchedule"
 
       resources:
         requests:
@@ -54,8 +59,8 @@ spec:
       minAvailable: 1
 
       autoscaling:
-        enabled: true
-        minReplicas: 2
+        enabled: false
+        minReplicas: 1
         maxReplicas: 40
         targetCPUUtilizationPercentage: 50
         targetMemoryUtilizationPercentage: 50
@@ -74,6 +79,19 @@ spec:
           # kube-proxy doesn't seem to work right on Windows nodes.
           "service.beta.kubernetes.io/aws-load-balancer-target-node-labels": "kubernetes.io/os=linux"
 
+
+          service.beta.kubernetes.io/aws-load-balancer-name: spack-eks-load-balancer
+          service.beta.kubernetes.io/aws-load-balancer-type: nlb
+          # Disable client IP preservation - see https://docs.aws.amazon.com/elasticloadbalancing/latest/network/load-balancer-troubleshooting.html#intermittent-connection-failure
+          service.beta.kubernetes.io/aws-load-balancer-target-group-attributes: "preserve_client_ip.enabled=false"
+          service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing
+          service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip
+
+          # service.beta.kubernetes.io/aws-load-balancer-target-node-labels: enable-nginx-ingress=true
+          # service.beta.kubernetes.io/aws-load-balancer-healthcheck-protocol: http
+          # service.beta.kubernetes.io/aws-load-balancer-healthcheck-path: /healthz
+          # service.beta.kubernetes.io/aws-load-balancer-healthcheck-port: 10254
+
         labels: {}
 
         enableHttp: true
@@ -103,7 +121,7 @@ spec:
 
     ## Default 404 backend
     defaultBackend:
-      enabled: true
+      enabled: false
       replicaCount: 3
       minAvailable: 1
       nodeSelector: