Migrate to AWS Load Balancer Controller

k8s/production/aws-load-balancer-controller/release.yaml

@@ -0,0 +1,40 @@
+---
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: aws-load-balancer-controller
+  namespace: kube-system
+spec:
+  interval: 10m
+  url: https://aws.github.io/eks-charts
+
+---
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: aws-load-balancer-controller
+  namespace: kube-system
+spec:
+  interval: 10m
+  chart:
+    spec:
+      chart: aws-load-balancer-controller
+      version: 1.9.0  # [email protected]
+      sourceRef:
+        kind: HelmRepository
+        name: aws-load-balancer-controller
+  install:
+    crds: CreateReplace
+  upgrade:
+    crds: CreateReplace
+  valuesFrom:
+    # See terraform/modules/spack/eks.tf
+    - kind: ConfigMap
+      name: aws-lb-controller-config
+      valuesKey: values.yaml
+  values:
+    tolerations:
+      # The LB Controller should be scheduled on the initial managed nodegroup pods
+      # so that it is not dependent on Karpenter to be scheduled.
+      - key: "CriticalAddonsOnly"
+        operator: "Exists"

k8s/production/ingress-nginx/release.yaml

@@ -43,7 +43,12 @@ spec:
       #   configMapNamespace: ""
 
       nodeSelector:
-        spack.io/node-pool: beefy
+        spack.io/node-pool: nginx
+
+      tolerations:
+        - key: "nginx"
+          operator: "Exists"
+          effect: "NoSchedule"
 
       resources:
         requests:
@@ -54,8 +59,8 @@ spec:
       minAvailable: 1
 
       autoscaling:
-        enabled: true
-        minReplicas: 2
+        enabled: false
+        minReplicas: 1
         maxReplicas: 40
         targetCPUUtilizationPercentage: 50
         targetMemoryUtilizationPercentage: 50
@@ -74,6 +79,19 @@ spec:
           # kube-proxy doesn't seem to work right on Windows nodes.
           "service.beta.kubernetes.io/aws-load-balancer-target-node-labels": "kubernetes.io/os=linux"
 
+
+          service.beta.kubernetes.io/aws-load-balancer-name: spack-eks-load-balancer
+          service.beta.kubernetes.io/aws-load-balancer-type: nlb
+          # Disable client IP preservation - see https://docs.aws.amazon.com/elasticloadbalancing/latest/network/load-balancer-troubleshooting.html#intermittent-connection-failure
+          service.beta.kubernetes.io/aws-load-balancer-target-group-attributes: "preserve_client_ip.enabled=false"
+          service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing
+          service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip
+
+          # service.beta.kubernetes.io/aws-load-balancer-target-node-labels: enable-nginx-ingress=true
+          # service.beta.kubernetes.io/aws-load-balancer-healthcheck-protocol: http
+          # service.beta.kubernetes.io/aws-load-balancer-healthcheck-path: /healthz
+          # service.beta.kubernetes.io/aws-load-balancer-healthcheck-port: 10254
+
         labels: {}
 
         enableHttp: true
@@ -103,7 +121,7 @@ spec:
 
     ## Default 404 backend
     defaultBackend:
-      enabled: true
+      enabled: false
       replicaCount: 3
       minAvailable: 1
       nodeSelector:

k8s/production/sealed-secrets/release.yaml

@@ -28,5 +28,5 @@ spec:
       spack.io/node-pool: base
     tolerations:
       - key: CriticalAddonsOnly
-        operator: Equal
+        operator: Exists
         effect: NoSchedule

terraform/modules/spack_aws_k8s/binary_mirrors.tf

@@ -25,7 +25,7 @@ resource "aws_cloudfront_cache_policy" "min_ttl_zero" {
 }
 
 module "pr_binary_mirror" {
-  source = "./modules/binary_mirror"
+  source = "../binary_mirror"
 
   bucket_iam_username = "pull-requests-binary-mirror${local.bucket_name_suffix}"
   bucket_name         = "spack-binaries-prs${local.bucket_name_suffix}"
@@ -38,7 +38,7 @@ module "pr_binary_mirror" {
 }
 
 module "protected_binary_mirror" {
-  source = "./modules/binary_mirror"
+  source = "../binary_mirror"
 
   bucket_iam_username = "protected-binary-mirror${local.bucket_name_suffix}"
   bucket_name         = "spack-binaries${local.bucket_name_suffix}"