Overhaul k8s configs (things working)

dcramer · Dec 22, 2023 · d764db5 · d764db5
1 parent 28face6
commit d764db5
Show file tree

Hide file tree

Showing 9 changed files with 127 additions and 77 deletions.
diff --git a/terraform/gke.tf b/terraform/gke.tf
@@ -48,64 +48,27 @@ module "gke" {
 
   node_pools = [
     {
-      name                      = "default-node-pool"
-      machine_type              = "e2-highcpu-2"
+      name                      = "default-pool"
+      machine_type              = "e2-standard-2"
       node_locations            = join(", ", var.zones)
       min_count                 = 1
-      max_count                 = 2
+      max_count                 = 4
       local_ssd_count           = 0
       spot                      = false
       local_ssd_ephemeral_count = 0
-      disk_size_gb              = 10
+      disk_size_gb              = 100
       disk_type                 = "pd-standard"
       image_type                = "COS_CONTAINERD"
       enable_gcfs               = false
       enable_gvnic              = false
       logging_variant           = "DEFAULT"
       auto_repair               = true
       auto_upgrade              = true
-      # service_account           = "gke-node-sa@${data.google_project.project.number}.iam.gserviceaccount.com"
       preemptible               = false
       initial_node_count        = 1
     },
   ]
 
-  node_pools_labels = {
-    all = {}
-
-    default-node-pool = {
-      default-node-pool = true
-    }
-  }
-
-  node_pools_metadata = {
-    all = {}
-
-    default-node-pool = {
-      node-pool-metadata-custom-value = "default-node-pool"
-    }
-  }
-
-  node_pools_taints = {
-    all = []
-
-    default-node-pool = [
-      {
-        key    = "default-node-pool"
-        value  = true
-        effect = "PREFER_NO_SCHEDULE"
-      },
-    ]
-  }
-
-  node_pools_tags = {
-    all = []
-
-    default-node-pool = [
-      "default-node-pool",
-    ]
-  }
-
   depends_on = [
     module.gcp-network,
   ]

diff --git a/terraform/modules/faktory/variables.tf b/terraform/modules/faktory/variables.tf
@@ -30,5 +30,5 @@ variable "cpu" {
 
 variable "ephemeral_storage" {
   type    = string
-  default = "250m"
+  default = "1Gi"
 }
diff --git a/terraform/modules/service/main.tf b/terraform/modules/service/main.tf
@@ -3,7 +3,7 @@ locals {
   cloud_sql_http_port  = 9801
   cloud_sql_admin_port = 9092
   # memory should scale based on pg pool size
-  cloud_sql_memory = "1Gi"
+  cloud_sql_memory = "512Mi"
   # cant seem to adjust this to less than 1
   cloud_sql_cpu     = "1"
   cloud_sql_storage = "1Gi"
@@ -214,6 +214,60 @@ resource "kubernetes_deployment_v1" "default" {
           }
         }
 
+        # dynamic "container" {
+        #   for_each = var.containers
+
+        #   content {
+        #     name  = container.name
+        #     image = container.image
+
+
+        #     dynamic "env" {
+        #       for_each = var.env
+        #       content {
+        #         name  = env.key
+        #         value = env.value
+        #       }
+        #     }
+
+        #     dynamic "port" {
+        #       for_each = container.value.port != 0 ? [container.value.port] : []
+        #       content {
+        #         container_port = port.value
+        #       }
+        #     }
+
+        #     resources {
+        #       requests = {
+        #         cpu               = container.value.cpu
+        #         memory            = container.memory
+        #         ephemeral-storage = container.ephemeral_storage
+        #       }
+
+        #       limits = {
+        #         cpu               = container.cpu
+        #         memory            = container.memory
+        #         ephemeral-storage = container.ephemeral_storage
+        #       }
+        #     }
+
+        #     security_context {
+        #       allow_privilege_escalation = false
+        #       privileged                 = false
+        #       read_only_root_filesystem  = false
+        #       run_as_non_root            = false
+
+        #       capabilities {
+        #         add = []
+        #         drop = [
+        #           "NET_RAW"
+        #         ]
+        #       }
+        #     }
+        #   }
+        # }
+
+
         # https://github.com/GoogleCloudPlatform/cloud-sql-proxy/blob/main/examples/k8s-health-check/proxy_with_http_health_check.yaml
         dynamic "container" {
           for_each = local.cloud_sql_instance

diff --git a/terraform/modules/service/variables.tf b/terraform/modules/service/variables.tf
@@ -44,15 +44,29 @@ variable "k8s_service_account" {
 
 variable "memory" {
   type    = string
-  default = "500m"
+  default = "256Mi"
 }
 
 variable "cpu" {
   type    = string
-  default = "250m"
+  default = "100m"
 }
 
 variable "ephemeral_storage" {
   type    = string
-  default = "250m"
+  default = "1Gi"
 }
+
+
+# variable "containers" {
+#   type = list(object({
+#     name = string
+#     image = string
+#     port = number
+#     cpu = string
+#     memory = string
+#     storage = string
+#   }))
+
+#   default = []
+# }
diff --git a/terraform/server.tf b/terraform/server.tf
@@ -6,8 +6,8 @@ module "peated-api-service" {
   domains = ["api.peated.com", "api.staging.peated.com", "api.peated.app", "api.staging.peated.app"]
   port    = 4000
 
-  cpu = "500m"
-  memory = "500m"
+  cpu = "250m"
+  memory = "512Mi"
 
   healthcheck = {
     path = "/_health"
@@ -31,7 +31,23 @@ module "peated-api-service" {
     NODE_NO_WARNINGS     = "1"
     JWT_SECRET           = data.google_secret_manager_secret_version.jwt_secret.secret_data
     FAKTORY_URL          = "tcp://:${data.google_secret_manager_secret_version.faktory_password.secret_data}@${var.faktory_host}:7419"
+
+    NODE_NO_WARNINGS = "1"
+
+    # this is prob a bad idea
+    OPENAI_API_KEY  = data.google_secret_manager_secret_version.openai_api_key.secret_data
+    ACCESS_TOKEN    = data.google_secret_manager_secret_version.api_access_token.secret_data
+    DISCORD_WEBHOOK = data.google_secret_manager_secret_version.discord_webhook.secret_data
   }
 
+  # containers = [{
+  #   name = "peated-worked"
+  #   image  = "us-central1-docker.pkg.dev/${data.google_project.project.project_id}/${google_artifact_registry_repository.peated.name}/worker"
+  #   port = 0
+  #   cpu = "250m"
+  #   memory = "500m"
+  #   ephemeral_storage = "1Gi"
+  # }]
+
   depends_on = [module.db-main, module.faktory]
 }
diff --git a/terraform/terraform.tfvars b/terraform/terraform.tfvars
@@ -1,5 +1,6 @@
-region           = "us-central1"
 project_id       = "cask-382601"
+region           = "us-central1"
+zones            = ["us-central1-a"]
 project_number   = "721909483682"
 cluster_name     = "default"
 google_client_id = "721909483682-uk3befic1j1krv3drig2puu30v1i4v48.apps.googleusercontent.com"

diff --git a/terraform/variables.tf b/terraform/variables.tf
@@ -11,6 +11,11 @@ variable "region" {
   description = "GCP region"
 }
 
+variable "zones" {
+  type    = list(string)
+  default = ["us-central1-a"]
+}
+
 variable "cluster_name" {
   type        = string
   description = "The name for the GKE cluster"
@@ -45,11 +50,6 @@ variable "ip_range_services_name" {
   default     = "ip-range-services"
 }
 
-variable "zones" {
-  type    = list(string)
-  default = ["us-central1-a"]
-}
-
 variable "google_client_id" {
   type = string
 }

diff --git a/terraform/web.tf b/terraform/web.tf
@@ -4,7 +4,7 @@ module "peated-web-service" {
   image  = "us-central1-docker.pkg.dev/${data.google_project.project.project_id}/${google_artifact_registry_repository.peated.name}/web"
 
   cpu = "250m"
-  memory = "500m"
+  memory = "512Mi"
 
   domains = ["peated.com", "staging.peated.com", "www.peated.com", "peated.app", "staging.peated.app"]
   port    = 3000

diff --git a/terraform/worker.tf b/terraform/worker.tf
@@ -1,27 +1,29 @@
-module "peated-worker-service" {
-  source = "./modules/service"
-  name   = "peated-worker"
-  image  = "us-central1-docker.pkg.dev/${data.google_project.project.project_id}/${google_artifact_registry_repository.peated.name}/worker"
+# module "peated-worker-service" {
+#   source = "./modules/service"
+#   name   = "peated-worker"
+#   image  = "us-central1-docker.pkg.dev/${data.google_project.project.project_id}/${google_artifact_registry_repository.peated.name}/worker"
 
-  cpu = "500m"
-  memory = "500m"
+#   cpu = "250m"
+#   memory = "500m"
 
-  k8s_service_account = module.gke_workload_identity.k8s_service_account_name
+#   k8s_service_account = module.gke_workload_identity.k8s_service_account_name
 
-  cloud_sql_instance = module.db-main.connection_name
+#   cloud_sql_instance = module.db-main.connection_name
 
-  env = {
-    DATABASE_URL     = "postgresql://peated:[email protected]/peated"
-    GOOGLE_CLIENT_ID = var.google_client_id
-    SENTRY_DSN       = var.sentry_dsn
-    API_SERVER       = "https://api.peated.com"
-    NODE_NO_WARNINGS = "1"
-    # this is prob a bad idea
-    OPENAI_API_KEY  = data.google_secret_manager_secret_version.openai_api_key.secret_data
-    ACCESS_TOKEN    = data.google_secret_manager_secret_version.api_access_token.secret_data
-    FAKTORY_URL     = "tcp://:${data.google_secret_manager_secret_version.faktory_password.secret_data}@${var.faktory_host}:7419"
-    DISCORD_WEBHOOK = data.google_secret_manager_secret_version.discord_webhook.secret_data
-  }
+#   env = {
+#     DATABASE_URL     = "postgresql://peated:[email protected]/peated"
+#     GOOGLE_CLIENT_ID = var.google_client_id
+#     SENTRY_DSN       = var.sentry_dsn
+#     API_SERVER       = "https://api.peated.com"
+#     NODE_NO_WARNINGS = "1"
+#     # this is prob a bad idea
+#     OPENAI_API_KEY  = data.google_secret_manager_secret_version.openai_api_key.secret_data
+#     ACCESS_TOKEN    = data.google_secret_manager_secret_version.api_access_token.secret_data
+#     FAKTORY_URL     = "tcp://:${data.google_secret_manager_secret_version.faktory_password.secret_data}@${var.faktory_host}:7419"
+#     DISCORD_WEBHOOK = data.google_secret_manager_secret_version.discord_webhook.secret_data
+#   }
 
-  depends_on = [module.db-main, module.faktory]
-}
+#   depends_on = [module.db-main, module.faktory]
+
+
+# }