Merge pull request #23 from entechlog/develop

Snowflake Example Release - 0.1.6
entechlog · Jun 21, 2022 · 921d87c · 921d87c
2 parents 7aaccd3 + 605cbfc
commit 921d87c
Show file tree

Hide file tree

Showing 7 changed files with 364 additions and 240 deletions.
diff --git a/terraform/modules/01-snowflake-config.tf b/terraform/modules/01-snowflake-config.tf
@@ -0,0 +1,42 @@
+terraform {
+  backend "remote" {
+    organization = "entechlog"
+    workspaces {
+      prefix = "snowflake-"
+    }
+  }
+}
+
+locals {
+  create_in_dev_map = {
+    snowflake-dev = 1
+    snowflake-stg = 0
+    snowflake-prd = 0
+  }
+
+  create_in_prod_map = {
+    snowflake-dev = 0
+    snowflake-stg = 0
+    snowflake-prd = 1
+  }
+
+  enable_in_dev_flag  = local.create_in_dev_map[terraform.workspace]
+  enable_in_prod_flag = local.create_in_prod_map[terraform.workspace]
+}
+
+terraform {
+  required_providers {
+    snowflake = {
+      source  = "Snowflake-Labs/snowflake"
+      version = "0.35.0"
+    }
+  }
+}
+
+provider "snowflake" {
+  account  = var.snowflake_account
+  region   = var.snowflake_region
+  username = var.snowflake_user
+  password = var.snowflake_password
+  role     = var.snowflake_role
+}
diff --git a/terraform/modules/02-users-and-roles.tf b/terraform/modules/02-users-and-roles.tf
@@ -0,0 +1,101 @@
+//***************************************************************************//
+// Create Snowflake service accounts using modules. We will have one service account in each enviroment with different roles and each with different level of access
+//***************************************************************************//
+
+module "all_service_accounts" {
+  source = "./user"
+  user_map = {
+    "${lower(var.env_code)}_entechlog_dbt_user" : { "first_name" = "dbt", "last_name" = "User" },
+    "${lower(var.env_code)}_entechlog_atlan_user" : { "first_name" = "Atlan", "last_name" = "User" },
+    "${lower(var.env_code)}_entechlog_kafka_user" : { "first_name" = "Kafka", "last_name" = "User" }
+  }
+}
+
+//***************************************************************************//
+// Create Snowflake user accounts using modules. We will have only one user account for all enviroment
+//***************************************************************************//
+
+module "all_user_accounts" {
+  source = "./user"
+  count  = local.enable_in_dev_flag
+  user_map = {
+    "[email protected]" : { "first_name" = "Siva", "last_name" = "Nadesan", "email" = "[email protected]" }
+  }
+}
+
+//***************************************************************************//
+// Create service roles using modules. We will have one service role in each enviroment with different level of access
+//***************************************************************************//
+
+module "entechlog_dbt_role" {
+  source       = "./roles"
+  role_name    = "${upper(var.env_code)}_ENTECHLOG_DBT_ROLE"
+  role_comment = "Snowflake role used by dbt in ${var.env_code}"
+
+  roles = ["SYSADMIN"]
+  users = [lower("${var.env_code}_entechlog_dbt_user")]
+
+  depends_on = [module.all_service_accounts]
+}
+
+module "entechlog_atlan_role" {
+  source       = "./roles"
+  role_name    = "${upper(var.env_code)}_ENTECHLOG_ATLAN_ROLE"
+  role_comment = "Snowflake role used by Atlan in ${var.env_code}"
+
+  roles = ["SYSADMIN"]
+  users = [lower("${var.env_code}_entechlog_atlan_user")]
+
+  depends_on = [module.all_service_accounts]
+}
+
+module "entechlog_kafka_role" {
+  source       = "./roles"
+  role_name    = "${upper(var.env_code)}_ENTECHLOG_KAFKA_ROLE"
+  role_comment = "Snowflake role used by Kafka in ${var.env_code}"
+
+  roles = ["SYSADMIN"]
+  users = [lower("${var.env_code}_entechlog_kafka_user")]
+
+  depends_on = [module.all_service_accounts]
+}
+
+//***************************************************************************//
+// Create user roles using modules. We will have one user role for all enviroment with different level of access
+//***************************************************************************//
+
+module "entechlog_analyst_role" {
+  source       = "./roles"
+  count        = local.enable_in_dev_flag
+  role_name    = "ENTECHLOG_ANALYST_ROLE"
+  role_comment = "Snowflake role used by Analyst"
+
+  roles = ["SYSADMIN"]
+  users = [lower("[email protected]")]
+
+  depends_on = [module.all_user_accounts]
+}
+
+module "entechlog_developer_role" {
+  source       = "./roles"
+  count        = local.enable_in_dev_flag
+  role_name    = "ENTECHLOG_DEVELOPER_ROLE"
+  role_comment = "Snowflake role used by Developers"
+
+  roles = ["SYSADMIN"]
+  users = [lower("[email protected]")]
+
+  depends_on = [module.all_user_accounts]
+}
+
+// Output block starts here
+
+output "all_service_accounts" {
+  value     = module.all_service_accounts
+  sensitive = true
+}
+
+output "all_user_accounts" {
+  value     = module.all_user_accounts
+  sensitive = true
+}
diff --git a/terraform/modules/03-database-objects.tf b/terraform/modules/03-database-objects.tf
@@ -0,0 +1,176 @@
+//***************************************************************************//
+// Create Snowflake warehouse using modules
+//***************************************************************************//
+
+module "entechlog_dbt_wh_xs" {
+  source                 = "./warehouse"
+  warehouse_name         = "${upper(var.env_code)}_ENTECHLOG_DBT_WH_XS"
+  warehouse_size         = "XSMALL"
+  warehouse_auto_suspend = 30
+  warehouse_grant_roles = {
+    "OWNERSHIP" = ["SYSADMIN"]
+    "MODIFY"    = [var.snowflake_role]
+    "USAGE"     = (upper(var.env_code) == "DEV" ? [module.entechlog_dbt_role.role.name, "ENTECHLOG_DEVELOPER_ROLE"] : [module.entechlog_dbt_role.role.name])
+    "MONITOR"   = [module.entechlog_dbt_role.role.name]
+  }
+
+  depends_on = [module.entechlog_dbt_role.role, module.entechlog_developer_role.role]
+}
+
+module "entechlog_query_wh_xs" {
+  source                 = "./warehouse"
+  count                  = local.enable_in_dev_flag
+  warehouse_name         = "ALL_ENTECHLOG_QUERY_WH_XS"
+  warehouse_size         = "XSMALL"
+  warehouse_auto_suspend = 30
+  warehouse_grant_roles = {
+    "OWNERSHIP" = ["SYSADMIN"]
+    "MODIFY"    = [var.snowflake_role]
+    "USAGE"     = [module.entechlog_analyst_role[0].role.name, module.entechlog_developer_role[0].role.name]
+    "MONITOR"   = [module.entechlog_dbt_role.role.name]
+  }
+
+  depends_on = [module.entechlog_analyst_role.role, module.entechlog_developer_role.role]
+}
+
+//***************************************************************************//
+// Create Snowflake database and schema using modules
+//***************************************************************************//
+
+// RAW Layer
+
+module "entechlog_raw_db" {
+  source = "./database"
+
+  db_name    = "${upper(var.env_code)}_ENTECHLOG_RAW_DB"
+  db_comment = "Database to store the ingested RAW data"
+
+  db_grant_roles = {
+    "OWNERSHIP"     = ["SYSADMIN"]
+    "CREATE SCHEMA" = [var.snowflake_role]
+    "USAGE"         = [module.entechlog_dbt_role.role.name, "ENTECHLOG_DEVELOPER_ROLE"]
+  }
+
+  schemas = ["FACEBOOK", "GOOGLE"]
+  schema_grant = {
+    "FACEBOOK OWNERSHIP"    = { "roles" = ["SYSADMIN"] },
+    "FACEBOOK USAGE"        = { "roles" = [module.entechlog_dbt_role.role.name, module.entechlog_atlan_role.role.name, module.entechlog_kafka_role.role.name, "ENTECHLOG_DEVELOPER_ROLE"] },
+    "FACEBOOK CREATE TABLE" = { "roles" = [module.entechlog_dbt_role.role.name] },
+    "FACEBOOK CREATE VIEW"  = { "roles" = [module.entechlog_dbt_role.role.name] },
+    "GOOGLE OWNERSHIP"      = { "roles" = ["SYSADMIN"] },
+    "GOOGLE USAGE"          = { "roles" = [module.entechlog_dbt_role.role.name, module.entechlog_atlan_role.role.name, module.entechlog_kafka_role.role.name] },
+    "GOOGLE CREATE TABLE"   = { "roles" = [module.entechlog_dbt_role.role.name] },
+    "GOOGLE CREATE VIEW"    = { "roles" = [module.entechlog_dbt_role.role.name] }
+  }
+
+  table_grant = {
+    "FACEBOOK SELECT" = { "roles" = [module.entechlog_atlan_role.role.name, "ENTECHLOG_DEVELOPER_ROLE"] },
+    "GOOGLE SELECT"   = { "roles" = [module.entechlog_atlan_role.role.name, "ENTECHLOG_DEVELOPER_ROLE"] }
+  }
+
+  depends_on = [module.entechlog_dbt_role.role, module.entechlog_atlan_role.role, module.entechlog_kafka_role.role]
+}
+
+// Staging Layer, No user access other than dbt roles and developer role
+
+module "entechlog_staging_db" {
+  source = "./database"
+
+  db_name    = "${upper(var.env_code)}_ENTECHLOG_STAGING_DB"
+  db_comment = "Database to store the standardized data"
+
+  db_grant_roles = {
+    "OWNERSHIP"     = ["SYSADMIN"]
+    "CREATE SCHEMA" = [var.snowflake_role, module.entechlog_dbt_role.role.name]
+    "USAGE"         = [module.entechlog_dbt_role.role.name, "ENTECHLOG_DEVELOPER_ROLE"]
+  }
+
+  schemas = ["DIM", "FACT", "UTILS"]
+
+  schema_grant = {
+    "DIM OWNERSHIP"    = { "roles" = ["SYSADMIN"] },
+    "DIM USAGE"        = { "roles" = [module.entechlog_dbt_role.role.name, "ENTECHLOG_DEVELOPER_ROLE"] },
+    "DIM CREATE TABLE" = { "roles" = [module.entechlog_dbt_role.role.name] },
+    "DIM CREATE VIEW"  = { "roles" = [module.entechlog_dbt_role.role.name] },
+
+    "FACT OWNERSHIP"    = { "roles" = ["SYSADMIN"] },
+    "FACT USAGE"        = { "roles" = [module.entechlog_dbt_role.role.name, "ENTECHLOG_DEVELOPER_ROLE"] },
+    "FACT CREATE TABLE" = { "roles" = [module.entechlog_dbt_role.role.name] },
+    "FACT CREATE VIEW"  = { "roles" = [module.entechlog_dbt_role.role.name] }
+
+    "UTILS OWNERSHIP"    = { "roles" = ["SYSADMIN"] },
+    "UTILS USAGE"        = { "roles" = [module.entechlog_dbt_role.role.name, "ENTECHLOG_DEVELOPER_ROLE"] },
+    "UTILS CREATE TABLE" = { "roles" = [module.entechlog_dbt_role.role.name] },
+    "UTILS CREATE VIEW"  = { "roles" = [module.entechlog_dbt_role.role.name] }
+  }
+
+  table_grant = {
+    "DIM SELECT"   = { "roles" = [module.entechlog_dbt_role.role.name, "ENTECHLOG_DEVELOPER_ROLE"] },
+    "FACT SELECT"  = { "roles" = [module.entechlog_dbt_role.role.name, "ENTECHLOG_DEVELOPER_ROLE"] },
+    "UTILS SELECT" = { "roles" = [module.entechlog_dbt_role.role.name, "ENTECHLOG_DEVELOPER_ROLE"] }
+  }
+
+  depends_on = [module.entechlog_dbt_role.role, module.entechlog_developer_role.role]
+}
+
+// DW Layer, This is the only layer an end user should have access
+
+module "entechlog_dw_db" {
+  source = "./database"
+
+  db_name    = "${upper(var.env_code)}_ENTECHLOG_DW_DB"
+  db_comment = "Database to store the DW data"
+
+  db_grant_roles = {
+    "OWNERSHIP"     = ["SYSADMIN"]
+    "CREATE SCHEMA" = [var.snowflake_role, module.entechlog_dbt_role.role.name]
+    "USAGE"         = [module.entechlog_dbt_role.role.name, "ENTECHLOG_DEVELOPER_ROLE", "ENTECHLOG_ANALYST_ROLE"]
+  }
+
+  schemas = ["DIM", "FACT", "UTILS", "COMPLIANCE"]
+
+  schema_grant = {
+    "DIM OWNERSHIP"    = { "roles" = ["SYSADMIN"] },
+    "DIM USAGE"        = { "roles" = [module.entechlog_dbt_role.role.name, "ENTECHLOG_DEVELOPER_ROLE", "ENTECHLOG_ANALYST_ROLE"] },
+    "DIM CREATE TABLE" = { "roles" = [module.entechlog_dbt_role.role.name] },
+    "DIM CREATE VIEW"  = { "roles" = [module.entechlog_dbt_role.role.name] },
+
+    "FACT OWNERSHIP"    = { "roles" = ["SYSADMIN"] },
+    "FACT USAGE"        = { "roles" = [module.entechlog_dbt_role.role.name, "ENTECHLOG_DEVELOPER_ROLE", "ENTECHLOG_ANALYST_ROLE"] },
+    "FACT CREATE TABLE" = { "roles" = [module.entechlog_dbt_role.role.name] },
+    "FACT CREATE VIEW"  = { "roles" = [module.entechlog_dbt_role.role.name] }
+
+    "UTILS OWNERSHIP"    = { "roles" = ["SYSADMIN"] },
+    "UTILS USAGE"        = { "roles" = [module.entechlog_dbt_role.role.name, "ENTECHLOG_DEVELOPER_ROLE", "ENTECHLOG_ANALYST_ROLE"] },
+    "UTILS CREATE TABLE" = { "roles" = [module.entechlog_dbt_role.role.name] },
+    "UTILS CREATE VIEW"  = { "roles" = [module.entechlog_dbt_role.role.name] }
+
+    "COMPLIANCE OWNERSHIP"    = { "roles" = ["SYSADMIN"] },
+    "COMPLIANCE USAGE"        = { "roles" = [module.entechlog_dbt_role.role.name] },
+    "COMPLIANCE CREATE TABLE" = { "roles" = [module.entechlog_dbt_role.role.name] },
+    "COMPLIANCE CREATE VIEW"  = { "roles" = [module.entechlog_dbt_role.role.name] }
+  }
+
+  table_grant = {
+    "DIM SELECT"        = { "roles" = [module.entechlog_dbt_role.role.name, "ENTECHLOG_DEVELOPER_ROLE", "ENTECHLOG_ANALYST_ROLE"] },
+    "FACT SELECT"       = { "roles" = [module.entechlog_dbt_role.role.name, "ENTECHLOG_DEVELOPER_ROLE", "ENTECHLOG_ANALYST_ROLE"] },
+    "UTILS SELECT"      = { "roles" = [module.entechlog_dbt_role.role.name, "ENTECHLOG_DEVELOPER_ROLE", "ENTECHLOG_ANALYST_ROLE"] },
+    "COMPLIANCE SELECT" = { "roles" = [module.entechlog_dbt_role.role.name] }
+  }
+
+  depends_on = [module.entechlog_dbt_role.role, module.entechlog_developer_role.role]
+}
+
+// Output block starts here
+
+output "entechlog_raw_db" {
+  value = module.entechlog_raw_db
+}
+
+output "entechlog_staging_db" {
+  value = module.entechlog_staging_db
+}
+
+output "entechlog_dw_db" {
+  value = module.entechlog_dw_db
+}
diff --git a/terraform/modules/04-schema-objects.tf b/terraform/modules/04-schema-objects.tf
@@ -0,0 +1,37 @@
+//***************************************************************************//
+// Create masking policy using modules
+//***************************************************************************//
+
+module "mp_encrypt_email" {
+  source                   = "./masking-policy"
+  count                    = local.enable_in_prod_flag
+  masking_policy_name      = "MP_ENCRYPT_EMAIL"
+  masking_policy_database  = module.entechlog_dw_db.database.name
+  masking_policy_schema    = module.entechlog_dw_db.schema["COMPLIANCE"].name
+  masking_value_data_type  = "VARCHAR"
+  masking_expression       = "CASE WHEN CURRENT_ROLE() IN ('SYSADMIN') THEN val ELSE '**********' END"
+  masking_return_data_type = "VARCHAR(16777216)"
+
+  masking_grants = {
+    "OWNERSHIP" = ["SYSADMIN"]
+    "APPLY"     = [module.entechlog_dbt_role.role.name]
+  }
+
+}
+
+//***************************************************************************//
+// Create storage integration using modules
+//***************************************************************************//
+
+module "entechlog_str_s3_intg" {
+  source                    = "./storage-integration"
+  count                     = local.enable_in_prod_flag
+  name                      = "ENTECHLOG_STR_S3_INTG"
+  comment                   = ""
+  storage_provider          = "S3"
+  enabled                   = true
+  storage_allowed_locations = ["s3://entechlog-demo/kafka-snowpipe-demo/"]
+  storage_blocked_locations = ["s3://entechlog-demo/secure/"]
+  storage_aws_role_arn      = "arn:aws:iam::001234567890:role/myrole"
+  roles                     = [module.entechlog_dbt_role.role.name]
+}
diff --git a/terraform/modules/database/main.tf b/terraform/modules/database/main.tf
@@ -20,14 +20,16 @@ resource "snowflake_database_grant" "database_grant" {
   privilege         = each.key
   roles             = each.value
   with_grant_option = false
+  depends_on        = [snowflake_database.database]
 }
 
 resource "snowflake_schema" "schema" {
 
   for_each = toset(var.schemas)
 
-  database = snowflake_database.database.name
-  name     = each.key
+  database   = snowflake_database.database.name
+  name       = each.key
+  depends_on = [snowflake_database_grant.database_grant]
 }
 
 resource "snowflake_schema_grant" "schema_grant" {