Change dbt authentication method (#254) (#257)

* replace authentication method * remove variables related to old auth method * correct variable name Co-authored-by: lucas zanotelli <[email protected]>
stellar · Nov 28, 2023 · 4796703 · 4796703
1 parent dfa47f1
commit 4796703
Show file tree

Hide file tree

Showing 3 changed files with 2 additions and 50 deletions.
diff --git a/airflow_variables_dev.json b/airflow_variables_dev.json
@@ -84,11 +84,6 @@
     "config_settings": ["last_modified_ledger"],
     "expiration": ["last_modified_ledger"]
   },
-  "dbt_auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
-  "dbt_auth_uri": "https://accounts.google.com/o/oauth2/auth",
-  "dbt_client_email": "[email protected]",
-  "dbt_client_id": 101879863915888261202,
-  "dbt_client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/803774498738-compute%40developer.gserviceaccount.com",
   "dbt_dataset": "test_sdf",
   "dbt_full_refresh_models": {
     "partnership_assets__account_holders_activity_fact": true,
@@ -97,15 +92,11 @@
   "dbt_image_name": "stellar/stellar-dbt:1e160fe",
   "dbt_job_execution_timeout_seconds": 300,
   "dbt_job_retries": 1,
-  "dbt_keyfile_profile": "",
   "dbt_mart_dataset": "test_sdf_marts",
   "dbt_maximum_bytes_billed": 250000000000,
-  "dbt_private_key": "dummy",
-  "dbt_private_key_id": "dummy",
   "dbt_project": "test-hubble-319619",
   "dbt_target": "dev_airflow",
   "dbt_threads": 1,
-  "dbt_token_uri": "https://oauth2.googleapis.com/token",
   "gcs_exported_data_bucket_name": "us-central1-hubble-1pt5-dev-7db0e004-bucket",
   "gcs_exported_object_prefix": "dag-exported",
   "image_name": "stellar/stellar-etl:08fd8de",

diff --git a/airflow_variables_prod.json b/airflow_variables_prod.json
@@ -97,11 +97,6 @@
     "config_settings": ["last_modified_ledger"],
     "expiration": ["last_modified_ledger"]
   },
-  "dbt_auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
-  "dbt_auth_uri": "https://accounts.google.com/o/oauth2/auth",
-  "dbt_client_email": "[email protected]",
-  "dbt_client_id": 102744410146098624313,
-  "dbt_client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/471979297599-compute%40developer.gserviceaccount.com",
   "dbt_dataset": "sdf",
   "dbt_full_refresh_models": {
     "history_assets": false,
@@ -119,15 +114,11 @@
   "dbt_image_name": "stellar/stellar-dbt:1e160fe",
   "dbt_job_execution_timeout_seconds": 6000,
   "dbt_job_retries": 1,
-  "dbt_keyfile_profile": "",
   "dbt_mart_dataset": "sdf_marts",
   "dbt_maximum_bytes_billed": 100000000000000,
-  "dbt_private_key": "",
-  "dbt_private_key_id": "",
   "dbt_project": "hubble-261722",
   "dbt_target": "prod",
   "dbt_threads": 1,
-  "dbt_token_uri": "https://oauth2.googleapis.com/token",
   "gcs_exported_data_bucket_name": "us-central1-hubble-2-d948d67b-bucket",
   "gcs_exported_object_prefix": "dag-exported",
   "image_name": "stellar/stellar-etl:08fd8de",

diff --git a/dags/stellar_etl_airflow/build_dbt_task.py b/dags/stellar_etl_airflow/build_dbt_task.py
@@ -19,14 +19,6 @@ def create_dbt_profile(project="prod"):
     dbt_job_retries = Variable.get("dbt_job_retries")
     dbt_project = Variable.get("dbt_project")
     dbt_threads = Variable.get("dbt_threads")
-    dbt_private_key_id = Variable.get("dbt_private_key_id")
-    dbt_private_key = Variable.get("dbt_private_key")
-    dbt_client_email = Variable.get("dbt_client_email")
-    dbt_client_id = Variable.get("dbt_client_id")
-    dbt_auth_uri = Variable.get("dbt_auth_uri")
-    dbt_token_uri = Variable.get("dbt_token_uri")
-    dbt_auth_provider_x509_cert_url = Variable.get("dbt_auth_provider_x509_cert_url")
-    dbt_client_x509_cert_url = Variable.get("dbt_client_x509_cert_url")
     if project == "pub":
         dbt_project = Variable.get("public_project")
         dbt_dataset = Variable.get("public_dataset")
@@ -41,21 +33,10 @@ def create_dbt_profile(project="prod"):
       job_execution_timeout_seconds: {dbt_job_execution_timeout_seconds}
       job_retries: {dbt_job_retries}
       location: us
-      method: service-account-json
+      method: oauth
       project: "{dbt_project}"
       threads: {dbt_threads}
       type: bigquery
-      keyfile_json:
-        type: "service_account"
-        project_id: "{dbt_project}"
-        private_key_id: "{dbt_private_key_id}"
-        private_key: "{dbt_private_key}"
-        client_email: "{dbt_client_email}"
-        client_id: "{dbt_client_id}"
-        auth_uri: "{dbt_auth_uri}"
-        token_uri: "{dbt_token_uri}"
-        auth_provider_x509_cert_url: "{dbt_auth_provider_x509_cert_url}"
-        client_x509_cert_url: "{dbt_client_x509_cert_url}"
 elementary:
   outputs:
     default:
@@ -64,21 +45,10 @@ def create_dbt_profile(project="prod"):
       job_execution_timeout_seconds: {dbt_job_execution_timeout_seconds}
       job_retries: {dbt_job_retries}
       location: us
-      method: service-account-json
+      method: oauth
       project: "{dbt_project}"
       threads: {dbt_threads}
       type: bigquery
-      keyfile_json:
-        type: "service_account"
-        project_id: "{dbt_project}"
-        private_key_id: "{dbt_private_key_id}"
-        private_key: "{dbt_private_key}"
-        client_email: "{dbt_client_email}"
-        client_id: "{dbt_client_id}"
-        auth_uri: "{dbt_auth_uri}"
-        token_uri: "{dbt_token_uri}"
-        auth_provider_x509_cert_url: "{dbt_auth_provider_x509_cert_url}"
-        client_x509_cert_url: "{dbt_client_x509_cert_url}"
 """
 
     create_dbt_profile_cmd = f"echo '{profiles_yml}' > profiles.yml;"