feat: Filter prod deployment events to persist in table

bq-workers/gitlab-parser/main.py

@@ -18,6 +18,7 @@
 import json
 
 import shared
+import logging
 
 from flask import Flask, request
 
@@ -44,6 +45,8 @@ def index():
     if "attributes" not in msg:
         raise Exception("Missing pubsub attributes")
 
+    metadata = get_metadata(msg)
+
     try:
         attr = msg["attributes"]
 
@@ -55,15 +58,20 @@ def index():
             if "X-Gitlab-Event" in headers:
                 event = process_gitlab_event(headers, msg)
 
-        shared.insert_row_into_bigquery(event)
+        if metadata["object_kind"] == "deployment":
+            if check_if_is_prod_environment(metadata):
+                shared.insert_row_into_bigquery(event)
+        else:
+            shared.insert_row_into_bigquery(event)
+
 
     except Exception as e:
         entry = {
-                "severity": "WARNING",
-                "msg": "Data not saved to BigQuery",
-                "errors": str(e),
-                "json_payload": envelope
-            }
+            "severity": "WARNING",
+            "msg": "Data not saved to BigQuery",
+            "errors": str(e),
+            "json_payload": envelope
+        }
         print(json.dumps(entry))
 
     return "", 204
@@ -82,7 +90,7 @@ def process_gitlab_event(headers, msg):
              "pipeline", "job", "deployment",
              "build"}
 
-    metadata = json.loads(base64.b64decode(msg["data"]).decode("utf-8").strip())
+    metadata = get_metadata(msg)
 
     event_type = metadata["object_kind"]
 
@@ -99,15 +107,15 @@ def process_gitlab_event(headers, msg):
         event_object = metadata["object_attributes"]
         e_id = event_object["id"]
         time_created = (
-            event_object.get("updated_at") or
-            event_object.get("finished_at") or
-            event_object.get("created_at"))
+                event_object.get("updated_at") or
+                event_object.get("finished_at") or
+                event_object.get("created_at"))
 
     if event_type in ("job"):
         e_id = metadata["build_id"]
         time_created = (
-            event_object.get("finished_at") or
-            event_object.get("started_at"))
+                event_object.get("finished_at") or
+                event_object.get("started_at"))
 
     if event_type in ("deployment"):
         e_id = metadata["deployment_id"]
@@ -116,9 +124,9 @@ def process_gitlab_event(headers, msg):
     if event_type in ("build"):
         e_id = metadata["build_id"]
         time_created = (
-            metadata.get("build_finished_at") or
-            metadata.get("build_started_at") or
-            metadata.get("build_created_at"))
+                metadata.get("build_finished_at") or
+                metadata.get("build_started_at") or
+                metadata.get("build_created_at"))
 
     # Some timestamps come in a format like "2021-04-28 21:50:00 +0200"
     # BigQuery does not accept this as a valid format
@@ -146,6 +154,13 @@ def process_gitlab_event(headers, msg):
     return gitlab_event
 
 
+def check_if_is_prod_environment(metadata):
+    return "prod" in metadata["environment"].lower()
+
+def get_metadata(msg):
+    return json.loads(base64.b64decode(msg["data"]).decode("utf-8").strip())
+
+
 if __name__ == "__main__":
     PORT = int(os.getenv("PORT")) if os.getenv("PORT") else 8080
 

bq-workers/gitlab-parser/main_test.py

@@ -101,6 +101,7 @@ def test_timestamp_timezone_event_processed(client):
                        "short_sha": "279484c0",
                        "status_changed_at": "2021-04-28 21:50:00 +0200",
                        "deployment_id": 15,
+                       "environment": "Production",
                        }).encode("utf-8")
 
     pubsub_msg = {
@@ -132,3 +133,44 @@ def test_timestamp_timezone_event_processed(client):
 
     shared.insert_row_into_bigquery.assert_called_with(event)
     assert r.status_code == 204
+
+def test_ignore_non_production_environment_event(client):
+    headers = {"X-Gitlab-Event": "deployment", "X-Gitlab-Token": "foo"}
+    data = json.dumps({"object_kind": "deployment",
+                       "short_sha": "279484c0",
+                       "status_changed_at": "2021-04-28 21:50:00 +0200",
+                       "deployment_id": 15,
+                       "environment": "development",
+                       }).encode("utf-8")
+
+    pubsub_msg = {
+        "message": {
+            "data": base64.b64encode(data).decode("utf-8"),
+            "attributes": {"headers": json.dumps(headers)},
+            "message_id": "foobar",
+            "publishTime": 1,
+        },
+    }
+
+    event = {
+        "event_type": "deployment",
+        "id": 15,
+        "metadata": data.decode(),
+        "time_created": "2021-04-28 21:50:00",
+        "signature": shared.create_unique_id(pubsub_msg["message"]),
+        "msg_id": "foobar",
+        "source": "gitlab",
+    }
+
+    shared.insert_row_into_bigquery = mock.MagicMock()
+
+    r = client.post(
+        "/",
+        data=json.dumps(pubsub_msg),
+        headers={"Content-Type": "application/json"},
+    )
+
+    shared.insert_row_into_bigquery.assert_not_called()
+    assert r.status_code == 204
+
+

queries/deployments.sql

@@ -19,7 +19,9 @@ WITH deploys_cloudbuild_github_gitlab AS (# Cloud Build, Github, Gitlab pipeline
       CASE WHEN source LIKE "github%" THEN ARRAY(
                 SELECT JSON_EXTRACT_SCALAR(string_element, '$')
                 FROM UNNEST(JSON_EXTRACT_ARRAY(metadata, '$.deployment.additional_sha')) AS string_element)
-           ELSE ARRAY<string>[] end as additional_commits
+           ELSE ARRAY<string>[] end as additional_commits,
+      JSON_EXTRACT_SCALAR(metadata, '$.project.id') as project_id,    
+      JSON_EXTRACT_SCALAR(metadata, '$.environment') as environment
       FROM four_keys.events_raw 
       WHERE (
       # Cloud Build Deployments
@@ -33,6 +35,7 @@ WITH deploys_cloudbuild_github_gitlab AS (# Cloud Build, Github, Gitlab pipeline
       # ArgoCD Deployments
       OR (source = "argocd" AND JSON_EXTRACT_SCALAR(metadata, '$.status') = "SUCCESS")
       )
+      AND JSON_EXTRACT_SCALAR(metadata, '$.environment') like "%prod%"
     ),
     deploys_tekton AS (# Tekton Pipelines
       SELECT