[urlscan-enrichment] Make indicator creation optional

internal-enrichment/urlscan-enrichment/README.md

@@ -45,6 +45,7 @@ Below are the parameters you'll need to set for URLScan Enrichment connector:
 | URLScan Enr. Visibility              | visibility              | `URLSCAN_ENRICHMENT_VISIBILITY`                   | `public`  | Yes        | URLScan offers several levels of visibility for submitted scans: `public`, `unlisted`, `private`                        |
 | URLScan Enr. Search filtered by date | search_filtered_by_date | `URLSCAN_ENRICHMENT_SEARCH_FILTERED_BY_DATE`      | `>now-1y` | Yes        | Allows you to filter by date available: `>now-1h`, `>now-1d`, `>now-1y`, `[2022 TO 2023]`, `[2022/01/01 TO 2023/12/01]` |
 | URLScan Enr. Max TLP                 | max_tlp                 | `URLSCAN_ENRICHMENT_MAX_TLP`                      | /         | Yes        | Do not send any data to URLScan if the TLP of the observable is greater than MAX_TLP                                    |
+| URLScan Enr. Create Indicator        | create_indicator        | `URLSCAN_ENRICHMENT_CREATE_INDICATOR`             | `true`    | No         | Decide whether or not to create an indicator based on this observable
 
 
 ## Deployment

internal-enrichment/urlscan-enrichment/docker-compose.yml

@@ -19,4 +19,5 @@ services:
       - URLSCAN_ENRICHMENT_VISIBILITY=public # Available values : public, unlisted, private
       - URLSCAN_ENRICHMENT_SEARCH_FILTERED_BY_DATE=>now-1y # Available : ">now-1h", ">now-1d", ">now-1y", "[2022 TO 2023]", "[2022/01/01 TO 2023/12/01]"
       - URLSCAN_ENRICHMENT_MAX_TLP=TLP:AMBER # Required, Available values: TLP:CLEAR, TLP:WHITE, TLP:GREEN, TLP:AMBER, TLP:AMBER+STRICT, TLP:RED
+      - URLSCAN_ENRICHMENT_CREATE_INDICATOR=true
     restart: always

internal-enrichment/urlscan-enrichment/src/config.yml.sample

@@ -16,4 +16,5 @@ urlscan_enrichment:
   import_screenshot: false
   visibility: "public" # Available values : public, unlisted, private
   search_filtered_by_date: ">now-2d" # Available : ">now-1d", ">now-1y", "[2022 TO 2023]", "[2022/01/01 TO 2023/12/01"
-  max_tlp: "TLP:AMBER" # Required, Available values: TLP:CLEAR, TLP:WHITE, TLP:GREEN, TLP:AMBER, TLP:AMBER+STRICT, TLP:RED
+  max_tlp: "TLP:AMBER" # Required, Available values: TLP:CLEAR, TLP:WHITE, TLP:GREEN, TLP:AMBER, TLP:AMBER+STRICT, TLP:RED
+  create_indicator: true

internal-enrichment/urlscan-enrichment/src/main.py

@@ -197,26 +197,32 @@ def _generate_stix_bundle(
 
                     if data_stat["domains"][0] in stix_entity["value"]:
 
-                        stix_indicator = (
-                            self.converter.upsert_stix_indicator_with_relationship(
-                                data,
-                                stix_entity,
-                                external_reference,
-                                labels,
-                                prepared_file_png,
+                        if self.config.create_indicator:
+                            stix_indicator = (
+                                self.converter.upsert_stix_indicator_with_relationship(
+                                    data,
+                                    stix_entity,
+                                    external_reference,
+                                    labels,
+                                    prepared_file_png,
+                                )
                             )
-                        )
-                        self.stix_objects.extend(stix_indicator)
+                            self.stix_objects.extend(stix_indicator)
 
                         for index, ip in enumerate(data_stat["ips"]):
                             if ip is None:
                                 continue
 
-                            # Generate Relationship : Indicator -> "based-on" -> obs_ip
-                            indicator_to_ip = self.converter.generate_stix_relationship(
-                                stix_indicator[0].id, "based-on", stix_obs_ip[index].id
-                            )
-                            self.stix_objects.append(indicator_to_ip)
+                            if self.config.create_indicator:
+                                # Generate Relationship : Indicator -> "based-on" -> obs_ip
+                                indicator_to_ip = (
+                                    self.converter.generate_stix_relationship(
+                                        stix_indicator[0].id,
+                                        "based-on",
+                                        stix_obs_ip[index].id,
+                                    )
+                                )
+                                self.stix_objects.append(indicator_to_ip)
 
                             # Generate Relationship : Observable -> "related-to" -> obs_ip
                             observable_to_ip = (

internal-enrichment/urlscan-enrichment/src/urlscan_enrichment_services/config_variables.py

@@ -79,3 +79,10 @@ def _initialize_configurations(self) -> None:
         self.max_tlp = get_config_variable(
             "URLSCAN_ENRICHMENT_MAX_TLP", ["urlscan_enrichment", "max_tlp"], self.load
         )
+
+        self.create_indicator = get_config_variable(
+            "URLSCAN_ENRICHMENT_CREATE_INDICATOR",
+            ["urlscan_enrichment", "create_indicator"],
+            self.load,
+            default="true",
+        )