fix: update categories (#1049)

Co-authored-by: Raphael Odini <[email protected]>

.gitignore

@@ -1,4 +1,5 @@
 node_modules
 yarn-error.log
 dist
-.vscode
+.vscode
+.venv

data/categories/filter_categories.py

@@ -29,6 +29,7 @@
     "en:pumpkins",  # 9
     "en:dried-mushrooms",  # 7
     "en:textured-vegetable-protein",  # 2
+    "en:squash-pulp",
 ]
 
 EXTRA_CHILDREN = [
@@ -46,6 +47,16 @@
 
 EXCLUDE_LIST = ["Cooked", "Fresh", "Frozen", "Canned", "Prepacked"]
 
+EXCLUDE_LIST_NODE_IDS = [
+    "en:vegetables-from-germany",
+    "en:vegetables-from-the-netherlands",
+    "en:spanish-vegetables",
+    "en:vegetables-from-spain",
+    "en:vegetables-from-italy",
+    "en:nut-macarons",
+    "en:red-kuri-pulp",
+]
+
 script_path = os.path.dirname(os.path.abspath(__file__))
 
 
@@ -179,7 +190,7 @@ def compare_new_categories_with_old_categories():
         CATEGORIES_FULL, PARENT_CATEGORIES_ID
     )
     print(
-        "Filter with the following parent categories:",
+        "Filter on the following parent categories:",
         [node.id for node in PARENT_CATEGORIES],
     )
 
@@ -193,31 +204,37 @@ def compare_new_categories_with_old_categories():
     categories_filtered.extend(
         get_taxonomy_node_list_by_id_list(CATEGORIES_FULL, EXTRA_CHILDREN)
     )
-    # Step 2c: keep only nodes starting with "en:"
+    # Step 2c: exlude
+    # - keep only nodes starting with "en:"
+    # - remove nodes in EXCLUDE_LIST_NODE_IDS
+    # - remove nodes containing some strings in EXCLUDE_LIST
     categories_filtered = [
-        node for node in categories_filtered if node.id.startswith("en:")
+        node
+        for node in categories_filtered
+        if node.id.startswith("en:")
     ]
-    # Step 2d: remove nodes containg some strings
-    print("Additional filtering on:", EXCLUDE_LIST)
+    categories_filtered = [node for node in categories_filtered if node.id not in EXCLUDE_LIST_NODE_IDS]
     categories_filtered = filter_node_list_by_exclude_string_list(
         categories_filtered, EXCLUDE_LIST
     )
+    print("Finished filtering:", len(categories_filtered))
 
-    # Deduplicate
+    # Step 3: deduplicate
     categories_filtered_deduped = []
     seen = set()
     for category in categories_filtered:
         if category.id not in seen:
             categories_filtered_deduped.append(category)
             seen.add(category.id)
+    print("Finished deduplicating")
 
-    # Step 3: transform to dict list
+    # Step 4: transform to dict list
     categories_filtered_to_dict_list = [
         {"id": node.id, **node.to_dict()} for node in categories_filtered_deduped
     ]
     print("Categories remaining:", len(categories_filtered_to_dict_list))
 
-    # Step 4: write to files (1 per language)
+    # Step 5: write to files (1 per language)
     write_categories_to_files(categories_filtered_to_dict_list, delete_parents=True)
     print("Wrote to language files")