feat: script to filter and generate labels (with translations) (#1215)

data/labels/generate_labels_translations.py

@@ -0,0 +1,121 @@
+import json
+from openfoodfacts.taxonomy import get_taxonomy
+
+
+OLD_LABELS_FILE = "src/data/labels-tags.json"
+
+EXTRA = [
+    "en:organic",
+]
+
+
+def get_languages():
+    with open("src/i18n/data/languages.json") as f:
+        return json.load(f)
+
+
+def get_label_taxonomy():
+    return get_taxonomy("label")
+
+
+def get_taxonomy_node_by_id(taxonomy, node_id):
+    return next((node for node in taxonomy.iter_nodes() if node.id == node_id), None)
+
+
+def get_taxonomy_node_list_by_id_list(taxonomy, node_id_list):
+    node_list = list()
+    for node_id in node_id_list:
+        taxonomy_node = get_taxonomy_node_by_id(taxonomy, node_id)
+        if taxonomy_node:
+            node_list.append(taxonomy_node)
+    return node_list
+
+
+def taxonomy_node_list_to_dict_list(node_list, delete_parents=False):
+    node_dict_list = list()
+    for node in node_list:
+        node_dict = { "id": node.id, **node.to_dict() }
+        if delete_parents:
+            del node_dict["parents"]
+        node_dict_list.append(node_dict)
+    return node_dict_list
+
+
+def get_taxonomy_node_children_full_list(taxonomy, node_parent):
+    children_node_list = list()
+    for node in taxonomy.iter_nodes():
+        node_parents = node.get_parents_hierarchy()
+        if next((n for n in node_parents if n == node_parent), None):
+            children_node_list.append(node)
+    return children_node_list
+
+
+def filter_labels(taxonomy):
+    node_list = list()
+    for node in taxonomy.iter_nodes():
+        if node.id in EXTRA:
+            node_list.append(node)
+    return node_list
+
+
+def write_labels_to_files(origins):
+    languages = get_languages()
+    for language in languages:
+        language_code = language['code']
+        language_labels = list()
+        # for each origin, get translation (or default to en)
+        for origin in origins:
+            language_origin_name = origin['name'][language_code] if (language_code in origin['name']) else origin['name']['en']
+            language_labels.append({"id": origin['id'], "name": language_origin_name})
+        # order by name
+        language_labels = sorted(language_labels, key=lambda x: x['name'])
+        # write to file
+        with open(f"src/data/labels/{language['code']}.json", "w") as f:
+            json.dump(language_labels, f, ensure_ascii=False)
+
+
+def compare_new_labels_with_old_labels():
+    with open("src/data/labels-tags.json") as f:
+        old_labels = json.load(f)
+    print("old_labels", len(old_labels))
+
+    with open("src/data/labels/en.json") as f:
+        new_labels = json.load(f)
+    print("new_labels", len(new_labels))
+
+    # check missing in new
+    origin_missing_in_new_list = list()
+    for origin in old_labels:
+        found = next((c for c in new_labels if c['id'] == origin['id']), None)
+        if not found:
+            origin_missing_in_new_list.append(origin)
+    print("missing in new", len(origin_missing_in_new_list))
+    print(origin_missing_in_new_list)
+
+    # check missing in old
+    origin_missing_in_old_list = list()
+    for origin in old_labels:
+        found = next((c for c in old_labels if c['id'] == origin['id']), None)
+        if not found:
+            origin_missing_in_old_list.append(origin)
+    print("missing in old", len(origin_missing_in_old_list))
+
+
+if __name__ == "__main__":
+    """
+    How-to run ?
+    > pip install openfoodfacts
+    > python data/labels/generate_labels_translations.py
+    """
+    # init
+    LABELS_FULL = get_label_taxonomy()
+    print("Total number of labels:", len(LABELS_FULL))
+
+    labels_filtered = filter_labels(LABELS_FULL)
+    labels_filtered_to_dict_list = taxonomy_node_list_to_dict_list(list(labels_filtered), delete_parents=True)
+    print("Labels remaining:", len(labels_filtered_to_dict_list))
+
+    write_labels_to_files(labels_filtered_to_dict_list)
+    print("Wrote to language files")
+
+    compare_new_labels_with_old_labels()

src/data/labels/aa.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/ach.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/af.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/ak.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/am.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/ar.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/as.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/ast.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/az.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/be.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/ber.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/bg.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Био"}]

src/data/labels/bm.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/bn.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/bo.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/br.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/bs.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/ca.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Orgànic"}]

src/data/labels/ce.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/chr.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/co.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/crs.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/cs.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Bio"}]

src/data/labels/cv.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/cy.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/da.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Økologisk"}]

src/data/labels/de.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Bio"}]

src/data/labels/el.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/en.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/en_AU.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/en_GB.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/eo.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/es.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Ecológico"}]

src/data/labels/et.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/eu.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/fa.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/fi.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Luomu"}]

src/data/labels/fil.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/fo.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/fr.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Bio"}]

src/data/labels/ga.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/gd.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/gl.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/gu.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/ha.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/he.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "אורגני"}]

src/data/labels/hi.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/hr.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Ekološki uzgoj"}]

src/data/labels/ht.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/hu.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Bio"}]

src/data/labels/hy.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/id.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/ii.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/is.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/it.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Biologico"}]

src/data/labels/iu.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/ja.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/jv.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/ka.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/kab.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/kk.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/km.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/kmr_TR.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/kn.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/ko.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "유기농"}]

src/data/labels/kw.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/ky.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/la.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/lb.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/lo.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/lt.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Ekologiškas"}]

src/data/labels/lv.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/mg.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/mi.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/ml.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/mn.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/mr.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/ms.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/mt.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/my.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/nb.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/ne.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/nl_BE.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/nl_NL.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/nn.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/no.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/nr.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/oc.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/or.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/pa.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/pl.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Ekologiczny"}]

src/data/labels/pt_BR.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]