add invalid languages filtering

EclipseMenu · Jan 21, 2025 · a682573 · a682573
1 parent 713cfa7
commit a682573
Show file tree

Hide file tree

Showing 4 changed files with 121 additions and 3 deletions.
diff --git a/.github/workflows/refresh-meta.yml b/.github/workflows/refresh-meta.yml
@@ -18,8 +18,7 @@ jobs:
           fetch-depth: 0
 
       - name: Generate metadata.json
-        run: |
-          echo "{\"timestamp\": $(date +%s), \"languages\": [$(ls translations/*.lang.json | sed 's/translations\///;s/.lang.json//;s/^/"/;s/$/"/' | paste -sd,)]}" > ../metadata.json
+        run: ./generate_meta.sh ../metadata.json
 
       - name: Update metadata branch
         run: |
@@ -32,7 +31,7 @@ jobs:
           fi
           mv ../metadata.json metadata.json
           git add metadata.json
-          git config --global user.email "bot@eclipse.menu"
+          git config --global user.email "github@eclipse.menu"
           git config --global user.name "Eclipse Menu Bot"
           git commit -m "Update metadata"
           git push origin metadata

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -1,5 +1,6 @@
 cmake_minimum_required(VERSION 3.21)
 
+# Copies translations from the translations repo to the resources folder on every build
 function(setup_translations target)
     if (DEFINED TRANSLATIONS_PATH)
         message(STATUS "Using translations from: ${TRANSLATIONS_PATH}.")
@@ -17,6 +18,14 @@ function(setup_translations target)
         COMMENT "Copying translations..."
     )
 
+    # Cleanup invalid translations after copying
+    add_custom_command(
+        TARGET ${target} PRE_LINK
+        COMMAND ${CMAKE_COMMAND} -P ${TRANSLATIONS_PATH}/cmake/VerifyLanguages.cmake
+        WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/resources/translations
+        COMMENT "Cleaning up translations..."
+    )
+
     # If using a local repo, set timestamp to a high value to disable translations updates
     if (DEFINED ENV{ECLIPSE_TRANSLATIONS_REPO_PATH})
         set(TRANSLATIONS_UNIX_TIMESTAMP 9999999999)

diff --git a/cmake/VerifyLanguages.cmake b/cmake/VerifyLanguages.cmake
@@ -0,0 +1,50 @@
+cmake_minimum_required(VERSION 3.21)
+
+message(STATUS "Verifying translation files...")
+file(GLOB_RECURSE TRANSLATIONS_FILES "*.lang.json")
+foreach(TRANSLATION_FILE ${TRANSLATIONS_FILES})
+    file(READ ${TRANSLATION_FILE} TRANSLATION_CONTENT)
+    string(FIND ${TRANSLATION_CONTENT} "\"language-name\"" LANGUAGE_NAME_INDEX)
+    string(FIND ${TRANSLATION_CONTENT} "\"language-native\"" LANGUAGE_NATIVE_INDEX)
+    if (LANGUAGE_NAME_INDEX EQUAL -1 OR LANGUAGE_NATIVE_INDEX EQUAL -1)
+        message(NOTICE "Translation file ${TRANSLATION_FILE} is missing 'language-name' or 'language-native' keys.")
+        file(REMOVE ${TRANSLATION_FILE})
+        continue()
+    endif()
+    string(FIND ${TRANSLATION_CONTENT} "\"language-charset\"" LANGUAGE_CHARSET_INDEX)
+    if (LANGUAGE_CHARSET_INDEX GREATER -1)
+        string(FIND ${TRANSLATION_CONTENT} "\"language-charset\": \"default\"" LANGUAGE_CHARSET_DEFAULT)
+        string(FIND ${TRANSLATION_CONTENT} "\"language-charset\": \"greek\"" LANGUAGE_CHARSET_GREEK)
+        string(FIND ${TRANSLATION_CONTENT} "\"language-charset\": \"korean\"" LANGUAGE_CHARSET_KOREAN)
+        string(FIND ${TRANSLATION_CONTENT} "\"language-charset\": \"japanese\"" LANGUAGE_CHARSET_JAPANESE)
+        string(FIND ${TRANSLATION_CONTENT} "\"language-charset\": \"chinese-full\"" LANGUAGE_CHARSET_CHINESE_FULL)
+        string(FIND ${TRANSLATION_CONTENT} "\"language-charset\": \"chinese-simplified\"" LANGUAGE_CHARSET_CHINESE_SIMPLIFIED)
+        string(FIND ${TRANSLATION_CONTENT} "\"language-charset\": \"cyrillic\"" LANGUAGE_CHARSET_CYRILLIC)
+        string(FIND ${TRANSLATION_CONTENT} "\"language-charset\": \"thai\"" LANGUAGE_CHARSET_THAI)
+        string(FIND ${TRANSLATION_CONTENT} "\"language-charset\": \"vietnamese\"" LANGUAGE_CHARSET_VIETNAMESE)
+        if (LANGUAGE_CHARSET_DEFAULT EQUAL -1 AND LANGUAGE_CHARSET_GREEK EQUAL -1 AND LANGUAGE_CHARSET_KOREAN EQUAL -1 AND LANGUAGE_CHARSET_JAPANESE EQUAL -1 AND LANGUAGE_CHARSET_CHINESE_FULL EQUAL -1 AND LANGUAGE_CHARSET_CHINESE_SIMPLIFIED EQUAL -1 AND LANGUAGE_CHARSET_CYRILLIC EQUAL -1 AND LANGUAGE_CHARSET_THAI EQUAL -1 AND LANGUAGE_CHARSET_VIETNAMESE EQUAL -1)
+            message(NOTICE "Translation file ${TRANSLATION_FILE} has an invalid 'language-charset' value.")
+            file(REMOVE ${TRANSLATION_FILE})
+            continue()
+        endif()
+    endif()
+    string(FIND ${TRANSLATION_CONTENT} "\"language-rtl\"" LANGUAGE_RTL_INDEX)
+    if (LANGUAGE_RTL_INDEX GREATER -1)
+        string(FIND ${TRANSLATION_CONTENT} "\"language-rtl\": true" LANGUAGE_RTL_TRUE)
+        string(FIND ${TRANSLATION_CONTENT} "\"language-rtl\": false" LANGUAGE_RTL_FALSE)
+        if (LANGUAGE_RTL_TRUE EQUAL -1 AND LANGUAGE_RTL_FALSE EQUAL -1)
+            message(NOTICE "Translation file ${TRANSLATION_FILE} has an invalid 'language-rtl' value.")
+            file(REMOVE ${TRANSLATION_FILE})
+            continue()
+        endif()
+    endif()
+    string(FIND ${TRANSLATION_CONTENT} "\"language-fallback\"" LANGUAGE_FALLBACK_INDEX)
+    if (LANGUAGE_FALLBACK_INDEX GREATER -1)
+        string(SUBSTRING ${TRANSLATION_CONTENT} ${LANGUAGE_FALLBACK_INDEX} 100 LANGUAGE_FALLBACK_STRING)
+        string(REGEX REPLACE "\"language-fallback\": \"([a-zA-Z_]+)\".*" "\\1" LANGUAGE_FALLBACK_CODE ${LANGUAGE_FALLBACK_STRING})
+        if (NOT EXISTS "${LANGUAGE_FALLBACK_CODE}.lang.json")
+            message(NOTICE "Translation file ${TRANSLATION_FILE} has 'language-fallback' value that doesn't exist: ${LANGUAGE_FALLBACK_CODE}.")
+            file(REMOVE ${TRANSLATION_FILE})
+        endif()
+    endif()
+endforeach()
diff --git a/generate_meta.sh b/generate_meta.sh
@@ -0,0 +1,60 @@
+#!/bin/bash
+# This script will generate a metadata.json file with the current timestamp and a list of all the languages available that pass the validation check
+
+function validate_json {
+  jq empty $1 > /dev/null
+  if [ $? -ne 0 ]; then
+    echo "Error: $1 is not a valid JSON file"
+    return 1
+  fi
+
+  jq -e 'has("language-name") and has("language-native")' $1 > /dev/null
+  if [ $? -ne 0 ]; then
+    echo "Error: $1 is missing required fields"
+    return 1
+  fi
+
+  jq -e '.["language-name"] != "" and .["language-native"] != ""' $1 > /dev/null
+  if [ $? -ne 0 ]; then
+    echo "Error: $1 has empty language-name or language-native"
+    return 1
+  fi
+
+  jq -e 'has("language-charset")' $1 > /dev/null
+  if [ $? -eq 0 ]; then
+    jq -e '.["language-charset"] | match("^(default|greek|korean|japanese|chinese-full|chinese-simplified|cyrillic|thai|vietnamese)$")' $1 > /dev/null
+    if [ $? -ne 0 ]; then
+      echo "Error: $1 has an invalid language-charset"
+      return 1
+    fi
+  fi
+
+  jq -e 'has("language-fallback")' $1 > /dev/null
+  if [ $? -eq 0 ]; then
+    fallback=$(jq -r '.["language-fallback"]' $1)
+    if [ ! -f translations/$fallback.lang.json ]; then
+      echo "Error: $1 has an invalid language-fallback"
+      return 1
+    fi
+  fi
+
+  return 0
+}
+
+TARGET_FILE=$1
+rm -f $TARGET_FILE
+
+echo -n "{\"timestamp\": $(date +%s), \"languages\": [" > $TARGET_FILE
+
+for file in translations/*.lang.json; do
+  validate_json $file
+  if [ $? -ne 0 ]; then
+    echo "Skipping $file"
+    continue
+  fi
+  echo "Adding $(basename $file .lang.json) to metadata.json"
+  echo -n "\"$(basename $file .lang.json)\"," >> $TARGET_FILE
+done
+
+sed -i '$ s/.$//' $TARGET_FILE
+echo -n "]}" >> $TARGET_FILE