From d92f5bbb4f5bb46ac62a3d3e287b702a6f56c88b Mon Sep 17 00:00:00 2001 From: JoJo Date: Fri, 10 May 2024 11:00:13 -0700 Subject: [PATCH 1/2] Utilize AniDb online cache --- modules/anidb.py | 228 +++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 191 insertions(+), 37 deletions(-) diff --git a/modules/anidb.py b/modules/anidb.py index 3aec97847..cad7cab5e 100644 --- a/modules/anidb.py +++ b/modules/anidb.py @@ -8,6 +8,7 @@ builders = ["anidb_id", "anidb_relation", "anidb_popular", "anidb_tag"] base_url = "https://anidb.net" api_url = "http://api.anidb.net:9001/httpapi" +cache_url = "https://raw.githubusercontent.com/notseteve/AnimeAggregations/main/anime" urls = { "anime": f"{base_url}/anime", "popular": f"{base_url}/latest/anime/popular/?h=1", @@ -16,12 +17,151 @@ "login": f"{base_url}/perl-bin/animedb.pl" } weights = {"anidb": 1000, "anidb_3_0": 600, "anidb_2_5": 500, "anidb_2_0": 400, "anidb_1_5": 300, "anidb_1_0": 200, "anidb_0_5": 100} +language_matcher = { + "af" : "AFRIKAANS", + "sq" : "ALBANIAN", + "al" : "ALBANIAN", + "ar" : "ARABIC", + "bn" : "BENGALI", + "bd" : "BENGALI", + "bs" : "BOSNIAN", + "bg" : "BULGARIAN", + "my" : "BURMESE", + "zh" : "CHINESE", + "zh-nan" : "CHINESE_TAIWAN", + "zh-Hant" : "CHINESE_TRADITIONAL", + "x-zht" : "CHINESE_TRANSLITERATED", + "zh-Hans" : "CHINESE_SIMPLIFIED", + "zh-cmn" : "CHINESE_SIMPLIFIED", + "hr" : "CROATIAN", + "cs" : "CZECH", + "da" : "DANISH", + "nl" : "DUTCH", + "en" : "ENGLISH", + "eo" : "ESPERANTO", + "et" : "ESTONIAN", + "fil" : "FILIPINO", + "fi" : "FINNISH", + "fr" : "FRENCH", + "ka" : "GEORGIAN", + "de" : "GERMAN", + "el" : "GREEK", + "grc" : "GREEK_ANCIENT", + "ht" : "HAITIAN", + "he" : "HEBREW", + "hi" : "HINDI", + "hu" : "HUNGARIAN", + "is" : "ICELANDIC", + "id" : "INDONESIAN", + "it" : "ITALIAN", + "ja" : "JAPANESE", + "x-jat" : "JAPANESE_TRANSLITERATED", + "jv" : "JAVANESE", + "ko" : "KOREAN", + "x-kot" : "KOREAN_TRANSLITERATED", + "la" : "LATIN", + "lv" : "LATVIAN", + "lt" : "LITHUANIAN", + "ms" : "MALAY", + "mn" : "MONGOLIAN", + "ne" : "NEPALI", + "no" : "NORWEGIAN", + "fa" : "PERSIAN", + "pl" : "POLISH", + "pt" : "PORTUGUESE", + "pt-BR" : "PORTUGUESE_BRAZIL", + "ro" : "ROMANIAN", + "ru" : "RUSSIAN", + "sr" : "SERBIAN", + "si" : "SINHALA", + "sk" : "SLOVAK", + "sl" : "SLOVENIAN", + "es" : "SPANISH", + "es-PV" : "SPANISH_BASQUE", + "es-CT" : "SPANISH_CATALAN", + "es-CA" : "SPANISH_CATALAN", + "es-GA" : "SPANISH_GALICIA", + "es-419" : "SPANISH_LATIN", + "sv" : "SWEDISH", + "tl" : "TAGALOG", + "ta" : "TAMIL", + "tt" : "TATAR", + "te" : "TELUGU", + "th" : "THAI", + "x-tht" : "THAI_TRANSLITERATED", + "tr" : "TURKISH", + "uk" : "UKRAINIAN", + "ur" : "URDU", + "vi" : "VIETNAMESE", + "x-unk" : "UNKNOWN", + "x-other" : "OTHER" +} + class AniDBObj: - def __init__(self, anidb, anidb_id, data): + def __init__(self, anidb, anidb_id, data, from_json=False): self._anidb = anidb self.anidb_id = anidb_id self._data = data + self.main_title = None + self.titles = [] + self.official_title = None + self.studio = None + self.rating = None + self.average = None + self.score = None + self.released = None + self.tags = {} + self.mal_id = None + self.imdb_id = None + self.tmdb_id = None + self.tmdb_type = None + + def _parseJSON(self): + self.all_titles = data["titles"] + for title in self.all_titles: + if title["type"] == "MAIN": + self.main_title = title["title"] + if title["type"] == "OFFICIAL": + self.titles.append(title) + if title["type"] == "OFFICIAL" and (title["language"] == self._anidb.language.upper() or title["language"] == language_matcher[self._anidb.language]): + self.official_title = title["title"] + if not self.official_title: + self.official_title = self.main_title + + if "creators" in data: + for creator_id, creators in data["creators"].items(): + for creator in creators: + if creator["type"] == "ANIMATION_WORK": + self.studio = creator["name"] + + if "ratings" in data and "PERMANENT" in data["ratings"]: + self.rating = float(data["ratings"]["PERMANENT"]["rating"]) + if "ratings" in data and "TEMPORARY" in data["ratings"]: + self.average = float(data["ratings"]["TEMPORARY"]["rating"]) + if "ratings" in data and "REVIEW" in data["ratings"]: + self.score = float(data["ratings"]["REVIEW"]["rating"]) + if "start_date" in data: + self.released = datetime.strptime(data["start_date"], "%Y-%m-%d") + + if "tags" in data: + for tag_id, tag in data["tags"].items(): + if "info_box" in tag and (tag["info_box"] or tag["info_box"] == "true"): + self.tags[tag["name"]] = 1001 + elif "weight" in tag: + self.tags[tag["name"]] = float(tag["weight"]) + + if "resources" in data and "MAL" in data["resources"]: + self.mal_id = int(data["resources"]["MAL"][0]) + if "resources" in data and "IMDB" in data["resources"]: + self.imdb_id = data["resources"]["IMDB"][0] + if "resources" in data and "TMDB" in data["resources"]: + if data["resources"]["TMDB"][0].startswith("movie/"): + self.tmdb_id = int(data["resources"]["TMDB"][0][6:]) + self.tmdb_type = "movie" + elif data["resources"]["TMDB"][0].startswith("tv/"): + self.tmdb_id = int(data["resources"]["TMDB"][0][3:]) + self.tmdb_type = "show" def _parse(attr, xpath, is_list=False, is_dict=False, is_int=False, is_float=False, is_date=False, fail=False): try: @@ -63,30 +203,35 @@ def _parse(attr, xpath, is_list=False, is_dict=False, is_int=False, is_float=Fal else: return None - self.main_title = _parse("main_title", "//anime/titles/title[@type='main']/text()", fail=True) - self.titles = _parse("titles", "//anime/titles/title[@type='official']", is_dict=True) - self.official_title = self.titles[self._anidb.language] if self._anidb.language in self.titles else self.main_title - self.studio = _parse("studio", "//anime/creators/name[@type='Animation Work']/text()") - self.rating = _parse("rating", "//anime/ratings/permanent/text()", is_float=True) - self.average = _parse("average", "//anime/ratings/temporary/text()", is_float=True) - self.score = _parse("score", "//anime/ratings/review/text()", is_float=True) - self.released = _parse("released", "//anime/startdate/text()", is_date=True) - self.tags = _parse("tags", "//anime/tags/tag", is_dict=True) - self.mal_id = _parse("mal_id", "//anime/resources/resource[@type='2']/externalentity/identifier/text()", is_int=True) - self.imdb_id = _parse("imdb_id", "//anime/resources/resource[@type='43']/externalentity/identifier/text()") - if isinstance(data, dict): - self.tmdb_id = _parse("tmdb_id", "", is_int=True) - self.tmdb_type = _parse("tmdb_type", "") - else: - tmdb = _parse("tmdb", "//anime/resources/resource[@type='44']/externalentity/identifier/text()", is_list=True) - self.tmdb_id = None - self.tmdb_type = None - for i in tmdb: - try: - self.tmdb_id = int(i) - except ValueError: - self.tmdb_type = i + def _parseXML(self): + self.main_title = _parse("main_title", "//anime/titles/title[@type='main']/text()", fail=True) + self.titles = _parse("titles", "//anime/titles/title[@type='official']", is_dict=True) + self.official_title = self.titles[self._anidb.language] if self._anidb.language in self.titles else self.main_title + self.studio = _parse("studio", "//anime/creators/name[@type='Animation Work']/text()") + self.rating = _parse("rating", "//anime/ratings/permanent/text()", is_float=True) + self.average = _parse("average", "//anime/ratings/temporary/text()", is_float=True) + self.score = _parse("score", "//anime/ratings/review/text()", is_float=True) + self.released = _parse("released", "//anime/startdate/text()", is_date=True) + self.tags = _parse("tags", "//anime/tags/tag", is_dict=True) + self.mal_id = _parse("mal_id", "//anime/resources/resource[@type='2']/externalentity/identifier/text()", is_int=True) + self.imdb_id = _parse("imdb_id", "//anime/resources/resource[@type='43']/externalentity/identifier/text()") + if isinstance(data, dict): + self.tmdb_id = _parse("tmdb_id", "", is_int=True) + self.tmdb_type = _parse("tmdb_type", "") + else: + tmdb = _parse("tmdb", "//anime/resources/resource[@type='44']/externalentity/identifier/text()", is_list=True) + self.tmdb_id = None + self.tmdb_type = None + for i in tmdb: + try: + self.tmdb_id = int(i) + except ValueError: + self.tmdb_type = i + if from_json: + _parseJSON(self) + else: + _parseXML(self) class AniDB: def __init__(self, config, data): @@ -184,22 +329,31 @@ def _tag(self, tag, limit): def get_anime(self, anidb_id, ignore_cache=False): expired = None anidb_dict = None + from_json = False if self.config.Cache and not ignore_cache: anidb_dict, expired = self.config.Cache.query_anidb(anidb_id, self.expiration) if expired or not anidb_dict: - time_check = time.time() - if self._delay is not None: - while time_check - self._delay < 2: - time_check = time.time() - anidb_dict = self._request(api_url, params={ - "client": self.client, - "clientver": self.version, - "protover": 1, - "request": "anime", - "aid": anidb_id - }) - self._delay = time.time() - obj = AniDBObj(self, anidb_id, anidb_dict) + if not ignore_cache: + try: + anidb_dict = self.config.get_json(f"{cache_url}/{anidb_id}.json") + if anidb_dict: + from_json = True + except ValueError: + pass + if not from_json: + time_check = time.time() + if self._delay is not None: + while time_check - self._delay < 2: + time_check = time.time() + anidb_dict = self._request(api_url, params={ + "client": self.client, + "clientver": self.version, + "protover": 1, + "request": "anime", + "aid": anidb_id + }) + self._delay = time.time() + obj = AniDBObj(self, anidb_id, anidb_dict, from_json) if self.config.Cache and not ignore_cache: self.config.Cache.update_anidb(expired, anidb_id, obj, self.expiration) return obj From c65cd6d9026ef51f04f45c40c70f0fd7f82684f1 Mon Sep 17 00:00:00 2001 From: JoJo Date: Fri, 10 May 2024 13:42:30 -0700 Subject: [PATCH 2/2] Make language case-insensitive --- modules/anidb.py | 159 +++++++++++++++++++++++------------------------ 1 file changed, 79 insertions(+), 80 deletions(-) diff --git a/modules/anidb.py b/modules/anidb.py index cad7cab5e..33cf430e4 100644 --- a/modules/anidb.py +++ b/modules/anidb.py @@ -17,87 +17,86 @@ "login": f"{base_url}/perl-bin/animedb.pl" } weights = {"anidb": 1000, "anidb_3_0": 600, "anidb_2_5": 500, "anidb_2_0": 400, "anidb_1_5": 300, "anidb_1_0": 200, "anidb_0_5": 100} -language_matcher = { - "af" : "AFRIKAANS", - "sq" : "ALBANIAN", - "al" : "ALBANIAN", - "ar" : "ARABIC", - "bn" : "BENGALI", - "bd" : "BENGALI", - "bs" : "BOSNIAN", - "bg" : "BULGARIAN", - "my" : "BURMESE", - "zh" : "CHINESE", - "zh-nan" : "CHINESE_TAIWAN", - "zh-Hant" : "CHINESE_TRADITIONAL", - "x-zht" : "CHINESE_TRANSLITERATED", - "zh-Hans" : "CHINESE_SIMPLIFIED", - "zh-cmn" : "CHINESE_SIMPLIFIED", - "hr" : "CROATIAN", - "cs" : "CZECH", - "da" : "DANISH", - "nl" : "DUTCH", - "en" : "ENGLISH", - "eo" : "ESPERANTO", - "et" : "ESTONIAN", - "fil" : "FILIPINO", - "fi" : "FINNISH", - "fr" : "FRENCH", - "ka" : "GEORGIAN", - "de" : "GERMAN", - "el" : "GREEK", - "grc" : "GREEK_ANCIENT", - "ht" : "HAITIAN", - "he" : "HEBREW", - "hi" : "HINDI", - "hu" : "HUNGARIAN", - "is" : "ICELANDIC", - "id" : "INDONESIAN", - "it" : "ITALIAN", - "ja" : "JAPANESE", - "x-jat" : "JAPANESE_TRANSLITERATED", - "jv" : "JAVANESE", - "ko" : "KOREAN", - "x-kot" : "KOREAN_TRANSLITERATED", - "la" : "LATIN", - "lv" : "LATVIAN", - "lt" : "LITHUANIAN", - "ms" : "MALAY", - "mn" : "MONGOLIAN", - "ne" : "NEPALI", - "no" : "NORWEGIAN", - "fa" : "PERSIAN", - "pl" : "POLISH", - "pt" : "PORTUGUESE", - "pt-BR" : "PORTUGUESE_BRAZIL", - "ro" : "ROMANIAN", - "ru" : "RUSSIAN", - "sr" : "SERBIAN", - "si" : "SINHALA", - "sk" : "SLOVAK", - "sl" : "SLOVENIAN", - "es" : "SPANISH", - "es-PV" : "SPANISH_BASQUE", - "es-CT" : "SPANISH_CATALAN", - "es-CA" : "SPANISH_CATALAN", - "es-GA" : "SPANISH_GALICIA", - "es-419" : "SPANISH_LATIN", - "sv" : "SWEDISH", - "tl" : "TAGALOG", - "ta" : "TAMIL", - "tt" : "TATAR", - "te" : "TELUGU", - "th" : "THAI", - "x-tht" : "THAI_TRANSLITERATED", - "tr" : "TURKISH", - "uk" : "UKRAINIAN", - "ur" : "URDU", - "vi" : "VIETNAMESE", - "x-unk" : "UNKNOWN", - "x-other" : "OTHER" +LANGUAGE_MATCHER = { + "AF" : "AFRIKAANS", + "SQ" : "ALBANIAN", + "AL" : "ALBANIAN", + "AR" : "ARABIC", + "BN" : "BENGALI", + "BD" : "BENGALI", + "BS" : "BOSNIAN", + "BG" : "BULGARIAN", + "MY" : "BURMESE", + "ZH" : "CHINESE", + "ZH-NAN" : "CHINESE_TAIWAN", + "ZH-HANT" : "CHINESE_TRADITIONAL", + "X-ZHT" : "CHINESE_TRANSLITERATED", + "ZH-HANS" : "CHINESE_SIMPLIFIED", + "ZH-CMN" : "CHINESE_SIMPLIFIED", + "HR" : "CROATIAN", + "CS" : "CZECH", + "DA" : "DANISH", + "NL" : "DUTCH", + "EN" : "ENGLISH", + "EO" : "ESPERANTO", + "ET" : "ESTONIAN", + "FIL" : "FILIPINO", + "FI" : "FINNISH", + "FR" : "FRENCH", + "KA" : "GEORGIAN", + "DE" : "GERMAN", + "EL" : "GREEK", + "GRC" : "GREEK_ANCIENT", + "HT" : "HAITIAN", + "HE" : "HEBREW", + "HI" : "HINDI", + "HU" : "HUNGARIAN", + "IS" : "ICELANDIC", + "ID" : "INDONESIAN", + "IT" : "ITALIAN", + "JA" : "JAPANESE", + "X-JAT" : "JAPANESE_TRANSLITERATED", + "JV" : "JAVANESE", + "KO" : "KOREAN", + "X-KOT" : "KOREAN_TRANSLITERATED", + "LA" : "LATIN", + "LV" : "LATVIAN", + "LT" : "LITHUANIAN", + "MS" : "MALAY", + "MN" : "MONGOLIAN", + "NE" : "NEPALI", + "NO" : "NORWEGIAN", + "FA" : "PERSIAN", + "PL" : "POLISH", + "PT" : "PORTUGUESE", + "PT-BR" : "PORTUGUESE_BRAZIL", + "RO" : "ROMANIAN", + "RU" : "RUSSIAN", + "SR" : "SERBIAN", + "SI" : "SINHALA", + "SK" : "SLOVAK", + "SL" : "SLOVENIAN", + "ES" : "SPANISH", + "ES-PV" : "SPANISH_BASQUE", + "ES-CT" : "SPANISH_CATALAN", + "ES-CA" : "SPANISH_CATALAN", + "ES-GA" : "SPANISH_GALICIA", + "ES-419" : "SPANISH_LATIN", + "SV" : "SWEDISH", + "TL" : "TAGALOG", + "TA" : "TAMIL", + "TT" : "TATAR", + "TE" : "TELUGU", + "TH" : "THAI", + "X-THT" : "THAI_TRANSLITERATED", + "TR" : "TURKISH", + "UK" : "UKRAINIAN", + "UR" : "URDU", + "VI" : "VIETNAMESE", + "X-UNK" : "UNKNOWN", + "X-OTHER" : "OTHER" } - class AniDBObj: def __init__(self, anidb, anidb_id, data, from_json=False): self._anidb = anidb @@ -124,7 +123,7 @@ def _parseJSON(self): self.main_title = title["title"] if title["type"] == "OFFICIAL": self.titles.append(title) - if title["type"] == "OFFICIAL" and (title["language"] == self._anidb.language.upper() or title["language"] == language_matcher[self._anidb.language]): + if title["type"] == "OFFICIAL" and self._anidb.language and (title["language"] == self._anidb.language.upper() or title["language"] == language_matcher[self._anidb.language.upper()]): self.official_title = title["title"] if not self.official_title: self.official_title = self.main_title