diff --git a/docs/supportedsites.md b/docs/supportedsites.md index e3b4ea7ef5..df61f1223a 100644 --- a/docs/supportedsites.md +++ b/docs/supportedsites.md @@ -1371,6 +1371,12 @@ Consider all listed sites to potentially be NSFW. Posts, Tag Searches + + Rule34Hentai + https://rule34hentai.net/ + Posts, Tag Searches + + szurubooru Instances diff --git a/gallery_dl/extractor/shimmie2.py b/gallery_dl/extractor/shimmie2.py index 912e6013fb..725d05d255 100644 --- a/gallery_dl/extractor/shimmie2.py +++ b/gallery_dl/extractor/shimmie2.py @@ -35,6 +35,8 @@ def _init(self): if self.category == "giantessbooru": self.posts = self._posts_giantessbooru + elif self.category == "rule34hentai": + self.posts = self._posts_rule34hentai def items(self): data = self.metadata() @@ -85,6 +87,10 @@ def posts(self): "pattern": r"booru\.cavemanon\.xyz", "file_url": "{0}/index.php?q=image/{2}.{4}", }, + "rule34hentai": { + "root": "https://rule34hentai.net", + "pattern": r"rule34hentai\.net", + }, } BASE_PATTERN = Shimmie2Extractor.update(INSTANCES) + r"/(?:index\.php\?q=/?)?" @@ -187,6 +193,56 @@ def _posts_giantessbooru(self): if not extr('/{}">{}<'.format(pnum, pnum), ">"): return + def _posts_rule34hentai(self): + pnum = text.parse_int(self.page, 1) + file_url_fmt = self.file_url_fmt.format + + init = True + mime = "" + + while True: + url = "{}/post/list/{}/{}".format(self.root, self.tags, pnum) + page = self.request(url).text + extr = text.extract_from(page) + + if init: + init = False + has_mime = ("data-mime=\"" in page) + has_pid = ("data-post-id=\"" in page) + + while True: + if has_mime: + mime = extr("data-mime=\"", "\"") + if has_pid: + pid = extr("data-post-id=\"", "\"") + else: + pid = extr("href='/post/view/", "?") + + if not pid: + break + + tags, dimensions, size, ext = extr( + "title=\"", "\"").split(" // ") + width, _, height = dimensions.partition("x") + md5 = extr("/_thumbs/", "/") + + yield { + "file_url": file_url_fmt( + self.root, md5, pid, text.quote(tags), + mime.rpartition("/")[2] if mime else "jpg"), + "id": pid, + "md5": md5, + "tags": tags, + "width": width, + "height": height, + "size": text.parse_bytes(size[:-1]), + } + + pnum += 1 + if not extr(">Next<", ">"): + if not extr("/{}'>{}<".format(pnum, pnum), ">"): + return + class Shimmie2PostExtractor(Shimmie2Extractor): """Extractor for single shimmie2 posts""" @@ -234,3 +290,25 @@ def _posts_giantessbooru(self): "height" : 0, "size" : 0, },) + + def _posts_rule34hentai(self): + url = "{}/post/view/{}".format(self.root, self.post_id) + extr = text.extract_from(self.request(url).text) + + post = { + "id" : self.post_id, + "tags" : extr(": ", "<").partition(" - ")[0].rstrip(")"), + "md5" : extr("/_thumbs/", "/"), + "file_url": self.root + ( + extr('id="main_image" src="', '"') or + extr('").partition( + " ")[0].strip("\"'"), + "size" : 0, + } + + if not post["md5"]: + post["md5"] = text.extr(post["file_url"], "/_images/", "/") + + return (post,) diff --git a/scripts/supportedsites.py b/scripts/supportedsites.py index 2a625e7680..11473b53ff 100755 --- a/scripts/supportedsites.py +++ b/scripts/supportedsites.py @@ -106,6 +106,7 @@ "redgifs" : "RedGIFs", "rozenarcana" : "Rozen Arcana", "rule34" : "Rule 34", + "rule34hentai" : "Rule34Hentai", "rule34us" : "Rule 34", "sankaku" : "Sankaku Channel", "sankakucomplex" : "Sankaku Complex", diff --git a/test/results/rule34hentai.py b/test/results/rule34hentai.py new file mode 100644 index 0000000000..1d3cb29122 --- /dev/null +++ b/test/results/rule34hentai.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import shimmie2 + + +__tests__ = ( +{ + "#url" : "https://rule34hentai.net/post/list/mizuki_kotora/1", + "#category": ("shimmie2", "rule34hentai", "tag"), + "#class" : shimmie2.Shimmie2TagExtractor, + "#urls" : ( + "https://rule34hentai.net/_images/7f3a411263d0f6de936e47ae8f9d35fb/332%20-%20Darkstalkers%20Felicia%20mizuki_kotora.jpeg", + "https://rule34hentai.net/_images/1a8eca7c04f8bf325bc993c5751a91c4/264%20-%20Darkstalkers%20Felicia%20mizuki_kotora.jpeg", + "https://rule34hentai.net/_images/09511511c4c9e9e1f9b795e059a60832/259%20-%20Darkstalkers%20Felicia%20mizuki_kotora.jpeg", + ), + + "extension" : "jpeg", + "file_url" : r"re:https://rule34hentai.net/_images/.+\.jpeg", + "filename" : r"re:\d+ - \w+", + "height" : range(496, 875), + "id" : range(259, 332), + "md5" : r"re:^[0-9a-f]{32}$", + "search_tags": "mizuki_kotora", + "size" : int, + "tags" : str, + "width" : range(500, 850), +}, + +{ + "#url" : "https://rule34hentai.net/post/view/264", + "#category": ("shimmie2", "rule34hentai", "post"), + "#class" : shimmie2.Shimmie2PostExtractor, + "#urls" : "https://rule34hentai.net/_images/1a8eca7c04f8bf325bc993c5751a91c4/264%20-%20Darkstalkers%20Felicia%20mizuki_kotora.jpg", + "#sha1_content": "6c23780bb78673cbff1bca9accb77ea11ec734f3", + + "extension": "jpg", + "file_url" : "https://rule34hentai.net/_images/1a8eca7c04f8bf325bc993c5751a91c4/264%20-%20Darkstalkers%20Felicia%20mizuki_kotora.jpg", + "filename" : "264 - Darkstalkers Felicia mizuki_kotora", + "height" : 875, + "id" : 264, + "md5" : "1a8eca7c04f8bf325bc993c5751a91c4", + "size" : 0, + "tags" : "Darkstalkers Felicia mizuki_kotora", + "width" : 657, +}, + +)