Skip to content

Commit

Permalink
[nijie] fix image URLs of multi-image posts (#4876)
Browse files Browse the repository at this point in the history
  • Loading branch information
@mikf
mikf committed Dec 5, 2023
1 parent a4e6ea6 commit 4eb3590
Show file tree
Hide file tree
Showing 3 changed files with 108 additions and 20 deletions.
34 changes: 16 additions & 18 deletions gallery_dl/extractor/nijie.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,11 @@ def items(self):
data["user_name"] = data["artist_name"]
yield Message.Directory, data

for image in self._extract_images(page):
for num, url in enumerate(self._extract_images(image_id, page)):
image = text.nameext_from_url(url, {
"num": num,
"url": "https:" + url,
})
image.update(data)
if not image["extension"]:
image["extension"] = "jpg"
Expand All @@ -72,7 +76,7 @@ def _extract_data(page):
extr = text.extract_from(page)
keywords = text.unescape(extr(
'name="keywords" content="', '" />')).split(",")
data = {
return {
"title" : keywords[0].strip(),
"description": text.unescape(extr(
'"description": "', '"').replace("&", "&")),
Expand All @@ -82,15 +86,14 @@ def _extract_data(page):
"artist_name": keywords[1],
"tags" : keywords[2:-1],
}
return data

@staticmethod
def _extract_data_horne(page):
"""Extract image metadata from 'page'"""
extr = text.extract_from(page)
keywords = text.unescape(extr(
'name="keywords" content="', '" />')).split(",")
data = {
return {
"title" : keywords[0].strip(),
"description": text.unescape(extr(
'property="og:description" content="', '"')),
Expand All @@ -101,21 +104,16 @@ def _extract_data_horne(page):
"itemprop='datePublished' content=", "<").rpartition(">")[2],
"%Y-%m-%d %H:%M:%S", 9),
}
return data

@staticmethod
def _extract_images(page):
"""Extract image URLs from 'page'"""
images = text.extract_iter(page, "/view_popup.php", "</a>")
for num, image in enumerate(images):
src = text.extr(image, 'src="', '"')
if not src:
continue
url = ("https:" + src).replace("/__rs_l120x120/", "/")
yield text.nameext_from_url(url, {
"num": num,
"url": url,
})
def _extract_images(self, image_id, page):
if '&#diff_1" ' in page:
# multiple images
url = "{}/view_popup.php?id={}".format(self.root, image_id)
page = self.request(url).text
yield from text.extract_iter(
page, 'href="javascript:void(0);"><img src="', '"')
else:
yield text.extr(page, 'itemprop="image" src="', '"')

@staticmethod
def _extract_user_name(page):
Expand Down
36 changes: 35 additions & 1 deletion test/results/horne.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,10 @@
"#url" : "https://horne.red/members.php?id=58000",
"#category": ("Nijie", "horne", "user"),
"#class" : nijie.NijieUserExtractor,
"#urls" : (
"https://horne.red/members_illust.php?id=58000",
"https://horne.red/members_dojin.php?id=58000",
),
},

{
Expand Down Expand Up @@ -71,11 +75,41 @@
"#class" : nijie.NijieFollowedExtractor,
},

{
"#url" : "https://horne.red/view.php?id=8708",
"#category": ("Nijie", "horne", "image"),
"#class" : nijie.NijieImageExtractor,
"#urls" : "https://pic.nijie.net/07/horne/18/00/58000/illust/0_0_c8f715a8f3d53943_db6231.png",

"artist_id" : 58000,
"artist_name": "のえるわ",
"date" : "dt:2018-01-29 14:25:39",
"description": "前回とシチュがまるかぶり　\r\n竿野郎は塗るのだるかった",
"extension" : "png",
"filename" : "0_0_c8f715a8f3d53943_db6231",
"image_id" : 8708,
"num" : 0,
"tags" : [
"男の娘",
"オリキャラ",
"うちのこ",
],
"title" : "うちのこえっち",
"url" : "https://pic.nijie.net/07/horne/18/00/58000/illust/0_0_c8f715a8f3d53943_db6231.png",
"user_id" : 58000,
"user_name" : "のえるわ",
},

{
"#url" : "https://horne.red/view.php?id=8716",
"#category": ("Nijie", "horne", "image"),
"#class" : nijie.NijieImageExtractor,
"#count" : 4,
"#urls" : (
"https://pic.nijie.net/07/horne/18/00/58000/illust/0_0_b4ffb4b6f7ec6d51_1a32c0.png",
"https://pic.nijie.net/07/horne/18/00/58000/illust/8716_0_2690972a4f6270bb_85ed8f.png",
"https://pic.nijie.net/07/horne/18/00/58000/illust/8716_1_09348508d8b76f36_f6cf47.png",
"https://pic.nijie.net/08/horne/18/00/58000/illust/8716_2_5151d956d3789277_d76e75.png",
),

"artist_id" : 58000,
"artist_name": "のえるわ",
Expand Down
58 changes: 57 additions & 1 deletion test/results/nijie.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,13 +14,20 @@
"#url" : "https://nijie.info/members.php?id=44",
"#category": ("Nijie", "nijie", "user"),
"#class" : nijie.NijieUserExtractor,
"#urls" : (
"https://nijie.info/members_illust.php?id=44",
"https://nijie.info/members_dojin.php?id=44",
),
},

{
"#url" : "https://nijie.info/members_illust.php?id=44",
"#category": ("Nijie", "nijie", "illustration"),
"#class" : nijie.NijieIllustrationExtractor,
"#sha1_url": "1553e5144df50a676f5947d02469299b401ad6c0",
"#urls": (
"https://pic.nijie.net/04/nijie/14/44/44/illust/0_0_f46c08462568c2f1_be95d7.jpg",
"https://pic.nijie.net/06/nijie/14/44/44/illust/0_0_28e8c02d921bee33_9222d3.jpg",
),

"artist_id" : 44,
"artist_name": "ED",
Expand Down Expand Up @@ -93,9 +100,58 @@
"#url" : "https://nijie.info/view.php?id=70720",
"#category": ("Nijie", "nijie", "image"),
"#class" : nijie.NijieImageExtractor,
"#urls" : "https://pic.nijie.net/06/nijie/14/44/44/illust/0_0_28e8c02d921bee33_9222d3.jpg",
"#sha1_url" : "3d654e890212ba823c9647754767336aebc0a743",
"#sha1_metadata": "41da5d0e178b04f01fe72460185df52fadc3c91b",
"#sha1_content" : "d85e3ea896ed5e4da0bca2390ad310a4df716ca6",

"artist_id" : 44,
"artist_name": "ED",
"date" : "dt:2014-01-18 19:58:21",
"description": "租絵にてお邪魔いたし候\r\n是非ともこの”おっぱい”をご高覧賜りたく馳せ参じた次第\r\n長文にて失礼仕る\r\n\r\nまず全景でありますが、首を右に傾けてみて頂きたい\r\nこの絵図は茶碗を眺めていた私が思わぬ美しさにて昇天したときのものを、筆をとり、したためたものである（トレースではない）\r\n筆は疾風の如く走り、半刻過ぎには私好みの”おっぱい”になっていたのである！\r\n次に細部をみて頂きたい\r\n絵図を正面から見直して頂くと、なんとはんなりと美しいお椀型をしたおっぱいであろうか　　右手から緩やかに生まれる曲線は左手に進むにつれて、穏やかな歪みを含み流れる　　これは所謂轆轤目であるが三重の紐でおっぱいをぐるぐると巻きつけた情景そのままであり、この歪みから茶碗の均整は崩れ、たぷんたぷんのおっぱいの重量感を醸し出している！\r\nさらに左手に進めば梅花皮（カイラギ）を孕んだ高大が現れる　今回は点線にて表現するが、その姿は乳首から母乳が噴出するが如く　或は精子をぶっかけられたが如く　白くとろっとした釉薬の凝固が素晴しい景色をつくりだしているのである！\r\n最後には極めつけ、すくっと螺旋を帯びながらそそり立つ兜巾（ときん）！この情景はまさしく乳首である！　　全体をふんわりと盛り上げさせる乳輪にちょこっと存在する乳頭はぺろりと舌で確かめ勃起させたくなる風情がある！\r\n\r\nこれを”おっぱい”と呼ばずなんと呼ぼうや！？\r\n\r\n興奮のあまり失礼致した\r\n御免",
"extension" : "jpg",
"filename" : "0_0_28e8c02d921bee33_9222d3",
"image_id" : 70720,
"num" : 0,
"tags" : ["おっぱい"],
"title" : "俺好高麗井戸茶碗　銘おっぱい",
"url" : "https://pic.nijie.net/06/nijie/14/44/44/illust/0_0_28e8c02d921bee33_9222d3.jpg",
"user_id" : 44,
"user_name" : "ED",
},

{
"#url" : "https://nijie.info/view.php?id=594044",
"#category": ("Nijie", "nijie", "image"),
"#class" : nijie.NijieImageExtractor,
"#urls": (
"https://pic.nijie.net/02/nijie/23m12/09/49509/illust/0_0_63568cc428259d50_45ca51.jpg",
"https://pic.nijie.net/01/nijie/23m12/09/49509/illust/594044_0_1c94b7cc4503589f_79c66c.jpg",
"https://pic.nijie.net/02/nijie/23m12/09/49509/illust/594044_1_9f4737ad48bf43c7_8f1e8e.jpg",
"https://pic.nijie.net/01/nijie/23m12/09/49509/illust/594044_2_a162861fac970a45_38c5f8.jpg",
),

"artist_id" : 49509,
"artist_name": "黒川　竜",
"date" : "dt:2023-12-02 04:19:29",
"description": "【DLサイトコム】ウィンターセール　30%OFF\r\n期間：2024年2月14日まで\r\n【toloveるドリンク】\r\nhttps://www.dlsite.com/maniax/work/=/product_id/RJ042727.html\r\n【toloveるドリンク２】\r\nhttps://www.dlsite.com/maniax/work/=/product_id/RJ043289.html\r\n【クランクランBIG】\r\nhttps://www.dlsite.com/maniax/work/=/product_id/RJ043564.html",
"image_id" : 594044,
"num" : range(0, 3),
"tags" : [
"オリジナル",
"漫画",
"中出し",
"爆乳",
"巨乳",
"ToLOVEる",
"宣伝",
"クラン・クラン",
"マクロスF",
],
"title" : "【DLサイトコム】ウィンターセール",
"url" : str,
"user_id" : 49509,
"user_name" : "黒川　竜",
},

{
Expand Down

0 comments on commit 4eb3590

Please sign in to comment.