diff --git a/client.ts b/client.ts index 8f3ce2f..707b68c 100644 --- a/client.ts +++ b/client.ts @@ -8,15 +8,23 @@ import { RecoverableError } from "./task_manager.ts"; export type GID = [number, string]; export class Client { - cookies; - host; - ua; + cfg; + get cookies() { + return this.cfg.cookies; + } + get host() { + return this.cfg.ex ? "exhentai.org" : "e-hentai.org"; + } + get ua() { + return this.cfg.ua || + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36"; + } + get timeout() { + return this.cfg.fetch_timeout; + } signal; constructor(cfg: Config, signal?: AbortSignal) { - this.cookies = cfg.cookies; - this.ua = cfg.ua || - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36"; - this.host = cfg.ex ? "exhentai.org" : "e-hentai.org"; + this.cfg = cfg; this.signal = signal; } get( @@ -94,6 +102,15 @@ export class Client { if (!d.signal && this.signal) { d.signal = this.signal; } + const osignal = d.signal; + const abortController = new AbortController(); + const timeout = setTimeout(() => { + abortController.abort(); + }, this.timeout); + osignal?.addEventListener("abort", () => { + abortController.abort(osignal?.reason); + }); + d.signal = abortController.signal; try { return fetch(url, d); } catch (e) { @@ -101,6 +118,8 @@ export class Client { throw new RecoverableError(e.message, { cause: e.cause }); } throw e; + } finally { + clearTimeout(timeout); } } } diff --git a/config.ts b/config.ts index ef2411f..fecf356 100644 --- a/config.ts +++ b/config.ts @@ -26,6 +26,8 @@ export type ConfigType = { meili_hosts?: Record; cors_credentials_hosts: Array; flutter_frontend?: string; + fetch_timeout: number; + download_timeout: number; }; export enum ThumbnailMethod { @@ -164,6 +166,12 @@ export class Config { get flutter_frontend() { return this._return_string("flutter_frontend"); } + get fetch_timeout() { + return this._return_number("fetch_timeout") || 10000; + } + get download_timeout() { + return this._return_number("download_timeout") || 10000; + } to_json(): ConfigType { return { cookies: typeof this.cookies === "string", @@ -190,6 +198,8 @@ export class Config { meili_hosts: this.meili_hosts, cors_credentials_hosts: this.cors_credentials_hosts, flutter_frontend: this.flutter_frontend, + fetch_timeout: this.fetch_timeout, + download_timeout: this.download_timeout, }; } } diff --git a/server/utils.ts b/server/utils.ts index 5abf173..eecbc07 100644 --- a/server/utils.ts +++ b/server/utils.ts @@ -22,7 +22,7 @@ function gen_response( status = (d.status >= 400 && d.status < 600) ? d.status : 400; } const h = new Headers(headers); - h.set("Content-Type", "application/json"); + h.set("Content-Type", "application/json; charset=UTF-8"); return new Response(JSON.stringify(d), { status, headers: h }); } @@ -57,7 +57,7 @@ export function gen_error( export function return_json(data: T, status = 200) { return new Response(JSON.stringify(data), { status, - headers: { "Content-Type": "application/json" }, + headers: { "Content-Type": "application/json; chatset=UTF-8" }, }); } diff --git a/tasks/download.ts b/tasks/download.ts index b090c27..a11d517 100644 --- a/tasks/download.ts +++ b/tasks/download.ts @@ -16,6 +16,7 @@ import { PromiseStatus, sleep, sure_dir, + TimeoutError, } from "../utils.ts"; import { join, resolve } from "std/path/mod.ts"; import { exists } from "std/fs/exists.ts"; @@ -109,7 +110,14 @@ class DownloadManager { return re; } add_new_details(d: TaskDownloadSingleProgress) { - this.#progress.details.push(d); + const index = this.#progress.details.findIndex((v) => + v.index === d.index + ); + if (index !== -1) { + this.#progress.details[index] = d; + } else { + this.#progress.details.push(d); + } this.#sendEvent(); } async add_new_task(f: () => Promise) { @@ -165,6 +173,7 @@ interface Image { is_original: boolean | undefined; name: string; page_token: string; + redirected_url: string | undefined; sampled_name: string; get_file(path: string): EhFile | undefined; get_original_file(path: string): EhFile | undefined; @@ -260,126 +269,189 @@ export async function download_task( return; } } + let load_times = 0; function load() { return new Promise((resolve, reject) => { const errors: unknown[] = []; - function try_load(a: number) { - if (a >= max_retry_count) reject(errors); + function try_load() { + if (load_times >= max_retry_count) reject(errors); i.load().then(resolve).catch((e) => { if (force_abort.aborted) { throw Error("aborted."); } errors.push(e); - try_load(a + 1); + load_times += 1; + try_load(); }); } - try_load(0); + try_load(); }); } - await load(); - assert(i.data); - const pmeta = i.to_pmeta(); - if (pmeta) db.add_pmeta(pmeta); - const download_original = download_original_img && - !i.is_original; - const is_sampled = !download_original_img && !i.is_original; - let path = resolve( - join(base_path, is_sampled ? i.sampled_name : i.name), - ); - if (names[i.name] > 1) { - path = add_suffix_to_path(path, i.page_token); - console.log("Changed path to", path); - } - const f = download_original - ? i.get_original_file(path) - : i.get_file(path); - if (f === undefined) throw Error("Failed to get file."); - m.add_new_details({ - downloaded: 0, - height: f.height, - index: i.index, - is_original: f.is_original, - name: i.name, - token: i.page_token, - total: 0, - width: f.width, - started: 0, - }); - function download_img() { - return new Promise((resolve, reject) => { - async function download() { - const re = await (download_original - ? i.load_original_image() - : i.load_image()); - if (re === undefined) { - throw Error("Failed to fetch image."); - } - m.set_details_started(i.index); - const len = re.headers.get("Content-Length"); - if (len) { - const tmp = parseInt(len); - if (!isNaN(tmp)) { - m.set_details_total(i.index, tmp); + async function deal_with_img() { + assert(i.data); + const pmeta = i.to_pmeta(); + if (pmeta) db.add_pmeta(pmeta); + const download_original = download_original_img && + !i.is_original; + const is_sampled = !download_original_img && !i.is_original; + let path = resolve( + join(base_path, is_sampled ? i.sampled_name : i.name), + ); + if (names[i.name] > 1) { + path = add_suffix_to_path(path, i.page_token); + console.log("Changed path to", path); + } + const f = download_original + ? i.get_original_file(path) + : i.get_file(path); + if (f === undefined) throw Error("Failed to get file."); + m.add_new_details({ + downloaded: 0, + height: f.height, + index: i.index, + is_original: f.is_original, + name: i.name, + token: i.page_token, + total: 0, + width: f.width, + started: 0, + }); + function download_img() { + return new Promise((resolve, reject) => { + async function download() { + const re = await (download_original + ? i.load_original_image() + : i.load_image()); + if (re === undefined) { + throw Error("Failed to fetch image."); } - } - if (re.body === null) { - throw Error("Response don't have a body."); - } - const pr = new ProgressReadable(re.body); - pr.addEventListener("progress", (e) => { - m.set_details_downloaded(i.index, e.detail); - }); - pr.addEventListener("finished", () => { - m.remove_details(i.index); - }); - try { - const f = await Deno.open(path, { - create: true, - write: true, - truncate: true, + m.set_details_started(i.index); + const len = re.headers.get("Content-Length"); + if (len) { + const tmp = parseInt(len); + if (!isNaN(tmp)) { + m.set_details_total(i.index, tmp); + } + } + if (re.body === null) { + throw Error("Response don't have a body."); + } + const pr = new ProgressReadable( + re.body, + cfg.download_timeout, + force_abort, + ); + pr.addEventListener("progress", (e) => { + m.set_details_downloaded(i.index, e.detail); + }); + pr.addEventListener("finished", () => { + m.remove_details(i.index); }); try { - await pr.readable.pipeTo(f.writable, { - signal: force_abort, - preventClose: true, + const f = await Deno.open(path, { + create: true, + write: true, + truncate: true, }); - if (pr.error) { - throw (pr.error); + try { + await pr.readable.pipeTo(f.writable, { + signal: pr.signal, + preventClose: true, + }); + if (pr.error) { + throw (pr.error); + } + } finally { + try { + f.close(); + } catch (_) { + null; + } + } + } catch (e) { + if (pr.is_timeout) { + throw new TimeoutError(); } + throw e; } finally { try { - f.close(); + pr.readable.cancel(); } catch (_) { null; } } - } finally { - try { - pr.readable.cancel(); - } catch (_) { - null; + } + const errors: unknown[] = []; + function try_download(a: number) { + if (a >= max_retry_count) { + m.remove_details(i.index); + reject(errors); } + download().then(resolve).catch((e) => { + if (force_abort.aborted) { + throw Error("aborted."); + } + if (e instanceof DOMException) { + if (e.name == "AbortError") { + reject(new TimeoutError()); + return; + } + } + if (e instanceof TimeoutError) { + reject(e); + return; + } + errors.push(e); + try_download(a + 1); + }); } + try_download(0); + }); + } + await download_img(); + db.add_file(f); + } + let retry = 0; + function try_deal_with_img() { + return new Promise((resolve, reject) => { + function try_() { + load().then(() => { + deal_with_img().then(resolve).catch((e) => { + console.log("Failed to download, retry: ", e); + retry += 1; + if (retry >= max_retry_count) { + reject(e); + return; + } + const download_original = download_original_img && + !i.is_original; + if (download_original) { + i.redirected_url = undefined; + try2_(); + } else try_(); + }); + }).catch(reject); } - const errors: unknown[] = []; - function try_download(a: number) { - if (a >= max_retry_count) { - m.remove_details(i.index); - reject(errors); - } - download().then(resolve).catch((e) => { - if (force_abort.aborted) { - throw Error("aborted."); + function try2_() { + deal_with_img().then(resolve).catch((e) => { + console.log("Failed to download, retry: ", e); + retry += 1; + if (retry >= max_retry_count) { + reject(e); + return; } - errors.push(e); - try_download(a + 1); + const download_original = download_original_img && + !i.is_original; + if (download_original) { + i.redirected_url = undefined; + try2_(); + } else try_(); }); } - try_download(0); + try_(); }); } - await download_img(); - db.add_file(f); + await try_deal_with_img(); return; } if (mpv_enabled || g.mpv_enabled) { diff --git a/thumbnail/ffmpeg_binary.ts b/thumbnail/ffmpeg_binary.ts index 2f0ea47..663ea13 100644 --- a/thumbnail/ffmpeg_binary.ts +++ b/thumbnail/ffmpeg_binary.ts @@ -18,6 +18,7 @@ export async function fb_generate_thumbnail( cfg: ThumbnailConfig, ) { const args = [ + "-n", "-i", i, "-vf", diff --git a/utils.ts b/utils.ts index 51b1d49..264054a 100644 --- a/utils.ts +++ b/utils.ts @@ -264,3 +264,9 @@ export function map( } return re; } + +export class TimeoutError extends Error { + constructor() { + super("Timeout"); + } +} diff --git a/utils/progress_readable.ts b/utils/progress_readable.ts index a44d843..1c8a1bd 100644 --- a/utils/progress_readable.ts +++ b/utils/progress_readable.ts @@ -7,10 +7,31 @@ export class ProgressReadable extends EventTarget { readable: ReadableStream; readed: number; error?: unknown; - constructor(readable: ReadableStream) { + timeout: number; + get signal() { + return this.#controller.signal; + } + get is_timeout() { + return this.#is_timeout; + } + #controller: AbortController; + #is_timeout: boolean; + #timeout?: number; + constructor( + readable: ReadableStream, + timeout: number, + originalSignal?: AbortSignal, + ) { super(); this.readed = 0; + this.timeout = timeout; + this.#is_timeout = false; const reader = readable.getReader(); + this.#controller = new AbortController(); + originalSignal?.addEventListener("abort", () => { + this.#controller.abort(); + this.#clearTimeout(); + }); this.readable = new ReadableStream({ pull: (c) => { if (c.byobRequest) { @@ -20,27 +41,51 @@ export class ProgressReadable extends EventTarget { if (v.done) { this.dispatchEvent("finished", this.readed); c.close(); + this.#clearTimeout(); return; } else { this.readed += v.value.byteLength; this.dispatchEvent("progress", this.readed); c.enqueue(v.value); + if (v.value.byteLength != 0) { + this.#clearTimeout(); + this.#setTimeout(); + } } }).catch((e) => { - c.close(); + try { + c.close(); + } catch (_) { + null; + } this.error = e; + this.#clearTimeout(); }); } }, cancel: (reason) => { try { - readable.cancel(reason); + if (!readable.locked) readable.cancel(reason); + this.#clearTimeout(); } catch (_) { null; } }, type: "bytes", }); + this.#setTimeout(); + } + #clearTimeout() { + if (this.#timeout) { + clearTimeout(this.#timeout); + } + } + #setTimeout() { + this.#timeout = setTimeout(() => { + this.#is_timeout = true; + this.#controller.abort(); + console.log("aborted"); + }, this.timeout); } // @ts-ignore Checked type addEventListener(