diff --git a/web/package-lock.json b/web/package-lock.json index c6615f0afc979..782971cffbd2d 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -30,8 +30,7 @@ "svelte-gestures": "^5.1.3", "svelte-i18n": "^4.0.1", "svelte-local-storage-store": "^0.6.4", - "svelte-maplibre": "^0.9.13", - "thumbhash": "^0.1.1" + "svelte-maplibre": "^0.9.13" }, "devDependencies": { "@eslint/eslintrc": "^3.1.0", @@ -8209,11 +8208,6 @@ "integrity": "sha512-AUwVmViIEUgBwxJJ7stnF0NkPpZxx1aZ6WiAbQ/Qq61h6I9UR4grXtZDmO8mnlaNORhHnIBlXJ1uBxILEKuVyw==", "license": "MIT" }, - "node_modules/thumbhash": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/thumbhash/-/thumbhash-0.1.1.tgz", - "integrity": "sha512-kH5pKeIIBPQXAOni2AiY/Cu/NKdkFREdpH+TLdM0g6WA7RriCv0kPLgP731ady67MhTAqrVG/4mnEeibVuCJcg==" - }, "node_modules/timers-ext": { "version": "0.1.7", "resolved": "https://registry.npmjs.org/timers-ext/-/timers-ext-0.1.7.tgz", diff --git a/web/package.json b/web/package.json index 7396e1288ad6e..3fd4396e21189 100644 --- a/web/package.json +++ b/web/package.json @@ -86,8 +86,7 @@ "svelte-gestures": "^5.1.3", "svelte-i18n": "^4.0.1", "svelte-local-storage-store": "^0.6.4", - "svelte-maplibre": "^0.9.13", - "thumbhash": "^0.1.1" + "svelte-maplibre": "^0.9.13" }, "volta": { "node": "22.14.0" diff --git a/web/src/lib/actions/thumbhash.ts b/web/src/lib/actions/thumbhash.ts index e49f04dbee546..d3d4e369af8d9 100644 --- a/web/src/lib/actions/thumbhash.ts +++ b/web/src/lib/actions/thumbhash.ts @@ -1,5 +1,4 @@ import { decodeBase64 } from '$lib/utils'; -import { thumbHashToRGBA } from 'thumbhash'; /** * Renders a thumbnail onto a canvas from a base64 encoded hash. @@ -7,13 +6,132 @@ import { thumbHashToRGBA } from 'thumbhash'; * @param param1 object containing the base64 encoded hash (base64Thumbhash: yourString) */ export function thumbhash(canvas: HTMLCanvasElement, { base64ThumbHash }: { base64ThumbHash: string }) { - const ctx = canvas.getContext('2d'); + const ctx = canvas.getContext('bitmaprenderer'); if (ctx) { const { w, h, rgba } = thumbHashToRGBA(decodeBase64(base64ThumbHash)); - const pixels = ctx.createImageData(w, h); - canvas.width = w; - canvas.height = h; - pixels.data.set(rgba); - ctx.putImageData(pixels, 0, 0); + void createImageBitmap(new ImageData(rgba, w, h)).then((bitmap) => ctx.transferFromImageBitmap(bitmap)); } } + +// This copyright notice applies to the below code +// It is a modified version of the original that uses `Uint8ClampedArray` instead of `UInt8Array` and has some trivial typing/linting changes + +/* Copyright (c) 2023 Evan Wallace +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.*/ +/** + * Decodes a ThumbHash to an RGBA image. RGB is not be premultiplied by A. + * + * @param hash The bytes of the ThumbHash. + * @returns The width, height, and pixels of the rendered placeholder image. + */ +export function thumbHashToRGBA(hash: Uint8Array) { + const { PI, max, cos, round } = Math; + + // Read the constants + const header24 = hash[0] | (hash[1] << 8) | (hash[2] << 16); + const header16 = hash[3] | (hash[4] << 8); + const l_dc = (header24 & 63) / 63; + const p_dc = ((header24 >> 6) & 63) / 31.5 - 1; + const q_dc = ((header24 >> 12) & 63) / 31.5 - 1; + const l_scale = ((header24 >> 18) & 31) / 31; + const hasAlpha = header24 >> 23; + const p_scale = ((header16 >> 3) & 63) / 63; + const q_scale = ((header16 >> 9) & 63) / 63; + const isLandscape = header16 >> 15; + const lx = max(3, isLandscape ? (hasAlpha ? 5 : 7) : header16 & 7); + const ly = max(3, isLandscape ? header16 & 7 : hasAlpha ? 5 : 7); + const a_dc = hasAlpha ? (hash[5] & 15) / 15 : 1; + const a_scale = (hash[5] >> 4) / 15; + + // Read the varying factors (boost saturation by 1.25x to compensate for quantization) + const ac_start = hasAlpha ? 6 : 5; + let ac_index = 0; + const decodeChannel = (nx: number, ny: number, scale: number) => { + const ac = []; + for (let cy = 0; cy < ny; cy++) { + for (let cx = cy ? 0 : 1; cx * ny < nx * (ny - cy); cx++) { + ac.push((((hash[ac_start + (ac_index >> 1)] >> ((ac_index++ & 1) << 2)) & 15) / 7.5 - 1) * scale); + } + } + return ac; + }; + const l_ac = decodeChannel(lx, ly, l_scale); + const p_ac = decodeChannel(3, 3, p_scale * 1.25); + const q_ac = decodeChannel(3, 3, q_scale * 1.25); + const a_ac = hasAlpha ? decodeChannel(5, 5, a_scale) : null; + + // Decode using the DCT into RGB + const ratio = thumbHashToApproximateAspectRatio(hash); + const w = round(ratio > 1 ? 32 : 32 * ratio); + const h = round(ratio > 1 ? 32 / ratio : 32); + const rgba = new Uint8ClampedArray(w * h * 4), + fx = [], + fy = []; + for (let y = 0, i = 0; y < h; y++) { + for (let x = 0; x < w; x++, i += 4) { + let l = l_dc, + p = p_dc, + q = q_dc, + a = a_dc; + + // Precompute the coefficients + for (let cx = 0, n = max(lx, hasAlpha ? 5 : 3); cx < n; cx++) { + fx[cx] = cos((PI / w) * (x + 0.5) * cx); + } + for (let cy = 0, n = max(ly, hasAlpha ? 5 : 3); cy < n; cy++) { + fy[cy] = cos((PI / h) * (y + 0.5) * cy); + } + + // Decode L + for (let cy = 0, j = 0; cy < ly; cy++) { + for (let cx = cy ? 0 : 1, fy2 = fy[cy] * 2; cx * ly < lx * (ly - cy); cx++, j++) { + l += l_ac[j] * fx[cx] * fy2; + } + } + + // Decode P and Q + for (let cy = 0, j = 0; cy < 3; cy++) { + for (let cx = cy ? 0 : 1, fy2 = fy[cy] * 2; cx < 3 - cy; cx++, j++) { + const f = fx[cx] * fy2; + p += p_ac[j] * f; + q += q_ac[j] * f; + } + } + + // Decode A + if (a_ac !== null) { + for (let cy = 0, j = 0; cy < 5; cy++) { + for (let cx = cy ? 0 : 1, fy2 = fy[cy] * 2; cx < 5 - cy; cx++, j++) { + a += a_ac[j] * fx[cx] * fy2; + } + } + } + + // Convert to RGB + const b = l - (2 / 3) * p; + const r = (3 * l - b + q) / 2; + const g = r - q; + rgba[i] = 255 * r; + rgba[i + 1] = 255 * g; + rgba[i + 2] = 255 * b; + rgba[i + 3] = 255 * a; + } + } + return { w, h, rgba }; +} + +/** + * Extracts the approximate aspect ratio of the original image. + * + * @param hash The bytes of the ThumbHash. + * @returns The approximate aspect ratio (i.e. width / height). + */ +export function thumbHashToApproximateAspectRatio(hash: Uint8Array) { + const header = hash[3]; + const hasAlpha = hash[2] & 0x80; + const isLandscape = hash[4] & 0x80; + const lx = isLandscape ? (hasAlpha ? 5 : 7) : header & 7; + const ly = isLandscape ? header & 7 : hasAlpha ? 5 : 7; + return lx / ly; +} diff --git a/web/src/lib/components/assets/thumbnail/thumbnail.svelte b/web/src/lib/components/assets/thumbnail/thumbnail.svelte index c381574277ae8..775c59bb23f1d 100644 --- a/web/src/lib/components/assets/thumbnail/thumbnail.svelte +++ b/web/src/lib/components/assets/thumbnail/thumbnail.svelte @@ -39,6 +39,7 @@ thumbnailSize?: number | undefined; thumbnailWidth?: number | undefined; thumbnailHeight?: number | undefined; + eagerThumbhash?: boolean; selected?: boolean; selectionCandidate?: boolean; disabled?: boolean; @@ -71,6 +72,7 @@ thumbnailSize = undefined, thumbnailWidth = undefined, thumbnailHeight = undefined, + eagerThumbhash = true, selected = false, selectionCandidate = false, disabled = false, @@ -113,7 +115,6 @@ let width = $derived(thumbnailSize || thumbnailWidth || 235); let height = $derived(thumbnailSize || thumbnailHeight || 235); - let display = $derived(intersecting); const onIconClickedHandler = (e?: MouseEvent) => { e?.stopPropagation(); @@ -207,7 +208,11 @@ ? 'bg-gray-300' : 'bg-immich-primary/20 dark:bg-immich-dark-primary/20'}" > - {#if !loaded && asset.thumbhash} + + {#if eagerThumbhash && !loaded && asset.thumbhash} {/if} - {#if display} + {#if intersecting} + {#if !eagerThumbhash && !loaded && asset.thumbhash} + + {/if} +
{#each dateGroup.assets as asset, index (asset.id)} {@const box = dateGroup.geometry.boxes[index]} + {@const isSmallGroup = dateGroup.assets.length <= SMALL_GROUP_THRESHOLD}
{/each} diff --git a/web/src/lib/components/photos-page/asset-grid.svelte b/web/src/lib/components/photos-page/asset-grid.svelte index fd98f7e6a3a10..dfbfc2a25f092 100644 --- a/web/src/lib/components/photos-page/asset-grid.svelte +++ b/web/src/lib/components/photos-page/asset-grid.svelte @@ -20,7 +20,7 @@ } from '$lib/utils/timeline-util'; import { TUNABLES } from '$lib/utils/tunables'; import type { AlbumResponseDto, AssetResponseDto, PersonResponseDto } from '@immich/sdk'; - import { throttle } from 'lodash-es'; + import { debounce, throttle } from 'lodash-es'; import { onDestroy, onMount, type Snippet } from 'svelte'; import Portal from '../shared-components/portal/portal.svelte'; import Scrubber from '../shared-components/scrubber/scrubber.svelte'; @@ -81,8 +81,9 @@ let { isViewing: showAssetViewer, asset: viewingAsset, preloadAssets, gridScrollTarget } = assetViewingStore; - const viewport: ViewportXY = $state({ width: 0, height: 0, x: 0, y: 0 }); - const safeViewport: ViewportXY = $state({ width: 0, height: 0, x: 0, y: 0 }); + // this does *not* need to be reactive and making it reactive causes expensive repeated updates + // svelte-ignore non_reactive_update + let safeViewport: ViewportXY = { width: 0, height: 0, x: 0, y: 0 }; const componentId = generateId(); let element: HTMLElement | undefined = $state(); @@ -103,7 +104,7 @@ let leadout = $state(false); const { - ASSET_GRID: { NAVIGATE_ON_ASSET_IN_VIEW }, + ASSET_GRID: { NAVIGATE_ON_ASSET_IN_VIEW, LARGE_BUCKET_THRESHOLD, LARGE_BUCKET_DEBOUNCE_MS }, BUCKET: { INTERSECTION_ROOT_TOP: BUCKET_INTERSECTION_ROOT_TOP, INTERSECTION_ROOT_BOTTOM: BUCKET_INTERSECTION_ROOT_BOTTOM, @@ -114,14 +115,6 @@ }, } = TUNABLES; - const isViewportOrigin = () => { - return viewport.height === 0 && viewport.width === 0; - }; - - const isEqual = (a: ViewportXY, b: ViewportXY) => { - return a.height == b.height && a.width == b.width && a.x === b.x && a.y === b.y; - }; - const completeNav = () => { navigating = false; if (internalScroll) { @@ -235,6 +228,14 @@ }; onMount(() => { + if (element) { + const rect = element.getBoundingClientRect(); + safeViewport.height = rect.height; + safeViewport.width = rect.width; + safeViewport.x = rect.x; + safeViewport.y = rect.y; + } + void $assetStore .init({ bucketListener }) .then(() => ($assetStore.connect(), $assetStore.updateViewport(safeViewport))); @@ -259,8 +260,6 @@ } return offset; } - const _updateViewport = () => void $assetStore.updateViewport(safeViewport); - const updateViewport = throttle(_updateViewport, 16); const getMaxScrollPercent = () => ($assetStore.timelineHeight + bottomSectionHeight + topSectionHeight - safeViewport.height) / @@ -744,23 +743,8 @@ } }); - $effect(() => { - if (element && isViewportOrigin()) { - const rect = element.getBoundingClientRect(); - viewport.height = rect.height; - viewport.width = rect.width; - viewport.x = rect.x; - viewport.y = rect.y; - } - if (!isViewportOrigin() && !isEqual(viewport, safeViewport)) { - safeViewport.height = viewport.height; - safeViewport.width = viewport.width; - safeViewport.x = viewport.x; - safeViewport.y = viewport.y; - updateViewport(); - } - }); - + let largeBucketMode = false; + let updateViewport = debounce(() => $assetStore.updateViewport(safeViewport), 8); let shortcutList = $derived( (() => { if ($isSearchEnabled || $showAssetViewer) { @@ -843,7 +827,21 @@ id="asset-grid" class="scrollbar-hidden h-full overflow-y-auto outline-none {isEmpty ? 'm-0' : 'ml-4 tall:ml-0 mr-[60px]'}" tabindex="-1" - use:resizeObserver={({ height, width }) => ((viewport.width = width), (viewport.height = height))} + use:resizeObserver={({ width, height }) => { + if (!largeBucketMode && assetStore.maxBucketAssets >= LARGE_BUCKET_THRESHOLD) { + largeBucketMode = true; + // Each viewport update causes each asset to re-decode both the thumbhash and the thumbnail. + // This is because the thumbnail components are destroyed and re-mounted, possibly because of the intersection observer. + // For larger buckets, this can lead to freezing and a poor user experience. + // As a mitigation, we aggressively debounce the viewport update to reduce the number of these events. + updateViewport = debounce(() => $assetStore.updateViewport(safeViewport), LARGE_BUCKET_DEBOUNCE_MS, { + leading: false, + trailing: true, + }); + } + safeViewport = { width, height, x: safeViewport.x, y: safeViewport.y }; + void updateViewport(); + }} bind:this={element} onscroll={() => ((assetStore.lastScrollTime = Date.now()), handleTimelineScroll())} > diff --git a/web/src/lib/stores/assets.store.ts b/web/src/lib/stores/assets.store.ts index 215707543c01f..99a5b9af630fb 100644 --- a/web/src/lib/stores/assets.store.ts +++ b/web/src/lib/stores/assets.store.ts @@ -232,6 +232,7 @@ export class AssetStore { albumAssets: Set = new Set(); pendingScrollBucket: AssetBucket | undefined; pendingScrollAssetId: string | undefined; + maxBucketAssets = 0; listeners: BucketListener[] = []; @@ -560,6 +561,7 @@ export class AssetStore { bucket.assets = assets; bucket.dateGroups = splitBucketIntoDateGroups(bucket, get(locale)); + this.maxBucketAssets = Math.max(this.maxBucketAssets, assets.length); this.updateGeometry(bucket, true); this.timelineHeight = this.buckets.reduce((accumulator, b) => accumulator + b.bucketHeight, 0); bucket.loaded(); diff --git a/web/src/lib/utils/tunables.ts b/web/src/lib/utils/tunables.ts index e21c30de77b2a..8ffa767dc8d69 100644 --- a/web/src/lib/utils/tunables.ts +++ b/web/src/lib/utils/tunables.ts @@ -40,6 +40,8 @@ export const TUNABLES = { }, ASSET_GRID: { NAVIGATE_ON_ASSET_IN_VIEW: getBoolean(localStorage.getItem('ASSET_GRID.NAVIGATE_ON_ASSET_IN_VIEW'), false), + LARGE_BUCKET_THRESHOLD: getNumber(localStorage.getItem('ASSET_GRID.LARGE_BUCKET_THRESHOLD'), 3000), + LARGE_BUCKET_DEBOUNCE_MS: getNumber(localStorage.getItem('ASSET_GRID.LARGE_BUCKET_DEBOUNCE_MS'), 200), }, BUCKET: { PRIORITY: getNumber(localStorage.getItem('BUCKET.PRIORITY'), 2), @@ -51,6 +53,7 @@ export const TUNABLES = { INTERSECTION_DISABLED: getBoolean(localStorage.getItem('DATEGROUP.INTERSECTION_DISABLED'), false), INTERSECTION_ROOT_TOP: localStorage.getItem('DATEGROUP.INTERSECTION_ROOT_TOP') || '150%', INTERSECTION_ROOT_BOTTOM: localStorage.getItem('DATEGROUP.INTERSECTION_ROOT_BOTTOM') || '150%', + SMALL_GROUP_THRESHOLD: getNumber(localStorage.getItem('DATEGROUP.SMALL_GROUP_THRESHOLD'), 100), }, THUMBNAIL: { PRIORITY: getNumber(localStorage.getItem('THUMBNAIL.PRIORITY'), 8),