From ee2a34739ca15e59870e77b117a512efff9becd7 Mon Sep 17 00:00:00 2001 From: Erwan Leboucher Date: Sat, 22 Aug 2026 17:35:36 +0200 Subject: [PATCH] fix(gifs): send gifs through the media proxy instead of uploading them --- .changeset/configurable-gif-provider.md | 2 +- config.json | 1 + src/app/features/room/RoomInput.test.tsx | 4 +- src/app/features/room/RoomInput.tsx | 6 +- src/app/features/room/msgContent.test.ts | 98 ++++++++++++------------ src/app/features/room/msgContent.ts | 92 +++++++--------------- src/app/utils/gifProviders.test.ts | 60 +++++++++++++++ src/app/utils/gifProviders.ts | 52 +++++++++++-- 8 files changed, 195 insertions(+), 120 deletions(-) diff --git a/.changeset/configurable-gif-provider.md b/.changeset/configurable-gif-provider.md index 624efe76e..c05f06433 100644 --- a/.changeset/configurable-gif-provider.md +++ b/.changeset/configurable-gif-provider.md @@ -2,4 +2,4 @@ default: minor --- -Add a configurable gif provider (Tenor, Giphy or Klipy), selectable in settings, and send picked gifs as uploaded images again +Add a configurable gif provider (Tenor, Giphy or Klipy) and send gifs through the media proxy diff --git a/config.json b/config.json index 7b997a09f..feac16efd 100644 --- a/config.json +++ b/config.json @@ -47,6 +47,7 @@ "gifs": { "provider": "tenor", + "proxyUrl": "gifs.sable.moe", "tenorApiKey": "AIzaSyCZt6SSh5VgVPzD9fhyzG1DprdPRhtoaR4", "klipyApiKey": "pmpZyPifwSulBfELHCbpaOllUfgsjqt9yeImc2XWIcHSWjnAUBw9oueRf4kD5r25", "giphyApiKey": "Gc7131jiJuvI7IdN0HZ1D7nh0ow5BU6g" diff --git a/src/app/features/room/RoomInput.test.tsx b/src/app/features/room/RoomInput.test.tsx index 37e26e2b3..3d9cdd29a 100644 --- a/src/app/features/room/RoomInput.test.tsx +++ b/src/app/features/room/RoomInput.test.tsx @@ -255,10 +255,10 @@ vi.mock('$components/upload-card', () => ({ })); vi.mock('./msgContent', async (importOriginal) => ({ ...(await importOriginal()), - getGifMsgContent: (_mx: unknown, gif: { title: string }) => ({ + getGifMsgContent: (gif: { title: string }) => ({ msgtype: 'm.image', body: `${gif.title}.gif`, - url: 'mxc://server/gif', + url: 'mxc://gifs.sable.moe/tenor_QWJDZEVmMTIz', info: { w: 320, h: 240, mimetype: 'image/gif' }, }), })); diff --git a/src/app/features/room/RoomInput.tsx b/src/app/features/room/RoomInput.tsx index a8b89f9e1..97b27b28d 100644 --- a/src/app/features/room/RoomInput.tsx +++ b/src/app/features/room/RoomInput.tsx @@ -194,6 +194,7 @@ import { buildGalleryContent, getGalleryItemContent, } from './msgContent'; +import { useClientConfig } from '$hooks/useClientConfig'; import { CommandAutocomplete } from './CommandAutocomplete'; import type { AudioMessageRecorderHandle, @@ -321,6 +322,7 @@ export const RoomInput = forwardRef( // don't clobber the main room draft (and vice versa). const draftKey = threadRootId ?? roomId; const mx = useMatrixClient(); + const gifProxyUrl = useClientConfig().gifs?.proxyUrl; const useAuthentication = useMediaAuthentication(); const [enterForNewline] = useSetting(settingsAtom, 'enterForNewline'); const [editorOldAddFile] = useSetting(settingsAtom, 'editorOldAddFile'); @@ -1900,8 +1902,8 @@ export const RoomInput = forwardRef( const submission = takeSubmission({ clearEditor: false }); return composerControllerRef.current?.enqueue(async (isLive) => { try { - const content = await getGifMsgContent(mx, gif, { - encrypt: room.hasEncryptionStateEvent(), + const content = await getGifMsgContent(gif, { + proxyUrl: gifProxyUrl, spoiler, }); if (!content) throw new Error('Unsendable GIF content'); diff --git a/src/app/features/room/msgContent.test.ts b/src/app/features/room/msgContent.test.ts index f77175439..dd0d14760 100644 --- a/src/app/features/room/msgContent.test.ts +++ b/src/app/features/room/msgContent.test.ts @@ -77,75 +77,79 @@ describe('TGS message content', () => { }); }); +const proxyId = (prefix: string, payload: string) => + `mxc://gifs.sable.moe/${prefix}_${btoa(payload).replaceAll('+', '-').replaceAll('/', '_').replace(/=+$/, '')}`; + describe('GIF message content', () => { const searchResult = { id: 'gif-id', title: 'Reaction', shareUrl: 'https://tenor.com/view/gif-id', - mediaUrl: 'https://media.tenor.com/gif-id/reaction.gif', + mediaUrl: 'https://media.tenor.com/AbCdEf123/reaction.gif', width: 480, height: 270, mimetype: 'image/gif', }; - it('uploads the gif and sends it as an image event', async () => { - fetchMock.mockResolvedValue(new Response('gif-bytes', { status: 200 })); - uploadMock.mockResolvedValue({ content_uri: 'mxc://server/uploaded' }); + it('sends a proxy mxc url without fetching or uploading the gif', async () => { + const content = await getGifMsgContent(searchResult, { proxyUrl: 'gifs.sable.moe' }); - const content = await getGifMsgContent({} as MatrixClient, searchResult, { encrypt: false }); - - expect(fetchMock).toHaveBeenCalledWith(searchResult.mediaUrl); + expect(fetchMock).not.toHaveBeenCalled(); + expect(uploadMock).not.toHaveBeenCalled(); expect(encryptFileMock).not.toHaveBeenCalled(); expect(content).toEqual({ msgtype: MsgType.Image, body: 'Reaction.gif', - url: 'mxc://server/uploaded', - info: { w: 480, h: 270, mimetype: 'image/gif', size: 9 }, - }); - }); - - it('encrypts the upload for encrypted rooms', async () => { - fetchMock.mockResolvedValue(new Response('gif-bytes', { status: 200 })); - uploadMock.mockResolvedValue({ content_uri: 'mxc://server/encrypted' }); - encryptFileMock.mockImplementation(async (file: File) => ({ - file, - encInfo: { key: { k: 'secret' } }, - })); - - const content = await getGifMsgContent({} as MatrixClient, searchResult, { - encrypt: true, - spoiler: true, - }); - - expect(content?.url).toBeUndefined(); - expect(content?.file).toEqual({ key: { k: 'secret' }, url: 'mxc://server/encrypted' }); - expect(content?.[MATRIX_UNSTABLE_SPOILER_PROPERTY_NAME]).toBe(true); - }); - - it('sends favorited homeserver gifs without re-uploading', async () => { - const content = await getGifMsgContent( - {} as MatrixClient, - { ...searchResult, mediaUrl: 'mxc://matrix.example/media-id' }, - { encrypt: false } - ); - - expect(fetchMock).not.toHaveBeenCalled(); - expect(content).toMatchObject({ - msgtype: MsgType.Image, - body: 'Reaction.gif', - url: 'mxc://matrix.example/media-id', + url: proxyId('tenor', 'AbCdEf123'), info: { w: 480, h: 270, mimetype: 'image/gif' }, }); }); - it('refuses media URLs outside the configured providers', async () => { + it('marks Giphy gifs as webp because that is what the proxy serves', async () => { + const content = await getGifMsgContent( + { + ...searchResult, + id: 'l0MYt5jPR6QX5pnqM', + mediaUrl: 'https://media0.giphy.com/media/l0MYt5jPR6QX5pnqM/giphy.gif', + }, + { proxyUrl: 'gifs.sable.moe' } + ); + + expect(content).toMatchObject({ + body: 'Reaction.webp', + url: proxyId('giphy', 'l0MYt5jPR6QX5pnqM'), + info: { mimetype: 'image/webp' }, + }); + }); + + it('encodes the Klipy cdn path after /ii/', async () => { + const content = await getGifMsgContent( + { ...searchResult, mediaUrl: 'https://static.klipy.com/ii/abc/e8/3a/x.gif' }, + { proxyUrl: 'gifs.sable.moe' } + ); + + expect(content?.url).toBe(proxyId('klipy', 'abc/e8/3a/x.gif')); + }); + + it('sends favorited homeserver gifs unchanged', async () => { + const content = await getGifMsgContent( + { ...searchResult, mediaUrl: 'mxc://matrix.example/media-id' }, + { proxyUrl: 'gifs.sable.moe', spoiler: true } + ); + + expect(content).toMatchObject({ + url: 'mxc://matrix.example/media-id', + [MATRIX_UNSTABLE_SPOILER_PROPERTY_NAME]: true, + }); + }); + + it('refuses to send when no proxy is configured or the host is unknown', async () => { + await expect(getGifMsgContent(searchResult, {})).resolves.toBeUndefined(); await expect( getGifMsgContent( - {} as MatrixClient, - { ...searchResult, mediaUrl: 'https://media.tenor.com.attacker.example/a.gif' }, - { encrypt: false } + { ...searchResult, mediaUrl: 'https://media.tenor.com.attacker.example/a/x.gif' }, + { proxyUrl: 'gifs.sable.moe' } ) ).resolves.toBeUndefined(); - expect(fetchMock).not.toHaveBeenCalled(); }); }); diff --git a/src/app/features/room/msgContent.ts b/src/app/features/room/msgContent.ts index 27288034c..6aab46989 100644 --- a/src/app/features/room/msgContent.ts +++ b/src/app/features/room/msgContent.ts @@ -24,8 +24,7 @@ import type { GifData } from '$components/emoji-board/types'; import { encodeBlurHashAsync } from '$utils/blurHash'; import { scaleYDimension } from '$utils/common'; import { createLogger } from '$utils/debug'; -import { isAllowedGifMediaUrl } from '$utils/gifProviders'; -import { fetch } from '$utils/fetch'; +import { getProxiedGif, isAllowedGifMediaUrl } from '$utils/gifProviders'; import { MATRIX_UNSTABLE_BLUR_HASH_PROPERTY_NAME, MATRIX_UNSTABLE_SPOILER_PROPERTY_NAME, @@ -270,78 +269,45 @@ export const getFileMsgContent = (item: TUploadItem, mxc: string): IContent => { return content; }; -export const getGifMsgContent = async ( - mx: MatrixClient, - gif: GifData, - options: { encrypt: boolean; spoiler?: boolean } -): Promise => { - const mimetype = gif.mimetype ?? 'image/gif'; - const ext = mimeTypeToExt(mimetype); - const body = gif.title.endsWith(`.${ext}`) ? gif.title : `${gif.title}.${ext}`; - const spoiler = options.spoiler ? { [MATRIX_UNSTABLE_SPOILER_PROPERTY_NAME]: true } : undefined; +const getGifBlurHash = async (gif: GifData): Promise => { + const source = + gif.preview_url && isAllowedGifMediaUrl(gif.preview_url) ? gif.preview_url : gif.mediaUrl; + if (!isAllowedGifMediaUrl(source) || !gif.width || !gif.height) return undefined; - // Favorites saved from a sent message already live on a homeserver. - if (gif.mediaUrl.startsWith('mxc://')) { - return { - msgtype: MsgType.Image, - body, - url: gif.mediaUrl, - info: { - w: gif.width, - h: gif.height, - mimetype, - ...(gif.size ? { size: gif.size } : {}), - }, - ...spoiler, - }; - } - - if (!isAllowedGifMediaUrl(gif.mediaUrl)) return undefined; - - const response = await fetch(gif.mediaUrl); - if (!response.ok) throw new Error(`Failed to fetch GIF: HTTP ${response.status}`); - const blob = await response.blob(); - const file = new File([blob], body, { type: mimetype }); - - const encData = options.encrypt ? await encryptFile(file) : undefined; - const uploadData = await uploadContentToServer(mx, encData?.file ?? file); - const mxc = uploadData?.content_uri; - if (!mxc) throw new Error('Failed when uploading GIF!'); - - const objectUrl = URL.createObjectURL(blob); - let imgEl: HTMLImageElement | undefined; try { - imgEl = await loadImageElement(objectUrl); + const imgEl = await loadImageElement(source, 'anonymous'); + return await encodeBlurHashAsync(imgEl, 32, scaleYDimension(gif.width, 32, gif.height)); } catch (e) { - log.warn('Failed to load GIF for blurhash, falling back to basic metadata:', e); - } finally { - URL.revokeObjectURL(objectUrl); + log.warn('Failed to load GIF for blurhash:', e); + return undefined; } +}; - const blurHash = imgEl - ? await encodeBlurHashAsync(imgEl, 512, scaleYDimension(imgEl.width, 512, imgEl.height)) - : undefined; +export const getGifMsgContent = async ( + gif: GifData, + options: { proxyUrl?: string; spoiler?: boolean } +): Promise => { + const proxied = gif.mediaUrl.startsWith('mxc://') + ? { mxcUrl: gif.mediaUrl, mimetype: gif.mimetype ?? 'image/gif' } + : getProxiedGif(gif, options.proxyUrl); + if (!proxied) return undefined; - const content: IContent = { + const ext = mimeTypeToExt(proxied.mimetype); + const blurHash = await getGifBlurHash(gif); + + return { msgtype: MsgType.Image, - body, + body: gif.title.endsWith(`.${ext}`) ? gif.title : `${gif.title}.${ext}`, + url: proxied.mxcUrl, info: { - w: imgEl?.width ?? gif.width, - h: imgEl?.height ?? gif.height, - mimetype, - size: blob.size, + w: gif.width, + h: gif.height, + mimetype: proxied.mimetype, + ...(gif.size && proxied.mimetype === (gif.mimetype ?? 'image/gif') ? { size: gif.size } : {}), ...(blurHash ? { [MATRIX_UNSTABLE_BLUR_HASH_PROPERTY_NAME]: blurHash } : {}), }, - ...spoiler, + ...(options.spoiler ? { [MATRIX_UNSTABLE_SPOILER_PROPERTY_NAME]: true } : {}), }; - - if (encData?.encInfo) { - content.file = { ...encData.encInfo, url: mxc }; - } else { - content.url = mxc; - } - - return content; }; const swapMsgTypeToItemType = ( diff --git a/src/app/utils/gifProviders.test.ts b/src/app/utils/gifProviders.test.ts index 4ca9ce5f5..af85918ca 100644 --- a/src/app/utils/gifProviders.test.ts +++ b/src/app/utils/gifProviders.test.ts @@ -3,6 +3,7 @@ import { GIF_PROVIDERS, getGifProvider, getGifProviderOptions, + getProxiedGif, isAllowedGifMediaUrl, } from './gifProviders'; @@ -193,3 +194,62 @@ describe('search response parsing', () => { expect(GIF_PROVIDERS.klipy.parseResults({ data: {} })).toEqual([]); }); }); + +describe('proxy mxc minting', () => { + const gif = { id: 'id', title: 'GIF', shareUrl: '', mediaUrl: '', width: 1, height: 1 }; + + it('uses the url-safe base64 alphabet with no padding', () => { + const proxied = getProxiedGif( + { ...gif, mediaUrl: 'https://static.klipy.com/ii/a?b/c~d/e+f/g.gif' }, + 'gifs.sable.moe' + ); + + expect(proxied?.mxcUrl.split('klipy_')[1]).toMatch(/^[\w-]+$/); + }); + + it('round-trips through the decoder the proxy uses', () => { + const path = 'ffd4ac143e6335ac68951b787d3c1902/e8/3a/5LM0jRpL.gif'; + const proxied = getProxiedGif( + { ...gif, mediaUrl: `https://static.klipy.com/ii/${path}` }, + 'gifs.sable.moe' + ); + const encoded = proxied?.mxcUrl.slice(proxied.mxcUrl.indexOf('_') + 1) ?? ''; + const normalized = encoded.replaceAll('-', '+').replaceAll('_', '/'); + + expect(atob(normalized.padEnd(Math.ceil(normalized.length / 4) * 4, '='))).toBe(path); + }); + + it('encodes the Tenor media id, not the /m/ path prefix', () => { + const full = getProxiedGif( + { ...gif, mediaUrl: 'https://media1.tenor.com/m/lfDATg4Bhc0AAAAC/happy-cat.gif' }, + 'gifs.sable.moe' + ); + const preview = getProxiedGif( + { ...gif, mediaUrl: 'https://media.tenor.com/lfDATg4Bhc0AAAAM/happy-cat.gif' }, + 'gifs.sable.moe' + ); + + expect(full?.mxcUrl).toBe('mxc://gifs.sable.moe/tenor_bGZEQVRnNEJoYzBBQUFBQw'); + expect(preview?.mxcUrl).toBe('mxc://gifs.sable.moe/tenor_bGZEQVRnNEJoYzBBQUFBTQ'); + expect(full?.mxcUrl).not.toBe('mxc://gifs.sable.moe/tenor_bQ'); + }); + + it('encodes the Giphy media id rather than the rendition path', () => { + const proxied = getProxiedGif( + { + ...gif, + id: 'tphCApwvdtC1VJabZ1', + mediaUrl: 'https://media4.giphy.com/media/v1.Y2lkPWVjZjA1ZTQ3/tphCApwvdtC1VJabZ1/giphy.gif', + }, + 'gifs.sable.moe' + ); + + expect(proxied?.mxcUrl).toBe('mxc://gifs.sable.moe/giphy_dHBoQ0Fwd3ZkdEMxVkphYlox'); + }); + + it('needs a configured proxy host', () => { + expect( + getProxiedGif({ ...gif, mediaUrl: 'https://media.tenor.com/abc/x.gif' }, ' ') + ).toBeUndefined(); + }); +}); diff --git a/src/app/utils/gifProviders.ts b/src/app/utils/gifProviders.ts index d85c0267a..c5944f8c4 100644 --- a/src/app/utils/gifProviders.ts +++ b/src/app/utils/gifProviders.ts @@ -6,6 +6,7 @@ export type GifProviderId = (typeof GIF_PROVIDER_IDS)[number]; export type GifsConfig = { provider?: GifProviderId; + proxyUrl?: string; klipyApiKey?: string; tenorApiKey?: string; giphyApiKey?: string; @@ -19,6 +20,8 @@ export type GifProvider = { buildSearchUrl: (apiKey: string, query: string) => string; parseResults: (payload: unknown) => GifData[]; isMediaUrlAllowed: (url: URL) => boolean; + getProxyPayload: (url: URL, gif: GifData) => string | undefined; + proxyMimetype: string; }; const RESULT_LIMIT = 50; // TODO: infinite scroll? @@ -35,6 +38,9 @@ const toPositiveInt = (value: unknown): number | undefined => { : undefined; }; +const idBeforeFilename = (url: URL): string | undefined => + url.pathname.split('/').filter(Boolean).at(-2); + const isPlainHttpsUrl = (url: URL): boolean => url.protocol === 'https:' && url.port === '' && url.username === '' && url.password === ''; @@ -172,11 +178,10 @@ const parseGiphyResults = (payload: unknown): GifData[] => { const images = isRecord(result.images) ? result.images : {}; const preview = parseGiphyRendition(images, 'fixed_width') ?? parseGiphyRendition(images, 'preview_gif'); - const fullRes = pickFullRes([ - parseGiphyRendition(images, 'original'), - parseGiphyRendition(images, 'downsized'), - preview, - ]); + const fullRes = + parseGiphyRendition(images, 'original') ?? + parseGiphyRendition(images, 'downsized') ?? + preview; return toGifData( typeof result.id === 'string' ? result.id : '', @@ -203,6 +208,8 @@ export const GIF_PROVIDERS: Record = { parseResults: parseKlipyResults, isMediaUrlAllowed: (url) => isPlainHttpsUrl(url) && url.hostname === 'static.klipy.com' && /^\/ii\/.+/.test(url.pathname), + getProxyPayload: (url) => url.pathname.slice('/ii/'.length) || undefined, + proxyMimetype: 'image/gif', }, tenor: { id: 'tenor', @@ -221,6 +228,8 @@ export const GIF_PROVIDERS: Record = { parseResults: parseTenorResults, isMediaUrlAllowed: (url) => isPlainHttpsUrl(url) && /^(?:c|media\d*)\.tenor\.com$/.test(url.hostname), + getProxyPayload: (url) => idBeforeFilename(url), + proxyMimetype: 'image/gif', }, giphy: { id: 'giphy', @@ -237,6 +246,8 @@ export const GIF_PROVIDERS: Record = { parseResults: parseGiphyResults, isMediaUrlAllowed: (url) => isPlainHttpsUrl(url) && /^(?:i|media\d*)\.giphy\.com$/.test(url.hostname), + getProxyPayload: (url, gif) => gif.id || idBeforeFilename(url), + proxyMimetype: 'image/webp', }, }; @@ -271,6 +282,37 @@ export const getGifProviderOptions = ( })), ]; +const toBase64Url = (value: string): string => { + const bytes = new TextEncoder().encode(value); + let binary = ''; + for (const byte of bytes) binary += String.fromCodePoint(byte); + return btoa(binary).replaceAll('+', '-').replaceAll('/', '_').replaceAll(/=+$/g, ''); +}; + +export type ProxiedGif = { mxcUrl: string; mimetype: string }; + +export const getProxiedGif = (gif: GifData, proxyUrl?: string): ProxiedGif | undefined => { + if (!proxyUrl?.trim()) return undefined; + + let url: URL; + try { + url = new URL(gif.mediaUrl); + } catch { + return undefined; + } + + const provider = Object.values(GIF_PROVIDERS).find((candidate) => + candidate.isMediaUrlAllowed(url) + ); + const payload = provider?.getProxyPayload(url, gif); + if (!provider || !payload) return undefined; + + return { + mxcUrl: `mxc://${proxyUrl.trim()}/${provider.id}_${toBase64Url(payload)}`, + mimetype: provider.proxyMimetype, + }; +}; + export const isAllowedGifMediaUrl = (value: string | URL): boolean => { try { const url = typeof value === 'string' ? new URL(value) : value;