diff --git a/app/javascript/mastodon/actions/importer/emoji.ts b/app/javascript/mastodon/actions/importer/emoji.ts index e9356ab621..36fb04b51e 100644 --- a/app/javascript/mastodon/actions/importer/emoji.ts +++ b/app/javascript/mastodon/actions/importer/emoji.ts @@ -18,7 +18,7 @@ export async function importCustomEmoji(emojis: ApiCustomEmojiJSON[]) { ); // If there's a mismatch, re-import all custom emojis. - if (existingEmojis.length < emojis.length) { + if (existingEmojis.length > 0 && existingEmojis.length < emojis.length) { await clearCache('custom'); await loadCustomEmoji(); diff --git a/app/javascript/mastodon/features/emoji/database.test.ts b/app/javascript/mastodon/features/emoji/database.test.ts index e272ec2dea..cbbbf9f165 100644 --- a/app/javascript/mastodon/features/emoji/database.test.ts +++ b/app/javascript/mastodon/features/emoji/database.test.ts @@ -22,6 +22,10 @@ function rawEmojiFactory(data: Partial = {}): CompactEmoji { } describe('emoji database', () => { + beforeEach(async () => { + await testGet(); // Loads the database schema. + }); + afterEach(() => { testClear(); indexedDB = new IDBFactory(); diff --git a/app/javascript/mastodon/features/emoji/database.ts b/app/javascript/mastodon/features/emoji/database.ts index 3dd2f95858..2ba20029bf 100644 --- a/app/javascript/mastodon/features/emoji/database.ts +++ b/app/javascript/mastodon/features/emoji/database.ts @@ -143,7 +143,28 @@ export async function search({ return intersection; }) .values(), - ).toSorted((a, b) => a.score - b.score); + ); + + // If there are no results, try a cursor-based custom emoji search instead. + if (results.length === 0) { + const trx = db.transaction('custom', 'readonly'); + const foundEmojis = new Set(); + for await (const cursor of trx.store) { + const emoji = cursor.value; + const score = getScoreForEmoji(emoji, query, false); + if (score === null || foundEmojis.has(emoji.shortcode)) { + continue; + } + + results.push({ ...emoji, score }); + foundEmojis.add(emoji.shortcode); + } + log('cursor search found %d results for "%s"', foundEmojis.size, query); + await trx.done; + } + + // Sort by score, descending. + results.sort((a, b) => a.score - b.score); const time = performance.measure('emoji-search-end', 'emoji-search-start'); log( @@ -159,14 +180,19 @@ export async function search({ return results; } -function getScoreForEmoji(emoji: AnyEmojiData, query: string) { +function getScoreForEmoji( + emoji: AnyEmojiData, + query: string, + checkTokens = true, +) { const id = 'shortcode' in emoji ? emoji.shortcode : emoji.label; if (id === query) { return 0; } let index = 1; - for (const token of [id, ...emoji.tokens]) { + const searchTokens = checkTokens ? [id, ...emoji.tokens] : [id]; + for (const token of searchTokens) { const tokenIndex = token.indexOf(query); if (tokenIndex !== -1) { return index + tokenIndex / token.length; @@ -246,6 +272,13 @@ export async function clearCache(key: CacheKey) { log('Cleared cache for %s', key); } +export async function resetDatabase() { + const db = await loadDB(); + const storeNames = [...db.objectStoreNames]; + await Promise.all(storeNames.map((storeName) => db.clear(storeName))); + log(storeNames, 'Reset emoji database stores:'); +} + export async function loadEmojiByHexcode( hexcode: string, localeString: string, diff --git a/app/javascript/mastodon/features/emoji/db-schema.ts b/app/javascript/mastodon/features/emoji/db-schema.ts index 7f4d09fd0b..f31ac96f81 100644 --- a/app/javascript/mastodon/features/emoji/db-schema.ts +++ b/app/javascript/mastodon/features/emoji/db-schema.ts @@ -10,6 +10,7 @@ import type { StoreNames, } from 'idb'; +import { resetDatabase } from './database'; import type { CustomEmojiData, CacheKey, UnicodeEmojiData } from './types'; import { emojiLogger } from './utils'; @@ -57,7 +58,7 @@ type Transaction = export type Database = IDBPDatabase; -const SCHEMA_VERSION = 3; +const SCHEMA_VERSION = 4; export async function openEmojiDB() { const db = await openDB('mastodon-emoji', SCHEMA_VERSION, { @@ -98,6 +99,8 @@ export async function openEmojiDB() { }); deleteOldIndexes(shortcodeTable, ['hexcode']); + void resetDatabase(); + log( 'Upgraded emoji database from version %d to %d', oldVersion, diff --git a/app/javascript/mastodon/features/emoji/emoji_picker.tsx b/app/javascript/mastodon/features/emoji/emoji_picker.tsx index 233c451aad..3661a7193b 100644 --- a/app/javascript/mastodon/features/emoji/emoji_picker.tsx +++ b/app/javascript/mastodon/features/emoji/emoji_picker.tsx @@ -57,6 +57,8 @@ export const Emoji: FC = ({ const { mode } = useEmojiAppState(); return ( 0) { + log('loaded %d custom emojis', customEmojis.length); + await reloadCustomEmojis(); + } const shortcodes = await importLegacyShortcodes(); if (shortcodes?.length) { log('loaded %d legacy shortcodes', shortcodes.length); } - await loadEmojiLocale(userLocale); -} -async function loadEmojiLocale(localeString: string) { - const locale = toSupportedLocale(localeString); - const { importEmojiData } = await import('./loader'); - - if (worker) { - log('asking worker to load locale %s', locale); - messageWorker(locale); - } else { - const emojis = await importEmojiData(locale); - if (emojis) { - log('loaded %d emojis to locale %s', emojis.length, locale); - } + const emojis = await importEmojiData(userLocale); + if (emojis) { + log('loaded %d emojis to locale %s', emojis.length, userLocale); } } @@ -96,11 +97,16 @@ export async function loadCustomEmoji() { } } -function messageWorker( - locale: LocaleOrCustom | typeof EMOJI_DB_NAME_SHORTCODES, -) { +function messageWorker(data: EmojiWorkerMessage | string) { if (!worker) { return; } - worker.postMessage({ locale }); + if (typeof data === 'string') { + worker.postMessage({ + type: 'load', + storeName: data, + } satisfies EmojiWorkerMessage); + } else { + worker.postMessage(data); + } } diff --git a/app/javascript/mastodon/features/emoji/normalize.test.ts b/app/javascript/mastodon/features/emoji/normalize.test.ts index 8c6346709b..08587138c6 100644 --- a/app/javascript/mastodon/features/emoji/normalize.test.ts +++ b/app/javascript/mastodon/features/emoji/normalize.test.ts @@ -4,7 +4,7 @@ import { basename, resolve } from 'path'; import { flattenEmojiData } from 'emojibase'; import unicodeRawEmojis from 'emojibase-data/en/data.json'; -import { unicodeToTwemojiHex } from './normalize'; +import { extractTokens, unicodeToTwemojiHex } from './normalize'; const emojiSVGFiles = await readdir( // This assumes tests are run from project root @@ -33,3 +33,32 @@ describe('unicodeToTwemojiHex', () => { expect(svgFileNamesWithoutBorder).toContain(result); }); }); + +describe('extractTokens', () => { + test('returns an empty array for blank input', () => { + expect(extractTokens(' ', null)).toEqual([]); + }); + + test('check token word breaking with Intl.Segmenter', () => { + const segmenter = new Intl.Segmenter('en', { granularity: 'word' }); + + expect( + extractTokens('thumbs_up smiling-face camelCase', segmenter), + ).toEqual(['thumbs', 'up', 'smiling', 'face', 'camel', 'case']); + }); + + test('check token word breaking with regex', () => { + expect(extractTokens('Smile_face joy-test A ok 7 z', null)).toEqual([ + 'smile', + 'face', + 'joy', + 'test', + 'ok', + ]); + }); + + test('ensure +1 and -1 are preserved', () => { + expect(extractTokens('+1', null)).toEqual(['+1']); + expect(extractTokens('-1', null)).toEqual(['-1']); + }); +}); diff --git a/app/javascript/mastodon/features/emoji/normalize.ts b/app/javascript/mastodon/features/emoji/normalize.ts index 257cddfcb6..3583d178eb 100644 --- a/app/javascript/mastodon/features/emoji/normalize.ts +++ b/app/javascript/mastodon/features/emoji/normalize.ts @@ -214,6 +214,11 @@ export function extractTokens( } const tokens: string[] = []; + // Handle the edge case of thumbs up and down emoticons. + if (input === '+1' || input === '-1') { + return [input]; + } + // Prefer to use Intl.Segmenter if available for better locale support. if (segmenter) { for (const { isWordLike, segment } of segmenter.segment( diff --git a/app/javascript/mastodon/features/emoji/picker.ts b/app/javascript/mastodon/features/emoji/picker.ts index fccf6769c3..7acf878f51 100644 --- a/app/javascript/mastodon/features/emoji/picker.ts +++ b/app/javascript/mastodon/features/emoji/picker.ts @@ -3,6 +3,7 @@ import { useEffect, useState } from 'react'; import type { CategoryName, CustomEmoji } from 'emoji-mart'; import { autoPlayGif } from '@/mastodon/initial_state'; +import { createLimitedCache } from '@/mastodon/utils/cache'; import { emojiLogger } from './utils'; @@ -21,6 +22,8 @@ let customCategories = [ 'flags', ] as CategoryName[]; +const searchCache = createLimitedCache({ maxSize: 10, log }); + export async function fetchCustomEmojiData() { if (customEmojis !== null) { return customEmojis; @@ -89,6 +92,7 @@ export async function reloadCustomEmojis() { await import('@/mastodon/hooks/useCustomEmojis'); await Promise.all([fetchCustomEmojiData(), loadEmojisIntoCache()]); + searchCache.clear(); } // Replicates the old legacy search function. @@ -102,16 +106,25 @@ export async function emojiMartSearch( return []; } + const cacheKey = `${query}|${locale}|${limit}`; + const cachedResult = searchCache.get(cacheKey); + if (cachedResult) { + return cachedResult; + } + const { search } = await import('./database'); const results = await search({ query, locale, limit }); - return results.map((emoji) => + const legacyResults = results.map((emoji) => 'shortcode' in emoji - ? { id: emoji.shortcode, custom: true } + ? ({ id: emoji.shortcode, custom: true } as const) : { id: emoji.label.replaceAll(' ', '_').toLowerCase(), native: emoji.unicode, }, ); + searchCache.set(cacheKey, legacyResults); + + return legacyResults; } export function usePickerEmojis() { diff --git a/app/javascript/mastodon/features/emoji/types.ts b/app/javascript/mastodon/features/emoji/types.ts index 2eb0c0f232..5c9dc6e5c6 100644 --- a/app/javascript/mastodon/features/emoji/types.ts +++ b/app/javascript/mastodon/features/emoji/types.ts @@ -80,3 +80,13 @@ export type ExtraCustomEmojiMap = Record< string, Pick >; + +export type EmojiWorkerMessage = + | { + type: 'load'; + storeName: string; + } + | { + type: 'debug'; + debugValue: string; + }; diff --git a/app/javascript/mastodon/features/emoji/utils.ts b/app/javascript/mastodon/features/emoji/utils.ts index 670e63a3c3..1722cde08c 100644 --- a/app/javascript/mastodon/features/emoji/utils.ts +++ b/app/javascript/mastodon/features/emoji/utils.ts @@ -5,6 +5,9 @@ import { emojiRegexPolyfill } from '@/mastodon/polyfills'; import { VARIATION_SELECTOR_CODE } from './constants'; export function emojiLogger(segment: string) { + if (typeof window === 'undefined') { + return debug(`emojis:worker:${segment}`); + } return debug(`emojis:${segment}`); } diff --git a/app/javascript/mastodon/features/emoji/worker.ts b/app/javascript/mastodon/features/emoji/worker.ts index 5602577dbe..d4f002daf6 100644 --- a/app/javascript/mastodon/features/emoji/worker.ts +++ b/app/javascript/mastodon/features/emoji/worker.ts @@ -1,31 +1,36 @@ +import debug from 'debug'; + import { EMOJI_DB_NAME_SHORTCODES, EMOJI_TYPE_CUSTOM } from './constants'; import { importCustomEmojiData, importEmojiData, importLegacyShortcodes, } from './loader'; +import type { EmojiWorkerMessage } from './types'; addEventListener('message', handleMessage); self.postMessage('ready'); // After the worker is ready, notify the main thread -function handleMessage(event: MessageEvent<{ locale: string }>) { - const { - data: { locale }, - } = event; - void loadData(locale); +function handleMessage(event: MessageEvent) { + const { data } = event; + if (data.type === 'debug') { + debug.enable(data.debugValue); + } else { + void loadData(data.storeName); + } } -async function loadData(locale: string) { +async function loadData(storeName: string) { let importCount: number | undefined; - if (locale === EMOJI_TYPE_CUSTOM) { + if (storeName === EMOJI_TYPE_CUSTOM) { importCount = (await importCustomEmojiData())?.length; - } else if (locale === EMOJI_DB_NAME_SHORTCODES) { + } else if (storeName === EMOJI_DB_NAME_SHORTCODES) { importCount = (await importLegacyShortcodes())?.length; } else { - importCount = (await importEmojiData(locale))?.length; + importCount = (await importEmojiData(storeName))?.length; } if (importCount) { - self.postMessage(`loaded ${importCount} emojis into ${locale}`); + self.postMessage(`loaded ${importCount} emojis into ${storeName}`); } } diff --git a/app/javascript/mastodon/utils/__tests__/cache.test.ts b/app/javascript/mastodon/utils/__tests__/cache.test.ts index 340a51fdb4..8d9d9c78ad 100644 --- a/app/javascript/mastodon/utils/__tests__/cache.test.ts +++ b/app/javascript/mastodon/utils/__tests__/cache.test.ts @@ -40,11 +40,13 @@ describe('createCache', () => { }); test('removes oldest item cached if it exceeds a set size', () => { - const cache = createLimitedCache({ maxSize: 1 }); + const cache = createLimitedCache({ maxSize: 2 }); cache.set('test1', 1); cache.set('test2', 2); + cache.set('test3', 3); expect(cache.get('test1')).toBeUndefined(); expect(cache.get('test2')).toBe(2); + expect(cache.get('test3')).toBe(3); }); test('retrieving a value bumps up last access', () => { @@ -63,13 +65,13 @@ describe('createCache', () => { const cache = createLimitedCache({ maxSize: 1, log }); cache.set('test1', 1); expect(log).toHaveBeenLastCalledWith( - 'Added %s to cache, now size %d', + 'Added %o to cache, now size %d', 'test1', 1, ); cache.set('test2', 1); expect(log).toHaveBeenLastCalledWith( - 'Added %s and deleted %s from cache, now size %d', + 'Added %o and deleted %o from cache, now size %d', 'test2', 'test1', 1, diff --git a/app/javascript/mastodon/utils/cache.ts b/app/javascript/mastodon/utils/cache.ts index 2e3d21bfed..d00b6fc9e0 100644 --- a/app/javascript/mastodon/utils/cache.ts +++ b/app/javascript/mastodon/utils/cache.ts @@ -43,13 +43,13 @@ export function createLimitedCache({ cacheMap.delete(lastKey); cacheKeys.delete(lastKey); log( - 'Added %s and deleted %s from cache, now size %d', + 'Added %o and deleted %o from cache, now size %d', key, lastKey, cacheMap.size, ); } else { - log('Added %s to cache, now size %d', key, cacheMap.size); + log('Added %o to cache, now size %d', key, cacheMap.size); } }, clear: () => {