Emoji substring search (#39353)

app/javascript/mastodon/actions/importer/emoji.ts

@@ -18,7 +18,7 @@ export async function importCustomEmoji(emojis: ApiCustomEmojiJSON[]) {
   );
 
   // If there's a mismatch, re-import all custom emojis.
-  if (existingEmojis.length < emojis.length) {
+  if (existingEmojis.length > 0 && existingEmojis.length < emojis.length) {
     await clearCache('custom');
     await loadCustomEmoji();
 

app/javascript/mastodon/features/emoji/database.test.ts

@@ -22,6 +22,10 @@ function rawEmojiFactory(data: Partial<CompactEmoji> = {}): CompactEmoji {
 }
 
 describe('emoji database', () => {
+  beforeEach(async () => {
+    await testGet(); // Loads the database schema.
+  });
+
   afterEach(() => {
     testClear();
     indexedDB = new IDBFactory();

app/javascript/mastodon/features/emoji/database.ts

@@ -143,7 +143,28 @@ export async function search({
         return intersection;
       })
       .values(),
-  ).toSorted((a, b) => a.score - b.score);
+  );
+
+  // If there are no results, try a cursor-based custom emoji search instead.
+  if (results.length === 0) {
+    const trx = db.transaction('custom', 'readonly');
+    const foundEmojis = new Set<string>();
+    for await (const cursor of trx.store) {
+      const emoji = cursor.value;
+      const score = getScoreForEmoji(emoji, query, false);
+      if (score === null || foundEmojis.has(emoji.shortcode)) {
+        continue;
+      }
+
+      results.push({ ...emoji, score });
+      foundEmojis.add(emoji.shortcode);
+    }
+    log('cursor search found %d results for "%s"', foundEmojis.size, query);
+    await trx.done;
+  }
+
+  // Sort by score, descending.
+  results.sort((a, b) => a.score - b.score);
 
   const time = performance.measure('emoji-search-end', 'emoji-search-start');
   log(
@@ -159,14 +180,19 @@ export async function search({
   return results;
 }
 
-function getScoreForEmoji(emoji: AnyEmojiData, query: string) {
+function getScoreForEmoji(
+  emoji: AnyEmojiData,
+  query: string,
+  checkTokens = true,
+) {
   const id = 'shortcode' in emoji ? emoji.shortcode : emoji.label;
   if (id === query) {
     return 0;
   }
 
   let index = 1;
-  for (const token of [id, ...emoji.tokens]) {
+  const searchTokens = checkTokens ? [id, ...emoji.tokens] : [id];
+  for (const token of searchTokens) {
     const tokenIndex = token.indexOf(query);
     if (tokenIndex !== -1) {
       return index + tokenIndex / token.length;
@@ -246,6 +272,13 @@ export async function clearCache(key: CacheKey) {
   log('Cleared cache for %s', key);
 }
 
+export async function resetDatabase() {
+  const db = await loadDB();
+  const storeNames = [...db.objectStoreNames];
+  await Promise.all(storeNames.map((storeName) => db.clear(storeName)));
+  log(storeNames, 'Reset emoji database stores:');
+}
+
 export async function loadEmojiByHexcode(
   hexcode: string,
   localeString: string,

app/javascript/mastodon/features/emoji/db-schema.ts

@@ -10,6 +10,7 @@ import type {
   StoreNames,
 } from 'idb';
 
+import { resetDatabase } from './database';
 import type { CustomEmojiData, CacheKey, UnicodeEmojiData } from './types';
 import { emojiLogger } from './utils';
 
@@ -57,7 +58,7 @@ type Transaction<Mode extends IDBTransactionMode = 'versionchange'> =
 
 export type Database = IDBPDatabase<EmojiDB>;
 
-const SCHEMA_VERSION = 3;
+const SCHEMA_VERSION = 4;
 
 export async function openEmojiDB() {
   const db = await openDB<EmojiDB>('mastodon-emoji', SCHEMA_VERSION, {
@@ -98,6 +99,8 @@ export async function openEmojiDB() {
       });
       deleteOldIndexes(shortcodeTable, ['hexcode']);
 
+      void resetDatabase();
+
       log(
         'Upgraded emoji database from version %d to %d',
         oldVersion,

app/javascript/mastodon/features/emoji/emoji_picker.tsx

@@ -57,6 +57,8 @@ export const Emoji: FC<EmojiProps> = ({
   const { mode } = useEmojiAppState();
   return (
     <EmojiRaw
+      // eslint-disable-next-line @typescript-eslint/no-deprecated -- In React props are frozen, but the library we use is old so doesn't respect that.
+      {...(EmojiRaw.defaultProps ?? {})}
       data={EmojiData}
       set={set}
       sheetSize={sheetSize}

app/javascript/mastodon/features/emoji/index.ts

@@ -1,9 +1,8 @@
 import { initialState } from '@/mastodon/initial_state';
 
-import type { EMOJI_DB_NAME_SHORTCODES } from './constants';
 import { toSupportedLocale } from './locale';
 import { reloadCustomEmojis } from './picker';
-import type { LocaleOrCustom } from './types';
+import type { EmojiWorkerMessage } from './types';
 import { emojiLogger } from './utils';
 
 const userLocale = toSupportedLocale(initialState?.meta.locale ?? 'en');
@@ -11,9 +10,10 @@ const userLocale = toSupportedLocale(initialState?.meta.locale ?? 'en');
 let worker: Worker | null = null;
 
 const log = emojiLogger('index');
+const workerLog = emojiLogger('worker');
 
 // This is too short, but better to fallback quickly than wait.
-const WORKER_TIMEOUT = 1_000;
+const WORKER_TIMEOUT = 2_000;
 
 export async function initializeEmoji() {
   log('initializing emojis');
@@ -43,43 +43,44 @@ export async function initializeEmoji() {
     const { data: message } = event;
 
     worker ??= tempWorker;
+    clearTimeout(timeoutId);
 
-    if (message === 'ready') {
+    if (message !== 'ready') {
-      log('worker ready, loading data');
+      workerLog(message);
-      clearTimeout(timeoutId);
+      return;
-      messageWorker('shortcodes');
-      void loadCustomEmoji();
-      void loadEmojiLocale(userLocale);
-    } else {
-      log('got worker message: %s', message);
     }
+
+    const debugValue = localStorage.getItem('debug');
+    if (debugValue) {
+      messageWorker({ type: 'debug', debugValue });
+    }
+
+    workerLog('loading data');
+    messageWorker(userLocale);
+    messageWorker('custom');
+    messageWorker('shortcodes');
   });
 }
 
 async function fallbackLoad() {
   log('falling back to main thread for loading');
 
-  await loadCustomEmoji();
+  const { importCustomEmojiData, importLegacyShortcodes, importEmojiData } =
-  const { importLegacyShortcodes } = await import('./loader');
+    await import('./loader');
+
+  const customEmojis = await importCustomEmojiData();
+  if (customEmojis && customEmojis.length > 0) {
+    log('loaded %d custom emojis', customEmojis.length);
+    await reloadCustomEmojis();
+  }
   const shortcodes = await importLegacyShortcodes();
   if (shortcodes?.length) {
     log('loaded %d legacy shortcodes', shortcodes.length);
   }
-  await loadEmojiLocale(userLocale);
-}
 
-async function loadEmojiLocale(localeString: string) {
+  const emojis = await importEmojiData(userLocale);
-  const locale = toSupportedLocale(localeString);
+  if (emojis) {
-  const { importEmojiData } = await import('./loader');
+    log('loaded %d emojis to locale %s', emojis.length, userLocale);
-
-  if (worker) {
-    log('asking worker to load locale %s', locale);
-    messageWorker(locale);
-  } else {
-    const emojis = await importEmojiData(locale);
-    if (emojis) {
-      log('loaded %d emojis to locale %s', emojis.length, locale);
-    }
   }
 }
 
@@ -96,11 +97,16 @@ export async function loadCustomEmoji() {
   }
 }
 
-function messageWorker(
+function messageWorker(data: EmojiWorkerMessage | string) {
-  locale: LocaleOrCustom | typeof EMOJI_DB_NAME_SHORTCODES,
-) {
   if (!worker) {
     return;
   }
-  worker.postMessage({ locale });
+  if (typeof data === 'string') {
+    worker.postMessage({
+      type: 'load',
+      storeName: data,
+    } satisfies EmojiWorkerMessage);
+  } else {
+    worker.postMessage(data);
+  }
 }

app/javascript/mastodon/features/emoji/normalize.test.ts

@@ -4,7 +4,7 @@ import { basename, resolve } from 'path';
 import { flattenEmojiData } from 'emojibase';
 import unicodeRawEmojis from 'emojibase-data/en/data.json';
 
-import { unicodeToTwemojiHex } from './normalize';
+import { extractTokens, unicodeToTwemojiHex } from './normalize';
 
 const emojiSVGFiles = await readdir(
   // This assumes tests are run from project root
@@ -33,3 +33,32 @@ describe('unicodeToTwemojiHex', () => {
     expect(svgFileNamesWithoutBorder).toContain(result);
   });
 });
+
+describe('extractTokens', () => {
+  test('returns an empty array for blank input', () => {
+    expect(extractTokens('   ', null)).toEqual([]);
+  });
+
+  test('check token word breaking with Intl.Segmenter', () => {
+    const segmenter = new Intl.Segmenter('en', { granularity: 'word' });
+
+    expect(
+      extractTokens('thumbs_up smiling-face camelCase', segmenter),
+    ).toEqual(['thumbs', 'up', 'smiling', 'face', 'camel', 'case']);
+  });
+
+  test('check token word breaking with regex', () => {
+    expect(extractTokens('Smile_face joy-test A ok 7 z', null)).toEqual([
+      'smile',
+      'face',
+      'joy',
+      'test',
+      'ok',
+    ]);
+  });
+
+  test('ensure +1 and -1 are preserved', () => {
+    expect(extractTokens('+1', null)).toEqual(['+1']);
+    expect(extractTokens('-1', null)).toEqual(['-1']);
+  });
+});

app/javascript/mastodon/features/emoji/normalize.ts

@@ -214,6 +214,11 @@ export function extractTokens(
   }
   const tokens: string[] = [];
 
+  // Handle the edge case of thumbs up and down emoticons.
+  if (input === '+1' || input === '-1') {
+    return [input];
+  }
+
   // Prefer to use Intl.Segmenter if available for better locale support.
   if (segmenter) {
     for (const { isWordLike, segment } of segmenter.segment(

app/javascript/mastodon/features/emoji/picker.ts

@@ -3,6 +3,7 @@ import { useEffect, useState } from 'react';
 import type { CategoryName, CustomEmoji } from 'emoji-mart';
 
 import { autoPlayGif } from '@/mastodon/initial_state';
+import { createLimitedCache } from '@/mastodon/utils/cache';
 
 import { emojiLogger } from './utils';
 
@@ -21,6 +22,8 @@ let customCategories = [
   'flags',
 ] as CategoryName[];
 
+const searchCache = createLimitedCache<LegacyEmoji[]>({ maxSize: 10, log });
+
 export async function fetchCustomEmojiData() {
   if (customEmojis !== null) {
     return customEmojis;
@@ -89,6 +92,7 @@ export async function reloadCustomEmojis() {
     await import('@/mastodon/hooks/useCustomEmojis');
 
   await Promise.all([fetchCustomEmojiData(), loadEmojisIntoCache()]);
+  searchCache.clear();
 }
 
 // Replicates the old legacy search function.
@@ -102,16 +106,25 @@ export async function emojiMartSearch(
     return [];
   }
 
+  const cacheKey = `${query}|${locale}|${limit}`;
+  const cachedResult = searchCache.get(cacheKey);
+  if (cachedResult) {
+    return cachedResult;
+  }
+
   const { search } = await import('./database');
   const results = await search({ query, locale, limit });
-  return results.map((emoji) =>
+  const legacyResults = results.map((emoji) =>
     'shortcode' in emoji
-      ? { id: emoji.shortcode, custom: true }
+      ? ({ id: emoji.shortcode, custom: true } as const)
       : {
           id: emoji.label.replaceAll(' ', '_').toLowerCase(),
           native: emoji.unicode,
         },
   );
+  searchCache.set(cacheKey, legacyResults);
+
+  return legacyResults;
 }
 
 export function usePickerEmojis() {

app/javascript/mastodon/features/emoji/types.ts

@@ -80,3 +80,13 @@ export type ExtraCustomEmojiMap = Record<
   string,
   Pick<CustomEmojiData, 'shortcode' | 'static_url' | 'url'>
 >;
+
+export type EmojiWorkerMessage =
+  | {
+      type: 'load';
+      storeName: string;
+    }
+  | {
+      type: 'debug';
+      debugValue: string;
+    };

app/javascript/mastodon/features/emoji/utils.ts

@@ -5,6 +5,9 @@ import { emojiRegexPolyfill } from '@/mastodon/polyfills';
 import { VARIATION_SELECTOR_CODE } from './constants';
 
 export function emojiLogger(segment: string) {
+  if (typeof window === 'undefined') {
+    return debug(`emojis:worker:${segment}`);
+  }
   return debug(`emojis:${segment}`);
 }
 

app/javascript/mastodon/features/emoji/worker.ts

@@ -1,31 +1,36 @@
+import debug from 'debug';
+
 import { EMOJI_DB_NAME_SHORTCODES, EMOJI_TYPE_CUSTOM } from './constants';
 import {
   importCustomEmojiData,
   importEmojiData,
   importLegacyShortcodes,
 } from './loader';
+import type { EmojiWorkerMessage } from './types';
 
 addEventListener('message', handleMessage);
 self.postMessage('ready'); // After the worker is ready, notify the main thread
 
-function handleMessage(event: MessageEvent<{ locale: string }>) {
+function handleMessage(event: MessageEvent<EmojiWorkerMessage>) {
-  const {
+  const { data } = event;
-    data: { locale },
+  if (data.type === 'debug') {
-  } = event;
+    debug.enable(data.debugValue);
-  void loadData(locale);
+  } else {
+    void loadData(data.storeName);
+  }
 }
 
-async function loadData(locale: string) {
+async function loadData(storeName: string) {
   let importCount: number | undefined;
-  if (locale === EMOJI_TYPE_CUSTOM) {
+  if (storeName === EMOJI_TYPE_CUSTOM) {
     importCount = (await importCustomEmojiData())?.length;
-  } else if (locale === EMOJI_DB_NAME_SHORTCODES) {
+  } else if (storeName === EMOJI_DB_NAME_SHORTCODES) {
     importCount = (await importLegacyShortcodes())?.length;
   } else {
-    importCount = (await importEmojiData(locale))?.length;
+    importCount = (await importEmojiData(storeName))?.length;
   }
 
   if (importCount) {
-    self.postMessage(`loaded ${importCount} emojis into ${locale}`);
+    self.postMessage(`loaded ${importCount} emojis into ${storeName}`);
   }
 }

app/javascript/mastodon/utils/__tests__/cache.test.ts

@@ -40,11 +40,13 @@ describe('createCache', () => {
   });
 
   test('removes oldest item cached if it exceeds a set size', () => {
-    const cache = createLimitedCache({ maxSize: 1 });
+    const cache = createLimitedCache({ maxSize: 2 });
     cache.set('test1', 1);
     cache.set('test2', 2);
+    cache.set('test3', 3);
     expect(cache.get('test1')).toBeUndefined();
     expect(cache.get('test2')).toBe(2);
+    expect(cache.get('test3')).toBe(3);
   });
 
   test('retrieving a value bumps up last access', () => {
@@ -63,13 +65,13 @@ describe('createCache', () => {
     const cache = createLimitedCache({ maxSize: 1, log });
     cache.set('test1', 1);
     expect(log).toHaveBeenLastCalledWith(
-      'Added %s to cache, now size %d',
+      'Added %o to cache, now size %d',
       'test1',
       1,
     );
     cache.set('test2', 1);
     expect(log).toHaveBeenLastCalledWith(
-      'Added %s and deleted %s from cache, now size %d',
+      'Added %o and deleted %o from cache, now size %d',
       'test2',
       'test1',
       1,

app/javascript/mastodon/utils/cache.ts

@@ -43,13 +43,13 @@ export function createLimitedCache<CacheValue, CacheKey = string>({
         cacheMap.delete(lastKey);
         cacheKeys.delete(lastKey);
         log(
-          'Added %s and deleted %s from cache, now size %d',
+          'Added %o and deleted %o from cache, now size %d',
           key,
           lastKey,
           cacheMap.size,
         );
       } else {
-        log('Added %s to cache, now size %d', key, cacheMap.size);
+        log('Added %o to cache, now size %d', key, cacheMap.size);
       }
     },
     clear: () => {