Improve emoji search (#39815)

app/javascript/mastodon/actions/compose.js

@@ -572,7 +572,7 @@ const fetchComposeSuggestionsTags = throttle((dispatch, token) => {
 }, 200, { leading: true, trailing: true });
 
 export function fetchComposeSuggestions(token) {
-  return (dispatch, getState) => {
+  return (dispatch) => {
     switch (token[0]) {
     case ':':
       void fetchComposeSuggestionsEmojis(dispatch, token);

app/javascript/mastodon/components/autosuggest/utils.test.ts

@@ -10,6 +10,14 @@ describe('textAtCursorMatchesToken', () => {
       ['#hash tag', 8, ['#']],
       [1, '#hash tag'],
     ],
+    [
+      [':+1', 2, [':']],
+      [1, ':+1'],
+    ],
+    [
+      [':-1', 2, [':']],
+      [1, ':-1'],
+    ],
     [
       ['#ハッシュタグ', 6, ['#']],
       [1, '#ハッシュタグ'],

app/javascript/mastodon/components/autosuggest/utils.ts

@@ -8,7 +8,7 @@ export const textAtCursorMatchesToken = (
   let word: string;
 
   const regex = new RegExp(
-    `[${searchTokens.join('')}${WORD}]+(\\s[${WORD}]+)?$`,
+    `[${searchTokens.join('')}${WORD}+-]+(\\s[${WORD}]+)?$`,
     'iu',
   );
   const left = str.slice(0, caretPosition).search(regex);

app/javascript/mastodon/features/emoji/database.test.ts

@@ -5,7 +5,6 @@ import { customEmojiFactory, unicodeEmojiFactory } from '@/testing/factories';
 
 import {
   putEmojiData,
-  search,
   loadEmojiByHexcode,
   testClear,
   testGet,
@@ -15,10 +14,11 @@ import {
 } from './database';
 
 function rawEmojiFactory(data: Partial<CompactEmoji> = {}): CompactEmoji {
+  const factory = unicodeEmojiFactory();
   return {
-    ...unicodeEmojiFactory(),
+    ...factory,
-    tags: ['test', 'emoji'],
     ...data,
+    tags: data.tags ?? factory.tokens,
   };
 }
 
@@ -32,108 +32,6 @@ describe('emoji database', () => {
     indexedDB = new IDBFactory();
   });
 
-  describe('search', () => {
-    beforeEach(async () => {
-      await putEmojiData([], 'en');
-    });
-
-    test('test no query tokens', async () => {
-      await putEmojiData([rawEmojiFactory()], 'en');
-      await expect(search({ query: '   ', locale: 'en' })).resolves.toEqual([]);
-    });
-
-    test('unicode results', async () => {
-      await putEmojiData(
-        [
-          rawEmojiFactory({
-            hexcode: 'unicode_hex',
-            label: 'Party Popper',
-            shortcodes: ['party_popper'],
-            unicode: '🎉',
-          }),
-        ],
-        'en',
-      );
-
-      await expect(
-        search({ query: 'party', locale: 'en' }),
-      ).resolves.toContainEqual(
-        expect.objectContaining({
-          hexcode: 'unicode_hex',
-        }),
-      );
-    });
-
-    test('custom results', async () => {
-      await putCustomEmojiData({
-        emojis: [customEmojiFactory({ shortcode: 'party_custom' })],
-      });
-
-      await expect(
-        search({ query: 'party', locale: 'en' }),
-      ).resolves.toContainEqual(
-        expect.objectContaining({
-          shortcode: 'party_custom',
-        }),
-      );
-    });
-
-    test('shortcode results', async () => {
-      await putEmojiData([rawEmojiFactory()], 'en');
-      await putLegacyShortcodes({
-        test: ['legacy_smile'],
-      });
-
-      await expect(
-        search({ query: 'legacy', locale: 'en' }),
-      ).resolves.toContainEqual(
-        expect.objectContaining({
-          hexcode: 'test',
-        }),
-      );
-    });
-
-    test('full custom emoji search', async () => {
-      await putCustomEmojiData({
-        emojis: [
-          customEmojiFactory({ shortcode: 'arrow' }),
-          customEmojiFactory({ shortcode: 'party_parrot' }),
-        ],
-      });
-
-      const result = await search({ query: 'arro', locale: 'en' });
-      expect(result).toContainEqual(
-        // Test for ordinary IDB search
-        expect.objectContaining({
-          shortcode: 'arrow',
-        }),
-      );
-      expect(result).toContainEqual(
-        // Test for manual iteration search
-        expect.objectContaining({
-          shortcode: 'party_parrot',
-        }),
-      );
-    });
-
-    test('limit test', async () => {
-      await putCustomEmojiData({
-        emojis: [
-          customEmojiFactory({ shortcode: 'limit' }),
-          customEmojiFactory({ shortcode: 'limit_extra' }),
-        ],
-      });
-
-      await expect(
-        search({ query: 'limit', locale: 'en', limit: 1 }),
-      ).resolves.toEqual([
-        expect.objectContaining({
-          shortcode: 'limit',
-        }),
-      ]);
-    });
-  });
-
   describe('putEmojiData', () => {
     test('adds to loaded locales', async () => {
       const { loadedLocales } = await testGet();

app/javascript/mastodon/features/emoji/database.ts

@@ -9,12 +9,11 @@ import type { Database } from './db-schema';
 import { importEmojiData } from './loader';
 import { localeToSegmenter, toSupportedLocale } from './locale';
 import {
-  extractTokens,
   skinHexcodeToEmoji,
   transformCustomEmojiData,
   transformEmojiData,
 } from './normalize';
-import type { AnyEmojiData, CacheKey, CustomEmojiData } from './types';
+import type { CacheKey } from './types';
 import { emojiLogger } from './utils';
 
 const loadedLocales = new Set<Locale>();
@@ -43,223 +42,22 @@ const loadDB = (() => {
   return loadPromise;
 })();
 
-type ScoreMap = Map<string, AnyEmojiData & { score: number }>;
+export async function rawSearch(query: string, locale: Locale, prefix = true) {
-
+  await toLoadedLocale(locale);
-export async function search({
-  query,
-  locale: localeString,
-  limit = 0,
-}: {
-  query: string;
-  locale: string;
-  limit?: number;
-}) {
-  performance.mark('emoji-search-start');
-
-  // Get the locale, and extract tokens from the query.
-  const locale = await toLoadedLocale(localeString);
-  const segmenter = localeToSegmenter(locale);
-  const queryTokens = extractTokens(query, segmenter);
-
-  if (queryTokens.length === 0) {
-    log('no tokens extracted from query "%s"', query);
-    return [];
-  }
-  const lastToken = queryTokens.at(-1);
-  if (!lastToken) {
-    throw new Error('Missing tokens from query');
-  }
-
-  log('searching for tokens %o in locale %s', queryTokens, locale);
-
-  // Create an array of emoji results
   const db = await loadDB();
-  const resultArrays: ScoreMap[] = [];
+  const range = prefix
-  const existingCustomShortcodes = new Set<string>();
+    ? IDBKeyRange.lowerBound(query)
-
+    : IDBKeyRange.only(query);
-  for (let i = 0; i < queryTokens.length; i++) {
+  const [unicodeResults, customResults, shortcodeResults] = await Promise.all([
-    const token = queryTokens[i];
+    db.getAllFromIndex(locale, 'tokens', range),
-    if (!token) continue;
+    db.getAllFromIndex('custom', 'tokens', range),
-
+    db.getAllFromIndex('shortcodes', 'shortcodes', range),
-    // Only query the range for the last token to allow partial matches.
+  ]);
-    const range =
+  return {
-      i === queryTokens.length - 1
+    unicodeResults,
-        ? IDBKeyRange.lowerBound(token)
+    customResults,
-        : IDBKeyRange.only(token);
+    shortcodeResults,
-
+  };
-    const [unicodeResults, customResults, shortcodeResults] = await Promise.all(
-      [
-        db.getAllFromIndex(locale, 'tokens', range),
-        db.getAllFromIndex('custom', 'tokens', range),
-        db.getAllFromIndex('shortcodes', 'shortcodes', range),
-      ],
-    );
-    const resultMap: ScoreMap = new Map();
-
-    for (const emoji of unicodeResults) {
-      const score = getScoreForEmoji(emoji, token);
-      if (score === null) {
-        continue;
-      }
-      resultMap.set(emoji.hexcode, { ...emoji, score });
-    }
-
-    for (const emoji of customResults) {
-      const score = getScoreForEmoji(emoji, token);
-      if (score === null) {
-        continue;
-      }
-      existingCustomShortcodes.add(emoji.shortcode);
-      resultMap.set(emoji.shortcode, { ...emoji, score });
-    }
-
-    for (const shortcodeResult of shortcodeResults) {
-      if (resultMap.has(shortcodeResult.hexcode)) {
-        continue;
-      }
-      const emoji = await db.get(locale, shortcodeResult.hexcode);
-      if (!emoji) {
-        continue;
-      }
-      // Score the emoji with the legacy shortcode, even though it's not part of the emoji.
-      const score = getScoreForEmoji(
-        {
-          ...emoji,
-          shortcodes: [...shortcodeResult.shortcodes, ...emoji.shortcodes],
-        },
-        token,
-      );
-      if (score === null) {
-        continue;
-      }
-      resultMap.set(emoji.hexcode, { ...emoji, score });
-    }
-
-    log('found %d results for token "%s"', resultMap.size, token);
-    resultArrays.push(resultMap);
-  }
-
-  // Utilize maps to find the intersection of all result sets.
-  const results = Array.from(
-    resultArrays
-      .reduce((prev, curr) => {
-        const intersection: ScoreMap = new Map();
-        for (const [code, emoji] of prev) {
-          if (curr.has(code)) {
-            intersection.set(code, emoji);
-          }
-        }
-        return intersection;
-      })
-      .values(),
-  );
-
-  // If there are no results, try a cursor-based custom emoji search instead.
-  if (results.length === 0 || results.length < limit) {
-    const customEmojisFound = await fullCustomSearch(
-      query,
-      existingCustomShortcodes,
-    );
-    if (customEmojisFound.length > 0) {
-      log(
-        'cursor search found %d results for "%s"',
-        customEmojisFound.length,
-        query,
-      );
-      results.push(...customEmojisFound);
-    }
-  }
-
-  // Sort by score, descending.
-  results.sort((a, b) => a.score - b.score);
-
-  const time = performance.measure('emoji-search-end', 'emoji-search-start');
-  log(
-    'search for "%s" in locale %s returned %d results and took %dms',
-    query,
-    locale,
-    results.length,
-    time.duration,
-  );
-  if (limit > 0) {
-    return results.slice(0, limit);
-  }
-  return results;
-}
-
-function getScoreForEmoji(
-  emoji: AnyEmojiData,
-  query: string,
-  checkTokens = true,
-) {
-  const id = 'shortcode' in emoji ? emoji.shortcode : emoji.label;
-  if (id === query) {
-    return 0;
-  }
-
-  let index = 1;
-  const searchTokens = [id];
-  if (checkTokens) {
-    // Check shortcodes before tokens as they are more important.
-    if ('shortcodes' in emoji) {
-      searchTokens.push(...emoji.shortcodes);
-    }
-    searchTokens.push(...emoji.tokens);
-  }
-  for (const token of searchTokens) {
-    const tokenIndex = token.indexOf(query);
-    if (tokenIndex !== -1) {
-      return index + tokenIndex / token.length;
-    }
-    index++;
-  }
-
-  return null;
-}
-
-async function fullCustomSearch(query: string, existing = new Set<string>()) {
-  const db = await loadDB();
-  const trx = db.transaction('custom', 'readonly');
-  const foundEmojis = new Set<string>();
-
-  // First iterate over chunks of 1,000 custom emoji keys and find any matches.
-  const chunkSize = 1_000;
-  let lastKey: string | null = null;
-  let keys: string[] = [];
-  do {
-    const keyRange = lastKey ? IDBKeyRange.lowerBound(lastKey, true) : null;
-    keys = await trx.store.getAllKeys(keyRange, chunkSize);
-
-    if (keys.length === 0) {
-      break;
-    }
-    log('cursor search got batch of %d emojis', keys.length);
-    lastKey = keys.at(-1) ?? null;
-
-    for (const key of keys) {
-      if (!foundEmojis.has(key) && !existing.has(key) && key.includes(query)) {
-        foundEmojis.add(key);
-      }
-    }
-  } while (keys.length === chunkSize);
-
-  // Next get the full emojis for all matches.
-  const emojis = await Promise.all(
-    foundEmojis.keys().map((key) => trx.store.get(key)),
-  );
-  const results: (CustomEmojiData & { score: number })[] = [];
-  for (const emoji of emojis) {
-    if (emoji) {
-      const score = getScoreForEmoji(emoji, query, false);
-      if (score && score > 0) {
-        results.push({
-          score,
-          ...emoji,
-        });
-      }
-    }
-  }
-  return results;
 }
 
 export async function putEmojiData(emojis: CompactEmoji[], locale: Locale) {
@@ -369,6 +167,9 @@ export async function loadCustomEmojiByShortcode(shortcode: string) {
 }
 
 export async function searchCustomEmojisByShortcodes(shortcodes: string[]) {
+  if (shortcodes.length === 0) {
+    return [];
+  }
   const db = await loadDB();
   const sortedCodes = shortcodes.toSorted();
   const results = await db.getAll(
@@ -378,6 +179,15 @@ export async function searchCustomEmojisByShortcodes(shortcodes: string[]) {
   return results.filter((emoji) => shortcodes.includes(emoji.shortcode));
 }
 
+export async function loadCustomEmojiKeys(
+  query?: string | null,
+  chunkSize = 1_000,
+) {
+  const db = await loadDB();
+  const keyRange = query ? IDBKeyRange.lowerBound(query, true) : null;
+  return db.getAllKeys('custom', keyRange, chunkSize);
+}
+
 export async function loadAllCustomEmoji() {
   const db = await loadDB();
   const cacheValue = await db.get('etags', 'custom');

app/javascript/mastodon/features/emoji/db-schema.ts

@@ -64,7 +64,7 @@ type Transaction<Mode extends IDBTransactionMode = 'versionchange'> =
 export type Database = IDBPDatabase<EmojiDB>;
 
 const DATABASE_NAME = 'mastodon-emoji';
-const SCHEMA_VERSION = 4;
+const SCHEMA_VERSION = 5;
 
 export async function openEmojiDB() {
   try {

app/javascript/mastodon/features/emoji/normalize.ts

@@ -60,7 +60,7 @@ export function transformEmojiData(
       ...extract(label),
       ...(normalizedEmoticons ?? []),
     ]),
-  ].sort((a, b) => a.localeCompare(b));
+  ];
 
   const res: UnicodeEmojiData = {
     tokens,

app/javascript/mastodon/features/emoji/picker.ts

@@ -7,6 +7,7 @@ import {
 } from '@/mastodon/store/typed_functions';
 import { createLimitedCache } from '@/mastodon/utils/cache';
 
+import { search } from './search';
 import { emojiLogger } from './utils';
 
 const log = emojiLogger('picker');
@@ -37,7 +38,6 @@ export async function emojiMartSearch(
     return cachedResult;
   }
 
-  const { search } = await import('./database');
   const results = await search({ query, locale, limit });
   const legacyResults = results.map((emoji) =>
     'shortcode' in emoji

app/javascript/mastodon/features/emoji/search.test.ts

@@ -0,0 +1,188 @@
+import type { CompactEmoji } from 'emojibase';
+
+import { unicodeEmojiFactory, customEmojiFactory } from '@/testing/factories';
+
+import {
+  putEmojiData,
+  putCustomEmojiData,
+  putLegacyShortcodes,
+  testGet,
+  testClear,
+} from './database';
+import { search } from './search';
+
+function rawEmojiFactory(data: Partial<CompactEmoji> = {}): CompactEmoji {
+  const factory = unicodeEmojiFactory();
+  return {
+    ...factory,
+    ...data,
+    tags: data.tags ?? factory.tokens,
+  };
+}
+
+describe('search', () => {
+  beforeEach(async () => {
+    await testGet(); // Loads the database schema.
+    await putEmojiData([], 'en');
+  });
+
+  afterEach(() => {
+    testClear();
+    indexedDB = new IDBFactory();
+  });
+
+  test('test no query tokens', async () => {
+    await putEmojiData([rawEmojiFactory()], 'en');
+    await expect(search({ query: '   ', locale: 'en' })).resolves.toEqual([]);
+  });
+
+  test('unicode results', async () => {
+    await putEmojiData(
+      [
+        rawEmojiFactory({
+          hexcode: 'unicode_hex',
+          label: 'Party Popper',
+          shortcodes: ['party_popper'],
+          unicode: '🎉',
+        }),
+      ],
+      'en',
+    );
+
+    await expect(
+      search({ query: 'party', locale: 'en' }),
+    ).resolves.toContainEqual(
+      expect.objectContaining({
+        hexcode: 'unicode_hex',
+      }),
+    );
+  });
+
+  test('custom results', async () => {
+    await putCustomEmojiData({
+      emojis: [customEmojiFactory({ shortcode: 'party_custom' })],
+    });
+
+    await expect(
+      search({ query: 'party', locale: 'en' }),
+    ).resolves.toContainEqual(
+      expect.objectContaining({
+        shortcode: 'party_custom',
+      }),
+    );
+  });
+
+  test('shortcode results', async () => {
+    await putEmojiData([rawEmojiFactory()], 'en');
+    await putLegacyShortcodes({
+      test: ['legacy_smile'],
+    });
+
+    await expect(
+      search({ query: 'legacy', locale: 'en' }),
+    ).resolves.toContainEqual(
+      expect.objectContaining({
+        hexcode: 'test',
+      }),
+    );
+  });
+
+  test('full custom emoji search', async () => {
+    await putCustomEmojiData({
+      emojis: [
+        customEmojiFactory({ shortcode: 'arrow' }),
+        customEmojiFactory({ shortcode: 'party_parrot' }),
+      ],
+    });
+
+    const result = await search({ query: 'arro', locale: 'en' });
+    expect(result).toContainEqual(
+      // Test for ordinary IDB search
+      expect.objectContaining({
+        shortcode: 'arrow',
+      }),
+    );
+    expect(result).toContainEqual(
+      // Test for manual iteration search
+      expect.objectContaining({
+        shortcode: 'party_parrot',
+      }),
+    );
+  });
+
+  test('limit test', async () => {
+    await putCustomEmojiData({
+      emojis: [
+        customEmojiFactory({ shortcode: 'limit' }),
+        customEmojiFactory({ shortcode: 'limit_extra' }),
+      ],
+    });
+
+    await expect(
+      search({ query: 'limit', locale: 'en', limit: 1 }),
+    ).resolves.toEqual([
+      expect.objectContaining({
+        shortcode: 'limit',
+      }),
+    ]);
+  });
+
+  test('prefix matches', async () => {
+    await putCustomEmojiData({
+      emojis: [
+        customEmojiFactory({ shortcode: 'sob_other' }),
+        customEmojiFactory({ shortcode: 'meow_sob' }),
+      ],
+    });
+    await putEmojiData(
+      [
+        rawEmojiFactory({
+          label: 'loudly crying face',
+          hexcode: '1F62D',
+          shortcodes: ['loudly_crying_face'],
+          tags: ['bawling', 'cry', 'sad', 'sob', 'tear', 'tears', 'unhappy'],
+          emoticon: ":'o",
+          unicode: '😭',
+        }),
+      ],
+      'en',
+    );
+
+    const results = await search({ query: 'sob', locale: 'en' });
+
+    expect(results).toHaveLength(3);
+    expect(results).toEqual([
+      expect.objectContaining({ shortcode: 'sob_other' }),
+      expect.objectContaining({ shortcode: 'meow_sob' }),
+      expect.objectContaining({ hexcode: '1F62D' }),
+    ]);
+  });
+
+  test('shortcode matches', async () => {
+    await putLegacyShortcodes({ '1F62D': 'sob' });
+    await putEmojiData(
+      [
+        rawEmojiFactory({
+          label: 'loudly crying face',
+          hexcode: '1F62D',
+          shortcodes: ['loudly_crying_face'],
+          tags: ['bawling', 'cry', 'sad', 'sob', 'tear', 'tears', 'unhappy'],
+          emoticon: ":'o",
+          unicode: '😭',
+        }),
+      ],
+      'en',
+    );
+    await putCustomEmojiData({
+      emojis: [customEmojiFactory({ shortcode: 'sob_other' })],
+    });
+
+    const results = await search({ query: 'sob', locale: 'en' });
+
+    expect(results).toHaveLength(2);
+    expect(results).toEqual([
+      expect.objectContaining({ hexcode: '1F62D' }),
+      expect.objectContaining({ shortcode: 'sob_other' }),
+    ]);
+  });
+});

app/javascript/mastodon/features/emoji/search.ts

@@ -0,0 +1,407 @@
+import { log } from 'debug';
+import type { ArrayValues, KeysOfUnion } from 'type-fest';
+
+import {
+  loadCustomEmojiKeys,
+  loadEmojiByHexcode,
+  rawSearch,
+  searchCustomEmojisByShortcodes,
+} from './database';
+import { localeToSegmenter, toSupportedLocale } from './locale';
+import { extractTokens } from './normalize';
+import type { AnyEmojiData, CustomEmojiData } from './types';
+
+/*
+Emoji search logic:
+1. When provided a string, extract all tokens and iterate over each.
+2. For each token, do an IDB lookup to get any matches on emojis and shortcodes.
+3. Score each emoji found (see below).
+4. Create a final list of emojis that is the intersection of all token results, with the scores combined.
+5. If not enough emoji are found, do a cursor search on custom emojis.
+6. Sort and return the emoji by ranked score, stopping when the optional limit is reached.
+
+Scoring functions as follows:
+- Go over every scoreRanking field in order and extract score data.
+- Scores prefer exact match, then prefix match, and lastly substring match.
+- When sorting, prefer identifier fields first, then score, then category, and lastly prioritize Unicode emojis.
+*/
+
+const scoreRanking = [
+  'label',
+  'shortcode',
+  'emoticons',
+  'shortcodes',
+  'tokens',
+] as const satisfies KeysOfUnion<AnyEmojiData>[];
+type ScoreRankingKeys = ArrayValues<typeof scoreRanking>;
+type ScoreRanking = Record<ScoreRankingKeys, number>;
+
+const identifierFields = new Set<ScoreRankingKeys>([
+  'label',
+  'shortcode',
+  'emoticons',
+  'shortcodes',
+]);
+
+interface BestRank {
+  categoryWeight: number;
+  score: number;
+  fieldWeight: number;
+}
+type ScoredEmoji = AnyEmojiData & { scores: ScoreRanking };
+type RankedEmoji = AnyEmojiData & { rank: BestRank };
+type ScoreMap = Map<string, ScoredEmoji>;
+
+export async function search({
+  query: rawQuery,
+  locale: localeString,
+  limit = 0,
+}: {
+  query: string;
+  locale: string;
+  limit?: number;
+}) {
+  performance.mark('emoji-search-start');
+
+  // Get the locale, and extract tokens from the query.
+  const locale = toSupportedLocale(localeString);
+  const segmenter = localeToSegmenter(locale);
+  const query = rawQuery.toLowerCase();
+  const queryTokens = extractTokens(query, segmenter);
+
+  if (queryTokens.length === 0) {
+    log('no tokens extracted from query "%s"', query);
+    return [];
+  }
+  const lastToken = queryTokens.at(-1);
+  if (!lastToken) {
+    throw new Error('Missing tokens from query');
+  }
+
+  log('searching for tokens %o in locale %s', queryTokens, locale);
+
+  // Create an array of emoji results
+  const resultArrays: ScoreMap[] = [];
+
+  for (let i = 0; i < queryTokens.length; i++) {
+    const token = queryTokens[i];
+    if (!token) continue;
+
+    // Only query the range for the last token to allow partial matches.
+    const { unicodeResults, customResults, shortcodeResults } = await rawSearch(
+      token,
+      locale,
+      i === queryTokens.length - 1,
+    );
+    const resultMap: ScoreMap = new Map();
+    const checkedSet = new Set<string>();
+
+    // Score unicode results.
+    for (const emoji of unicodeResults) {
+      if (checkedSet.has(emoji.hexcode)) {
+        continue;
+      }
+      checkedSet.add(emoji.hexcode);
+      const scores = getScoreForEmoji(emoji, token);
+      if (scores) {
+        resultMap.set(emoji.hexcode, { ...emoji, scores });
+      }
+    }
+
+    // Score custom emojis.
+    for (const emoji of customResults) {
+      if (checkedSet.has(emoji.shortcode)) {
+        continue;
+      }
+      checkedSet.add(emoji.shortcode);
+      const scores = getScoreForEmoji(emoji, token);
+      if (scores) {
+        resultMap.set(emoji.shortcode, { ...emoji, scores });
+      }
+    }
+
+    // Score based on legacy shortcodes, using the higher score if there's a match.
+    for (const shortcodeResult of shortcodeResults) {
+      const emoji =
+        resultMap.get(shortcodeResult.hexcode) ??
+        (await loadEmojiByHexcode(shortcodeResult.hexcode, locale));
+      if (!emoji || !('hexcode' in emoji)) {
+        continue;
+      }
+
+      const newScores = getScoreForEmoji(
+        {
+          ...emoji,
+          shortcodes: shortcodeResult.shortcodes,
+        },
+        token,
+      );
+      if (!newScores) {
+        continue;
+      }
+      const oldScores = resultMap.get(emoji.hexcode)?.scores;
+      const scores = oldScores
+        ? combineEmojiScores(oldScores, newScores)
+        : newScores;
+      resultMap.set(emoji.hexcode, {
+        ...emoji,
+        shortcodes: [...shortcodeResult.shortcodes, ...emoji.shortcodes],
+        scores,
+      });
+    }
+
+    log('found %d results for token "%s"', resultMap.size, token);
+    resultArrays.push(resultMap);
+  }
+
+  // Iterate over all maps, getting a combined score for emojis that exist in all results.
+  const allEmojiIds = resultArrays.reduce((prev, map) => {
+    if (prev.size === 0) {
+      return new Set(map.keys());
+    }
+    return new Set(map.keys()).intersection(prev);
+  }, new Set<string>());
+
+  const finalMap: ScoreMap = new Map();
+  for (const resultArray of resultArrays) {
+    for (const [id, emoji] of resultArray.entries()) {
+      if (!allEmojiIds.has(id)) {
+        continue;
+      }
+      const existingEmoji = finalMap.get(id);
+      if (!existingEmoji) {
+        finalMap.set(id, emoji);
+      } else {
+        finalMap.set(id, {
+          ...existingEmoji,
+          scores: combineEmojiScores(existingEmoji.scores, emoji.scores),
+        });
+      }
+    }
+  }
+  const mixedResults = Array.from(finalMap.values());
+
+  // If there are no results, try a cursor-based custom emoji search instead.
+  if (mixedResults.length === 0 || mixedResults.length < limit) {
+    const customEmojisFound = await fullCustomSearch(query, allEmojiIds);
+    if (customEmojisFound.length > 0) {
+      log(
+        'cursor search found %d results for "%s"',
+        customEmojisFound.length,
+        query,
+      );
+      mixedResults.push(...customEmojisFound);
+    }
+  }
+
+  const rankedEmojis = mixedResults.map(({ scores, ...emoji }) => ({
+    ...emoji,
+    rank: getBestRank(scores),
+  }));
+
+  const results: RankedEmoji[] = [];
+  const resultEmojis = new Set<string>();
+  for (const result of rankedEmojis.toSorted(compareRankedEmoji)) {
+    const id = getIdentifier(result);
+    if (resultEmojis.has(id)) {
+      continue;
+    }
+    results.push(result);
+    resultEmojis.add(id);
+    if (limit > 0 && results.length >= limit) {
+      break;
+    }
+  }
+
+  const time = performance.measure('emoji-search-end', 'emoji-search-start');
+  log(
+    'search for "%s" in locale %s returned %d results and took %dms',
+    query,
+    locale,
+    mixedResults.length,
+    time.duration,
+  );
+  return results;
+}
+
+function hasField(
+  emoji: AnyEmojiData,
+  field: string,
+): field is keyof typeof emoji {
+  return Object.hasOwn(emoji, field);
+}
+
+function getIdentifier(emoji: AnyEmojiData) {
+  return 'shortcode' in emoji ? emoji.shortcode : emoji.hexcode;
+}
+
+// Creates ranked scores for a given emoji.
+function getScoreForEmoji(emoji: AnyEmojiData, query: string) {
+  const scores = Object.fromEntries(
+    scoreRanking.map((field) => [field, -1]),
+  ) as ScoreRanking;
+  let hasScore = false;
+
+  for (const field of scoreRanking) {
+    if (hasField(emoji, field)) {
+      const value = emoji[field] as string | string[] | undefined;
+      if (value === undefined) {
+        continue;
+      }
+      const tokens = Array.isArray(value) ? value : [value];
+      const score = getScoreForEmojiTokens(
+        tokens.map((token) => token.toLowerCase()),
+        query,
+      );
+
+      if (score >= 0) {
+        scores[field] = score;
+        hasScore = true;
+      }
+    }
+  }
+
+  if (!hasScore) {
+    return null;
+  }
+
+  return scores;
+}
+
+function combineEmojiScores(a: ScoreRanking, b: ScoreRanking): ScoreRanking {
+  const scores = Object.fromEntries(
+    scoreRanking.map((field) => [field, -1]),
+  ) as ScoreRanking;
+
+  for (const rank of scoreRanking) {
+    if (a[rank] === -1) {
+      scores[rank] = b[rank];
+    } else if (b[rank] === -1) {
+      scores[rank] = a[rank];
+    } else {
+      scores[rank] = Math.min(a[rank], b[rank]);
+    }
+  }
+
+  return scores;
+}
+
+// Compares two scored emojis by getting the best rank.
+function compareRankedEmoji(a: RankedEmoji, b: RankedEmoji): number {
+  const rankA = a.rank;
+  const rankB = b.rank;
+
+  // Identifier matches always outrank token matches, regardless of score.
+  if (rankA.categoryWeight !== rankB.categoryWeight) {
+    return rankA.categoryWeight - rankB.categoryWeight;
+  }
+  // Within the same category, compare the scores directly.
+  if (rankA.score !== rankB.score) {
+    return rankA.score - rankB.score;
+  }
+  // If equal, compare field weights.
+  if (rankA.fieldWeight !== rankB.fieldWeight) {
+    return rankA.fieldWeight - rankB.fieldWeight;
+  }
+
+  // Lastly prioritize Unicode emojis.
+  const aIsCustom = hasField(a, 'shortcode');
+  const bIsCustom = hasField(b, 'shortcode');
+  if (aIsCustom !== bIsCustom) {
+    return aIsCustom ? -1 : 1;
+  }
+
+  return 0;
+}
+
+// Extracts the best rank for a score ranking.
+function getBestRank(scores: ScoreRanking): BestRank {
+  let best: BestRank | null = null;
+
+  // Use the index to determine field weight.
+  for (const [fieldWeight, field] of scoreRanking.entries()) {
+    const score = scores[field];
+    if (score < 0) {
+      continue;
+    }
+    // Also weight identifier fields over other fields.
+    const categoryWeight = identifierFields.has(field) ? 0 : 1;
+    if (
+      !best ||
+      categoryWeight < best.categoryWeight ||
+      (categoryWeight === best.categoryWeight &&
+        (score < best.score ||
+          (score === best.score && fieldWeight < best.fieldWeight)))
+    ) {
+      best = { categoryWeight, score, fieldWeight };
+    }
+  }
+
+  return (
+    best ?? {
+      categoryWeight: 2,
+      score: Infinity,
+      fieldWeight: scoreRanking.length,
+    }
+  );
+}
+
+function getScoreForEmojiTokens(tokens: string[], query: string) {
+  let lowestScore = -1;
+
+  for (const token of tokens) {
+    let score = -1;
+    // Priority: exact match, prefix match, substring,
+    if (token === query) {
+      score = 0;
+    } else if (token.startsWith(query)) {
+      score = 1 + query.length / token.length;
+    } else if (token.includes(query)) {
+      score = 2 + query.length / token.length;
+    } // TODO: Add fuzzy search if needed
+
+    if (score >= 0 && (score < lowestScore || lowestScore < 0)) {
+      lowestScore = score;
+    }
+  }
+
+  return lowestScore;
+}
+
+async function fullCustomSearch(query: string, existing = new Set<string>()) {
+  const foundEmojis = new Set<string>();
+
+  // First iterate over chunks of 1,000 custom emoji keys and find any matches.
+  const chunkSize = 1_000;
+  let lastKey: string | null = null;
+  let keys: string[] = [];
+  do {
+    keys = await loadCustomEmojiKeys(lastKey, chunkSize);
+
+    if (keys.length === 0) {
+      break;
+    }
+    log('cursor search got batch of %d emojis', keys.length);
+    lastKey = keys.at(-1) ?? null;
+
+    for (const key of keys) {
+      if (!foundEmojis.has(key) && !existing.has(key) && key.includes(query)) {
+        foundEmojis.add(key);
+      }
+    }
+  } while (keys.length === chunkSize);
+
+  // Next get the full emojis for all matches.
+  const emojis = await searchCustomEmojisByShortcodes(Array.from(foundEmojis));
+  const results: (CustomEmojiData & { scores: ScoreRanking })[] = [];
+  for (const emoji of emojis) {
+    const scores = getScoreForEmoji(emoji, query);
+    if (scores) {
+      results.push({
+        scores,
+        ...emoji,
+      });
+    }
+  }
+  return results;
+}

app/javascript/testing/factories.ts

@@ -126,11 +126,12 @@ export function unicodeEmojiFactory(
   data: Partial<UnicodeEmojiData> = {},
 ): UnicodeEmojiData {
   return {
+    emoticons: undefined,
     hexcode: 'test',
     label: 'Test',
     unicode: '🧪',
     shortcodes: ['test_emoji'],
-    tokens: ['emoji', 'test'],
+    tokens: ['test', 'emoji'],
     group: 1,
     order: 1,
     ...data,

package.json

@@ -121,6 +121,7 @@
     "tesseract.js": "^7.0.0",
     "tiny-queue": "^0.2.1",
     "twitter-text": "3.1.0",
+    "type-fest": "^5.8.0",
     "use-debounce": "^10.0.0",
     "vite": "^8.0.0",
     "vite-plugin-manifest-sri": "^0.2.0",

yarn.lock

@@ -3014,6 +3014,7 @@ __metadata:
     tesseract.js: "npm:^7.0.0"
     tiny-queue: "npm:^0.2.1"
     twitter-text: "npm:3.1.0"
+    type-fest: "npm:^5.8.0"
     typescript: "npm:~6.0.0"
     typescript-eslint: "npm:^8.55.0"
     typescript-plugin-css-modules: "npm:^5.2.0"
@@ -14009,6 +14010,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"type-fest@npm:^5.8.0":
+  version: 5.8.0
+  resolution: "type-fest@npm:5.8.0"
+  dependencies:
+    tagged-tag: "npm:^1.0.0"
+  checksum: 10c0/c8aae118a763d550a9552a511dff6b71840a23dab4edf693cf1c4df22596942794e6f6723389bd9036a90182249d915158bacf0815a1ae87f05f901b1d5f574e
+  languageName: node
+  linkType: hard
+
 "type-is@npm:^2.0.1":
   version: 2.0.1
   resolution: "type-is@npm:2.0.1"