Emojis: Fix bug with search + improve custom tokenization (#39167)

app/javascript/mastodon/actions/importer/emoji.ts

@@ -1,5 +1,8 @@
 import type { ApiCustomEmojiJSON } from '@/mastodon/api_types/custom_emoji';
 import { loadCustomEmoji } from '@/mastodon/features/emoji';
+import { emojiLogger } from '@/mastodon/features/emoji/utils';
+
+const log = emojiLogger('actions');
 
 export async function importCustomEmoji(emojis: ApiCustomEmojiJSON[]) {
   if (emojis.length === 0) {
@@ -18,5 +21,11 @@ export async function importCustomEmoji(emojis: ApiCustomEmojiJSON[]) {
   if (existingEmojis.length < emojis.length) {
     await clearCache('custom');
     await loadCustomEmoji();
+
+    const { reloadCustomEmojis } =
+      await import('@/mastodon/features/emoji/picker');
+    await reloadCustomEmojis();
+
+    log('Custom emojis updated, reloaded cache and picker data.');
   }
 }

app/javascript/mastodon/features/emoji/database.ts

@@ -85,13 +85,16 @@ export async function search({
     // Only query the range for the last token to allow partial matches.
     const range =
       i === queryTokens.length - 1
-        ? IDBKeyRange.bound(token, token + '\uffff')
+        ? IDBKeyRange.lowerBound(token)
         : IDBKeyRange.only(token);
 
-    const [unicodeResults, customResults] = await Promise.all([
-      db.getAllFromIndex(locale, 'tokens', range),
-      db.getAllFromIndex('custom', 'tokens', range),
-    ]);
+    const [unicodeResults, customResults, shortcodeResults] = await Promise.all(
+      [
+        db.getAllFromIndex(locale, 'tokens', range),
+        db.getAllFromIndex('custom', 'tokens', range),
+        db.getAllFromIndex('shortcodes', 'shortcodes', range),
+      ],
+    );
     const resultMap: ScoreMap = new Map();
     for (const emoji of unicodeResults) {
       const score = getScoreForEmoji(emoji, token);
@@ -107,6 +110,22 @@ export async function search({
       }
       resultMap.set(emoji.shortcode, { ...emoji, score });
     }
+
+    for (const shortcodeResult of shortcodeResults) {
+      if (resultMap.has(shortcodeResult.hexcode)) {
+        continue;
+      }
+      const emoji = await db.get(locale, shortcodeResult.hexcode);
+      if (!emoji) {
+        continue;
+      }
+      const score = getScoreForEmoji(emoji, token);
+      if (score === null) {
+        continue;
+      }
+      resultMap.set(emoji.hexcode, { ...emoji, score });
+    }
+
     log('found %d results for token "%s"', resultMap.size, token);
     resultArrays.push(resultMap);
   }
@@ -147,7 +166,7 @@ function getScoreForEmoji(emoji: AnyEmojiData, query: string) {
   }
 
   let index = 1;
-  for (const token of [id, emoji.tokens]) {
+  for (const token of [id, ...emoji.tokens]) {
     const tokenIndex = token.indexOf(query);
     if (tokenIndex !== -1) {
       return index + tokenIndex / token.length;

app/javascript/mastodon/features/emoji/index.ts

@@ -2,6 +2,7 @@ import { initialState } from '@/mastodon/initial_state';
 
 import type { EMOJI_DB_NAME_SHORTCODES } from './constants';
 import { toSupportedLocale } from './locale';
+import { reloadCustomEmojis } from './picker';
 import type { LocaleOrCustom } from './types';
 import { emojiLogger } from './utils';
 
@@ -90,6 +91,7 @@ export async function loadCustomEmoji() {
     const emojis = await importCustomEmojiData();
     if (emojis && emojis.length > 0) {
       log('loaded %d custom emojis', emojis.length);
+      await reloadCustomEmojis();
     }
   }
 }

app/javascript/mastodon/features/emoji/normalize.ts

@@ -14,6 +14,7 @@ import {
   EMOJIS_REQUIRING_INVERSION_IN_DARK_MODE,
   EMOJI_MIN_TOKEN_LENGTH,
 } from './constants';
+import { localeToSegmenter } from './locale';
 import type {
   CustomEmojiData,
   CustomEmojiMapArg,
@@ -92,10 +93,11 @@ export function transformEmojiData(
 export function transformCustomEmojiData(
   emoji: ApiCustomEmojiJSON,
 ): CustomEmojiData {
-  const tokens = emoji.shortcode
-    .split('_')
-    .filter((word) => word.length >= EMOJI_MIN_TOKEN_LENGTH)
-    .map((word) => word.toLowerCase());
+  const tokens = extractTokens(emoji.shortcode, localeToSegmenter('en'));
+  if (!tokens.includes(emoji.shortcode)) {
+    tokens.unshift(emoji.shortcode);
+  }
+
   return {
     ...emoji,
     tokens,
@@ -215,7 +217,9 @@ export function extractTokens(
   // Prefer to use Intl.Segmenter if available for better locale support.
   if (segmenter) {
     for (const { isWordLike, segment } of segmenter.segment(
-      input.replaceAll('_', ' '), // Handle underscores from shortcodes.
+      input
+        .replaceAll(/[_-]+/g, ' ') // Handle underscores from shortcodes.
+        .replaceAll(/([a-z])([A-Z])/g, '$1 $2'), // Handle camelCase.
     )) {
       if (isWordLike && segment.length >= EMOJI_MIN_TOKEN_LENGTH) {
         tokens.push(segment.toLowerCase());

app/javascript/mastodon/features/emoji/picker.ts

@@ -82,13 +82,22 @@ type LegacyEmoji =
       custom: true;
     };
 
+export async function reloadCustomEmojis() {
+  customEmojis = null;
+
+  const { loadEmojisIntoCache } =
+    await import('@/mastodon/hooks/useCustomEmojis');
+
+  await Promise.all([fetchCustomEmojiData(), loadEmojisIntoCache()]);
+}
+
 // Replicates the old legacy search function.
 export async function emojiMartSearch(
   token: string,
   locale: string,
   limit = 5,
 ): Promise<LegacyEmoji[]> {
-  const query = token.replace(':', '').toLowerCase().trim();
+  const query = token.replace(':', '').trim();
   if (!query.length) {
     return [];
   }

app/javascript/mastodon/hooks/useCustomEmojis.ts

@@ -20,7 +20,7 @@ export function useCustomEmojis() {
   return emojis;
 }
 
-async function loadEmojisIntoCache() {
+export async function loadEmojisIntoCache() {
   const { loadAllCustomEmoji } = await import('../features/emoji/database');
   const emojisRaw = await loadAllCustomEmoji();
   if (emojisRaw === null) {