[Glitch] Emoji: Adds search

Port 628fc9b95b064f7feba7f54f3c4146210828f18d to glitch-soc

Co-authored-by: diondiondion <mail@diondiondion.com>
Signed-off-by: Claire <claire.github-309c@sitedethib.com>

app/javascript/flavours/glitch/components/emoji/index.tsx

@@ -88,7 +88,10 @@ export const Emoji: FC<EmojiProps> = ({
     );
   }
 
-  const src = unicodeHexToUrl(state.code, appState.darkTheme);
+  const src = unicodeHexToUrl({
+    unicodeHex: state.code,
+    ...appState,
+  });
 
   return (
     <img

app/javascript/flavours/glitch/features/emoji/constants.ts

@@ -15,6 +15,8 @@ export const SKIN_TONE_CODES = [
   0x1f3ff, // Dark skin tone
 ] as const;
 
+export const EMOJI_MIN_TOKEN_LENGTH = 2;
+
 // Emoji rendering modes. A mode is what we are using to render emojis, a style is what the user has selected.
 export const EMOJI_MODE_NATIVE = 'native';
 export const EMOJI_MODE_NATIVE_WITH_FLAGS = 'native-flags';

app/javascript/flavours/glitch/features/emoji/database.test.ts

@@ -1,3 +1,4 @@
+import type { CompactEmoji } from 'emojibase';
 import { IDBFactory } from 'fake-indexeddb';
 
 import { customEmojiFactory, unicodeEmojiFactory } from '@/testing/factories';
@@ -6,8 +7,6 @@ import { EMOJI_DB_SHORTCODE_TEST } from './constants';
 import {
   putEmojiData,
   loadEmojiByHexcode,
-  searchEmojisByHexcodes,
-  searchEmojisByTag,
   testClear,
   testGet,
   putCustomEmojiData,
@@ -17,6 +16,14 @@ import {
   putLatestEtag,
 } from './database';
 
+function rawEmojiFactory(data: Partial<CompactEmoji> = {}): CompactEmoji {
+  return {
+    ...unicodeEmojiFactory(),
+    tags: ['test', 'emoji'],
+    ...data,
+  };
+}
+
 describe('emoji database', () => {
   afterEach(() => {
     testClear();
@@ -32,7 +39,7 @@ describe('emoji database', () => {
     });
 
     test('loads emoji into indexedDB', async () => {
-      await putEmojiData([unicodeEmojiFactory()], 'en');
+      await putEmojiData([rawEmojiFactory()], 'en');
       const { db } = await testGet();
       await expect(db.get('en', 'test')).resolves.toEqual(
         unicodeEmojiFactory(),
@@ -60,7 +67,7 @@ describe('emoji database', () => {
       });
       await expect(db.get('custom', 'emoji1')).resolves.toBeUndefined();
       await expect(db.get('custom', 'emoji2')).resolves.toEqual(
-        customEmojiFactory({ shortcode: 'emoji2' }),
+        customEmojiFactory({ shortcode: 'emoji2', tokens: ['emoji2'] }),
       );
     });
   });
@@ -79,12 +86,6 @@ describe('emoji database', () => {
   });
 
   describe('loadEmojiByHexcode', () => {
-    test('throws if the locale is not loaded', async () => {
-      await expect(loadEmojiByHexcode('en', 'test')).rejects.toThrowError(
-        'Locale en',
-      );
-    });
-
     test('retrieves the emoji', async () => {
       await putEmojiData([unicodeEmojiFactory()], 'en');
       await expect(loadEmojiByHexcode('test', 'en')).resolves.toEqual(
@@ -98,90 +99,6 @@ describe('emoji database', () => {
     });
   });
 
-  describe('searchEmojisByHexcodes', () => {
-    const data = [
-      unicodeEmojiFactory({ hexcode: 'not a number' }),
-      unicodeEmojiFactory({ hexcode: '1' }),
-      unicodeEmojiFactory({ hexcode: '2' }),
-      unicodeEmojiFactory({ hexcode: '3' }),
-      unicodeEmojiFactory({ hexcode: 'another not a number' }),
-    ];
-    beforeEach(async () => {
-      await putEmojiData(data, 'en');
-    });
-    test('finds emoji in consecutive range', async () => {
-      const actual = await searchEmojisByHexcodes(['1', '2', '3'], 'en');
-      expect(actual).toHaveLength(3);
-    });
-
-    test('finds emoji in split range', async () => {
-      const actual = await searchEmojisByHexcodes(['1', '3'], 'en');
-      expect(actual).toHaveLength(2);
-      expect(actual).toContainEqual(data.at(1));
-      expect(actual).toContainEqual(data.at(3));
-    });
-
-    test('finds emoji with non-numeric range', async () => {
-      const actual = await searchEmojisByHexcodes(
-        ['3', 'not a number', '1'],
-        'en',
-      );
-      expect(actual).toHaveLength(3);
-      expect(actual).toContainEqual(data.at(0));
-      expect(actual).toContainEqual(data.at(1));
-      expect(actual).toContainEqual(data.at(3));
-    });
-
-    test('not found emoji are not returned', async () => {
-      const actual = await searchEmojisByHexcodes(['not found'], 'en');
-      expect(actual).toHaveLength(0);
-    });
-
-    test('only found emojis are returned', async () => {
-      const actual = await searchEmojisByHexcodes(
-        ['another not a number', 'not found'],
-        'en',
-      );
-      expect(actual).toHaveLength(1);
-      expect(actual).toContainEqual(data.at(4));
-    });
-  });
-
-  describe('searchEmojisByTag', () => {
-    const data = [
-      unicodeEmojiFactory({ hexcode: 'test1', tags: ['test 1'] }),
-      unicodeEmojiFactory({
-        hexcode: 'test2',
-        tags: ['test 2', 'something else'],
-      }),
-      unicodeEmojiFactory({ hexcode: 'test3', tags: ['completely different'] }),
-    ];
-    beforeEach(async () => {
-      await putEmojiData(data, 'en');
-    });
-    test('finds emojis with tag', async () => {
-      const actual = await searchEmojisByTag('test 1', 'en');
-      expect(actual).toHaveLength(1);
-      expect(actual).toContainEqual(data.at(0));
-    });
-
-    test('finds emojis starting with tag', async () => {
-      const actual = await searchEmojisByTag('test', 'en');
-      expect(actual).toHaveLength(2);
-      expect(actual).not.toContainEqual(data.at(2));
-    });
-
-    test('does not find emojis ending with tag', async () => {
-      const actual = await searchEmojisByTag('else', 'en');
-      expect(actual).toHaveLength(0);
-    });
-
-    test('finds nothing with invalid tag', async () => {
-      const actual = await searchEmojisByTag('not found', 'en');
-      expect(actual).toHaveLength(0);
-    });
-  });
-
   describe('loadLegacyShortcodesByShortcode', () => {
     const data = {
       hexcode: 'test_hexcode',

app/javascript/flavours/glitch/features/emoji/database.ts

@@ -1,55 +1,25 @@
 import { SUPPORTED_LOCALES } from 'emojibase';
-import type { Locale, ShortcodesDataset } from 'emojibase';
+import type { CompactEmoji, Locale, ShortcodesDataset } from 'emojibase';
-import type { DBSchema, IDBPDatabase } from 'idb';
+
-import { openDB } from 'idb';
+import type { ApiCustomEmojiJSON } from '@/flavours/glitch/api_types/custom_emoji';
 
 import { EMOJI_DB_SHORTCODE_TEST } from './constants';
-import { toSupportedLocale, toSupportedLocaleOrCustom } from './locale';
+import { openEmojiDB } from './db-schema';
-import type { CustomEmojiData, UnicodeEmojiData, EtagTypes } from './types';
+import type { Database } from './db-schema';
+import {
+  localeToSegmenter,
+  toSupportedLocale,
+  toSupportedLocaleOrCustom,
+} from './locale';
+import {
+  extractTokens,
+  skinHexcodeToEmoji,
+  transformCustomEmojiData,
+  transformEmojiData,
+} from './normalize';
+import type { AnyEmojiData, EtagTypes } from './types';
 import { emojiLogger } from './utils';
 
-interface EmojiDB extends LocaleTables, DBSchema {
-  custom: {
-    key: string;
-    value: CustomEmojiData;
-    indexes: {
-      category: string;
-    };
-  };
-  shortcodes: {
-    key: string;
-    value: {
-      hexcode: string;
-      shortcodes: string[];
-    };
-    indexes: {
-      hexcode: string;
-      shortcodes: string[];
-    };
-  };
-  etags: {
-    key: EtagTypes;
-    value: string;
-  };
-}
-
-interface LocaleTable {
-  key: string;
-  value: UnicodeEmojiData;
-  indexes: {
-    group: number;
-    label: string;
-    order: number;
-    tags: string[];
-    shortcodes: string[];
-  };
-}
-type LocaleTables = Record<Locale, LocaleTable>;
-
-type Database = IDBPDatabase<EmojiDB>;
-
-const SCHEMA_VERSION = 2;
-
 const loadedLocales = new Set<Locale>();
 
 const log = emojiLogger('database');
@@ -60,75 +30,7 @@ const loadDB = (() => {
 
   // Actually load the DB.
   async function initDB() {
-    const db = await openDB<EmojiDB>('mastodon-emoji', SCHEMA_VERSION, {
+    const db = await openEmojiDB();
-      upgrade(database, oldVersion, newVersion, trx) {
-        if (!database.objectStoreNames.contains('custom')) {
-          const customTable = database.createObjectStore('custom', {
-            keyPath: 'shortcode',
-            autoIncrement: false,
-          });
-          customTable.createIndex('category', 'category');
-        }
-
-        if (!database.objectStoreNames.contains('etags')) {
-          database.createObjectStore('etags');
-        }
-
-        for (const locale of SUPPORTED_LOCALES) {
-          if (!database.objectStoreNames.contains(locale)) {
-            const localeTable = database.createObjectStore(locale, {
-              keyPath: 'hexcode',
-              autoIncrement: false,
-            });
-            localeTable.createIndex('group', 'group');
-            localeTable.createIndex('label', 'label');
-            localeTable.createIndex('order', 'order');
-            localeTable.createIndex('tags', 'tags', { multiEntry: true });
-            localeTable.createIndex('shortcodes', 'shortcodes', {
-              multiEntry: true,
-            });
-          }
-          // Added in version 2.
-          const localeTable = trx.objectStore(locale);
-          if (!localeTable.indexNames.contains('shortcodes')) {
-            localeTable.createIndex('shortcodes', 'shortcodes', {
-              multiEntry: true,
-            });
-          }
-        }
-
-        if (!database.objectStoreNames.contains('shortcodes')) {
-          const shortcodeTable = database.createObjectStore('shortcodes', {
-            keyPath: 'hexcode',
-            autoIncrement: false,
-          });
-          shortcodeTable.createIndex('hexcode', 'hexcode');
-          shortcodeTable.createIndex('shortcodes', 'shortcodes', {
-            multiEntry: true,
-          });
-        }
-
-        log(
-          'Upgraded emoji database from version %d to %d',
-          oldVersion,
-          newVersion,
-        );
-      },
-      blocked(currentVersion, blockedVersion) {
-        log(
-          'Emoji database upgrade from version %d to %d is blocked',
-          currentVersion,
-          blockedVersion,
-        );
-      },
-      blocking(currentVersion, blockedVersion) {
-        log(
-          'Emoji database upgrade from version %d is blocking upgrade to %d',
-          currentVersion,
-          blockedVersion,
-        );
-      },
-    });
     await syncLocales(db);
     log('Loaded database version %d', db.version);
     return db;
@@ -149,11 +51,128 @@ const loadDB = (() => {
   return loadPromise;
 })();
 
-export async function putEmojiData(emojis: UnicodeEmojiData[], locale: Locale) {
+export async function search({
+  query,
+  locale: localeString,
+  limit = 0,
+}: {
+  query: string;
+  locale: string;
+  limit?: number;
+}) {
+  performance.mark('emoji-search-start');
+
+  // Get the locale, and extract tokens from the query.
+  const locale = await toLoadedLocale(localeString);
+  const segmenter = localeToSegmenter(locale);
+  const queryTokens = extractTokens(query, segmenter);
+
+  if (queryTokens.length === 0) {
+    log('no tokens extracted from query "%s"', query);
+    return [];
+  }
+  const lastToken = queryTokens.at(-1);
+  if (!lastToken) {
+    throw new Error('Missing tokens from query');
+  }
+
+  log('searching for tokens %o in locale %s', queryTokens, locale);
+
+  // Create an array of emoji results
+  const db = await loadDB();
+  const resultArrays: Map<string, AnyEmojiData>[] = [];
+  for (let i = 0; i < queryTokens.length; i++) {
+    const token = queryTokens[i];
+    if (!token) continue;
+
+    // Only query the range for the last token to allow partial matches.
+    const range =
+      i === queryTokens.length - 1
+        ? IDBKeyRange.bound(token, token + '\uffff')
+        : IDBKeyRange.only(token);
+
+    const [unicodeResults, customResults] = await Promise.all([
+      db.getAllFromIndex(locale, 'tokens', range),
+      db.getAllFromIndex('custom', 'tokens', range),
+    ]);
+    const resultMap = new Map<string, AnyEmojiData>([
+      ...unicodeResults.map(
+        (emoji) => [emoji.hexcode, emoji] as [string, AnyEmojiData],
+      ),
+      ...customResults.map(
+        (emoji) => [emoji.shortcode, emoji] as [string, AnyEmojiData],
+      ),
+    ]);
+    log('found %d results for token "%s"', resultMap.size, token);
+    resultArrays.push(resultMap);
+  }
+
+  // Utilize maps to find the intersection of all result sets.
+  const results = Array.from(
+    resultArrays
+      .reduce((prev, curr) => {
+        const intersection = new Map<string, AnyEmojiData>();
+        for (const [code, emoji] of prev) {
+          if (curr.has(code)) {
+            intersection.set(code, emoji);
+          }
+        }
+        return intersection;
+      })
+      .values(),
+  );
+
+  results.sort((a, b) => {
+    // Checks if a or b has the last token exactly, or only a prefix.
+    const aHasToken = a.tokens.includes(lastToken);
+    const bHasToken = b.tokens.includes(lastToken);
+    if (aHasToken && !bHasToken) {
+      return -1;
+    } else if (!aHasToken && bHasToken) {
+      return 1;
+    }
+
+    // If one is a custom emoji, prioritize it over Unicode emojis.
+    if ('category' in a) {
+      return -1;
+    } else if ('category' in b) {
+      return 1;
+    }
+
+    // If both are Unicode emojis, prioritize by order.
+    if ('order' in a && 'order' in b) {
+      return (a.order ?? 0) - (b.order ?? 0); // If these are both Unicode emojis, sort by order.
+    }
+
+    // ¯\_(ツ)_/¯
+    return 0;
+  });
+
+  const time = performance.measure('emoji-search-end', 'emoji-search-start');
+  log(
+    'search for "%s" in locale %s returned %d results and took %dms',
+    query,
+    locale,
+    results.length,
+    time.duration,
+  );
+  if (limit > 0) {
+    return results.slice(0, limit);
+  }
+  return results;
+}
+
+export async function putEmojiData(emojis: CompactEmoji[], locale: Locale) {
   loadedLocales.add(locale);
   const db = await loadDB();
   const trx = db.transaction(locale, 'readwrite');
-  await Promise.all(emojis.map((emoji) => trx.store.put(emoji)));
+  await trx.store.clear();
+  const segmenter = localeToSegmenter(locale);
+  await Promise.all(
+    emojis
+      .sort((a, b) => (a.order ?? 0) - (b.order ?? 0))
+      .map((emoji) => trx.store.put(transformEmojiData(emoji, segmenter))),
+  );
   await trx.done;
 }
 
@@ -161,7 +180,7 @@ export async function putCustomEmojiData({
   emojis,
   clear = false,
 }: {
-  emojis: CustomEmojiData[];
+  emojis: ApiCustomEmojiJSON[];
   clear?: boolean;
 }) {
   const db = await loadDB();
@@ -173,7 +192,9 @@ export async function putCustomEmojiData({
     log('Cleared existing custom emojis in database');
   }
 
-  await Promise.all(emojis.map((emoji) => trx.store.put(emoji)));
+  await Promise.all(
+    emojis.map((emoji) => trx.store.put(transformCustomEmojiData(emoji))),
+  );
   await trx.done;
 
   log('Imported %d custom emojis into database', emojis.length);
@@ -210,32 +231,25 @@ export async function loadEmojiByHexcode(
   localeString: string,
 ) {
   const db = await loadDB();
-  const locale = toLoadedLocale(localeString);
+  const locale = await toLoadedLocale(localeString);
-  return db.get(locale, hexcode);
+  const result = await db.get(locale, hexcode);
-}
+  if (result) {
+    return result;
+  }
 
-export async function searchEmojisByHexcodes(
+  // If the emoji wasn't found, check if it's a skin tone variant.
-  hexcodes: string[],
+  const skinResult = await db.getFromIndex(
-  localeString: string,
-) {
-  const db = await loadDB();
-  const locale = toLoadedLocale(localeString);
-  const sortedCodes = hexcodes.toSorted();
-  const results = await db.getAll(
     locale,
-    IDBKeyRange.bound(sortedCodes.at(0), sortedCodes.at(-1)),
+    'skinHexcodes',
+    IDBKeyRange.only(hexcode),
   );
-  return results.filter((emoji) => hexcodes.includes(emoji.hexcode));
-}
 
-export async function searchEmojisByTag(tag: string, localeString: string) {
+  if (!skinResult) {
-  const db = await loadDB();
+    return skinResult;
-  const locale = toLoadedLocale(localeString);
+  }
-  const range = IDBKeyRange.bound(
+
-    tag.toLowerCase(),
+  // Reconstruct the full unicode string from the skin tone hexcode.
-    `${tag.toLowerCase()}\uffff`,
+  return skinHexcodeToEmoji(hexcode, skinResult);
-  );
-  return db.getAllFromIndex(locale, 'tags', range);
 }
 
 export async function loadCustomEmojiByShortcode(shortcode: string) {
@@ -301,13 +315,16 @@ async function syncLocales(db: Database) {
   log('Loaded %d locales: %o', loadedLocales.size, loadedLocales);
 }
 
-function toLoadedLocale(localeString: string) {
+async function toLoadedLocale(localeString: string) {
   const locale = toSupportedLocale(localeString);
   if (localeString !== locale) {
     log(`Locale ${locale} is different from provided ${localeString}`);
   }
   if (!loadedLocales.has(locale)) {
-    throw new LocaleNotLoadedError(locale);
+    log('Locale %s not loaded, importing...', locale);
+    const { importEmojiData } = await import('./loader');
+    await importEmojiData(locale);
+    return locale;
   }
   return locale;
 }

app/javascript/flavours/glitch/features/emoji/db-schema.ts

@@ -0,0 +1,198 @@
+import { SUPPORTED_LOCALES } from 'emojibase';
+import type { Locale } from 'emojibase';
+import { openDB } from 'idb';
+import type {
+  DBSchema,
+  IDBPDatabase,
+  IDBPObjectStore,
+  IDBPTransaction,
+  IndexNames,
+  StoreNames,
+} from 'idb';
+
+import type { CustomEmojiData, EtagTypes, UnicodeEmojiData } from './types';
+import { emojiLogger } from './utils';
+
+const log = emojiLogger('database');
+
+interface EmojiDB extends LocaleTables, DBSchema {
+  custom: {
+    key: string;
+    value: CustomEmojiData;
+    indexes: {
+      tokens: string[];
+      category: string;
+    };
+  };
+  shortcodes: {
+    key: string;
+    value: {
+      hexcode: string;
+      shortcodes: string[];
+    };
+    indexes: {
+      shortcodes: string[];
+    };
+  };
+  etags: {
+    key: EtagTypes;
+    value: string;
+  };
+}
+
+interface LocaleTable {
+  key: string;
+  value: UnicodeEmojiData;
+  indexes: {
+    shortcodes: string[];
+    groupOrder: [number, number];
+    tokens: string[];
+    skinHexcodes: string[];
+  };
+}
+type LocaleTables = Record<Locale, LocaleTable>;
+
+type Transaction<Mode extends IDBTransactionMode = 'versionchange'> =
+  IDBPTransaction<EmojiDB, StoreNames<EmojiDB>[], Mode>;
+
+export type Database = IDBPDatabase<EmojiDB>;
+
+const SCHEMA_VERSION = 3;
+
+export async function openEmojiDB() {
+  const db = await openDB<EmojiDB>('mastodon-emoji', SCHEMA_VERSION, {
+    upgrade(database, oldVersion, newVersion, trx) {
+      if (!database.objectStoreNames.contains('custom')) {
+        database.createObjectStore('custom', {
+          keyPath: 'shortcode',
+          autoIncrement: false,
+        });
+      }
+      maybeAddIndex({ trx, storeName: 'custom', indexName: 'category' });
+      maybeAddIndex({
+        trx,
+        storeName: 'custom',
+        indexName: 'tokens',
+        options: { multiEntry: true },
+      });
+
+      if (!database.objectStoreNames.contains('etags')) {
+        database.createObjectStore('etags');
+      }
+
+      SUPPORTED_LOCALES.forEach((locale) => {
+        createLocaleTable(locale, database, trx);
+      });
+
+      const shortcodeTable = database.objectStoreNames.contains('shortcodes')
+        ? trx.objectStore('shortcodes')
+        : database.createObjectStore('shortcodes', {
+            keyPath: 'hexcode',
+            autoIncrement: false,
+          });
+      maybeAddIndex({
+        trx,
+        storeName: 'shortcodes',
+        indexName: 'shortcodes',
+        options: { multiEntry: true },
+      });
+      deleteOldIndexes(shortcodeTable, ['hexcode']);
+
+      log(
+        'Upgraded emoji database from version %d to %d',
+        oldVersion,
+        newVersion,
+      );
+    },
+    blocked(currentVersion, blockedVersion) {
+      log(
+        'Emoji database upgrade from version %d to %d is blocked',
+        currentVersion,
+        blockedVersion,
+      );
+    },
+    blocking(currentVersion, blockedVersion) {
+      log(
+        'Emoji database upgrade from version %d is blocking upgrade to %d',
+        currentVersion,
+        blockedVersion,
+      );
+    },
+  });
+
+  return db;
+}
+
+function maybeAddIndex<StoreName extends StoreNames<EmojiDB>>({
+  trx,
+  storeName,
+  indexName,
+  keys,
+  options,
+}: {
+  trx: Transaction;
+  storeName: StoreName;
+  indexName: IndexNames<EmojiDB, StoreName>;
+  keys?: string | string[];
+  options?: IDBIndexParameters;
+}) {
+  const store = trx.objectStore(storeName);
+  if (!store.indexNames.contains(indexName)) {
+    store.createIndex(indexName, keys ?? indexName, options);
+  }
+}
+
+function createLocaleTable(
+  locale: Locale,
+  database: Database,
+  trx: Transaction,
+) {
+  if (!database.objectStoreNames.contains(locale)) {
+    database.createObjectStore(locale, {
+      keyPath: 'hexcode',
+      autoIncrement: false,
+    });
+  }
+
+  maybeAddIndex({
+    trx,
+    storeName: locale,
+    indexName: 'shortcodes',
+    options: { multiEntry: true },
+  });
+  maybeAddIndex({
+    trx,
+    storeName: locale,
+    indexName: 'groupOrder',
+    keys: ['group', 'order'],
+  });
+  maybeAddIndex({
+    trx,
+    storeName: locale,
+    indexName: 'tokens',
+    keys: 'tokens',
+    options: { multiEntry: true },
+  });
+  maybeAddIndex({
+    trx,
+    storeName: locale,
+    indexName: 'skinHexcodes',
+    keys: 'skinHexcodes',
+    options: { multiEntry: true },
+  });
+
+  const oldIndexes = ['group', 'order', 'tag', 'label'] as const;
+  deleteOldIndexes(trx.objectStore(locale), oldIndexes);
+}
+
+function deleteOldIndexes(
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any -- Type is too complex, so only any works here.
+  table: IDBPObjectStore<any, any, any, 'versionchange'>,
+  indexes: readonly string[],
+) {
+  for (const index of indexes) {
+    if (table.indexNames.contains(index)) {
+      table.deleteIndex(index);
+    }
+  }
+}

app/javascript/flavours/glitch/features/emoji/loader.ts

@@ -1,10 +1,5 @@
-import { flattenEmojiData } from 'emojibase';
+import { joinShortcodes } from 'emojibase';
-import type {
+import type { CompactEmoji, Locale, ShortcodesDataset } from 'emojibase';
-  CompactEmoji,
-  FlatCompactEmoji,
-  Locale,
-  ShortcodesDataset,
-} from 'emojibase';
 
 import {
   putEmojiData,
@@ -28,7 +23,7 @@ export async function importEmojiData(localeString: string, shortcodes = true) {
     shortcodes ? ' and shortcodes' : '',
   );
 
-  const emojis = await fetchAndCheckEtag<CompactEmoji[]>({
+  let emojis = await fetchAndCheckEtag<CompactEmoji[]>({
     etagString: locale,
     path: localeToEmojiPath(locale),
   });
@@ -49,12 +44,10 @@ export async function importEmojiData(localeString: string, shortcodes = true) {
     }
   }
 
-  const flattenedEmojis: FlatCompactEmoji[] = flattenEmojiData(
+  emojis = joinShortcodes(emojis, shortcodesData);
-    emojis,
+
-    shortcodesData,
+  await putEmojiData(emojis, locale);
-  );
+  return emojis;
-  await putEmojiData(flattenedEmojis, locale);
-  return flattenedEmojis;
 }
 
 export async function importCustomEmojiData() {
@@ -135,11 +128,11 @@ async function fetchAndCheckEtag<ResultType extends object[] | object>({
   checkEtag?: boolean;
 }): Promise<ResultType | null> {
   const etagName = toValidEtagName(etagString);
+  const oldEtag = checkEtag ? await loadLatestEtag(etagName) : null;
 
   // Use location.origin as this script may be loaded from a CDN domain.
   const url = new URL(path, location.origin);
 
-  const oldEtag = checkEtag ? await loadLatestEtag(etagName) : null;
   const response = await fetch(url, {
     headers: {
       'Content-Type': 'application/json',
@@ -148,6 +141,7 @@ async function fetchAndCheckEtag<ResultType extends object[] | object>({
   });
   // If not modified, return null
   if (response.status === 304) {
+    log('etag not modified for %s', etagName);
     return null;
   }
   if (!response.ok) {
@@ -163,6 +157,8 @@ async function fetchAndCheckEtag<ResultType extends object[] | object>({
   if (etag && checkEtag) {
     log(`storing new etag for ${etagName}: ${etag}`);
     await putLatestEtag(etag, etagName);
+  } else if (!etag) {
+    log(`no etag found in response for ${etagName}`);
   }
 
   return data;

app/javascript/flavours/glitch/features/emoji/locale.ts

@@ -32,6 +32,13 @@ export function toValidEtagName(input: string): EtagTypes {
   return toSupportedLocale(lower);
 }
 
+export function localeToSegmenter(locale: Locale): Intl.Segmenter | null {
+  if (typeof Intl.Segmenter === 'function') {
+    return new Intl.Segmenter(locale, { granularity: 'word' });
+  }
+  return null;
+}
+
 function isSupportedLocale(locale: string): locale is Locale {
   return SUPPORTED_LOCALES.includes(locale as Locale);
 }

app/javascript/flavours/glitch/features/emoji/mode.ts

@@ -2,6 +2,7 @@
 // See: https://github.com/nolanlawson/emoji-picker-element/blob/master/src/picker/utils/testColorEmojiSupported.js
 
 import { createAppSelector, useAppSelector } from '@/flavours/glitch/store';
+import { assetHost } from '@/flavours/glitch/utils/config';
 import { isDevelopment } from '@/flavours/glitch/utils/environment';
 import { isDarkMode } from '@/flavours/glitch/utils/theme';
 
@@ -29,6 +30,7 @@ export function useEmojiAppState(): EmojiAppState {
     locales: [locale],
     mode,
     darkTheme: isDarkMode(),
+    assetHost,
   };
 }
 

app/javascript/flavours/glitch/features/emoji/normalize.test.ts

@@ -4,11 +4,7 @@ import { basename, resolve } from 'path';
 import { flattenEmojiData } from 'emojibase';
 import unicodeRawEmojis from 'emojibase-data/en/data.json';
 
-import {
+import { unicodeToTwemojiHex } from './normalize';
-  twemojiToUnicodeInfo,
-  unicodeToTwemojiHex,
-  emojiToUnicodeHex,
-} from './normalize';
 
 const emojiSVGFiles = await readdir(
   // This assumes tests are run from project root
@@ -26,23 +22,6 @@ const svgFileNamesWithoutBorder = svgFileNames.filter(
 
 const unicodeEmojis = flattenEmojiData(unicodeRawEmojis);
 
-describe('emojiToUnicodeHex', () => {
-  test.concurrent.for([
-    ['🎱', '1F3B1'],
-    ['🐜', '1F41C'],
-    ['⚫', '26AB'],
-    ['🖤', '1F5A4'],
-    ['💀', '1F480'],
-    ['❤️', '2764'], // Checks for trailing variation selector removal.
-    ['💂‍♂️', '1F482-200D-2642-FE0F'],
-  ] as const)(
-    'emojiToUnicodeHex converts %s to %s',
-    ([emoji, hexcode], { expect }) => {
-      expect(emojiToUnicodeHex(emoji)).toBe(hexcode);
-    },
-  );
-});
-
 describe('unicodeToTwemojiHex', () => {
   test.concurrent.for(
     unicodeEmojis
@@ -54,26 +33,3 @@ describe('unicodeToTwemojiHex', () => {
     expect(svgFileNamesWithoutBorder).toContain(result);
   });
 });
-
-describe('twemojiToUnicodeInfo', () => {
-  const unicodeCodeSet = new Set(unicodeEmojis.map((emoji) => emoji.hexcode));
-
-  test.concurrent.for(svgFileNamesWithoutBorder)(
-    'verifying SVG file %s maps to Unicode emoji',
-    (svgFileName, { expect }) => {
-      assert(!!svgFileName);
-      const result = twemojiToUnicodeInfo(svgFileName);
-      const hexcode = typeof result === 'string' ? result : result.unqualified;
-      if (!hexcode) {
-        // No hexcode means this is a special case like the Shibuya 109 emoji
-        expect(result).toHaveProperty('label');
-        return;
-      }
-      assert(!!hexcode);
-      expect(
-        unicodeCodeSet.has(hexcode),
-        `${hexcode} (${svgFileName}) not found`,
-      ).toBeTruthy();
-    },
-  );
-});

app/javascript/flavours/glitch/features/emoji/normalize.ts

@@ -1,48 +1,124 @@
 import { isList } from 'immutable';
 
-import { assetHost } from '@/flavours/glitch/utils/config';
+import type { CompactEmoji, SkinTone } from 'emojibase';
+import { fromHexcodeToCodepoint } from 'emojibase';
+
+import type { ApiCustomEmojiJSON } from '@/flavours/glitch/api_types/custom_emoji';
 
 import {
   VARIATION_SELECTOR_CODE,
   KEYCAP_CODE,
-  GENDER_FEMALE_CODE,
-  GENDER_MALE_CODE,
-  SKIN_TONE_CODES,
   EMOJIS_WITH_DARK_BORDER,
   EMOJIS_WITH_LIGHT_BORDER,
   EMOJIS_REQUIRING_INVERSION_IN_LIGHT_MODE,
   EMOJIS_REQUIRING_INVERSION_IN_DARK_MODE,
+  EMOJI_MIN_TOKEN_LENGTH,
 } from './constants';
-import type { CustomEmojiMapArg, ExtraCustomEmojiMap } from './types';
+import type {
+  CustomEmojiData,
+  CustomEmojiMapArg,
+  ExtraCustomEmojiMap,
+  UnicodeEmojiData,
+} from './types';
+import { emojiToUnicodeHex } from './utils';
 
-// Misc codes that have special handling
+const SKIN_TONE_MAP: Record<number, SkinTone> = {
-const SKIER_CODE = 0x26f7;
+  0x1f3fb: 1, // Light skin tone
-const CHRISTMAS_TREE_CODE = 0x1f384;
+  0x1f3fc: 2, // Medium-light skin tone
-const MR_CLAUS_CODE = 0x1f385;
+  0x1f3fd: 3, // Medium skin tone
-const EYE_CODE = 0x1f441;
+  0x1f3fe: 4, // Medium-dark skin tone
-const LEVITATING_PERSON_CODE = 0x1f574;
+  0x1f3ff: 5, // Dark skin tone
-const SPEECH_BUBBLE_CODE = 0x1f5e8;
+};
-const MS_CLAUS_CODE = 0x1f936;
 
-export function emojiToUnicodeHex(emoji: string): string {
+export function transformEmojiData(
-  const codes: number[] = [];
+  emoji: CompactEmoji,
-  for (const char of emoji) {
+  segmenter: Intl.Segmenter | null,
-    const code = char.codePointAt(0);
+): UnicodeEmojiData {
-    if (code !== undefined) {
+  const {
-      codes.push(code);
+    shortcodes = [],
+    tags = [],
+    label,
+    emoticon,
+    hexcode,
+    unicode,
+    group,
+    order,
+    skins = [],
+  } = emoji;
+  const extract = (str: string) => extractTokens(str, segmenter);
+
+  let normalizedEmoticons: string[] | undefined = undefined;
+  if (emoticon) {
+    normalizedEmoticons = Array.isArray(emoticon) ? emoticon : [emoticon];
+  }
+
+  const tokens = [
+    ...new Set([
+      ...shortcodes.map(extract).flat(),
+      ...tags.map(extract).flat(),
+      ...extract(label),
+      ...(normalizedEmoticons ?? []),
+    ]),
+  ].sort((a, b) => a.localeCompare(b));
+
+  const res: UnicodeEmojiData = {
+    tokens,
+    shortcodes,
+    label,
+    emoticons: normalizedEmoticons,
+    hexcode,
+    unicode,
+    group,
+    order,
+  };
+
+  for (const skin of skins) {
+    res.skinHexcodes ??= [];
+    res.skinHexcodes.push(skin.hexcode);
+
+    res.skinTones ??= [];
+    for (const codePoint of skin.unicode) {
+      const tone = SKIN_TONE_MAP[codePoint.codePointAt(0) ?? 0];
+      if (tone) {
+        res.skinTones.push(tone);
+        break;
+      }
     }
   }
 
-  // Handles how Emojibase removes the variation selector for single code emojis.
+  return res;
-  // See: https://emojibase.dev/docs/spec/#merged-variation-selectors
-  if (codes.at(1) === VARIATION_SELECTOR_CODE && codes.length === 2) {
-    codes.pop();
-  }
-  return hexNumbersToString(codes);
 }
 
+export function transformCustomEmojiData(
+  emoji: ApiCustomEmojiJSON,
+): CustomEmojiData {
+  const tokens = emoji.shortcode
+    .split('_')
+    .filter((word) => word.length >= EMOJI_MIN_TOKEN_LENGTH)
+    .map((word) => word.toLowerCase());
+  return {
+    ...emoji,
+    tokens,
+  };
+}
+
+export function skinHexcodeToEmoji(
+  skinHexcode: string,
+  emoji: UnicodeEmojiData,
+): UnicodeEmojiData {
+  return {
+    ...emoji,
+    unicode: String.fromCodePoint(...fromHexcodeToCodepoint(skinHexcode)),
+    hexcode: skinHexcode,
+  };
+}
+
+// Misc codes that have special handling
+const EYE_CODE = 0x1f441;
+const SPEECH_BUBBLE_CODE = 0x1f5e8;
+
 export function unicodeToTwemojiHex(unicodeHex: string): string {
-  const codes = hexStringToNumbers(unicodeHex);
+  const codes = fromHexcodeToCodepoint(unicodeHex);
   const normalizedCodes: number[] = [];
   for (let i = 0; i < codes.length; i++) {
     const code = codes[i];
@@ -64,19 +140,28 @@ export function unicodeToTwemojiHex(unicodeHex: string): string {
     normalizedCodes.push(code);
   }
 
-  return hexNumbersToString(normalizedCodes, 0).toLowerCase();
+  return normalizedCodes
+    .map((code) => code.toString(16))
+    .join('-')
+    .toLowerCase();
 }
 
-export const CODES_WITH_DARK_BORDER =
+const CODES_WITH_DARK_BORDER = EMOJIS_WITH_DARK_BORDER.map(emojiToUnicodeHex);
-  EMOJIS_WITH_DARK_BORDER.map(emojiToUnicodeHex);
 
-export const CODES_WITH_LIGHT_BORDER =
+const CODES_WITH_LIGHT_BORDER = EMOJIS_WITH_LIGHT_BORDER.map(emojiToUnicodeHex);
-  EMOJIS_WITH_LIGHT_BORDER.map(emojiToUnicodeHex);
 
-export function unicodeHexToUrl(unicodeHex: string, darkMode: boolean): string {
+export function unicodeHexToUrl({
+  unicodeHex,
+  darkTheme,
+  assetHost,
+}: {
+  unicodeHex: string;
+  darkTheme: boolean;
+  assetHost: string;
+}): string {
   const normalizedHex = unicodeToTwemojiHex(unicodeHex);
   let url = `${assetHost}/emoji/${normalizedHex}`;
-  if (darkMode && CODES_WITH_LIGHT_BORDER.includes(normalizedHex)) {
+  if (darkTheme && CODES_WITH_LIGHT_BORDER.includes(normalizedHex)) {
     url += '_border';
   }
   if (CODES_WITH_DARK_BORDER.includes(normalizedHex)) {
@@ -86,78 +171,6 @@ export function unicodeHexToUrl(unicodeHex: string, darkMode: boolean): string {
   return url;
 }
 
-interface TwemojiSpecificEmoji {
-  unqualified?: string;
-  gender?: number;
-  skin?: number;
-  label?: string;
-}
-
-// Normalize man/woman to male/female
-const GENDER_CODES_MAP: Record<number, number> = {
-  [GENDER_FEMALE_CODE]: GENDER_FEMALE_CODE,
-  [GENDER_MALE_CODE]: GENDER_MALE_CODE,
-  // These are man/woman markers, but are used for gender sometimes.
-  [0x1f468]: GENDER_MALE_CODE,
-  [0x1f469]: GENDER_FEMALE_CODE,
-};
-
-const TWEMOJI_SPECIAL_CASES: Record<string, string | TwemojiSpecificEmoji> = {
-  '1F441-200D-1F5E8': '1F441-FE0F-200D-1F5E8-FE0F', // Eye in speech bubble
-  // An emoji that was never ported to the Unicode standard.
-  // See: https://emojipedia.org/shibuya
-  E50A: { label: 'Shibuya 109' },
-};
-
-export function twemojiToUnicodeInfo(
-  twemojiHex: string,
-): TwemojiSpecificEmoji | string {
-  const specialCase = TWEMOJI_SPECIAL_CASES[twemojiHex.toUpperCase()];
-  if (specialCase) {
-    return specialCase;
-  }
-  const codes = hexStringToNumbers(twemojiHex);
-  let gender: undefined | number;
-  let skin: undefined | number;
-  for (const code of codes) {
-    if (!gender && code in GENDER_CODES_MAP) {
-      gender = GENDER_CODES_MAP[code];
-    } else if (!skin && code in SKIN_TONE_CODES) {
-      skin = code;
-    }
-
-    // Exit if we have both skin and gender
-    if (skin && gender) {
-      break;
-    }
-  }
-
-  let mappedCodes: unknown[] = codes;
-
-  if (codes.at(-1) === CHRISTMAS_TREE_CODE && codes.length >= 3 && gender) {
-    // Twemoji uses the christmas tree with a ZWJ for Mr. and Mrs. Claus,
-    // but in Unicode that only works for Mx. Claus.
-    const START_CODE =
-      gender === GENDER_FEMALE_CODE ? MS_CLAUS_CODE : MR_CLAUS_CODE;
-    mappedCodes = [START_CODE, skin];
-  } else if (codes.at(-1) === KEYCAP_CODE && codes.length === 2) {
-    // For key emoji, insert the variation selector
-    mappedCodes = [codes[0], VARIATION_SELECTOR_CODE, KEYCAP_CODE];
-  } else if (
-    (codes.at(0) === SKIER_CODE || codes.at(0) === LEVITATING_PERSON_CODE) &&
-    codes.length > 1
-  ) {
-    // Twemoji offers more gender and skin options for the skier and levitating person emoji.
-    return {
-      unqualified: hexNumbersToString([codes.at(0)]),
-      skin,
-      gender,
-    };
-  }
-
-  return hexNumbersToString(mappedCodes);
-}
-
 export function emojiToInversionClassName(emoji: string): string | null {
   if (EMOJIS_REQUIRING_INVERSION_IN_DARK_MODE.includes(emoji)) {
     return 'invert-on-dark';
@@ -189,19 +202,37 @@ export function cleanExtraEmojis(extraEmojis?: CustomEmojiMapArg) {
   return extraEmojis;
 }
 
-function hexStringToNumbers(hexString: string): number[] {
+/**
-  return hexString
+ * Tokenizes an input string into words, using Intl.Segmenter if available.
-    .split('-')
+ * @param input Any input string.
-    .map((code) => Number.parseInt(code, 16))
+ * @param segmenter Segmenter, if available.
-    .filter((code) => !Number.isNaN(code));
+ * @returns Array of tokens in lowercase.
-}
+ */
+export function extractTokens(
+  input: string,
+  segmenter: Intl.Segmenter | null,
+): string[] {
+  if (!input.trim()) {
+    return [];
+  }
+  const tokens: string[] = [];
 
-function hexNumbersToString(codes: unknown[], padding = 4): string {
+  // Prefer to use Intl.Segmenter if available for better locale support.
-  return codes
+  if (segmenter) {
-    .filter(
+    for (const { isWordLike, segment } of segmenter.segment(
-      (code): code is number =>
+      input.replaceAll('_', ' '), // Handle underscores from shortcodes.
-        typeof code === 'number' && code > 0 && !Number.isNaN(code),
+    )) {
-    )
+      if (isWordLike && segment.length >= EMOJI_MIN_TOKEN_LENGTH) {
-    .map((code) => code.toString(16).padStart(padding, '0').toUpperCase())
+        tokens.push(segment.toLowerCase());
-    .join('-');
+      }
+    }
+  } else {
+    // Fallback to simple splitting.
+    input.split(/[\s_-]+/).forEach((word) => {
+      if (/\w/.test(word) && word.length >= EMOJI_MIN_TOKEN_LENGTH) {
+        tokens.push(word.toLowerCase());
+      }
+    });
+  }
+  return tokens;
 }

app/javascript/flavours/glitch/features/emoji/render.ts

@@ -4,7 +4,6 @@ import {
   EMOJI_TYPE_UNICODE,
   EMOJI_TYPE_CUSTOM,
 } from './constants';
-import { emojiToUnicodeHex } from './normalize';
 import type {
   EmojiLoadedState,
   EmojiMode,
@@ -16,6 +15,7 @@ import type {
 import {
   anyEmojiRegex,
   emojiLogger,
+  emojiToUnicodeHex,
   isCustomEmoji,
   isUnicodeEmoji,
   stringHasUnicodeFlags,
@@ -140,12 +140,7 @@ export async function loadEmojiDataToState(
     }
 
     // If not found, assume it's not an emoji and return null.
-    log(
+    log('Could not find emoji %s for locale %s', state.code, locale);
-      'Could not find emoji %s of type %s for locale %s',
-      state.code,
-      state.type,
-      locale,
-    );
     return null;
   } catch (err: unknown) {
     // If the locale is not loaded, load it and retry once.

app/javascript/flavours/glitch/features/emoji/types.ts

@@ -1,6 +1,6 @@
 import type { List as ImmutableList } from 'immutable';
 
-import type { FlatCompactEmoji, Locale } from 'emojibase';
+import type { CompactEmoji, Locale, SkinTone } from 'emojibase';
 
 import type { ApiCustomEmojiJSON } from '@/flavours/glitch/api_types/custom_emoji';
 import type { CustomEmoji } from '@/flavours/glitch/models/custom_emoji';
@@ -32,10 +32,20 @@ export interface EmojiAppState {
   currentLocale: Locale;
   mode: EmojiMode;
   darkTheme: boolean;
+  assetHost: string;
 }
 
-export type CustomEmojiData = ApiCustomEmojiJSON;
+export type CustomEmojiData = ApiCustomEmojiJSON & { tokens: string[] };
-export type UnicodeEmojiData = FlatCompactEmoji;
+export interface UnicodeEmojiData extends Omit<
+  CompactEmoji,
+  'emoticon' | 'skins' | 'tags'
+> {
+  shortcodes: string[];
+  tokens: string[];
+  emoticons?: string[];
+  skinHexcodes?: string[];
+  skinTones?: (SkinTone | SkinTone[])[];
+}
 export type AnyEmojiData = CustomEmojiData | UnicodeEmojiData;
 
 type CustomEmojiRenderFields = Pick<

app/javascript/flavours/glitch/features/emoji/utils.ts

@@ -1,4 +1,5 @@
 import debug from 'debug';
+import { fromUnicodeToHexcode } from 'emojibase';
 
 import { emojiRegexPolyfill } from '@/flavours/glitch/polyfills';
 
@@ -44,6 +45,10 @@ export function anyEmojiRegex() {
   );
 }
 
+export function emojiToUnicodeHex(emoji: string): string {
+  return fromUnicodeToHexcode(emoji, false);
+}
+
 function supportsRegExpSets() {
   return 'unicodeSets' in RegExp.prototype;
 }