[Glitch] Refactor emoji search

Port 34c91555ae to glitch-soc

Signed-off-by: Claire <claire.github-309c@sitedethib.com>

app/javascript/flavours/glitch/actions/compose.js

@@ -7,7 +7,7 @@ import api from 'flavours/glitch/api';
 import { browserHistory } from 'flavours/glitch/components/router';
 import { countableText } from 'flavours/glitch/features/compose/util/counter';
 import { tagHistory } from 'flavours/glitch/settings';
-import { fetchCustomEmojiData } from '@/flavours/glitch/features/emoji/picker';
+import { emojiMartSearch } from '@/flavours/glitch/features/emoji/picker';
 import { recoverHashtags } from 'flavours/glitch/utils/hashtag';
 
 import { showAlert, showAlertForError } from './alerts';
@@ -625,9 +625,9 @@ const fetchComposeSuggestionsAccounts = throttle((dispatch, token) => {
 }, 200, { leading: true, trailing: true });
 
 const fetchComposeSuggestionsEmojis = async (dispatch, token) => {
-  const custom = await fetchCustomEmojiData();
-  const { search } = await import('@/flavours/glitch/features/emoji/emoji_mart_search_light');
-  const results = search(token.replace(':', ''), { maxResults: 5, custom });
+  // Right now we are hard-coding the locale to English since the picker search only supports English.
+  // Once we replace the legacy picker we can remove this and use the actual locale of the user.
+  const results = await emojiMartSearch(token, 'en', 5);
   dispatch(readyComposeSuggestionsEmojis(token, results));
 };
 

app/javascript/flavours/glitch/components/autosuggest_emoji.tsx

@@ -5,7 +5,7 @@ import { useCustomEmojis } from '@/flavours/glitch/hooks/useCustomEmojis';
 import { Emoji } from './emoji';
 
 interface LegacyEmoji {
-  colons: string;
+  id: string;
   custom?: boolean;
   native?: string;
   imageUrl?: string;
@@ -13,10 +13,11 @@ interface LegacyEmoji {
 
 export const AutosuggestEmoji: FC<{ emoji: LegacyEmoji }> = ({ emoji }) => {
   const emojis = useCustomEmojis();
+  const colons = `:${emoji.id}:`;
   return (
     <div className='autosuggest-emoji'>
-      <Emoji code={emoji.native ?? emoji.colons} customEmoji={emojis} />
-      <div className='autosuggest-emoji__name'>{emoji.colons}</div>
+      <Emoji code={emoji.native ?? colons} customEmoji={emojis} />
+      <div className='autosuggest-emoji__name'>{colons}</div>
     </div>
   );
 };

app/javascript/flavours/glitch/features/emoji/database.ts

@@ -46,6 +46,8 @@ const loadDB = (() => {
   return loadPromise;
 })();
 
+type ScoreMap = Map<string, AnyEmojiData & { score: number }>;
+
 export async function search({
   query,
   locale: localeString,
@@ -75,7 +77,7 @@ export async function search({
 
   // Create an array of emoji results
   const db = await loadDB();
-  const resultArrays: Map<string, AnyEmojiData>[] = [];
+  const resultArrays: ScoreMap[] = [];
   for (let i = 0; i < queryTokens.length; i++) {
     const token = queryTokens[i];
     if (!token) continue;
@@ -90,14 +92,21 @@ export async function search({
       db.getAllFromIndex(locale, 'tokens', range),
       db.getAllFromIndex('custom', 'tokens', range),
     ]);
-    const resultMap = new Map<string, AnyEmojiData>([
-      ...unicodeResults.map(
-        (emoji) => [emoji.hexcode, emoji] as [string, AnyEmojiData],
-      ),
-      ...customResults.map(
-        (emoji) => [emoji.shortcode, emoji] as [string, AnyEmojiData],
-      ),
-    ]);
+    const resultMap: ScoreMap = new Map();
+    for (const emoji of unicodeResults) {
+      const score = getScoreForEmoji(emoji, token);
+      if (score === null) {
+        continue;
+      }
+      resultMap.set(emoji.hexcode, { ...emoji, score });
+    }
+    for (const emoji of customResults) {
+      const score = getScoreForEmoji(emoji, token);
+      if (score === null) {
+        continue;
+      }
+      resultMap.set(emoji.shortcode, { ...emoji, score });
+    }
     log('found %d results for token "%s"', resultMap.size, token);
     resultArrays.push(resultMap);
   }
@@ -106,7 +115,7 @@ export async function search({
   const results = Array.from(
     resultArrays
       .reduce((prev, curr) => {
-        const intersection = new Map<string, AnyEmojiData>();
+        const intersection: ScoreMap = new Map();
         for (const [code, emoji] of prev) {
           if (curr.has(code)) {
             intersection.set(code, emoji);
@@ -115,33 +124,7 @@ export async function search({
         return intersection;
       })
       .values(),
-  );
-
-  results.sort((a, b) => {
-    // Checks if a or b has the last token exactly, or only a prefix.
-    const aHasToken = a.tokens.includes(lastToken);
-    const bHasToken = b.tokens.includes(lastToken);
-    if (aHasToken && !bHasToken) {
-      return -1;
-    } else if (!aHasToken && bHasToken) {
-      return 1;
-    }
-
-    // If one is a custom emoji, prioritize it over Unicode emojis.
-    if ('category' in a) {
-      return -1;
-    } else if ('category' in b) {
-      return 1;
-    }
-
-    // If both are Unicode emojis, prioritize by order.
-    if ('order' in a && 'order' in b) {
-      return (a.order ?? 0) - (b.order ?? 0); // If these are both Unicode emojis, sort by order.
-    }
-
-    // ¯\_(ツ)_/¯
-    return 0;
-  });
+  ).toSorted((a, b) => a.score - b.score);
 
   const time = performance.measure('emoji-search-end', 'emoji-search-start');
   log(
@@ -157,6 +140,24 @@ export async function search({
   return results;
 }
 
+function getScoreForEmoji(emoji: AnyEmojiData, query: string) {
+  const id = 'shortcode' in emoji ? emoji.shortcode : emoji.label;
+  if (id === query) {
+    return 0;
+  }
+
+  let index = 1;
+  for (const token of [id, emoji.tokens]) {
+    const tokenIndex = token.indexOf(query);
+    if (tokenIndex !== -1) {
+      return index + tokenIndex / token.length;
+    }
+    index++;
+  }
+
+  return null;
+}
+
 export async function putEmojiData(emojis: CompactEmoji[], locale: Locale) {
   loadedLocales.add(locale);
   const db = await loadDB();
@@ -252,6 +253,12 @@ export async function loadEmojiByHexcode(
   return skinHexcodeToEmoji(hexcode, skinResult);
 }
 
+export async function loadAllUnicodeEmojis(localeString: string) {
+  const locale = await toLoadedLocale(localeString);
+  const db = await loadDB();
+  return db.getAll(locale);
+}
+
 export async function loadCustomEmojiByShortcode(shortcode: string) {
   const db = await loadDB();
   return db.get('custom', shortcode);
@@ -285,6 +292,11 @@ export async function loadLegacyShortcodesByShortcode(shortcode: string) {
   );
 }
 
+export async function loadAllShortcodes() {
+  const db = await loadDB();
+  return db.getAll('shortcodes');
+}
+
 // Private functions
 
 async function syncLocales(db: Database) {

app/javascript/flavours/glitch/features/emoji/emoji_compressed.d.mts

@@ -1,57 +0,0 @@
-import type { BaseEmoji, EmojiData, NimbleEmojiIndex } from 'emoji-mart';
-import type { Category, Data, Emoji } from 'emoji-mart/dist-es/utils/data';
-
-/*
- * The 'search' property, although not defined in the [`Emoji`]{@link node_modules/@types/emoji-mart/dist-es/utils/data.d.ts#Emoji} type,
- * is used in the application.
- * This could be due to an oversight by the library maintainer.
- * The `search` property is defined and used [here]{@link node_modules/emoji-mart/dist/utils/data.js#uncompress}.
- */
-export type Search = string;
-/*
- * The 'skins' property does not exist in the application data.
- * This could be a potential area of refactoring or error handling.
- * The non-existence of 'skins' property is evident at [this location]{@link app/javascript/flavours/glitch/features/emoji/emoji_compressed.js:121}.
- */
-type Skins = null;
-
-type Filename = string;
-type UnicodeFilename = string;
-export type FilenameData = [
-  filename: Filename,
-  unicodeFilename?: UnicodeFilename,
-][];
-export type ShortCodesToEmojiDataKey =
-  | EmojiData['id']
-  | BaseEmoji['native']
-  | keyof NimbleEmojiIndex['emojis'];
-
-type SearchData = [
-  BaseEmoji['native'],
-  Emoji['short_names'],
-  Search,
-  Emoji['unified'],
-];
-
-export type ShortCodesToEmojiData = Record<
-  ShortCodesToEmojiDataKey,
-  [FilenameData, SearchData]
->;
-type EmojisWithoutShortCodes = FilenameData;
-
-type EmojiCompressed = [
-  ShortCodesToEmojiData,
-  Skins,
-  Category[],
-  Data['aliases'],
-  EmojisWithoutShortCodes,
-];
-
-/*
- * `emoji_compressed.js` uses `babel-plugin-preval`, which makes it difficult to convert to TypeScript.
- * As a temporary solution, we are allowing a default export here to apply the TypeScript type `EmojiCompressed` to the JS file export.
- * - {@link app/javascript/flavours/glitch/features/emoji/emoji_compressed.js}
- */
-declare const emojiCompressed: EmojiCompressed;
-
-export default emojiCompressed; // eslint-disable-line import/no-default-export

app/javascript/flavours/glitch/features/emoji/emoji_compressed.mjs

@@ -1,138 +0,0 @@
-// @preval
-// http://www.unicode.org/Public/emoji/5.0/emoji-test.txt
-// This file contains the compressed version of the emoji data from
-// both emoji_map.json and from emoji-mart's emojiIndex and data objects.
-// It's designed to be emitted in an array format to take up less space
-// over the wire.
-
-// This version comment should be bumped each time the emoji data is changed
-// to ensure that the prevaled file is regenerated by Babel
-// version: 4
-
-import { NimbleEmojiIndex } from 'emoji-mart';
-import { uncompress as emojiMartUncompress } from 'emoji-mart/dist/utils/data';
-
-import data from './emoji_data.json';
-import emojiMap from './emoji_map.json';
-import { unicodeToFilename, unicodeToUnifiedName } from './unicode_utils';
-
-emojiMartUncompress(data);
-
-const emojiMartData = data;
-const emojiIndex = new NimbleEmojiIndex(emojiMartData);
-
-const excluded = ['®', '©', '™'];
-const skinTones = ['🏻', '🏼', '🏽', '🏾', '🏿'];
-const shortcodeMap = {};
-
-const shortCodesToEmojiData = {};
-const emojisWithoutShortCodes = [];
-
-Object.keys(emojiIndex.emojis).forEach((key) => {
-  let emoji = emojiIndex.emojis[key];
-
-  // Emojis with skin tone modifiers are stored like this
-  if (Object.hasOwn(emoji, '1')) {
-    emoji = emoji['1'];
-  }
-
-  shortcodeMap[emoji.native] = emoji.id;
-});
-
-const stripModifiers = (unicode) => {
-  skinTones.forEach((tone) => {
-    unicode = unicode.replace(tone, '');
-  });
-
-  return unicode;
-};
-
-Object.keys(emojiMap).forEach((key) => {
-  if (excluded.includes(key)) {
-    delete emojiMap[key];
-    return;
-  }
-
-  const normalizedKey = stripModifiers(key);
-  let shortcode = shortcodeMap[normalizedKey];
-
-  if (!shortcode) {
-    shortcode = shortcodeMap[normalizedKey + '\uFE0F'];
-  }
-
-  const filename = emojiMap[key];
-
-  const filenameData = [key];
-
-  if (unicodeToFilename(key) !== filename) {
-    // filename can't be derived using unicodeToFilename
-    filenameData.push(filename);
-  }
-
-  if (typeof shortcode === 'undefined') {
-    emojisWithoutShortCodes.push(filenameData);
-  } else {
-    if (!Array.isArray(shortCodesToEmojiData[shortcode])) {
-      shortCodesToEmojiData[shortcode] = [[]];
-    }
-
-    shortCodesToEmojiData[shortcode][0].push(filenameData);
-  }
-});
-
-Object.keys(emojiIndex.emojis).forEach((key) => {
-  let emoji = emojiIndex.emojis[key];
-
-  // Emojis with skin tone modifiers are stored like this
-  if (Object.hasOwn(emoji, '1')) {
-    emoji = emoji['1'];
-  }
-
-  const { native } = emoji;
-  let { short_names, search, unified } = emojiMartData.emojis[key];
-
-  if (short_names[0] !== key) {
-    throw new Error(
-      'The compressor expects the first short_code to be the ' +
-        'key. It may need to be rewritten if the emoji change such that this ' +
-        'is no longer the case.',
-    );
-  }
-
-  short_names = short_names.slice(1); // first short name can be inferred from the key
-
-  const searchData = [native, short_names, search];
-
-  if (unicodeToUnifiedName(native) !== unified) {
-    // unified name can't be derived from unicodeToUnifiedName
-    searchData.push(unified);
-  }
-
-  if (!Array.isArray(shortCodesToEmojiData[key])) {
-    shortCodesToEmojiData[key] = [[]];
-  }
-
-  shortCodesToEmojiData[key].push(searchData);
-});
-
-// JSON.parse/stringify is to emulate what @preval is doing and avoid any
-// inconsistent behavior in dev mode
-export default JSON.parse(
-  JSON.stringify([
-    shortCodesToEmojiData,
-    /*
-     * The property `skins` is not found in the current context.
-     * This could potentially lead to issues when interacting with modules or data structures
-     * that expect the presence of `skins` property.
-     * Currently, no definitions or references to `skins` property can be found in:
-     * - {@link node_modules/emoji-mart/dist/utils/data.js}
-     * - {@link node_modules/emoji-mart/data/all.json}
-     * - {@link app/javascript/flavours/glitch/features/emoji/emoji_compressed.d.ts#Skins}
-     * Future refactorings or updates should consider adding definitions or handling for `skins` property.
-     */
-    emojiMartData.skins,
-    emojiMartData.categories,
-    emojiMartData.aliases,
-    emojisWithoutShortCodes,
-  ]),
-);

app/javascript/flavours/glitch/features/emoji/emoji_map.json

@@ -1 +1 @@
-{"🩵":"1fa75","🦌":"1f98c","🦓":"1f993","🦄":"1f984","🐎":"1f40e","🫏":"1facf","🫎":"1face","🐴":"1f434","🐆":"1f406","🐅":"1f405","🐯":"1f42f","🦁":"1f981","🐈":"1f408","💜":"1f49c","🐱":"1f431","🦝":"1f99d","🦊":"1f98a","🐺":"1f43a","🐩":"1f429","🦮":"1f9ae","🐕":"1f415","🐶":"1f436","🦧":"1f9a7","🦍":"1f98d","🏄":"1f3c4","🆗":"1f197","🅾":"1f17e","🤷":"1f937","🫥":"1fae5","🆖":"1f196","🆕":"1f195","Ⓜ":"24c2","🆔":"1f194","ℹ":"2139","🆓":"1f193","🆒":"1f192","🆑":"1f191","🅱":"1f171","🆎":"1f18e","🅰":"1f170","🔤":"1f524","🔣":"1f523","🔢":"1f522","🔡":"1f521","🔠":"1f520","🔟":"1f51f","😃":"1f603","😏":"1f60f","™":"2122","®":"ae","©":"a9","❇":"2747","✴":"2734","✳":"2733","😒":"1f612","〽":"303d","➿":"27bf","➰":"27b0","❎":"274e","❌":"274c","✔":"2714","☑":"2611","🙄":"1f644","✅":"2705","⭕":"2b55","🔰":"1f530","📛":"1f4db","😬":"1f62c","🔱":"1f531","⚜":"269c","♻":"267b","😄":"1f604","⚕":"2695","💲":"1f4b2","💱":"1f4b1","〰":"3030","🤥":"1f925","❗":"2757","❕":"2755","❔":"2754","❓":"2753","🫨":"1fae8","⁉":"2049","‼":"203c","♾":"267e","🟰":"1f7f0","➗":"2797","➖":"2796","➕":"2795","✖":"2716","😁":"1f601","⚧":"26a7","♂":"2642","♀":"2640","📴":"1f4f4","📳":"1f4f3","🛜":"1f6dc","📶":"1f4f6","🔆":"1f506","🔅":"1f505","🎦":"1f3a6","⏏":"23cf","⏺":"23fa","⏹":"23f9","⏸":"23f8","😆":"1f606","⏬":"23ec","🔽":"1f53d","⏫":"23eb","🔼":"1f53c","⏮":"23ee","⏪":"23ea","◀":"25c0","⏯":"23ef","😌":"1f60c","⏭":"23ed","⏩":"23e9","▶":"25b6","😔":"1f614","🔂":"1f502","🔁":"1f501","🔀":"1f500","⛎":"26ce","😪":"1f62a","♓":"2653","♒":"2652","♑":"2651","♐":"2650","♏":"264f","🤤":"1f924","♎":"264e","♍":"264d","♌":"264c","♋":"264b","♊":"264a","😴":"1f634","♉":"2649","♈":"2648","🪯":"1faaf","🔯":"1f52f","🕎":"1f54e","😷":"1f637","☮":"262e","☪":"262a","☦":"2626","🤒":"1f912","✝":"271d","☯":"262f","🤕":"1f915","☸":"2638","✡":"2721","🕉":"1f549","🤢":"1f922","⚛":"269b","🛐":"1f6d0","🔝":"1f51d","🤮":"1f92e","🔜":"1f51c","🔛":"1f51b","🔚":"1f51a","🔙":"1f519","🔄":"1f504","🤧":"1f927","🔃":"1f503","⤵":"2935","⤴":"2934","🥵":"1f975","↪":"21aa","↩":"21a9","↔":"2194","🥶":"1f976","↕":"2195","↖":"2196","🥴":"1f974","⬅":"2b05","↙":"2199","⬇":"2b07","😵":"1f635","↘":"2198","➡":"27a1","😅":"1f605","↗":"2197","⬆":"2b06","☣":"2623","🤯":"1f92f","☢":"2622","🔞":"1f51e","📵":"1f4f5","🤠":"1f920","🚷":"1f6b7","🚱":"1f6b1","🚯":"1f6af","🚭":"1f6ad","🚳":"1f6b3","🥳":"1f973","🚫":"1f6ab","⛔":"26d4","🚸":"1f6b8","⚠":"26a0","🥸":"1f978","🛅":"1f6c5","🛄":"1f6c4","🛃":"1f6c3","🛂":"1f6c2","🚾":"1f6be","😎":"1f60e","🚼":"1f6bc","🚻":"1f6bb","🚺":"1f6ba","🚹":"1f6b9","♿":"267f","🤓":"1f913","🚰":"1f6b0","🚮":"1f6ae","🏧":"1f3e7","🪪":"1faaa","🪧":"1faa7","🧐":"1f9d0","🗿":"1f5ff","🪬":"1faac","🧿":"1f9ff","⚱":"26b1","😕":"1f615","🪦":"1faa6","⚰":"26b0","🚬":"1f6ac","🛒":"1f6d2","🫤":"1fae4","🧯":"1f9ef","🧽":"1f9fd","🪥":"1faa5","🫧":"1fae7","🧼":"1f9fc","😟":"1f61f","🪣":"1faa3","🧻":"1f9fb","🧺":"1f9fa","🧹":"1f9f9","🧷":"1f9f7","🙁":"1f641","🧴":"1f9f4","🪒":"1fa92","🪤":"1faa4","🛁":"1f6c1","🚿":"1f6bf","🪠":"1faa0","🚽":"1f6bd","🪑":"1fa91","🛋":"1f6cb","🛏":"1f6cf","🪟":"1fa9f","🪞":"1fa9e","🛗":"1f6d7","🚪":"1f6aa","🩻":"1fa7b","☹":"2639","🩺":"1fa7a","🩼":"1fa7c","🩹":"1fa79","💊":"1f48a","🩸":"1fa78","💉":"1f489","📡":"1f4e1","🔭":"1f52d","🔬":"1f52c","🧬":"1f9ec","🧫":"1f9eb","😮":"1f62e","🧪":"1f9ea","⚗":"2697","🪜":"1fa9c","🧲":"1f9f2","🧰":"1f9f0","🪝":"1fa9d","⛓":"26d3","🔗":"1f517","😯":"1f62f","🦯":"1f9af","⚖":"2696","🗜":"1f5dc","😲":"1f632","⚙":"2699","🔩":"1f529","🪛":"1fa9b","🔧":"1f527","😳":"1f633","🪚":"1fa9a","🛡":"1f6e1","🏹":"1f3f9","🪃":"1fa83","🥺":"1f97a","💣":"1f4a3","⚔":"2694","🗡":"1f5e1","🥹":"1f979","🛠":"1f6e0","⚒":"2692","⛏":"26cf","😦":"1f626","🪓":"1fa93","🔨":"1f528","🗝":"1f5dd","👮":"1f46e","🔑":"1f511","😧":"1f627","🔐":"1f510","🔏":"1f50f","🔓":"1f513","🔒":"1f512","🗑":"1f5d1","🗄":"1f5c4","🗃":"1f5c3","✂":"2702","😨":"1f628","📐":"1f4d0","📏":"1f4cf","🖇":"1f587","📎":"1f4ce","📍":"1f4cd","📌":"1f4cc","📋":"1f4cb","📊":"1f4ca","🕵":"1f575","📉":"1f4c9","📈":"1f4c8","📇":"1f4c7","😰":"1f630","🗓":"1f5d3","🗒":"1f5d2","📆":"1f4c6","📅":"1f4c5","🗂":"1f5c2","📂":"1f4c2","📁":"1f4c1","💼":"1f4bc","📝":"1f4dd","🖍":"1f58d","😥":"1f625","🖌":"1f58c","🖊":"1f58a","🖋":"1f58b","✒":"2712","✏":"270f","💂":"1f482","😢":"1f622","🗳":"1f5f3","📮":"1f4ee","Line truncated
+{"🐮":"1f42e","🦌":"1f98c","🦓":"1f993","🦄":"1f984","🐎":"1f40e","🫏":"1facf","🫎":"1face","🩵":"1fa75","🐴":"1f434","🐆":"1f406","🐅":"1f405","🐯":"1f42f","🦁":"1f981","🐈":"1f408","🐱":"1f431","🦝":"1f99d","🦊":"1f98a","🐺":"1f43a","🏄":"1f3c4","🆕":"1f195","Ⓜ":"24c2","🆔":"1f194","ℹ":"2139","🆓":"1f193","🆒":"1f192","🆑":"1f191","😃":"1f603","🅱":"1f171","🆎":"1f18e","🅰":"1f170","🔤":"1f524","🔣":"1f523","🔢":"1f522","🔡":"1f521","🔠":"1f520","🔟":"1f51f","😏":"1f60f","😒":"1f612","🫟":"1fadf","™":"2122","®":"ae","©":"a9","❇":"2747","✴":"2734","🙄":"1f644","✳":"2733","〽":"303d","➿":"27bf","➰":"27b0","❎":"274e","❌":"274c","✔":"2714","😬":"1f62c","☑":"2611","✅":"2705","⭕":"2b55","😄":"1f604","🔰":"1f530","📛":"1f4db","🔱":"1f531","⚜":"269c","🤥":"1f925","♻":"267b","⚕":"2695","💲":"1f4b2","🫨":"1fae8","💱":"1f4b1","〰":"3030","❗":"2757","❕":"2755","❔":"2754","❓":"2753","⁉":"2049","‼":"203c","😁":"1f601","♾":"267e","🟰":"1f7f0","➗":"2797","➖":"2796","➕":"2795","✖":"2716","⚧":"26a7","♂":"2642","♀":"2640","📴":"1f4f4","📳":"1f4f3","😆":"1f606","🛜":"1f6dc","📶":"1f4f6","🔆":"1f506","🔅":"1f505","🎦":"1f3a6","⏏":"23cf","⏺":"23fa","⏹":"23f9","😌":"1f60c","⏸":"23f8","⏬":"23ec","🔽":"1f53d","⏫":"23eb","🔼":"1f53c","⏮":"23ee","⏪":"23ea","◀":"25c0","😔":"1f614","⏯":"23ef","⏭":"23ed","😪":"1f62a","⏩":"23e9","▶":"25b6","🔂":"1f502","🔁":"1f501","🤤":"1f924","🔀":"1f500","⛎":"26ce","♓":"2653","♒":"2652","♑":"2651","😴":"1f634","♐":"2650","♏":"264f","♎":"264e","♍":"264d","♌":"264c","🫩":"1fae9","♋":"264b","♊":"264a","♉":"2649","♈":"2648","🪯":"1faaf","😷":"1f637","🔯":"1f52f","🕎":"1f54e","☮":"262e","☪":"262a","🤒":"1f912","☦":"2626","✝":"271d","🤕":"1f915","☯":"262f","☸":"2638","✡":"2721","🤢":"1f922","🕉":"1f549","⚛":"269b","🤮":"1f92e","🛐":"1f6d0","🔝":"1f51d","🔜":"1f51c","🔛":"1f51b","🔚":"1f51a","🤧":"1f927","🔙":"1f519","🔄":"1f504","🔃":"1f503","⤵":"2935","🥵":"1f975","⤴":"2934","↪":"21aa","↩":"21a9","🥶":"1f976","↔":"2194","↕":"2195","🥴":"1f974","↖":"2196","⬅":"2b05","↙":"2199","😵":"1f635","⬇":"2b07","↘":"2198","😅":"1f605","➡":"27a1","↗":"2197","⬆":"2b06","🤯":"1f92f","☣":"2623","☢":"2622","🤠":"1f920","🔞":"1f51e","📵":"1f4f5","🚷":"1f6b7","🚱":"1f6b1","🚯":"1f6af","🥳":"1f973","🚭":"1f6ad","🚳":"1f6b3","🚫":"1f6ab","⛔":"26d4","🚸":"1f6b8","🥸":"1f978","⚠":"26a0","🛅":"1f6c5","🛄":"1f6c4","🛃":"1f6c3","😎":"1f60e","🛂":"1f6c2","🚾":"1f6be","🚼":"1f6bc","🚻":"1f6bb","🚺":"1f6ba","🤓":"1f913","🚹":"1f6b9","♿":"267f","🚰":"1f6b0","🚮":"1f6ae","🏧":"1f3e7","🧐":"1f9d0","🪪":"1faaa","🪧":"1faa7","🗿":"1f5ff","🪬":"1faac","🧿":"1f9ff","😕":"1f615","⚱":"26b1","🪦":"1faa6","⚰":"26b0","😀":"1f600","🚬":"1f6ac","🛒":"1f6d2","🧯":"1f9ef","🧽":"1f9fd","🪥":"1faa5","🫤":"1fae4","🫧":"1fae7","🧼":"1f9fc","🪣":"1faa3","🧻":"1f9fb","🧺":"1f9fa","😟":"1f61f","🧹":"1f9f9","🧷":"1f9f7","🧴":"1f9f4","🪒":"1fa92","🪤":"1faa4","🛁":"1f6c1","🙁":"1f641","🚿":"1f6bf","🪠":"1faa0","🚽":"1f6bd","🪑":"1fa91","🛋":"1f6cb","🛏":"1f6cf","🪟":"1fa9f","🪞":"1fa9e","🛗":"1f6d7","🚪":"1f6aa","🩻":"1fa7b","🩺":"1fa7a","🩼":"1fa7c","🩹":"1fa79","💊":"1f48a","🩸":"1fa78","💉":"1f489","📡":"1f4e1","🔭":"1f52d","🔬":"1f52c","☹":"2639","🧬":"1f9ec","🧫":"1f9eb","🧪":"1f9ea","⚗":"2697","🪏":"1fa8f","🪜":"1fa9c","🧲":"1f9f2","🧰":"1f9f0","🪝":"1fa9d","😮":"1f62e","⛓":"26d3","🔗":"1f517","🦯":"1f9af","⚖":"2696","😯":"1f62f","🗜":"1f5dc","⚙":"2699","😲":"1f632","🔩":"1f529","🪛":"1fa9b","🔧":"1f527","🪚":"1fa9a","🛡":"1f6e1","😳":"1f633","🏹":"1f3f9","🪃":"1fa83","💣":"1f4a3","⚔":"2694","🥺":"1f97a","🗡":"1f5e1","🛠":"1f6e0","🥹":"1f979","⚒":"2692","⛏":"26cf","🪓":"1fa93","👮":"1f46e","🔨":"1f528","😦":"1f626","🗝":"1f5dd","🔑":"1f511","🔐":"1f510","🔏":"1f50f","🔓":"1f513","🔒":"1f512","🗑":"1f5d1","🗄":"1f5c4","😧":"1f627","🗃":"1f5c3","✂":"2702","📐":"1f4d0","📏":"1f4cf","🖇":"1f587","📎":"1f4ce","📍":"1f4cd","🕵":"1f575","📌":"1f4cc","📋":"1f4cb","📊":"1f4ca","😨":"1f628","📉":"1f4c9","📈":"1f4c8","📇":"1f4c7","🗓":"1f5d3","🗒":"1f5d2","📆":"1f4c6","📅":"1f4c5","🗂":"1f5c2","📂":"1f4c2","📁":"1f4c1","😰":"1f630","💼":"1f4bc","📝":"1f4dd","🖍":"1f58d","🖌":"1f58c","🖊":"1f58a","🖋":"1f58b","💂":"1f482","✒":"2712","😥":"1f625","✏":"270f","🗳":"1f5f3","📮":"1f4ee","📭":"1f4ed","📬":"1f4ec","📪":"1f4ea","📫":"1f4eb","📦":"1f4e6","😢":"1f622","📥":"1f4e5","📤":"1f4e4","Line truncated

app/javascript/flavours/glitch/features/emoji/emoji_mart_data_light.ts

@@ -1,50 +0,0 @@
-// The output of this module is designed to mimic emoji-mart's
-// "data" object, such that we can use it for a light version of emoji-mart's
-// emojiIndex.search functionality.
-import type { BaseEmoji } from 'emoji-mart';
-import type { Emoji } from 'emoji-mart/dist-es/utils/data';
-import emojiCompressed from 'virtual:mastodon-emoji-compressed';
-import type {
-  Search,
-  ShortCodesToEmojiData,
-} from 'virtual:mastodon-emoji-compressed';
-
-import { unicodeToUnifiedName } from './unicode_utils';
-
-type Emojis = Record<
-  NonNullable<keyof ShortCodesToEmojiData>,
-  {
-    native: BaseEmoji['native'];
-    search: Search;
-    short_names: Emoji['short_names'];
-    unified: Emoji['unified'];
-  }
->;
-
-const [
-  shortCodesToEmojiData,
-  _skins,
-  categories,
-  short_names,
-  _emojisWithoutShortCodes,
-] = emojiCompressed;
-
-const emojis: Emojis = {};
-
-// decompress
-Object.keys(shortCodesToEmojiData).forEach((shortCode) => {
-  const emojiData = shortCodesToEmojiData[shortCode];
-  if (!emojiData) return;
-
-  const [_filenameData, searchData] = emojiData;
-  const [native, short_names, search, unified] = searchData;
-
-  emojis[shortCode] = {
-    native,
-    search,
-    short_names: short_names ? [shortCode].concat(short_names) : undefined,
-    unified: unified ?? unicodeToUnifiedName(native),
-  };
-});
-
-export { emojis, categories, short_names };

app/javascript/flavours/glitch/features/emoji/emoji_mart_search_light.js

@@ -1,185 +0,0 @@
-// This code is largely borrowed from:
-// https://github.com/missive/emoji-mart/blob/5f2ffcc/src/utils/emoji-index.js
-
-import { emojis, categories } from './emoji_mart_data_light';
-import { getData, getSanitizedData, uniq, intersect } from './emoji_utils';
-
-let originalPool = {};
-let index = {};
-let emojisList = {};
-let emoticonsList = {};
-let customEmojisList = [];
-
-for (let emoji in emojis) {
-  let emojiData = emojis[emoji];
-  let { short_names, emoticons } = emojiData;
-  let id = short_names[0];
-
-  if (emoticons) {
-    emoticons.forEach(emoticon => {
-      if (emoticonsList[emoticon]) {
-        return;
-      }
-
-      emoticonsList[emoticon] = id;
-    });
-  }
-
-  emojisList[id] = getSanitizedData(id);
-  originalPool[id] = emojiData;
-}
-
-function clearCustomEmojis(pool) {
-  customEmojisList.forEach((emoji) => {
-    let emojiId = emoji.id || emoji.short_names[0];
-
-    delete pool[emojiId];
-    delete emojisList[emojiId];
-  });
-}
-
-function addCustomToPool(custom, pool) {
-  if (customEmojisList.length) clearCustomEmojis(pool);
-
-  custom.forEach((emoji) => {
-    let emojiId = emoji.id || emoji.short_names[0];
-
-    if (emojiId && !pool[emojiId]) {
-      pool[emojiId] = getData(emoji);
-      emojisList[emojiId] = getSanitizedData(emoji);
-    }
-  });
-
-  customEmojisList = custom;
-  index = {};
-}
-
-function search(value, { emojisToShowFilter, maxResults, include, exclude, custom } = {}) {
-  if (custom !== undefined) {
-    if (customEmojisList !== custom)
-      addCustomToPool(custom, originalPool);
-  } else {
-    custom = [];
-  }
-
-  maxResults = maxResults || 75;
-  include = include || [];
-  exclude = exclude || [];
-
-  let results = null,
-    pool = originalPool;
-
-  if (value.length) {
-    if (value === '-' || value === '-1') {
-      return [emojisList['-1']];
-    }
-
-    let values = value.toLowerCase().split(/[\s|,\-_]+/),
-      allResults = [];
-
-    if (values.length > 2) {
-      values = [values[0], values[1]];
-    }
-
-    if (include.length || exclude.length) {
-      pool = {};
-
-      categories.forEach(category => {
-        let isIncluded = include && include.length ? include.indexOf(category.name.toLowerCase()) > -1 : true;
-        let isExcluded = exclude && exclude.length ? exclude.indexOf(category.name.toLowerCase()) > -1 : false;
-        if (!isIncluded || isExcluded) {
-          return;
-        }
-
-        category.emojis.forEach(emojiId => pool[emojiId] = emojis[emojiId]);
-      });
-
-      if (custom.length) {
-        let customIsIncluded = include && include.length ? include.indexOf('custom') > -1 : true;
-        let customIsExcluded = exclude && exclude.length ? exclude.indexOf('custom') > -1 : false;
-        if (customIsIncluded && !customIsExcluded) {
-          addCustomToPool(custom, pool);
-        }
-      }
-    }
-
-    const searchValue = (value) => {
-      let aPool = pool,
-        aIndex = index,
-        length = 0;
-
-      for (let charIndex = 0; charIndex < value.length; charIndex++) {
-        const char = value[charIndex];
-        length++;
-
-        aIndex[char] = aIndex[char] || {};
-        aIndex = aIndex[char];
-
-        if (!aIndex.results) {
-          let scores = {};
-
-          aIndex.results = [];
-          aIndex.pool = {};
-
-          for (let id in aPool) {
-            let emoji = aPool[id],
-              { search } = emoji,
-              sub = value.slice(0, length),
-              subIndex = search.indexOf(sub);
-
-            if (subIndex !== -1) {
-              let score = subIndex + 1;
-              if (sub === id) score = 0;
-
-              aIndex.results.push(emojisList[id]);
-              aIndex.pool[id] = emoji;
-
-              scores[id] = score;
-            }
-          }
-
-          aIndex.results.sort((a, b) => {
-            let aScore = scores[a.id],
-              bScore = scores[b.id];
-
-            return aScore - bScore;
-          });
-        }
-
-        aPool = aIndex.pool;
-      }
-
-      return aIndex.results;
-    };
-
-    if (values.length > 1) {
-      results = searchValue(value);
-    } else {
-      results = [];
-    }
-
-    allResults = values.map(searchValue).filter(a => a);
-
-    if (allResults.length > 1) {
-      allResults = intersect.apply(null, allResults);
-    } else if (allResults.length) {
-      allResults = allResults[0];
-    }
-
-    results = uniq(results.concat(allResults));
-  }
-
-  if (results) {
-    if (emojisToShowFilter) {
-      results = results.filter((result) => emojisToShowFilter(emojis[result.id]));
-    }
-
-    if (results && results.length > maxResults) {
-      results = results.slice(0, maxResults);
-    }
-  }
-
-  return results;
-}
-
-export { search };

app/javascript/flavours/glitch/features/emoji/emoji_utils.js

@@ -1,217 +0,0 @@
-// This code is largely borrowed from:
-// https://github.com/missive/emoji-mart/blob/5f2ffcc/src/utils/index.js
-
-import * as data from './emoji_mart_data_light';
-
-const buildSearch = (data) => {
-  const search = [];
-
-  let addToSearch = (strings, split) => {
-    if (!strings) {
-      return;
-    }
-
-    (Array.isArray(strings) ? strings : [strings]).forEach((string) => {
-      (split ? string.split(/[-|_|\s]+/) : [string]).forEach((s) => {
-        s = s.toLowerCase();
-
-        if (search.indexOf(s) === -1) {
-          search.push(s);
-        }
-      });
-    });
-  };
-
-  addToSearch(data.short_names, true);
-  addToSearch(data.name, true);
-  addToSearch(data.keywords, false);
-  addToSearch(data.emoticons, false);
-
-  return search.join(',');
-};
-
-const _String = String;
-
-const stringFromCodePoint = _String.fromCodePoint || function () {
-  let MAX_SIZE = 0x4000;
-  let codeUnits = [];
-  let highSurrogate;
-  let lowSurrogate;
-  let index = -1;
-  let length = arguments.length;
-  if (!length) {
-    return '';
-  }
-  let result = '';
-  while (++index < length) {
-    let codePoint = Number(arguments[index]);
-    if (
-      !isFinite(codePoint) ||       // `NaN`, `+Infinity`, or `-Infinity`
-      codePoint < 0 ||              // not a valid Unicode code point
-      codePoint > 0x10FFFF ||       // not a valid Unicode code point
-      Math.floor(codePoint) !== codePoint // not an integer
-    ) {
-      throw RangeError('Invalid code point: ' + codePoint);
-    }
-    if (codePoint <= 0xFFFF) { // BMP code point
-      codeUnits.push(codePoint);
-    } else { // Astral code point; split in surrogate halves
-      // http://mathiasbynens.be/notes/javascript-encoding#surrogate-formulae
-      codePoint -= 0x10000;
-      highSurrogate = (codePoint >> 10) + 0xD800;
-      lowSurrogate = (codePoint % 0x400) + 0xDC00;
-      codeUnits.push(highSurrogate, lowSurrogate);
-    }
-    if (index + 1 === length || codeUnits.length > MAX_SIZE) {
-      result += String.fromCharCode.apply(null, codeUnits);
-      codeUnits.length = 0;
-    }
-  }
-  return result;
-};
-
-
-const _JSON = JSON;
-
-const COLONS_REGEX = /^(?::([^:]+):)(?::skin-tone-(\d):)?$/;
-const SKINS = [
-  '1F3FA', '1F3FB', '1F3FC',
-  '1F3FD', '1F3FE', '1F3FF',
-];
-
-function unifiedToNative(unified) {
-  let unicodes = unified.split('-'),
-    codePoints = unicodes.map((u) => `0x${u}`);
-
-  return stringFromCodePoint.apply(null, codePoints);
-}
-
-function sanitize(emoji) {
-  let { name, short_names, skin_tone, skin_variations, emoticons, unified, custom, imageUrl } = emoji,
-    id = emoji.id || short_names[0],
-    colons = `:${id}:`;
-
-  if (custom) {
-    return {
-      id,
-      name,
-      colons,
-      emoticons,
-      custom,
-      imageUrl,
-    };
-  }
-
-  if (skin_tone) {
-    colons += `:skin-tone-${skin_tone}:`;
-  }
-
-  return {
-    id,
-    name,
-    colons,
-    emoticons,
-    unified: unified.toLowerCase(),
-    skin: skin_tone || (skin_variations ? 1 : null),
-    native: unifiedToNative(unified),
-  };
-}
-
-function getSanitizedData() {
-  return sanitize(getData(...arguments));
-}
-
-function getData(emoji, skin, set) {
-  let emojiData = {};
-
-  if (typeof emoji === 'string') {
-    let matches = emoji.match(COLONS_REGEX);
-
-    if (matches) {
-      emoji = matches[1];
-
-      if (matches[2]) {
-        skin = parseInt(matches[2]);
-      }
-    }
-
-    if (Object.hasOwn(data.short_names, emoji)) {
-      emoji = data.short_names[emoji];
-    }
-
-    if (Object.hasOwn(data.emojis, emoji)) {
-      emojiData = data.emojis[emoji];
-    }
-  } else if (emoji.id) {
-    if (Object.hasOwn(data.short_names, emoji.id)) {
-      emoji.id = data.short_names[emoji.id];
-    }
-
-    if (Object.hasOwn(data.emojis, emoji.id)) {
-      emojiData = data.emojis[emoji.id];
-      skin = skin || emoji.skin;
-    }
-  }
-
-  if (!Object.keys(emojiData).length) {
-    emojiData = emoji;
-    emojiData.custom = true;
-
-    if (!emojiData.search) {
-      emojiData.search = buildSearch(emoji);
-    }
-  }
-
-  emojiData.emoticons = emojiData.emoticons || [];
-  emojiData.variations = emojiData.variations || [];
-
-  if (emojiData.skin_variations && skin > 1 && set) {
-    emojiData = JSON.parse(_JSON.stringify(emojiData));
-
-    let skinKey = SKINS[skin - 1],
-      variationData = emojiData.skin_variations[skinKey];
-
-    if (!variationData.variations && emojiData.variations) {
-      delete emojiData.variations;
-    }
-
-    if (variationData[`has_img_${set}`]) {
-      emojiData.skin_tone = skin;
-
-      for (let k in variationData) {
-        let v = variationData[k];
-        emojiData[k] = v;
-      }
-    }
-  }
-
-  if (emojiData.variations && emojiData.variations.length) {
-    emojiData = JSON.parse(_JSON.stringify(emojiData));
-    emojiData.unified = emojiData.variations.shift();
-  }
-
-  return emojiData;
-}
-
-function uniq(arr) {
-  return arr.reduce((acc, item) => {
-    if (acc.indexOf(item) === -1) {
-      acc.push(item);
-    }
-    return acc;
-  }, []);
-}
-
-function intersect(a, b) {
-  const uniqA = uniq(a);
-  const uniqB = uniq(b);
-
-  return uniqA.filter(item => uniqB.indexOf(item) >= 0);
-}
-
-export {
-  getData,
-  getSanitizedData,
-  uniq,
-  intersect,
-};

app/javascript/flavours/glitch/features/emoji/picker.ts

@@ -75,6 +75,36 @@ export async function fetchCustomEmojiData() {
   return customEmojis;
 }
 
+type LegacyEmoji =
+  | { id: string; custom?: false; native: string }
+  | {
+      id: string;
+      custom: true;
+    };
+
+// Replicates the old legacy search function.
+export async function emojiMartSearch(
+  token: string,
+  locale: string,
+  limit = 5,
+): Promise<LegacyEmoji[]> {
+  const query = token.replace(':', '').toLowerCase().trim();
+  if (!query.length) {
+    return [];
+  }
+
+  const { search } = await import('./database');
+  const results = await search({ query, locale, limit });
+  return results.map((emoji) =>
+    'shortcode' in emoji
+      ? { id: emoji.shortcode, custom: true }
+      : {
+          id: emoji.label.replaceAll(' ', '_').toLowerCase(),
+          native: emoji.unicode,
+        },
+  );
+}
+
 export function usePickerEmojis() {
   const [, setLoaded] = useState(customEmojis !== null);